asf 0.1.1__py3-none-any.whl

asf/cli/cli_train.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""CLI entry point for training selectors.
+
+This script provides a command-line interface for training model-based selectors.
+It allows users to specify the selector type, model, budget, and other parameters
+to train and save the selector model.
+"""
+
+import argparse
+from pathlib import Path
+from functools import partial
+from typing import Dict, Callable, List
+
+import pandas as pd
+
+from asf import selectors
+
+import sklearn
+
+# Mapping of file extensions to pandas read functions
+pandas_read_map: Dict[str, Callable] = {
+    ".csv": pd.read_csv,
+    ".parquet": pd.read_parquet,
+    ".json": pd.read_json,
+    ".feather": pd.read_feather,
+    ".hdf": pd.read_hdf,
+    ".html": pd.read_html,
+    ".xml": pd.read_xml,
+}
+
+
+def parser_function() -> argparse.ArgumentParser:
+    """Define command line arguments for the CLI.
+
+    Returns:
+        argparse.ArgumentParser: The argument parser with defined arguments.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--selector",
+        choices=selectors.__implemented__,
+        required=True,
+        help="Selector to train",
+    )
+    parser.add_argument(
+        "--model",
+        default="RandomForestClassifier",
+        help="Model to use for the selector. "
+        "Make sure to specify as an attribute of sklearn.ensemble.",
+    )
+    parser.add_argument(
+        "--budget",
+        type=int,
+        default=None,
+        required=False,
+        help="Budget for the solvers",
+    )
+    parser.add_argument(
+        "--maximize",
+        type=bool,
+        default=False,
+        required=False,
+        help="Maximize the objective",
+    )
+    parser.add_argument(
+        "--performance-metric",
+        type=str,
+        default="",
+        required=False,
+        help="Performance metric to optimize",
+    )
+    parser.add_argument(
+        "--feature-data",
+        type=Path,
+        required=True,
+        help="Path to feature data",
+    )
+    parser.add_argument(
+        "--performance-data",
+        type=Path,
+        required=True,
+        help="Path to performance data",
+    )
+    parser.add_argument(
+        "--model-path",
+        type=Path,
+        required=True,
+        help="Path to save model",
+    )
+    return parser
+
+
+def build_cli_command(
+    selector: selectors.AbstractModelBasedSelector,
+    feature_data: Path,
+    performance_data: Path,
+    destination: Path,
+) -> List[str]:
+    """Build a CLI command from variables for async jobs.
+
+    Args:
+        selector (selectors.AbstractModelBasedSelector): Selector to train.
+        feature_data (Path): Path to feature data DataFrame.
+        performance_data (Path): Path to performance data DataFrame.
+        destination (Path): Path to save the trained model.
+
+    Returns:
+        List[str]: A list of command-line arguments to execute the training job.
+    """
+    model_class = (
+        selector.model_class.args[0]
+        if isinstance(selector.model_class, partial)
+        else selector.model_class
+    )
+    return [
+        "python",
+        str(Path(__file__).absolute()),
+        "--selector",
+        type(selector).__name__,
+        "--model",
+        f"{model_class.__name__}",
+        "--budget",
+        str(selector.budget),
+        "--maximize",
+        str(selector.maximize),
+        "--performance-metric",
+        str(selector.performance_metric),
+        "--feature-data",
+        str(feature_data),
+        "--performance-data",
+        str(performance_data),
+        "--model-path",
+        str(destination),
+    ]
+
+
+if __name__ == "__main__":
+    parser = parser_function()
+    args = parser.parse_args()
+
+    # Parse selector into variable
+    selector_class = getattr(selectors, args.selector)
+    model_class = getattr(sklearn.ensemble, args.model)
+
+    # Parse training data into variables
+    features: pd.DataFrame = pandas_read_map[args.feature_data.suffix](
+        args.feature_data, index_col=0
+    )
+    performance_data: pd.DataFrame = pandas_read_map[args.performance_data.suffix](
+        args.performance_data, index_col=0
+    )
+
+    selector = selector_class(
+        model_class,
+        maximize=args.maximize,
+        budget=args.budget,
+    )
+    selector.fit(features, performance_data)
+
+    # Save the model to the specified path
+    selector.save(args.model_path)

asf/epm/__init__.py

@@ -0,0 +1,4 @@
+from asf.epm.epm import EPM
+from asf.epm.epm_tuner import tune_epm
+
+__all__ = ["EPM", "tune_epm"]

asf/epm/epm.py

@@ -0,0 +1,147 @@
+from functools import partial
+from typing import Type, Union, Optional
+
+import pandas as pd
+import numpy as np
+from sklearn.base import RegressorMixin
+
+from asf.preprocessing.performace_scaling import AbstractNormalization, LogNormalization
+from asf.predictors import SklearnWrapper
+from asf.preprocessing.sklearn_preprocessor import get_default_preprocessor
+from sklearn.base import TransformerMixin
+from asf.predictors.abstract_predictor import AbstractPredictor
+
+
+class EPM:
+    """
+    The EPM (Empirical Performance Model) class is a wrapper for machine learning models
+    that includes preprocessing, normalization, and optional inverse transformation of predictions.
+
+    Attributes:
+        predictor_class (Type[AbstractPredictor] | Type[RegressorMixin]): The class of the predictor to use.
+        normalization_class (Type[AbstractNormalization]): The normalization class to apply to the target variable.
+        transform_back (bool): Whether to apply inverse transformation to predictions.
+        features_preprocessing (Union[str, TransformerMixin]): Preprocessing pipeline for features.
+        predictor_config (Optional[dict]): Configuration for the predictor.
+        predictor_kwargs (Optional[dict]): Additional keyword arguments for the predictor.
+    """
+
+    def __init__(
+        self,
+        predictor_class: Union[Type[AbstractPredictor], Type[RegressorMixin]],
+        normalization_class: Type[AbstractNormalization] = LogNormalization,
+        transform_back: bool = True,
+        features_preprocessing: Union[str, TransformerMixin] = "default",
+        categorical_features: Optional[list] = None,
+        numerical_features: Optional[list] = None,
+        predictor_config: Optional[dict] = None,
+        predictor_kwargs: Optional[dict] = None,
+    ):
+        """
+        Initialize the EPM model.
+
+        Parameters:
+            predictor_class (Type[AbstractPredictor] | Type[RegressorMixin]): The class of the predictor to use.
+            normalization_class (Type[AbstractNormalization]): The normalization class to apply to the target variable.
+            transform_back (bool): Whether to apply inverse transformation to predictions.
+            features_preprocessing (Union[str, TransformerMixin]): Preprocessing pipeline for features.
+            categorical_features (Optional[list]): List of categorical feature names.
+            numerical_features (Optional[list]): List of numerical feature names.
+            predictor_config (Optional[dict]): Configuration for the predictor.
+            predictor_kwargs (Optional[dict]): Additional keyword arguments for the predictor.
+        """
+        if isinstance(predictor_class, type) and issubclass(
+            predictor_class, (RegressorMixin)
+        ):
+            self.model_class = partial(SklearnWrapper, predictor_class)
+        else:
+            self.model_class = predictor_class
+
+        self.predictor_class = predictor_class
+        self.normalization_class = normalization_class
+        self.transform_back = transform_back
+        self.predictor_config = predictor_config
+        self.predictor_kwargs = predictor_kwargs or {}
+        self.numpy = False
+
+        if features_preprocessing == "default":
+            self.features_preprocessing = get_default_preprocessor(
+                categorical_features=categorical_features,
+                numerical_features=numerical_features,
+            )
+        else:
+            self.features_preprocessing = features_preprocessing
+
+    def fit(
+        self,
+        X: Union[pd.DataFrame, pd.Series, list],
+        y: Union[pd.Series, list],
+        sample_weight: Optional[list] = None,
+    ) -> "EPM":
+        """
+        Fit the EPM model to the data.
+
+        Parameters:
+            X (Union[pd.DataFrame, pd.Series, list]): Features.
+            y (Union[pd.Series, list]): Target variable.
+            sample_weight (Optional[list]): Sample weights (optional).
+
+        Returns:
+            EPM: The fitted EPM model.
+        """
+        if isinstance(X, np.ndarray) and isinstance(y, np.ndarray):
+            X = pd.DataFrame(
+                X,
+                index=range(len(X)),
+                columns=[f"f_{i}" for i in range(X.shape[1])],
+            )
+            y = pd.Series(
+                y,
+                index=range(len(y)),
+            )
+            self.numpy = True
+
+        if self.features_preprocessing is not None:
+            X = self.features_preprocessing.fit_transform(X)
+
+        self.normalization = self.normalization_class()
+        self.normalization.fit(y)
+        y = self.normalization.transform(y)
+
+        if self.predictor_config is None:
+            self.predictor = self.predictor_class()
+        else:
+            self.predictor = self.predictor_class.get_from_configuration(
+                self.predictor_config, **self.predictor_kwargs
+            )()
+
+        self.predictor.fit(X, y, sample_weight=sample_weight)
+        return self
+
+    def predict(self, X: Union[pd.DataFrame, pd.Series, list]) -> list:
+        """
+        Predict using the fitted EPM model.
+
+        Parameters:
+            X (Union[pd.DataFrame, pd.Series, list]): Features.
+
+        Returns:
+            list: Predicted values.
+        """
+        if self.numpy:
+            if isinstance(X, np.ndarray):
+                X = pd.DataFrame(
+                    X,
+                    index=range(len(X)),
+                    columns=[f"f_{i}" for i in range(X.shape[1])],
+                )
+
+        if self.features_preprocessing is not None:
+            X = self.features_preprocessing.transform(X)
+
+        y_pred = self.predictor.predict(X)
+
+        if self.transform_back:
+            y_pred = self.normalization.inverse_transform(y_pred)
+
+        return y_pred

asf/epm/epm_tuner.py

@@ -0,0 +1,141 @@
+from typing import Type, Union, Optional
+
+import pandas as pd
+import numpy as np
+from sklearn.base import TransformerMixin
+from sklearn.metrics import mean_squared_error  # Fixed incorrect import
+from sklearn.model_selection import KFold
+from smac import HyperparameterOptimizationFacade, Scenario
+from asf.utils.groupkfoldshuffle import GroupKFoldShuffle
+
+from asf.epm.epm import EPM
+from asf.preprocessing.performace_scaling import AbstractNormalization, LogNormalization
+from asf.predictors.abstract_predictor import AbstractPredictor
+
+
+def tune_epm(
+    X: np.ndarray,
+    y: np.ndarray,
+    model_class: Type[AbstractPredictor],
+    normalization_class: Type[AbstractNormalization] = LogNormalization,
+    features_preprocessing: Union[str, TransformerMixin] = "default",
+    categorical_features: Optional[list] = None,
+    numerical_features: Optional[list] = None,
+    groups: Optional[np.ndarray] = None,
+    cv: int = 5,
+    timeout: int = 3600,
+    runcount_limit: int = 100,
+    output_dir: str = "./smac_output",
+    seed: int = 0,
+    smac_metric: callable = mean_squared_error,  # Fixed incorrect import
+    smac_scenario_kwargs: Optional[dict] = {},
+    smac_kwargs: Optional[dict] = {},
+    predictor_kwargs: Optional[dict] = {},
+) -> EPM:
+    """
+    Tune the Empirical Performance Model (EPM) using SMAC (Sequential Model-based Algorithm Configuration).
+
+    Parameters:
+    ----------
+    X : np.ndarray
+        Feature matrix for training and validation.
+    y : np.ndarray
+        Target values corresponding to the feature matrix.
+    model_class : Type[AbstractPredictor]
+        The predictor class to be tuned.
+    normalization_class : Type[AbstractNormalization], optional
+        The normalization class to be applied to the data. Defaults to LogNormalization.
+    features_preprocessing : Union[str, TransformerMixin], optional
+        Preprocessing method for features. Defaults to "default".
+    categorical_features : Optional[list], optional
+        List of categorical feature names. Defaults to None.
+    numerical_features : Optional[list], optional
+        List of numerical feature names. Defaults to None.
+    groups : Optional[np.ndarray], optional
+        Group labels for cross-validation. Defaults to None.
+    cv : int, optional
+        Number of cross-validation folds. Defaults to 5.
+    timeout : int, optional
+        Time limit for the tuning process in seconds. Defaults to 3600.
+    runcount_limit : int, optional
+        Maximum number of configurations to evaluate. Defaults to 100.
+    output_dir : str, optional
+        Directory to store SMAC output. Defaults to "./smac_output".
+    seed : int, optional
+        Random seed for reproducibility. Defaults to 0.
+    smac_metric : callable, optional
+        Metric function to evaluate model performance. Defaults to mean_squared_error.
+    smac_scenario_kwargs : Optional[dict], optional
+        Additional keyword arguments for the SMAC scenario. Defaults to None.
+    smac_kwargs : Optional[dict], optional
+        Additional keyword arguments for SMAC optimization. Defaults to None.
+    predictor_kwargs : Optional[dict], optional
+        Additional keyword arguments for the predictor. Defaults to None.
+
+    Returns:
+    -------
+    EPM
+        The tuned Empirical Performance Model instance.
+    """
+    if isinstance(X, np.ndarray) and isinstance(y, np.ndarray):
+        X = pd.DataFrame(
+            X,
+            index=range(len(X)),
+            columns=[f"f_{i}" for i in range(X.shape[1])],
+        )
+        y = pd.Series(
+            y,
+            index=range(len(y)),
+        )
+
+    scenario = Scenario(
+        configspace=model_class.get_configuration_space(),
+        n_trials=runcount_limit,
+        walltime_limit=timeout,
+        deterministic=True,
+        output_directory=output_dir,
+        seed=seed,
+        **smac_scenario_kwargs,
+    )
+
+    def target_function(config, seed):
+        if groups is not None:
+            kfold = GroupKFoldShuffle(n_splits=cv, shuffle=True, random_state=seed)
+        else:
+            kfold = KFold(n_splits=cv, shuffle=True, random_state=seed)
+
+        scores = []
+        for train_idx, test_idx in kfold.split(X, y, groups):
+            X_train, X_test = X.iloc[train_idx], X.iloc[test_idx]
+            y_train, y_test = y.iloc[train_idx], y.iloc[test_idx]
+
+            epm = EPM(
+                predictor_class=model_class,
+                normalization_class=normalization_class,
+                transform_back=True,
+                predictor_config=config,
+                predictor_kwargs=predictor_kwargs,
+                features_preprocessing=features_preprocessing,
+                categorical_features=categorical_features,
+                numerical_features=numerical_features,
+            )
+            epm.fit(X_train, y_train)
+
+            y_pred = epm.predict(X_test)
+            score = smac_metric(y_test, y_pred)
+            scores.append(score)
+
+        return np.mean(scores)
+
+    smac = HyperparameterOptimizationFacade(scenario, target_function, **smac_kwargs)
+    best_config = smac.optimize()
+
+    return EPM(
+        predictor_class=model_class,
+        normalization_class=normalization_class,
+        transform_back=True,
+        predictor_config=best_config,
+        features_preprocessing=features_preprocessing,
+        categorical_features=categorical_features,
+        numerical_features=numerical_features,
+    )

asf/metrics/__init__.py

@@ -0,0 +1,14 @@
+from asf.metrics.baselines import (
+    single_best_solver,
+    virtual_best_solver,
+    running_time_selector_performance,
+    running_time_closed_gap,
+)
+
+
+__all__ = [
+    "single_best_solver",
+    "virtual_best_solver",
+    "running_time_selector_performance",
+    "running_time_closed_gap",
+]

asf/metrics/baselines.py

@@ -0,0 +1,127 @@
+import pandas as pd
+from typing import Dict, List, Tuple, Optional, Union
+
+
+def single_best_solver(performance: pd.DataFrame, maximize: bool = False) -> float:
+    """
+    Selects the single best solver across all instances based on the aggregated performance.
+
+    Args:
+        schedules (pd.DataFrame): The schedules to evaluate (not used in this function).
+        performance (pd.DataFrame): The performance data for the algorithms.
+        maximize (bool): Whether to maximize or minimize the performance.
+
+    Returns:
+        float: The best aggregated performance value across all instances.
+    """
+    perf_sum = performance.sum(axis=0)
+    if maximize:
+        return perf_sum.max()
+    else:
+        return perf_sum.min()
+
+
+def virtual_best_solver(performance: pd.DataFrame, maximize: bool = False) -> float:
+    """
+    Selects the virtual best solver for each instance by choosing the best performance per instance.
+
+    Args:
+        schedules (pd.DataFrame): The schedules to evaluate (not used in this function).
+        performance (pd.DataFrame): The performance data for the algorithms.
+        maximize (bool): Whether to maximize or minimize the performance.
+
+    Returns:
+        float: The sum of the best performance values for each instance.
+    """
+    if maximize:
+        return performance.max(axis=1).sum()
+    else:
+        return performance.min(axis=1).sum()
+
+
+def running_time_selector_performance(
+    schedules: Dict[str, List[Tuple[str, float]]],
+    performance: pd.DataFrame,
+    budget: float = 5000,
+    par: float = 10,
+    feature_time: Optional[pd.DataFrame] = None,
+) -> Dict[str, Union[float, int]]:
+    """
+    Calculates the total running time for a selector based on the given schedules and performance data.
+
+    Args:
+        schedules (Dict[str, List[Tuple[str, float]]]): The schedules to evaluate, where each key is an instance
+            and the value is a list of tuples (algorithm, allocated budget).
+        performance (pd.DataFrame): The performance data for the algorithms.
+        budget (float): The budget for the scenario.
+        par (float): The penalization factor for unsolved instances.
+        feature_time (Optional[pd.DataFrame]): The feature time data for each instance. Defaults to zero if not provided.
+
+    Returns:
+        Dict[str, Union[float, int]]: A dictionary mapping each instance to its total running time.
+    """
+    if feature_time is None:
+        feature_time = pd.DataFrame(
+            0, index=performance.index, columns=["feature_time"]
+        )
+    total_time = {}
+    for instance, schedule in schedules.items():
+        allocated_times = {algorithm: 0 for algorithm in performance.columns}
+        solved = False
+        for algorithm, algo_budget in schedule:
+            remaining_budget = (
+                budget
+                - sum(allocated_times.values())
+                - feature_time.loc[instance].sum().item()
+            )
+            remaining_time_to_solve = performance.loc[instance, algorithm] - (
+                algo_budget + allocated_times[algorithm]
+            )
+            if remaining_time_to_solve < 0:
+                allocated_times[algorithm] = performance.loc[instance, algorithm]
+                solved = True
+                break
+            elif remaining_time_to_solve <= remaining_budget:
+                allocated_times[algorithm] += remaining_time_to_solve
+            else:
+                allocated_times[algorithm] += remaining_budget
+                break
+        if solved:
+            total_time[instance] = (
+                sum(allocated_times.values()) + feature_time.loc[instance].sum().item()
+            )
+        else:
+            total_time[instance] = budget * par
+
+    total_time = sum(list(total_time.values()))
+    return total_time
+
+
+def running_time_closed_gap(
+    schedules: Dict[str, List[Tuple[str, float]]],
+    performance: pd.DataFrame,
+    budget: float,
+    feature_time: pd.DataFrame,
+    par: float = 10,
+) -> float:
+    """
+    Calculates the closed gap metric for a given selector.
+
+    Args:
+        schedules (Dict[str, List[Tuple[str, float]]]): The schedules to evaluate.
+        performance (pd.DataFrame): The performance data for the algorithms.
+        budget (float): The budget for the scenario.
+        par (float): The penalization factor for unsolved instances.
+        feature_time (pd.DataFrame): The feature time data for each instance.
+
+    Returns:
+        float: The closed gap value, representing the improvement of the selector over the single best solver
+        relative to the virtual best solver.
+    """
+    sbs_val = single_best_solver(performance, False)
+    vbs_val = virtual_best_solver(performance, False)
+    s_val = running_time_selector_performance(
+        schedules, performance, budget, par, feature_time
+    )
+
+    return (sbs_val - s_val) / (sbs_val - vbs_val)

asf/pre_selector/__init__.py

@@ -0,0 +1,18 @@
+from asf.pre_selector.marginal_contribution_based import (
+    MarginalContributionBasedPreSelector,
+)
+from asf.pre_selector.optimize_pre_selection import OptimizePreSelection
+from asf.pre_selector.sbs_pre_selection import SBSPreSelector
+from asf.pre_selector.brute_force_pre_selection import BruteForcePreSelector
+from asf.pre_selector.beam_search_pre_selection import BeamSearchPreSelector
+from asf.pre_selector.knee_of_the_curve_pre_selector import KneeOfCurvePreSelector
+
+
+__all__ = [
+    "MarginalContributionBasedPreSelector",
+    "OptimizePreSelection",
+    "SBSPreSelector",
+    "BruteForcePreSelector",
+    "BeamSearchPreSelector",
+    "KneeOfCurvePreSelector",
+]

asf/pre_selector/abstract_pre_selector.py

@@ -0,0 +1,32 @@
+import pandas as pd
+import numpy as np
+from typing import Union, Optional
+
+
+class AbstractPreSelector:
+    """
+    Abstract class for pre-selectors.
+    """
+
+    def __init__(self, n_algorithms: Optional[int] = None):
+        """
+        Initialize the pre-selector with the given configuration.
+
+        Args:
+            config (dict): Configuration for the pre-selector.
+        """
+        self.n_algorithms = n_algorithms
+
+    def fit_transform(
+        self, performance: Union[pd.DataFrame, np.ndarray]
+    ) -> Union[pd.DataFrame, np.ndarray]:
+        """
+        Fit the pre-selector to the performance data and transform it.
+        Args:
+            performance (Union[pd.DataFrame, np.ndarray]): Performance data to fit and transform.
+        Returns:
+            Union[pd.DataFrame, np.ndarray]: Transformed performance data.
+        """
+        raise NotImplementedError(
+            "fit_transform method must be implemented in subclasses."
+        )