SKSurrogate 0.1.0__py3-none-any.whl

SKSurrogate/aml.py

@@ -0,0 +1,1156 @@
+"""
+Optimized Pipeline Detector
+===============================
+
+"""
+from sklearn.base import BaseEstimator, TransformerMixin
+from typing import Any
+
+
+class StackingEstimator(BaseEstimator, TransformerMixin):
+    """
+    Meta-transformer for adding predictions and/or class probabilities as synthetic feature(s).
+
+    :param estimator: object with fit, predict, and predict_proba methods. The estimator to generate
+        synthetic features from.
+    :param res: True (default), stacks the final result of estimator
+    :param probs: True (default), stacks probabilities calculated by estimator
+    :param decision: True (default), stacks the result of decision function of the estimator
+    :param cv: cross-validation splitter used to create out-of-fold training features; default=5
+    :param n_jobs: number of parallel jobs used for out-of-fold predictions; default=None
+    """
+
+    def __init__(self, estimator, res=True, probs=True, decision=True, cv=5, n_jobs=None):
+
+        self.estimator = estimator
+        self.res = res
+        self.probs = probs
+        self.decision = decision
+        self.cv = cv
+        self.n_jobs = n_jobs
+
+    def set_params(self, **params):
+        """
+        Sets the sklearn related parameters for the estimator
+
+        :param params: parameters to be bassed to the estimator
+        :return: `self`
+        """
+        estimator_params = {}
+        for key, value in params.items():
+            if key.startswith("estimator__"):
+                estimator_params[key[len("estimator__"):]] = value
+            elif key in {"estimator", "res", "probs", "decision"}:
+                setattr(self, key, value)
+        if estimator_params:
+            self.estimator.set_params(**estimator_params)
+        return self
+
+    def fit(self, X, y=None, **fit_params):
+        """
+        Fit the StackingEstimator meta-transformer.
+
+        :param X: array-like of shape (n_samples, n_features). The training input samples.
+        :param y: array-like, shape (n_samples,). The target values (integers that correspond to classes
+            in classification, real numbers in regression).
+        :param fit_params: Other estimator-specific parameters.
+        :return: self, object. Returns a copy of the estimator
+        """
+        self.estimator.fit(X, y, **fit_params)
+        return self
+
+    def transform(self, X):
+        """
+        Transform data by adding two synthetic feature(s).
+
+        :param X: numpy ndarray, {n_samples, n_components}. New data, where n_samples is the number of
+            samples and n_components is the number of components.
+        :return: X_transformed: array-like, shape (n_samples, n_features + 1) or
+            (n_samples, n_features + 1 + n_classes) for classifier with predict_proba attribute;
+            The transformed feature set.
+        """
+        import numpy as np
+        from sklearn.utils import check_array
+
+        X = check_array(X)
+        features = [X]
+        # add class probabilities as a synthetic feature
+        if self.probs and hasattr(self.estimator, "predict_proba"):
+            features.append(self.estimator.predict_proba(X))
+
+        # add class decision_function as a synthetic feature
+        if self.decision and hasattr(self.estimator, "decision_function"):
+            features.append(np.reshape(self.estimator.decision_function(X), (-1, 1)))
+
+        # add class prediction as a synthetic feature
+        if self.res:
+            features.append(np.reshape(self.estimator.predict(X), (-1, 1)))
+
+        return np.hstack(features)
+
+    def fit_transform(self, X, y=None, **fit_params):
+        """Fit the estimator and return leakage-safe out-of-fold features.
+
+        During pipeline training, predictions are generated by cloned estimators
+        on folds that did not contain each sample. The wrapped estimator is then
+        fitted on all training data for use by :meth:`transform` on later data.
+        """
+        import numpy as np
+        from sklearn.base import clone
+        from sklearn.model_selection import cross_val_predict
+
+        if y is None or self.cv is None:
+            return self.fit(X, y, **fit_params).transform(X)
+
+        predictions = {}
+        if self.probs and hasattr(self.estimator, "predict_proba"):
+            predictions["probs"] = cross_val_predict(
+                clone(self.estimator),
+                X,
+                y,
+                cv=self.cv,
+                method="predict_proba",
+                n_jobs=self.n_jobs,
+            )
+        if self.decision and hasattr(self.estimator, "decision_function"):
+            predictions["decision"] = cross_val_predict(
+                clone(self.estimator),
+                X,
+                y,
+                cv=self.cv,
+                method="decision_function",
+                n_jobs=self.n_jobs,
+            )
+        if self.res:
+            predictions["res"] = cross_val_predict(
+                clone(self.estimator),
+                X,
+                y,
+                cv=self.cv,
+                method="predict",
+                n_jobs=self.n_jobs,
+            )
+
+        self.fit(X, y, **fit_params)
+        features = [np.asarray(X)]
+        if self.probs and "probs" in predictions:
+            features.append(predictions["probs"])
+        if self.decision and "decision" in predictions:
+            decision = np.asarray(predictions["decision"])
+            features.append(np.reshape(decision, (-1, 1)) if decision.ndim == 1 else decision)
+        if self.res:
+            result = np.asarray(predictions["res"])
+            features.append(np.reshape(result, (-1, 1)))
+        return np.hstack(features)
+
+
+class Words(object):
+    """
+    This class takes a set as alphabet and generates words of a given length accordingly.
+    A `Words` instant accepts the following parameters:
+
+    :param letters: is a set of letters (symbols) to make up the words
+    :param last: a subset of `letters` that are allowed to appear at the end of a word
+    :param first: a set of words that can only appear at the beginning of a word
+    :param repeat: whether consecutive occurrence of a letter is allowed
+    """
+
+    def __init__(self, letters, last=None, first=None, repeat=False):
+        self.letters = letters
+        self.last = last
+        self.first = first
+        self.words = []
+        self.repeat = repeat
+
+    def _check_cons_repeat(self, o):
+        lng = len(o)
+        for i in range(1, lng):
+            if self.repeat:
+                return True
+            if o[i - 1] == o[i]:
+                return False
+        return True
+
+    def _check_mid_first(self, o):
+        if (self.first is None) or (self.first == []):
+            return True
+        n_ = len(o)
+        for i in range(1, n_):
+            if o[i] in self.first:
+                return False
+        return True
+
+    def Generate(self, l):
+        """
+        Generates the set of legitimate words of length `l`
+
+        :param l: int, the length of words
+        :return: set of all legitimate words of length `l`
+        """
+        from itertools import product
+
+        words = []
+        for o in product(self.letters, repeat=l):
+            if self.last is not None:
+                if o[-1] in self.last:
+                    if self._check_cons_repeat(o) and self._check_mid_first(o):
+                        words.append(o)
+            else:
+                if self._check_cons_repeat(o) and self._check_mid_first(o):
+                    words.append(o)
+        return words
+
+
+try:
+    from .structsearch import Real, Integer, Categorical, HDReal
+except ModuleNotFoundError:
+    Real = lambda a, b: None
+    Integer = lambda a, b: None
+    Categorical = lambda a: None
+    HDReal = lambda a, b: None
+except ImportError:
+    Real = lambda a, b: None
+    Integer = lambda a, b: None
+    Categorical = lambda a: None
+    HDReal = lambda a, b: None
+
+default_config = {
+        # Classifiers
+        "sklearn.naive_bayes.BernoulliNB": {
+            "alpha": Real(10.0e-5, 100.0),
+            "fit_prior": Categorical([True, False]),
+        },
+        "sklearn.naive_bayes.GaussianNB": {"var_smoothing": Real(1.0e-9, 2.0e-1)},
+        "sklearn.tree.DecisionTreeClassifier": {
+            "criterion": Categorical(["gini", "entropy"]),
+            "splitter": Categorical(["best", "random"]),
+            "min_samples_split": Integer(2, 10),
+            "min_samples_leaf": Integer(1, 10),
+            "class_weight": HDReal((1.0e-5, 1.0e-5), (20.0, 20.0)),
+        },
+        "sklearn.linear_model.LogisticRegression": {
+            "penalty": Categorical(["l1", "l2"]),
+            "C": Real(1.0e-6, 10.0),
+            "class_weight": HDReal((1.0e-5, 1.0e-5), (20.0, 20.0))
+            # 'dual': Categorical([True, False])
+        },
+        "sklearn.svm.SVC": {
+            "C": Real(1e-6, 20.0),
+            "gamma": Real(1e-6, 10.0),
+            "tol": Real(1e-6, 10.0),
+            "class_weight": HDReal((1.0e-5, 1.0e-5), (20.0, 20.0)),
+        },
+        "sklearn.ensemble.RandomForestClassifier": {
+            "n_estimators": Integer(10, 200),
+            "criterion": Categorical(["gini", "entropy"]),
+            "min_samples_split": Integer(2, 10),
+            "min_samples_leaf": Integer(1, 10),
+            "class_weight": HDReal((1.0e-5, 1.0e-5), (20.0, 20.0)),
+        },
+        "sklearn.ensemble.GradientBoostingClassifier": {
+            "loss": Categorical(["log_loss", "exponential"]),
+            "learning_rate": Real(1.0e-6, 1.0 - 1.0e-6),
+            "n_estimators": Integer(10, 500),
+            "subsample": Real(1.0e-6, 1.0),
+            "criterion": Categorical(["friedman_mse", "squared_error"]),
+            # "min_samples_split": Integer(2, 20),
+            # "min_samples_leaf": Integer(1, 20),
+            # "min_weight_fraction_leaf": Real(0., .5),
+            # "max_depth": Integer(2, 20),
+            # "min_impurity_decrease": Real(0., 10.),
+            # "min_impurity_split": Real(0., 10.),
+            # "max_features": Categorical(['auto', 'sqrt', 'log2', None]),
+            "tol": Real(1.0e-6, 0.1),
+        },
+        "lightgbm.LGBMClassifier": {
+            "boosting_type": Categorical(["gbdt", "dart", "goss", "rf"]),
+            "num_leaves": Integer(2, 100),
+            "learning_rate": Real(1.0e-7, 1.0 - 1.0e-6),
+            "n_estimators": Integer(5, 250),
+            "min_split_gain": Real(0.0, 1.0),
+            # "min_child_weight": Real(1.e-6, 1.),
+            # "min_child_samples": Integer(5, 200),
+            "subsample": Real(1.0e-6, 1.0),
+            # "subsample_freq": Integer(0, 10),
+            # "colsample_bytree": Real(1.e-5, 1.),
+            # "class_weight": HDReal((1.e-5, 1.e-5), (20., 20.)),
+            # "reg_alpha": Real(0., 10.),
+            # "reg_lambda": Real(0., 10.),
+            "importance_type": Categorical(["split", "gain"]),
+        },
+        "xgboost.XGBClassifier": {
+            "n_estimators": Integer(50, 400),
+            "max_depth": Integer(2, 20),
+            "max_leaves": Integer(0, 200),
+            "learning_rate": Real(1.0e-5, 1.0),
+            "objective": Categorical(
+                ["binary:logistic", "binary:logitraw", "binary:hinge"]
+            ),
+            "booster": Categorical(["gbtree", "gblinear", "dart"]),
+            "gamma": Real(0.0, 100.0),
+            # 'min_child_weight': Real(0., 100.),
+            # 'max_delta_step': Real(0., 100.),
+            "subsample": Real(1.0e-6, 1.0),
+            # 'colsample_bytree': Real(1.e-6, 1.),
+            # 'colsample_bylevel': Real(1.e-6, 1.),
+            # 'reg_alpha': Real(0., 10.),
+            # 'reg_lambda': Real(0., 10.),
+            # 'scale_pos_weight': Real(1.e-5, 1.e3),
+            # 'base_score': Real(1.e-5, .9999)
+        },
+        # 'sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis': {
+        #    'tol': Real(1.e-5, 2.)
+        # },
+        # Feature Selectors
+        "sklearn.feature_selection.VarianceThreshold": {"threshold": Real(0.0, 0.3)},
+        "sklearn.decomposition.PCA": {
+            # 'svd_solver': Categorical(['randomized']),
+            "iterated_power": Integer(1, 11),
+            "n_components": Integer(2, 30),
+        },
+        "sklearn.decomposition.TruncatedSVD": {
+            "n_components": Integer(2, 30),
+            "algorithm": Categorical(["randomized", "arpack"]),
+        },
+        # Preprocesssors
+        "sklearn.preprocessing.StandardScaler": {
+            "with_mean": Categorical([True, False]),
+            "with_std": Categorical([True, False]),
+        },
+        "sklearn.preprocessing.Normalizer": {"norm": Categorical(["l1", "l2", "max"])},
+        # Transformers
+        # 'sklearn.preprocessing.PowerTransformer': {
+        #    'method': Categorical(['yeo-johnson', 'box-cox']),
+        #    'standardize': Categorical([True, False])
+        # },
+        "sklearn.kernel_approximation.Nystroem": {
+            "kernel": Categorical(["rbf", "poly", "sigmoid"]),
+            "gamma": Real(1.0e-2, 10.0),
+            "n_components": Integer(10, 100),
+        },
+        "sklearn.kernel_approximation.RBFSampler": {
+            "gamma": Real(1.0e-3, 10.0),
+            "n_components": Integer(10, 100),
+        },
+        "sklearn.kernel_approximation.AdditiveChi2Sampler": {"sample_steps": Integer(1, 3)},
+        "sklearn.kernel_approximation.SkewedChi2Sampler": {
+            "skewedness": Real(0.1, 5.0),
+            "n_components": Integer(10, 100),
+        },
+        # Sensitivity Analysis
+        "sksurrogate.SensAprx": {
+            "n_features_to_select": Integer(2, 30),
+            "method": Categorical(["sobol", "morris", "delta-mmnt"]),
+            # 'regressor': Categorical([None, regressor])
+        },
+        # Manifold Learning
+        # 'sklearn.manifold.Isomap': {
+        #    'n_neighbors': Integer(2, 12),
+        #    'n_components': Integer(1, 10),
+        # },
+        # 'sklearn.manifold.LocallyLinearEmbedding': {
+        #    'n_neighbors': Integer(2, 12),
+        #    'n_components': Integer(1, 10),
+        #    'reg ': Real(1.e-5, .2),
+        #    'method': Categorical(['standard', 'ltsa'])
+        # },
+        # 'sklearn.manifold.MDS': {
+        #    'n_components': Integer(1, 10),
+        #    'n_init': Integer(2, 8),
+        # },
+        # 'sklearn.manifold.SpectralEmbedding': {
+        #    'n_components': Integer(1, 10),
+        #    'affinity': Categorical(['nearest_neighbors', 'rbf', 'precomputed'])
+        # },
+        # 'sklearn.manifold.TSNE': {
+        #    'n_components': Integer(1, 10),
+        #    'perplexity': Real(5., 50.),
+        #    'early_exaggeration': Real(5., 25.),
+        #    'learning_rate ': Real(10., 500.)
+        # },
+        # Sampler
+        "imblearn.over_sampling.SMOTE": {"k_neighbors": Integer(3, 10)},
+    }
+
+
+
+class AML(object):
+    """
+    A class that accepts a nested dictionary with machine learning libraries as its keys
+    and a dictionary of their parameters and their ranges as value of each key and finds
+    an optimum combination based on training data.
+
+    :param config: A dictionary whose keys are scikit-learn-style objects (as strings) and its corresponding
+        values are dictionaries of the parameters and their acceptable ranges/values
+    :param length: default=5; Maximum number of objects in generated pipelines
+    :param scoring: default='accuracy'; The scoring method to be optimized. Must follow the sklearn scoring signature
+    :param cat_cols: default=None; The list of indices of categorical columns
+    :param surrogates: default=None; A list of 4-tuples determining surrogates. The first entity of each tuple is
+        a scikit-learn regressor and the
+        2nd entity is the number of iterations that this surrogate needs to be estimated and optimized.
+        The 3rd is the sampling strategy and the
+        4th is the `scipy.optimize` solver
+    :param min_random_evals: default=15; Number of randomly sampled initial values for hyper parameters
+    :param cv: default=3; The cross-validation method. Integer values use
+        stratified folds for classification and regular folds for regression.
+    :param check_point: default='./'; The path where the optimization results will be stored
+    :param stack_res: default=True; `StackingEstimator`s `res`
+    :param stack_probs: default=True; `StackingEstimator`s `probs`
+    :param stack_decision: default=True; `StackingEstimator`s `decision`
+    :param stack_cv: default=5; cross-validation splitter used to build the out-of-fold
+        features of the automatic `StackingEstimator` wrappers
+    :param stack_n_jobs: default=None; parallel jobs used for those out-of-fold predictions
+    :param verbose: default=1; Level of output details
+    :param n_jobs: int, default=-1; number of processes to run in parallel
+    :param cpu_limit: Optional maximum CPU parallelism to respect for each surrogate search.
+    :param memory_limit: Optional memory limit descriptor carried in the summary metadata.
+    :param random_state: Optional seed for the default CV and evolutionary search.
+    :param time_limit: Optional wall-clock limit in seconds for each surrogate search.
+    :param max_evals: Optional maximum objective evaluations for each surrogate search.
+    :param forbidden: Iterable of forbidden parameter rules forwarded to every
+        ``SurrogateRandomCV`` (dicts mapping param name -> disallowed value, or callables).
+    :param conditional: Mapping of param name -> rule describing when the parameter is active;
+        inactive parameters are dropped from each candidate before evaluation.
+    """
+
+    def __init__(
+            self,
+            config=None,
+            length=5,
+            scoring="accuracy",
+            cat_cols=None,
+            surrogates=None,
+            min_random_evals=15,
+            cv=None,
+            check_point="./",
+            stack_res=True,
+            stack_probs=True,
+            stack_decision=True,
+            stack_cv=5,
+            stack_n_jobs=None,
+            verbose=1,
+            n_jobs=-1,
+            random_state=None,
+            time_limit=None,
+            max_evals=None,
+            cpu_limit=None,
+            memory_limit=None,
+            forbidden=(),
+            conditional=(),
+    ):
+        from collections import OrderedDict
+
+        self.config: dict = {}
+        if config is None:
+            self.config = default_config
+        else:
+            self.config = config
+        self.config_types = {}
+        self.couldBlast = []
+        self.couldBfirst = []
+        self.feature_selector = set()
+        self.known_feature_selectors = {
+            "sklearn.decomposition.FactorAnalysis",
+            "sklearn.decomposition.FastICA",
+            "sklearn.decomposition.IncrementalPCA",
+            "sklearn.decomposition.KernelPCA",
+            "sklearn.decomposition.LatentDirichletAllocation",
+            "sklearn.decomposition.MiniBatchDictionaryLearning",
+            "sklearn.decomposition.MiniBatchSparsePCA",
+            "sklearn.decomposition.NMF",
+            "sklearn.decomposition.PCA",
+            "sklearn.decomposition.SparsePCA",
+            "sklearn.decomposition.TruncatedSVD",
+            "sklearn.feature_selection.VarianceThreshold",
+            "sklearn.manifold.LocallyLinearEmbedding",
+            "sklearn.manifold.Isomap",
+            "sklearn.manifold.MDS",
+            "sklearn.manifold.SpectralEmbedding",
+            "sklearn.manifold.TSNE",
+            "sksurrogate.SensAprx",
+            "skrebate.ReliefF",
+            "skrebate.SURF",
+            "skrebate.SURFstar",
+            "skrebate.MultiSURF",
+            "skrebate.MultiSURFstar",
+            "skrebate.TuRF",
+        }
+        self.letters = self.config_types.keys()
+        self.length = length
+        self.scoring = scoring
+        self.cat_cols = cat_cols
+        self.surrogates = surrogates
+        self.check_point = check_point
+        self.min_random_evals = min_random_evals
+        self.stack_res = stack_res
+        self.stack_probs = stack_probs
+        self.stack_decision = stack_decision
+        self.stack_cv = stack_cv
+        self.stack_n_jobs = stack_n_jobs
+        self.verbose = verbose
+        self.num_features = 2
+        self.cpu_limit = cpu_limit
+        self.memory_limit = memory_limit
+        self.forbidden = tuple(forbidden)
+        self.conditional = dict(conditional)
+        self.n_jobs = n_jobs
+        self.random_state = random_state
+        self.time_limit = time_limit
+        self.max_evals = max_evals
+        self._apply_resource_limits()
+        if cv is None:
+            self.cv = 3
+        elif isinstance(cv, dict):
+            # Accept a JSON spec as produced by ``mltrace.cv_to_spec`` /
+            # ``mltrace.GetCVSpec`` and rebuild the splitter. A raw dict has no
+            # ``split`` method, so scikit-learn's ``check_cv`` would treat it as
+            # an iterable of pre-computed (train, test) folds and fail with a
+            # confusing "too many values to unpack" error.
+            from .mltrace import build_cv
+
+            rebuilt = build_cv(cv)
+            self.cv = 3 if rebuilt is None else rebuilt
+        else:
+            self.cv = cv
+        self.types()
+        self.words = Words(self.letters, last=self.couldBlast, first=self.couldBfirst)
+        self.models = OrderedDict([])
+        self.evaluation_history_ = []
+        self.last_candidate_duration_ = None
+        self.best_estimator_ = None
+        self.best_estimator_score = 0.0
+        self.termination_reason = None
+        self.summary_ = {}
+
+    def _apply_resource_limits(self):
+        """Clamp the effective parallelism to the configured CPU budget."""
+        if self.cpu_limit is None:
+            return
+        limit = int(self.cpu_limit)
+        if limit < 1:
+            raise ValueError("cpu_limit must be a positive integer")
+        if self.n_jobs == -1 or self.n_jobs > limit:
+            self.n_jobs = limit
+
+    def types(self):
+        """
+        Recognizes the type of each estimator to determine proper placement of each
+
+        :return: None
+        """
+        from importlib import import_module
+        from sklearn.base import is_classifier, is_regressor
+        from sklearn.feature_selection import SelectorMixin
+
+        for alg in self.config:
+            if alg != "sklearn.pipeline.FeatureUnion":
+                detail = alg.split(".")
+                module_str = ".".join(detail[:-1])
+                module = import_module(module_str)
+                clss = module.__getattribute__(detail[-1])
+                mdl = self._new_estimator(clss)
+                if is_classifier(mdl):
+                    self.config_types[alg] = "classifier"
+                    self.couldBlast.append(alg)
+                elif is_regressor(mdl):
+                    self.config_types[alg] = "regressor"
+                    self.couldBlast.append(alg)
+                elif hasattr(mdl, "_estimator_type") and mdl._estimator_type == "sampler":
+                    self.config_types[alg] = "sampler"
+                    self.couldBfirst.append(alg)
+                else:
+                    self.config_types[alg] = "transformer"
+                if (
+                        issubclass(clss, SelectorMixin)
+                        or alg in self.known_feature_selectors
+                ):
+                    self.feature_selector.add(alg)
+            else:
+                self.config_types[alg] = "transformer"
+
+    def _validate_sequence(self, seq):
+        """
+        Validates the structure of the building sequence
+
+        :param seq: a sequence of (genes) estimators
+        :return: True or False
+        """
+        # Check the validity of the final estimator
+        if self.config_types[seq[-1]] not in ["regressor", "classifier"]:
+            return False
+        n = len(seq) - 1
+        # The second to last estimator can't be `FeatureUnion`
+        if seq[n - 1] == "sklearn.pipeline.FeatureUnion":
+            return False
+        flag = False
+        cnt = 0
+        for idx in range(n):
+            gene = seq[idx]
+            if gene == "sklearn.pipeline.FeatureUnion":
+                flag = True
+            elif (
+                    (gene in self.feature_selector)
+                    or (self.config_types[gene] in ["regressor", "classifier"])
+            ) and flag:
+                cnt += 1
+            elif flag and not (
+                    (gene in self.feature_selector)
+                    or (self.config_types[gene] in ["regressor", "classifier"])
+            ):
+                if cnt == 0:
+                    return False
+                else:
+                    flag = False
+                    cnt = 0
+        return True
+
+    def add_surrogate(self, estimator, itrs, sampling=None, optim="L-BFGS-B"):
+        """
+        Adding a regressor for surrogate optimization procedure.
+
+        :param estimator: A scikit-learn style regressor
+        :param itrs: Number of iterations the `estimator` needs to be fitted and optimized
+        :param sampling: default= BoxSample; The sampling strategy (`CompactSample`, `BoxSample` or `SphereSample`)
+        :param optim: default='L-BFGS-B';`scipy.optimize` solver
+        :return: None
+        """
+        if self.surrogates is None:
+            self.surrogates = []
+        if sampling is None:
+            from .structsearch import BoxSample
+
+            sampling = BoxSample
+        self.surrogates.append((estimator, itrs, sampling, optim))
+
+    def _cast(self, n, X, y):
+        """
+        Evaluates and optimizes all acceptable combinations of length `n`
+
+        :param n: The length of pipelines
+        :param X: Training data
+        :param y: Observed values
+        :return: None
+        """
+        from .structsearch import BoxSample, CompactSample
+
+        # if self.couldBfirst == []:
+        #    from sklearn.pipeline import Pipeline
+        # else:
+        #    from imblearn.pipeline import Pipeline
+        from sklearn.model_selection import RandomizedSearchCV
+
+        if self.surrogates is None:
+            from numpy import logspace
+            from sklearn.gaussian_process import GaussianProcessRegressor
+            from sklearn.kernel_ridge import KernelRidge
+            from sklearn.gaussian_process.kernels import (
+                Matern,
+                Sum,
+                ExpSineSquared,
+                WhiteKernel,
+            )
+
+            param_grid_gpr = {
+                "alpha": logspace(-8, 1, 20),
+                "kernel": [
+                    Sum(Matern(length_scale=l, nu=p), WhiteKernel(noise_level=q))
+                    for l in logspace(-3, 3, 20)
+                    for p in [0.5, 1.5, 2.5]
+                    for q in logspace(-3, 1.5, 20)
+                ],
+            }
+            GPR = RandomizedSearchCV(
+                GaussianProcessRegressor(),
+                param_distributions=param_grid_gpr,
+                n_iter=20,
+                cv=2,
+            )
+            param_grid_krr = {
+                "alpha": logspace(-2, 0, 10),
+                "kernel": [
+                    Sum(Matern(), ExpSineSquared(l, p))
+                    for l in logspace(-2, 2, 20)
+                    for p in logspace(0, 2, 20)
+                ],
+            }
+            KRR = RandomizedSearchCV(
+                KernelRidge(), param_distributions=param_grid_krr, n_iter=30, cv=2
+            )
+            self.surrogates = [
+                (KRR, 35, CompactSample, "L-BFGS-B"),
+                (GPR, 50, BoxSample, "L-BFGS-B"),
+            ]
+            self.min_random_evals = 10
+        surrogates = self.surrogates
+        if surrogates is None:
+            raise RuntimeError("No surrogate regressors are configured")
+        Pop = []
+        candidates = self.words.Generate(n)
+        for cnddt in candidates:
+            if self._validate_sequence(cnddt):
+                Pop.append(cnddt)
+        for seq in Pop:
+            if not self._validate_sequence(seq):
+                continue
+            best_mdl, best_scr = self.optimize_pipeline(seq, X, y)
+            self.models[seq] = (best_mdl, best_scr)
+            self.evaluation_history_.append(
+                {"pipeline": seq, "estimator": best_mdl, "score": best_scr}
+            )
+            if self.verbose > 0:
+                print("score:%f" % best_scr)
+                print(best_mdl)
+
+    def fit(self, X: Any, y: Any):
+        """
+        Generates and optimizes all legitimate pipelines. The best pipeline can be retrieved from `self.best_estimator_`
+
+        :param X: Training data
+        :param y: Corresponding observations
+        :return: `self`
+        """
+        import numpy as np
+
+        self.models.clear()
+        self.evaluation_history_.clear()
+        self.termination_reason = None
+        self.summary_ = {}
+        _X, _y = X, y
+        if self.cat_cols is not None:
+            from category_encoders.one_hot import OneHotEncoder
+
+            enc = OneHotEncoder(
+                cols=self.cat_cols, return_df=False, handle_unknown="ignore"
+            )
+            enc.fit(X)
+            _X = enc.transform(X)
+
+        X_, y_ = _X, _y
+        self.num_features = np.asarray(X_).shape[1]
+        for l in range(1, self.length + 1):
+            self._cast(l, X_, y_)
+        self.best_estimator_ = list(self.get_top(1).items())[0][1][0]
+        self.best_estimator_score = list(self.get_top(1).items())[0][1][1]
+        return self
+
+    @staticmethod
+    def _get_class(library):
+        """
+        Takes a string that refers to a class in an accessible module and returns the associated class
+
+        :param library: string referring to a class
+        :return: the actual class
+        """
+        from importlib import import_module
+
+        detail = library.split(".")
+        module_str = ".".join(detail[:-1])
+        module = import_module(module_str)
+        clss = module.__getattribute__(detail[-1])
+        return clss
+
+    def _new_estimator(self, estimator_class):
+        """Instantiate an estimator and seed it when it exposes ``random_state``."""
+        estimator = estimator_class()
+        if estimator_class.__module__.startswith("lightgbm"):
+            estimator.set_params(verbosity=-1)
+        if self.random_state is not None and hasattr(estimator, "get_params"):
+            params = estimator.get_params(deep=False)
+            if "random_state" in params:
+                estimator.set_params(random_state=self.random_state)
+        return estimator
+
+    def eoa_fit(self, X: Any, y: Any, **kwargs):
+        """
+        Applies evolutionary optimization methods to find an optimum pipeline
+
+        :param X: Training data
+        :param y: Corresponding observations
+        :param kwargs: `EOA` parameters
+        :return: `self`
+        """
+        import numpy as np
+        from .structsearch import BoxSample, CompactSample
+        from .eoa import EOA
+
+        self.models.clear()
+        self.evaluation_history_.clear()
+        self.termination_reason = None
+        self.summary_ = {}
+        _X, _y = X, y
+        if self.cat_cols is not None:
+            from category_encoders.one_hot import OneHotEncoder
+
+            enc = OneHotEncoder(
+                cols=self.cat_cols, return_df=False, handle_unknown="ignore"
+            )
+            enc.fit(X)
+            _X = enc.transform(X)
+        X_, y_ = _X, _y
+        self.num_features = np.asarray(X_).shape[1]
+        Pop = []
+        for l in range(1, self.length + 1):
+            candidates = self.words.Generate(l)
+            for cnddt in candidates:
+                if self._validate_sequence(cnddt):
+                    Pop.append(cnddt)
+
+        def _eval(ppl):
+            # if self.couldBfirst == []:
+            #    from sklearn.pipeline import Pipeline
+            # else:
+            #    from imblearn.pipeline import Pipeline
+            from sklearn.model_selection import RandomizedSearchCV
+
+            if self.surrogates is None:
+                from numpy import logspace
+                from sklearn.gaussian_process import GaussianProcessRegressor
+                from sklearn.kernel_ridge import KernelRidge
+                from sklearn.gaussian_process.kernels import (
+                    Matern,
+                    Sum,
+                    ExpSineSquared,
+                    WhiteKernel,
+                )
+
+                param_grid_gpr = {
+                    "alpha": logspace(-8, 1, 20),
+                    "kernel": [
+                        Sum(Matern(length_scale=l_, nu=p), WhiteKernel(noise_level=q))
+                        for l_ in logspace(-3, 3, 20)
+                        for p in [0.5, 1.5, 2.5]
+                        for q in logspace(-3, 1.5, 20)
+                    ],
+                }
+                GPR = RandomizedSearchCV(
+                    GaussianProcessRegressor(),
+                    param_distributions=param_grid_gpr,
+                    n_iter=20,
+                    cv=2,
+                )
+                param_grid_krr = {
+                    "alpha": logspace(-4, 0, 10),
+                    "kernel": [
+                        Sum(Matern(), ExpSineSquared(l_, p))
+                        for l_ in logspace(-2, 2, 20)
+                        for p in logspace(0, 2, 20)
+                    ],
+                }
+                KRR = RandomizedSearchCV(
+                    KernelRidge(), param_distributions=param_grid_krr, n_iter=30, cv=2
+                )
+                self.surrogates = [
+                    (KRR, 35, CompactSample, "L-BFGS-B"),
+                    (GPR, 50, BoxSample, "L-BFGS-B"),
+                ]
+                self.min_random_evals = 10
+            from collections import OrderedDict
+
+            fitted = OrderedDict([])
+            for seq in ppl:
+                best_mdl, best_scr = self.optimize_pipeline(seq, X_, y_)
+                if seq not in self.models:
+                    self.models[seq] = (best_mdl, best_scr)
+                    self.evaluation_history_.append(
+                        {
+                            "pipeline": seq,
+                            "estimator": best_mdl,
+                            "score": best_scr,
+                            "duration": getattr(self, "last_candidate_duration_", None),
+                        }
+                    )
+                if self.verbose > 0:
+                    print("score:%f" % best_scr)
+                    print(best_mdl)
+                fitted[seq] = -best_scr
+            return fitted
+
+        num_parents = kwargs.pop("num_parents", 30)
+        mutation_prob = kwargs.pop("mutation_prob", 0.1)
+        _eoa = EOA(
+            population=Pop,
+            fitness=_eval,
+            num_parents=num_parents,
+            mutation_prob=mutation_prob,
+            term_genes=self.couldBlast,
+            init_genes=self.couldBfirst,
+            check_point=self.check_point,
+            random_state=self.random_state,
+            **kwargs
+        )
+        _eoa()
+        self.best_estimator_ = list(self.get_top(1).items())[0][1][0]
+        return self
+
+    def get_top(self, num=5):
+        """
+        Finds the top `n` pipelines
+
+        :param num: Number of pipelines to be returned
+        :return: An OrderedDict of top models
+        """
+        from collections import OrderedDict
+
+        return OrderedDict(sorted(self.models.items(), key=lambda x: x[1][1])[:num])
+
+    def pareto_frontier(self, score_key="score", cost_key="duration"):
+        """Return the non-dominated candidates across ``evaluation_history_``.
+
+        Scores are maximized and costs (per-candidate wall-clock duration) are
+        minimized, mirroring :meth:`SurrogateRandomCV.pareto_frontier`. Entries
+        missing either metric are skipped. Stored scores are the negated search
+        objective, so they are flipped back to the scorer's higher-is-better
+        units before dominance testing; the returned records carry that flipped
+        value under ``score_key``.
+        """
+        from types import SimpleNamespace
+
+        from .structsearch import SurrogateRandomCV
+
+        records = []
+        for entry in self.evaluation_history_:
+            raw_score = entry.get(score_key)
+            cost = entry.get(cost_key)
+            if raw_score is None or cost is None:
+                continue
+            record = dict(entry)
+            record[score_key] = -float(raw_score)
+            # AML history entries carry no status; mark them complete so the
+            # structsearch dominance filter (status == "complete") keeps them.
+            record.setdefault("status", "complete")
+            records.append(record)
+        holder = SimpleNamespace(evaluation_history_=records)
+        return SurrogateRandomCV.pareto_frontier(holder, score_key=score_key, cost_key=cost_key)
+
+    def _require_fitted(self):
+        if self.best_estimator_ is None:
+            raise RuntimeError("AML has not been fitted")
+        return self.best_estimator_
+
+    def predict(self, X):
+        """Predict targets with the best pipeline found by ``fit`` or ``eoa_fit``."""
+        return self._require_fitted().predict(X)
+
+    def predict_proba(self, X):
+        """Predict class probabilities with the best classifier pipeline."""
+        estimator = self._require_fitted()
+        if not hasattr(estimator, "predict_proba"):
+            raise AttributeError("The selected estimator does not provide predict_proba")
+        return estimator.predict_proba(X)
+
+    def score(self, X, y):
+        """Return the best pipeline's estimator score on ``X`` and ``y``."""
+        return self._require_fitted().score(X, y)
+
+    def get_params(self, deep=True):
+        """Return constructor parameters using the scikit-learn convention."""
+        return {
+            "config": self.config,
+            "length": self.length,
+            "scoring": self.scoring,
+            "cat_cols": self.cat_cols,
+            "surrogates": self.surrogates,
+            "min_random_evals": self.min_random_evals,
+            "cv": self.cv,
+            "check_point": self.check_point,
+            "stack_res": self.stack_res,
+            "stack_probs": self.stack_probs,
+            "stack_decision": self.stack_decision,
+            "stack_cv": self.stack_cv,
+            "stack_n_jobs": self.stack_n_jobs,
+            "verbose": self.verbose,
+            "n_jobs": self.n_jobs,
+            "cpu_limit": self.cpu_limit,
+            "memory_limit": self.memory_limit,
+            "random_state": self.random_state,
+            "time_limit": self.time_limit,
+            "max_evals": self.max_evals,
+            "forbidden": self.forbidden,
+            "conditional": self.conditional,
+        }
+
+    def set_params(self, **params):
+        """Set constructor parameters and return this AML instance."""
+        valid_params = self.get_params()
+        invalid = sorted(set(params) - set(valid_params))
+        if invalid:
+            raise ValueError("Invalid AML parameters: %s" % ", ".join(invalid))
+        for key, value in params.items():
+            setattr(self, key, value)
+        self._apply_resource_limits()
+        self.best_estimator_ = None
+        self.evaluation_history_ = []
+        self.termination_reason = None
+        self.summary_ = {}
+        return self
+
+    def optimize_pipeline(self, seq, X, y):
+        """
+        Constructs and optimizes a pipeline according to the steps passed through `seq` which is a tuple of
+        estimators and transformers.
+
+        :param seq: the tuple of steps of the pipeline to be optimized
+        :param X: numpy array of training features
+        :param y: numpy array of training values
+        :return: the optimized pipeline and its score
+        """
+        from .structsearch import SurrogateRandomCV
+        surrogates = self.surrogates
+        if surrogates is None:
+            raise RuntimeError("No surrogate regressors are configured")
+
+        if self.couldBfirst == []:
+            from sklearn.pipeline import Pipeline
+        else:
+            from imblearn.pipeline import Pipeline
+        OPTIM = None
+        n = len(seq)
+        idx = 0
+        ent_idx = 0
+        steps = []
+        config = {}
+        task_name = self.check_point + "_".join(seq)
+        while ent_idx < n:
+            est = seq[ent_idx]
+            clss = self._get_class(est)
+            pre = "stp_%d" % idx
+            step_param_prefix = ""
+            if (
+                    self.config_types[est] in ["regressor", "classifier"]
+                    and ent_idx < n - 1
+            ):
+                mdl = self._new_estimator(clss)
+                steps.append(
+                    (
+                        pre,
+                        StackingEstimator(
+                            mdl,
+                            res=self.stack_res,
+                            probs=self.stack_probs,
+                            decision=self.stack_decision,
+                            cv=self.stack_cv,
+                            n_jobs=self.stack_n_jobs,
+                        ),
+                    )
+                )
+                step_param_prefix = "estimator__"
+                ent_idx += 1
+            elif est == "sklearn.pipeline.FeatureUnion":
+                self.config[est] = dict()
+                int_idx = 1
+                int_steps = []
+                next_est = seq[ent_idx + int_idx]
+                # Find the rest of the sequence before the last or the next feature selector
+                while (
+                        (self.config_types[next_est] in ["regressor", "classifier"])
+                        or (next_est in self.known_feature_selectors)
+                ) and (ent_idx + int_idx < n - 1):
+                    int_pre = "int_%d" % int_idx
+                    if next_est in self.known_feature_selectors:
+                        int_mdl = self._new_estimator(self._get_class(next_est))
+                        # set the parameter's dictionary
+                        for kw in self.config[next_est]:
+                            self.config[est][int_pre + "__" + kw] = self.config[
+                                next_est
+                            ][kw]
+                    else:
+                        from eli5.sklearn import PermutationImportance
+                        from sklearn.feature_selection import SelectFromModel
+                        from numpy import inf
+
+                        int_est = self._new_estimator(self._get_class(next_est))
+                        int_mdl = SelectFromModel(
+                            PermutationImportance(int_est, scoring=self.scoring, cv=3),  # type: ignore[arg-type]
+                            threshold=-inf,
+                        )
+                        self.config[est][int_pre + "__" + "max_features"] = Integer(
+                            1, self.num_features
+                        )
+                        for kw in self.config[next_est]:
+                            self.config[est][
+                                int_pre + "__" + "estimator__estimator__" + kw
+                                ] = self.config[next_est][kw]
+                    int_steps.append((int_pre, int_mdl))
+                    int_idx += 1
+                    next_est = seq[ent_idx + int_idx]
+                if int_steps != []:
+                    mdl = clss(int_steps)
+                    steps.append((pre, mdl))
+                ent_idx += int_idx
+            else:
+                mdl = self._new_estimator(clss)
+                steps.append((pre, mdl))
+                ent_idx += 1
+            idx += 1
+            for kw in self.config[est]:
+                config[pre + "__" + step_param_prefix + kw] = self.config[est][kw]
+        ppln = Pipeline(steps)
+        if self.verbose > 0:
+            print("=" * 90)
+            print(seq)
+            print("-" * 90)
+        from time import perf_counter
+
+        started = perf_counter()
+        if not config:
+            from sklearn.model_selection import cross_val_score
+
+            scores = cross_val_score(
+                ppln,
+                X,
+                y,
+                scoring=self.scoring,
+                cv=self.cv,
+                n_jobs=self.n_jobs,
+            )
+            ppln.fit(X, y)
+            score = float(scores.mean())
+            self.termination_reason = "completed"
+            self.summary_ = {
+                "termination_reason": "completed",
+                "status": "completed",
+                "budget": {
+                    "time_limit": self.time_limit,
+                    "max_evals": self.max_evals,
+                    "cpu_limit": self.cpu_limit,
+                    "memory_limit": self.memory_limit,
+                    "elapsed_seconds": 0.0,
+                    "iterations": 0,
+                    "evaluations": 0,
+                },
+                "current_value": None,
+            }
+            self.last_candidate_duration_ = perf_counter() - started
+            return ppln, score
+        for srgt in surrogates:
+            OPTIM = SurrogateRandomCV(
+                ppln,
+                params=config,
+                max_iter=srgt[1],
+                min_evals=self.min_random_evals,
+                scoring=self.scoring,
+                cv=self.cv,
+                verbose=max(self.verbose - 1, 0),
+                sampling=srgt[2],
+                regressor=srgt[0],
+                scipy_solver=srgt[3],
+                task_name=task_name,
+                Continue=True,
+                warm_start=True,
+                n_jobs=self.n_jobs,
+                time_limit=self.time_limit,
+                max_evals=self.max_evals,
+                random_state=self.random_state,
+                forbidden=self.forbidden,
+                conditional=self.conditional,
+            )
+            OPTIM.fit(X, y)
+            if getattr(OPTIM, "summary_", None):
+                self.termination_reason = getattr(
+                    OPTIM, "termination_reason", self.termination_reason
+                )
+                self.summary_ = dict(OPTIM.summary_)
+        if OPTIM is None:
+            raise RuntimeError("No surrogate search was executed")
+        self.last_candidate_duration_ = perf_counter() - started
+        return OPTIM.best_estimator_, OPTIM.best_estimator_score