pybear-dask 0.2.0__py3-none-any.whl

pybear_dask/model_selection/GSTCV/_GSTCVDask/__init__.py

@@ -0,0 +1,5 @@
+# Author:
+#         Bill Sousa
+#
+# License: BSD 3 clause
+#

pybear_dask/model_selection/GSTCV/_GSTCVDask/_fit/__init__.py

@@ -0,0 +1,5 @@
+# Author:
+#         Bill Sousa
+#
+# License: BSD 3 clause
+#

pybear_dask/model_selection/GSTCV/_GSTCVDask/_fit/_estimator_fit_params_helper.py

@@ -0,0 +1,106 @@
+# Author:
+#         Bill Sousa
+#
+# License: BSD 3 clause
+#
+
+
+
+from typing_extensions import Any
+from .._type_aliases import DaskKFoldType
+
+from dask import compute
+
+from ._fold_splitter import _fold_splitter as _dask_fold_splitter
+from pybear.model_selection.GSTCV._GSTCV._fit._fold_splitter import \
+    _fold_splitter as _sk_fold_splitter
+
+
+
+def _estimator_fit_params_helper(
+    _data_len: int,
+    _fit_params: dict[str, Any],
+    _KFOLD: DaskKFoldType
+) -> dict[int, dict[str, Any]]:
+    """This module customizes the estimator's fit params for each pass
+    of cv, to be passed at fit time for the respective fold.
+
+    This is being done via a dictionary keyed by fold index, whose values
+    are dictionaries holding the respective fit params for that fold. In
+    particular, this is designed to perform splitting on any fit param
+    whose length matches the number of examples in the data, so that the
+    contents of that fit param are matched correctly to the train fold
+    of data concurrently being passed to fit. Other params that are not
+    split are simply replicated into each dictionary inside the helper.
+
+    Parameters
+    ----------
+    _data_len : int
+        The number of examples in the full data set.
+    _fit_params : dict[str, Any]
+        All the fit params passed to GSTCVDask fit for the estimator.
+    _KFOLD : DaskKFoldType
+        The KFold indices that were used to create the train / test
+        splits of data.
+
+    Returns
+    -------
+    _fit_params_helper : dict[int, dict[str, Any]]
+        A dictionary of customized fit params for each pass of cv.
+
+    """
+
+    # validation ** * ** * ** * ** * ** * ** * ** * ** * ** * ** * ** *
+    try:
+        float(_data_len)
+        if isinstance(_data_len, bool):
+            raise Exception
+        if not int(_data_len) == _data_len:
+            raise Exception
+        _data_len = int(_data_len)
+        if not _data_len > 0:
+            raise Exception
+    except:
+        raise TypeError(f"'data_len' must be an integer greater than 0")
+
+    assert isinstance(_fit_params, dict)
+    assert all(map(isinstance, list(_fit_params), (str for i in _fit_params)))
+
+    assert isinstance(_KFOLD, list), f"{type(_KFOLD)=}"
+    assert all(map(isinstance, _KFOLD, (tuple for _ in _KFOLD)))
+    # END validation ** * ** * ** * ** * ** * ** * ** * ** * ** * ** *
+
+
+    _fit_params_helper = {}
+
+    for f_idx, (train_idxs, test_idxs) in enumerate(_KFOLD):
+
+        _fit_params_helper[f_idx] = {}
+
+        for _fit_param_key, _fit_param_value in _fit_params.items():
+
+            try:
+                iter(_fit_param_value)
+                if isinstance(_fit_param_value, (dict, str)):
+                    raise Exception
+                if [*compute(len(_fit_param_value))][0] != _data_len:
+                    raise Exception
+                # remember we only care about the train fold.
+                # the fit_param may not be a dask object. try with
+                # dask _fold_splitter, if that fails, try sk _fold_splitter
+                try:
+                    _fit_params_helper[f_idx][_fit_param_key] = \
+                        _dask_fold_splitter(train_idxs, test_idxs, _fit_param_value)[0][0]
+                except:
+                    _fit_params_helper[f_idx][_fit_param_key] = \
+                        _sk_fold_splitter(train_idxs, test_idxs, _fit_param_value)[0][0]
+            except:
+                _fit_params_helper[f_idx][_fit_param_key] = _fit_param_value
+
+
+    return _fit_params_helper
+
+
+
+
+

pybear_dask/model_selection/GSTCV/_GSTCVDask/_fit/_fold_splitter.py

@@ -0,0 +1,86 @@
+# Author:
+#         Bill Sousa
+#
+# License: BSD 3 clause
+#
+
+
+
+from typing_extensions import Union
+
+from .._type_aliases import (
+    DaskSlicerType,
+    DaskSplitType,
+    DaskXType,
+    DaskYType
+)
+
+import dask.array as da
+import dask.dataframe as ddf
+
+
+
+def _fold_splitter(
+    train_idxs: DaskSlicerType,
+    test_idxs: DaskSlicerType,
+    *data_objects: Union[DaskXType, DaskYType]
+) -> tuple[DaskSplitType, ...]:
+    """Split given data objects into train / test pairs using the given
+    train and test indices.
+
+    The train and test indices independently slice the given data objects;
+    the entire data object need not be consumed in a train / test split
+    and the splits can also possibly share indices. Standard indexing
+    rules apply. Returns a tuple whose length is equal to the number of
+    data objects passed, holding tuples of the train / test splits for
+    the respective data objects. `train_idxs` and `test_idxs` must be 1D
+    vectors of indices, not booleans.
+
+    Parameters
+    ----------
+    train_idxs : DaskSlicerType
+        1D vector of row indices used to slice train sets out of every
+        given data object.
+    test_idxs : DaskSlicerType
+        1D vector of row indices used to slice test sets out of every
+        given data object.
+    *data_objects : Union[DaskXType, DaskYType]
+        The data objects to slice. Need not be of equal size, and need
+        not be completely consumed in the train / test splits. However,
+        standard indexing rules apply when slicing by `train_idxs` and
+        `test_idxs`.
+
+    Returns
+    -------
+    SPLITS : tuple[DaskSplitType, ...]
+        Return the train / test splits for the given data objects in the
+        order passed in a tuple of tuples, each inner tuple containing a
+        train/test pair.
+
+    """
+
+
+    SPLITS = []
+    for _data in data_objects:
+
+        # the compute()s need to be here for ddf slicing to work
+        if isinstance(_data, da.core.Array):
+            _data_train = _data[train_idxs]
+            _data_test = _data[test_idxs]
+        elif isinstance(_data, ddf.DataFrame):
+            _data_train = _data.loc[train_idxs.compute(), :]
+            _data_test = _data.loc[test_idxs.compute(), :]
+        elif isinstance(_data, ddf.Series):
+            _data_train = _data[train_idxs.compute()]
+            _data_test = _data[test_idxs.compute()]
+        else:
+             raise TypeError(f"disallowed container '{type(_data)}'")
+
+        SPLITS.append(tuple((_data_train, _data_test)))
+
+
+    return tuple(SPLITS)
+
+
+
+

pybear_dask/model_selection/GSTCV/_GSTCVDask/_fit/_get_kfold.py

@@ -0,0 +1,117 @@
+# Author:
+#         Bill Sousa
+#
+# License: BSD 3 clause
+#
+
+
+
+from typing import (
+    Iterator,
+    Optional
+)
+from .._type_aliases import (
+    DaskXType,
+    DaskYType,
+    DaskKFoldType
+)
+
+import time
+
+from dask_ml.model_selection import KFold as dask_KFold
+
+
+
+def _get_kfold(
+    _X: DaskXType,
+    _n_splits: int,
+    _iid: bool,
+    _verbose: int,
+    _y: Optional[DaskYType] = None
+) -> Iterator[DaskKFoldType]:
+    """Use dask_ml KFold to get train / test splits when cv is passed as
+    an integer.
+
+    KFold uses the number of rows in `_X` and `_n_splits` to determine
+    the indices in each train / test split. 'y' is optional in dask_ml
+    KFold. If passed, the number of rows in `_X` and `_y` must be equal.
+
+    *** IMPORTANT!!!
+    This function can be called multiple times within a single param grid
+    permutation, first to fit, again to get test score, then again if
+    `return_train_score`. Therefore, it must return the same indices for
+    each call. The only things that should cause indices to be different
+    are `_n_splits` and the number of rows in `_X`. Since this is dask
+    KFold, there is the wildcard of the `_iid` setting. If `_iid` is
+    False -- meaning the data is known to have some non-random grouping
+    along axis 0 -- via the 'shuffle' argument KFold will generate
+    indices that sample across chunks to randomize the data in the
+    splits. In that case, fix the 'random_state' parameter to make
+    selection repeatable. If `_iid` is True, 'shuffle' is False,
+    'random_state' can be None, and the splits should be repeatable.
+
+    Parameters
+    ----------
+    _X : DaskXType
+        The data to be split.
+    _n_splits : int
+        The number of splits to produce; the number of split pairs
+        yielded by the returned generator object.
+    _iid : bool
+        True, the examples in X are distributed randomly; False,
+        there is some kind of non-random ordering of the examples in X.
+    _verbose : int
+        A number from 0 to 10 indicating the amount of information to
+        display to screen during the grid search trials. 0 means no
+        output, 10 means full output.
+    _y : Optional[DaskYType]
+        The target the data is being fit against, to be split in the
+        same way as the data.
+
+    Returns
+    -------
+    KFOLD : Iterator[DaskKFoldType]
+        A generator object yielding pairs of train test indices as
+        da.core.Array[int].
+
+    """
+
+    # validation ** * ** * ** * ** * ** * ** * ** * ** * ** * ** * ** * ** *
+    # 25_04_29 NOT VALIDATING X & y HERE ANYMORE. LET KFold RAISE.
+    assert isinstance(_n_splits, int)
+    assert _n_splits > 1
+    assert isinstance(_iid, bool)
+    try:
+        float(_verbose)
+    except:
+        raise AssertionError(f"'_verbose' must be numeric")
+    assert _verbose >= 0
+    # END validation ** * ** * ** * ** * ** * ** * ** * ** * ** * ** * ** *
+
+
+    split_t0 = time.perf_counter()
+    # KFold keeps the same chunks ax X
+    # as of 25_04_29 dask_KFold only accepts da array for X
+    KFOLD = dask_KFold(
+        n_splits=_n_splits,
+        shuffle=not _iid,
+        random_state=7 if not _iid else None,
+        # shuffle is on if not iid. must use random_state so that later
+        # calls for train score get same splits.
+    ).split(_X, _y)
+
+
+    if _verbose >= 5:
+        print(f'split time = {time.perf_counter() - split_t0: ,.3g} s')
+
+    del split_t0
+
+    return KFOLD
+
+
+
+
+
+
+
+

pybear_dask/model_selection/GSTCV/_GSTCVDask/_fit/_parallelized_fit.py

@@ -0,0 +1,127 @@
+# Author:
+#         Bill Sousa
+#
+# License: BSD 3 clause
+#
+
+
+
+from typing import Literal
+from typing_extensions import (
+    Any,
+    Union
+)
+from .._type_aliases import (
+    DaskXType,
+    DaskYType
+)
+
+import numbers
+import time
+import warnings
+
+from pybear.model_selection.GSTCV._type_aliases import ClassifierProtocol
+
+
+
+def _parallelized_fit(
+    _f_idx: int,
+    _X_train: DaskXType,
+    _y_train: DaskYType,
+    _estimator: ClassifierProtocol,
+    _grid: dict[str, Any],
+    _error_score: Union[numbers.Real, Literal['raise']],
+    **_fit_params
+) -> tuple[ClassifierProtocol, float, bool]:
+    """Estimator fit method designed for dask parallelism.
+
+    Special exception handling on fit.
+
+    Parameters
+    ----------
+    _f_idx : int
+        The zero-based fold index of the train partition used in this
+        fit; parallelism occurs over the different folds.
+    _X_train : DaskXType
+        A train partition of the data being fit.
+    _y_train : DaskYType
+        The corresponding train partition  of the target for the X train
+        partition.
+    _estimator : ClassifierProtocol
+        Any scikit-style classifier, having `fit`, `predict_proba`,
+        `get_params`, and `set_params` methods (the `score` method is
+        not necessary, as GSTCVDask never calls it.) This includes,
+        but is not limited to, dask_ml, XGBoost dask, and LGBM dask
+        classifiers.
+    _grid : dict[str, Any]
+        The hyperparameter values to be used during this fit. One
+        permutation of all the grid search permutations.
+    _error_score : Union[numbers.Real, Literal['raise']]
+        If a training fold excepts during fitting, the exception can be
+        allowed to raise by passing the 'raise' literal. Otherwise,
+        passing a number-like will cause the exception to be handled,
+        allowing the grid search to proceed, and the given number carries
+        through scoring tabulations in place of the missing score(s).
+    **_fit_params : dict[str, Any]
+        Dictionary of fit_param:value pairs to be passed to the
+        estimator's `fit` method. `fit_params` must have been processed
+        by :func:`_estimator_fit_params_helper` so that any `fit_param`
+        that has length == (n_samples in X and y) is split in the same
+        way as X and y.
+
+    Returns
+    -------
+    __ : tuple[_estimator, _fit_time, _fit_excepted]
+        _estimator : EstimatorProtocol
+            The fit estimator
+        _fit_time : float
+            The seconds elapsed when performing the fit
+        _fit_excepted : bool
+            True if the fit excepted and '_error_score' was not 'raise';
+            False if the fit ran successfully.
+
+    """
+
+
+    # validation ** * ** * ** * ** * ** * ** * ** * ** * ** * ** * ** *
+
+    assert isinstance(_f_idx, int)
+    assert isinstance(_grid, dict)
+    assert isinstance(_error_score, (str, numbers.Real))
+
+    # END validation ** * ** * ** * ** * ** * ** * ** * ** * ** * ** * *
+
+
+    _fit_excepted = False
+
+    _X_train = _X_train.persist()
+    _y_train = _y_train.persist()
+
+    t0_fit = time.perf_counter()
+
+    try:
+        _estimator.fit(_X_train, _y_train, **_fit_params)
+    except BrokenPipeError:
+        raise BrokenPipeError  # FOR PYTEST ONLY
+    except Exception as f:
+        if _error_score == 'raise':
+            raise ValueError(
+                f"estimator excepted during fitting on {_grid}, cv fold "
+                f"index {_f_idx} --- {f}"
+            )
+        else:
+            _fit_excepted = True
+            warnings.warn(
+                f'\033[93mfit excepted on {_grid}, cv fold index {_f_idx}\033[0m'
+            )
+
+    _fit_time = time.perf_counter() - t0_fit
+
+    del t0_fit
+
+    return _estimator, _fit_time, _fit_excepted
+
+
+
+
+

pybear_dask/model_selection/GSTCV/_GSTCVDask/_fit/_parallelized_scorer.py

@@ -0,0 +1,228 @@
+# Author:
+#         Bill Sousa
+#
+# License: BSD 3 clause
+#
+
+
+
+from typing_extensions import Union
+from pybear.model_selection.GSTCV._type_aliases import (
+    ScorerWIPType,
+    ClassifierProtocol,
+    ThresholdsWIPType,
+    MaskedHolderType
+)
+from .._type_aliases import (
+    DaskXType,
+    DaskYType
+)
+
+import numbers
+import time
+
+import numpy as np
+
+from pybear.model_selection.GSTCV._GSTCVMixin._validation._predict_proba \
+    import _val_predict_proba
+
+
+
+def _parallelized_scorer(
+    _X_test: DaskXType,
+    _y_test: DaskYType,
+    _FIT_OUTPUT_TUPLE: tuple[ClassifierProtocol, float, bool],
+    _f_idx: int,
+    _SCORER_DICT: ScorerWIPType,
+    _THRESHOLDS: ThresholdsWIPType,
+    _error_score: Union[numbers.Real, None],
+    _verbose: int
+) -> tuple[MaskedHolderType, MaskedHolderType]:
+
+    # dont adjust the spacing, is congruent with train scorer
+
+    """Using the estimators fit on each train fold, use `predict_proba`
+    and _X_tests to generate _y_preds and score against the corresponding
+    _y_tests using all of the scorers and thresholds.
+
+    Builds one fold layer of the TEST_FOLD_x_THRESHOLD_x_SCORER__SCORE
+    and TEST_FOLD_x_THRESHOLD_x_SCORER__SCORE_TIME cubes.
+
+    Parameters
+    ----------
+    _X_test : DaskXType
+        A test partition of the data, matched up with the estimator that
+        was trained on the complementary train set.
+    _y_test : DaskYType
+        The corresponding test partition of the target for the `_X_test`
+        partition.
+    _FIT_OUTPUT_TUPLE : tuple[ClassifierProtocol, float, bool]
+        A tuple holding the fitted estimator, the fit time (not needed
+        here), and the fit_excepted boolean (needed here.)
+    _f_idx : int
+        The zero-based split index of the test partition used here;
+        parallelism occurs over the different splits.
+    _SCORER_DICT : ScorerWIPType
+        A dictionary with scorer name as keys and the scorer callables
+        as values. The scorer callables are scoring metrics (or similar),
+        not make_scorer.
+    _THRESHOLDS : ThresholdsWIPType
+        For the current search permutation, there was a mother param
+        grid that contained a 'thresholds' parameter, that was separated
+        from the mother before building cv_results. This is the vector
+        of thresholds from the mother that also mothered this search
+        permutation.
+    _error_score : Union[numbers.Real, Literal['raise']]
+        If the training fold complementing this test fold excepted during
+        fitting and `error_score` was set to the 'raise' literal, this
+        module cannot be reached. Otherwise, a number or number-like was
+        passed to `error_score`. If 'fit_excepted' is True, this module
+        puts the `error_score` value in every position of the
+        TEST_THRESHOLD_x_SCORER__SCORE_LAYER array. If `error_score` is
+        set to np.nan, that layer is also masked. Every value in
+        TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER is set to np.nan and
+        masked.
+    _verbose : int
+        A number from 0 to 10 that indicates the amount of information
+        to display to the screen during the grid search process. 0 means
+        no output, 10 means maximum output.
+
+    Returns
+    -------
+    __ : tuple[np.ma.MaskedArray, np.ma.MaskedArray]
+        TEST_THRESHOLD_x_SCORER__SCORE_LAYER : MaskedHolderType
+            Masked array of shape (n_thresholds, n_scorers) holding the
+            scores for each scorer on each threshold for one fold of
+            test data.
+
+        TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER : MaskedHolderType
+            Masked array of shape (n_thresholds, n_scorers) holding the
+            times to score each scorer on each threshold for one fold of
+            test data.
+
+    """
+
+    # validation ** * ** * ** * ** * ** * ** * ** * ** * ** * ** * ** *
+    assert isinstance(_FIT_OUTPUT_TUPLE, tuple)
+    assert isinstance(_f_idx, int)
+    assert isinstance(_SCORER_DICT, dict)
+    assert all(map(callable, _SCORER_DICT.values()))
+    assert isinstance(_THRESHOLDS, list)
+    assert isinstance(_verbose, (int, float))
+    # END validation ** * ** * ** * ** * ** * ** * ** * ** * ** * ** * *
+
+    if _verbose >= 5:
+        print(f"Start scoring fold {_f_idx + 1} test with different thresholds "
+              f"and scorers")
+
+    _estimator_, _fit_time, _fit_excepted = _FIT_OUTPUT_TUPLE
+
+    TEST_THRESHOLD_x_SCORER__SCORE_LAYER: MaskedHolderType = \
+        np.ma.zeros((len(_THRESHOLDS), len(_SCORER_DICT)), dtype=np.float64)
+    TEST_THRESHOLD_x_SCORER__SCORE_LAYER.mask = True
+
+    TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER: MaskedHolderType = \
+        np.ma.zeros((len(_THRESHOLDS), len(_SCORER_DICT)), dtype=np.float64)
+    TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER.mask = True
+
+    # IF A FOLD EXCEPTED DURING FIT, ALL THE THRESHOLDS AND SCORERS
+    # IN THAT FOLD LAYER GET SET TO error_score.
+    # SCORE TIME CANT BE TAKEN SINCE SCORING WONT BE DONE SO ALSO MASK THAT
+    if _fit_excepted:
+        TEST_THRESHOLD_x_SCORER__SCORE_LAYER[:, :] = _error_score
+        if _error_score is np.nan:
+            TEST_THRESHOLD_x_SCORER__SCORE_LAYER[:, :] = np.ma.masked
+
+        TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER[:, :] = np.nan
+        TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER[:, :] = np.ma.masked
+
+        if _verbose >= 5:
+            print(f'fold {_f_idx + 1} excepted during fit, unable to score')
+
+        return (TEST_THRESHOLD_x_SCORER__SCORE_LAYER,
+                    TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER)
+
+    # v v v only accessible if fit() did not except v v v
+
+    _X_test = _X_test.persist()
+    _y_test = _y_test.persist()
+
+    pp0_time = time.perf_counter()
+    _predict_proba = _estimator_.predict_proba(_X_test)[:, -1].ravel()
+    _val_predict_proba(
+        _predict_proba,
+        _X_test.shape[0] if hasattr(_X_test, 'shape') else len(_X_test)
+    )
+    pp_time = time.perf_counter() - pp0_time
+    del pp0_time
+
+    if _verbose >= 5:
+        print(f'fold {_f_idx + 1} test predict_proba time = {pp_time: ,.3g} s')
+    del pp_time
+
+    # GET SCORES FOR ALL SCORERS & THRESHOLDS ##########################
+
+    _test_fold_score_t0 = time.perf_counter()
+    for thresh_idx, _threshold in enumerate(_THRESHOLDS):
+
+        _y_pred_t0 = time.perf_counter()
+        _y_test_pred = (_predict_proba >= _threshold).astype(np.uint8)
+        _ypt = time.perf_counter() - _y_pred_t0
+        del _y_pred_t0
+        if _verbose == 10:
+            print(f"fold {_f_idx+1} test thresholding time = {_ypt: ,.3g} s")
+        del _ypt
+
+        for s_idx, scorer_key in enumerate(_SCORER_DICT):
+
+
+
+
+
+
+
+
+
+            test_scorer_t0 = time.perf_counter()
+            _score = _SCORER_DICT[scorer_key](_y_test, _y_test_pred)
+            test_scorer_score_time = time.perf_counter() - test_scorer_t0
+            del test_scorer_t0
+            TEST_THRESHOLD_x_SCORER__SCORE_LAYER[thresh_idx, s_idx] = _score
+            TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER[thresh_idx, s_idx] = \
+                test_scorer_score_time
+
+            if _verbose >= 8:
+                print(f"fold {_f_idx + 1} '{scorer_key}' test score time = "
+                      f"{test_scorer_score_time: ,.3g} s")
+
+        del _score, test_scorer_score_time
+
+
+    tfst = time.perf_counter() - _test_fold_score_t0
+    del _test_fold_score_t0
+
+    # END GET SCORE FOR ALL SCORERS & THRESHOLDS #######################
+
+    if _verbose >= 5:
+        print(f'End scoring fold {_f_idx + 1} test with different thresholds and '
+              f'scorers')
+        _ = _f_idx + 1
+
+        print(f'fold {_} total test thresh & score wall time = {tfst: ,.3g} s')
+        _ast = tfst / len(_SCORER_DICT) / len(_THRESHOLDS)
+        print(f'fold {_} avg test thresh & score wall time = {_ast: ,.3g} s')
+        del _ast
+
+        __ = TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER
+        print(f'fold {_} total test actual scoring time = {__.sum(): ,.3g} s')
+        print(f'fold {_} avg test actual scoring time = {__.mean(): ,.3g} s')
+        del _, __
+
+    del _X_test, _y_test, _predict_proba, _y_test_pred, tfst
+
+
+    return (TEST_THRESHOLD_x_SCORER__SCORE_LAYER,
+                TEST_THRESHOLD_x_SCORER__SCORE_TIME_LAYER)
+
+
+