cpgtools 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

impyute/ts/moving_window.py

@@ -0,0 +1,128 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+# pylint: disable=invalid-name, too-many-arguments, too-many-locals, too-many-branches, broad-except, len-as-condition
+
+@wrapper.wrappers
+@wrapper.checks
+def moving_window(data, nindex=None, wsize=5, errors="coerce", func=np.mean,
+        inplace=False):
+    """ Interpolate the missing values based on nearby values.
+
+    For example, with an array like this:
+
+        array([[-1.24940, -1.38673, -0.03214945,  0.08255145, -0.007415],
+               [ 2.14662,  0.32758 , -0.82601414,  1.78124027,  0.873998],
+               [-0.41400, -0.977629,         nan, -1.39255344,  1.680435],
+               [ 0.40975,  1.067599,  0.29152388, -1.70160145, -0.565226],
+               [-0.54592, -1.126187,  2.04004377,  0.16664863, -0.010677]])
+
+    Using a `k` or window size of 3. The one missing value would be set
+    to -1.18509122. The window operates on the horizontal axis.
+
+    Usage
+    -----
+
+    The parameters default the function to a moving mean. You may want to change
+    the default window size:
+
+        moving_window(data, wsize=10)
+
+    To only look at past data (null value is at the rightmost index in the window):
+
+        moving_window(data, nindex=-1)
+
+    To use a custom function:
+
+        moving_window(data, func=np.median)
+
+    You can also do something like take 1.5x the max of previous values in the window:
+
+        moving_window(data, func=lambda arr: max(arr) * 1.50, nindex=-1)
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        2D matrix to impute.
+    nindex: int
+        Null index. Index of the null value inside the moving average window.
+        Use cases: Say you wanted to make value skewed toward the left or right
+        side. 0 would only take the average of values from the right and -1
+        would only take the average of values from the left
+    wsize: int
+        Window size. Size of the moving average window/area of values being used
+        for each local imputation. This number includes the missing value.
+    errors: {"raise", "coerce", "ignore"}
+        Errors will occur with the indexing of the windows - for example if there
+        is a nan at data[x][0] and `nindex` is set to -1 or there is a nan at
+        data[x][-1] and `nindex` is set to 0. `"raise"` will raise an error,
+        `"coerce"` will try again using an nindex set to the middle and `"ignore"`
+        will just leave it as a nan.
+    inplace: {True, False}
+        Whether to return a copy or run on the passed-in array
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    if errors == "ignore":
+        raise Exception("`errors` value `ignore` not implemented yet. Sorry!")
+
+    if not inplace:
+        data = data.copy()
+
+    if nindex is None: # If using equal window side lengths
+        assert wsize % 2 == 1, "The parameter `wsize` should not be even "\
+        "if the value `nindex` is not set since it defaults to the midpoint "\
+        "and an even `wsize` makes the midpoint ambiguous"
+        wside_left = wsize // 2
+        wside_right = wsize // 2
+    else: # If using custom window side lengths
+        assert nindex < wsize, "The null index must be smaller than the window size"
+        if nindex == -1:
+            wside_left = wsize - 1
+            wside_right = 0
+        else:
+            wside_left = nindex
+            wside_right = wsize - nindex - 1
+
+    while True:
+        nan_xy = matrix.nan_indices(data)
+        n_nan_prev = len(nan_xy)
+        for x_i, y_i in nan_xy:
+            left_i = max(0, y_i-wside_left)
+            right_i = min(len(data), y_i+wside_right+1)
+            window = data[x_i, left_i: right_i]
+            window_not_null = window[~np.isnan(window)]
+
+            if len(window_not_null) > 0:
+                try:
+                    data[x_i][y_i] = func(window_not_null)
+                    continue
+                except Exception as e:
+                    if errors == "raise":
+                        raise e
+
+            if errors == "coerce":
+                # If either the window has a length of 0 or the aggregate function fails somehow,
+                # do a fallback of just trying the best we can by using it as the middle and trying
+                # to recalculate. Use temporary wside_left/wside_right, for only the calculation of
+                # this specific problamatic value
+                wside_left_tmp = wsize // 2
+                wside_right_tmp = wside_left_tmp
+
+                left_i_tmp = max(0, y_i-wside_left_tmp)
+                right_i_tmp = min(len(data), y_i+wside_right_tmp+1)
+
+                window = data[x_i, left_i_tmp:right_i_tmp]
+                window_not_null = window[~np.isnan(window)]
+                try:
+                    data[x_i][y_i] = func(window_not_null)
+                except Exception as e:
+                    print("Exception:", e)
+        if n_nan_prev == len(matrix.nan_indices(data)):
+            break
+
+    return data

missingpy/__init__.py

@@ -0,0 +1,4 @@
+#from .knnimpute import KNNImputer
+from .missforest import MissForest
+
+__all__ = ['MissForest']

missingpy/knnimpute.py

@@ -0,0 +1,328 @@
+"""KNN Imputer for Missing Data"""
+# Author: Ashim Bhattarai
+# License: GNU General Public License v3 (GPLv3)
+
+import warnings
+
+import numpy as np
+
+from sklearn.base import BaseEstimator, TransformerMixin
+from sklearn.utils import check_array
+from sklearn.utils.validation import check_is_fitted
+from sklearn.utils.validation import FLOAT_DTYPES
+from sklearn.neighbors.base import _check_weights
+from sklearn.neighbors.base import _get_weights
+
+from .pairwise_external import pairwise_distances
+from .pairwise_external import _get_mask
+from .pairwise_external import _MASKED_METRICS
+
+__all__ = [
+    'KNNImputer',
+]
+
+
+class KNNImputer(BaseEstimator, TransformerMixin):
+    """Imputation for completing missing values using k-Nearest Neighbors.
+
+    Each sample's missing values are imputed using values from ``n_neighbors``
+    nearest neighbors found in the training set. Each missing feature is then
+    imputed as the average, either weighted or unweighted, of these neighbors.
+    Note that if a sample has more than one feature missing, then the
+    neighbors for that sample can be different depending on the particular
+    feature being imputed. Finally, where the number of donor neighbors is
+    less than ``n_neighbors``, the training set average for that feature is
+    used during imputation.
+
+    Parameters
+    ----------
+    missing_values : integer or "NaN", optional (default = "NaN")
+        The placeholder for the missing values. All occurrences of
+        `missing_values` will be imputed. For missing values encoded as
+        ``np.nan``, use the string value "NaN".
+
+    n_neighbors : int, optional (default = 5)
+        Number of neighboring samples to use for imputation.
+
+    weights : str or callable, optional (default = "uniform")
+        Weight function used in prediction.  Possible values:
+
+        - 'uniform' : uniform weights.  All points in each neighborhood
+          are weighted equally.
+        - 'distance' : weight points by the inverse of their distance.
+          in this case, closer neighbors of a query point will have a
+          greater influence than neighbors which are further away.
+        - [callable] : a user-defined function which accepts an
+          array of distances, and returns an array of the same shape
+          containing the weights.
+
+    metric : str or callable, optional (default = "masked_euclidean")
+        Distance metric for searching neighbors. Possible values:
+        - 'masked_euclidean'
+        - [callable] : a user-defined function which conforms to the
+        definition of _pairwise_callable(X, Y, metric, **kwds). In other
+        words, the function accepts two arrays, X and Y, and a
+        ``missing_values`` keyword in **kwds and returns a scalar distance
+        value.
+
+    row_max_missing : float, optional (default = 0.5)
+        The maximum fraction of columns (i.e. features) that can be missing
+        before the sample is excluded from nearest neighbor imputation. It
+        means that such rows will not be considered a potential donor in
+        ``fit()``, and in ``transform()`` their missing feature values will be
+        imputed to be the column mean for the entire dataset.
+
+    col_max_missing : float, optional (default = 0.8)
+        The maximum fraction of rows (or samples) that can be missing
+        for any feature beyond which an error is raised.
+
+    copy : boolean, optional (default = True)
+        If True, a copy of X will be created. If False, imputation will
+        be done in-place whenever possible. Note that, if metric is
+        "masked_euclidean" and copy=False then missing_values in the
+        input matrix X will be overwritten with zeros.
+
+    Attributes
+    ----------
+    statistics_ : 1-D array of length {n_features}
+        The 1-D array contains the mean of each feature calculated using
+        observed (i.e. non-missing) values. This is used for imputing
+        missing values in samples that are either excluded from nearest
+        neighbors search because they have too many ( > row_max_missing)
+        missing features or because all of the sample's k-nearest neighbors
+        (i.e., the potential donors) also have the relevant feature value
+        missing.
+
+    References
+    ----------
+    * Olga Troyanskaya, Michael Cantor, Gavin Sherlock, Pat Brown, Trevor
+      Hastie, Robert Tibshirani, David Botstein and Russ B. Altman, Missing
+      value estimation methods for DNA microarrays, BIOINFORMATICS Vol. 17
+      no. 6, 2001 Pages 520-525.
+
+    Examples
+    --------
+    >>> from missingpy import KNNImputer
+    >>> nan = float("NaN")
+    >>> X = [[1, 2, nan], [3, 4, 3], [nan, 6, 5], [8, 8, 7]]
+    >>> imputer = KNNImputer(n_neighbors=2, weights="uniform")
+    >>> imputer.fit_transform(X)
+    array([[1. , 2. , 4. ],
+           [3. , 4. , 3. ],
+           [5.5, 6. , 5. ],
+           [8. , 8. , 7. ]])
+    """
+
+    def __init__(self, missing_values="NaN", n_neighbors=5,
+                 weights="uniform", metric="masked_euclidean",
+                 row_max_missing=0.5, col_max_missing=0.8, copy=True):
+
+        self.missing_values = missing_values
+        self.n_neighbors = n_neighbors
+        self.weights = weights
+        self.metric = metric
+        self.row_max_missing = row_max_missing
+        self.col_max_missing = col_max_missing
+        self.copy = copy
+
+    def _impute(self, dist, X, fitted_X, mask, mask_fx):
+        """Helper function to find and impute missing values"""
+
+        # For each column, find and impute
+        n_rows_X, n_cols_X = X.shape
+        for c in range(n_cols_X):
+            if not np.any(mask[:, c], axis=0):
+                continue
+
+            # Row index for receivers and potential donors (pdonors)
+            receivers_row_idx = np.where(mask[:, c])[0]
+            pdonors_row_idx = np.where(~mask_fx[:, c])[0]
+
+            # Impute using column mean if n_neighbors are not available
+            if len(pdonors_row_idx) < self.n_neighbors:
+                warnings.warn("Insufficient number of neighbors! "
+                              "Filling in column mean.")
+                X[receivers_row_idx, c] = self.statistics_[c]
+                continue
+
+            # Get distance from potential donors
+            dist_pdonors = dist[receivers_row_idx][:, pdonors_row_idx]
+            dist_pdonors = dist_pdonors.reshape(-1,
+                                                len(pdonors_row_idx))
+
+            # Argpartition to separate actual donors from the rest
+            pdonors_idx = np.argpartition(
+                dist_pdonors, self.n_neighbors - 1, axis=1)
+
+            # Get final donors row index from pdonors
+            donors_idx = pdonors_idx[:, :self.n_neighbors]
+
+            # Get weights or None
+            dist_pdonors_rows = np.arange(len(donors_idx))[:, None]
+            weight_matrix = _get_weights(
+                dist_pdonors[
+                    dist_pdonors_rows, donors_idx], self.weights)
+            donor_row_idx_ravel = donors_idx.ravel()
+
+            # Retrieve donor values and calculate kNN score
+            fitted_X_temp = fitted_X[pdonors_row_idx]
+            donors = fitted_X_temp[donor_row_idx_ravel, c].reshape(
+                (-1, self.n_neighbors))
+            donors_mask = _get_mask(donors, self.missing_values)
+            donors = np.ma.array(donors, mask=donors_mask)
+
+            # Final imputation
+            imputed = np.ma.average(donors, axis=1,
+                                    weights=weight_matrix)
+            X[receivers_row_idx, c] = imputed.data
+        return X
+
+    def fit(self, X, y=None):
+        """Fit the imputer on X.
+
+        Parameters
+        ----------
+        X : {array-like}, shape (n_samples, n_features)
+            Input data, where ``n_samples`` is the number of samples and
+            ``n_features`` is the number of features.
+
+        Returns
+        -------
+        self : object
+            Returns self.
+        """
+
+        # Check data integrity and calling arguments
+        force_all_finite = False if self.missing_values in ["NaN",
+                                                            np.nan] else True
+        if not force_all_finite:
+            if self.metric not in _MASKED_METRICS and not callable(
+                    self.metric):
+                raise ValueError(
+                    "The selected metric does not support NaN values.")
+        X = check_array(X, accept_sparse=False, dtype=np.float64,
+                        force_all_finite=force_all_finite, copy=self.copy)
+        self.weights = _check_weights(self.weights)
+
+        # Check for +/- inf
+        if np.any(np.isinf(X)):
+            raise ValueError("+/- inf values are not allowed.")
+
+        # Check if % missing in any column > col_max_missing
+        mask = _get_mask(X, self.missing_values)
+        if np.any(mask.sum(axis=0) > (X.shape[0] * self.col_max_missing)):
+            raise ValueError("Some column(s) have more than {}% missing values"
+                             .format(self.col_max_missing * 100))
+        X_col_means = np.ma.array(X, mask=mask).mean(axis=0).data
+
+        # Check if % missing in any row > row_max_missing
+        bad_rows = mask.sum(axis=1) > (mask.shape[1] * self.row_max_missing)
+        if np.any(bad_rows):
+            warnings.warn(
+                "There are rows with more than {0}% missing values. These "
+                "rows are not included as donor neighbors."
+                    .format(self.row_max_missing * 100))
+
+            # Remove rows that have more than row_max_missing % missing
+            X = X[~bad_rows, :]
+
+        # Check if sufficient neighboring samples available
+        if X.shape[0] < self.n_neighbors:
+            raise ValueError("There are only %d samples, but n_neighbors=%d."
+                             % (X.shape[0], self.n_neighbors))
+        self.fitted_X_ = X
+        self.statistics_ = X_col_means
+
+        return self
+
+    def transform(self, X):
+        """Impute all missing values in X.
+
+        Parameters
+        ----------
+        X : {array-like}, shape = [n_samples, n_features]
+            The input data to complete.
+
+        Returns
+        -------
+        X : {array-like}, shape = [n_samples, n_features]
+            The imputed dataset.
+        """
+
+        check_is_fitted(self, ["fitted_X_", "statistics_"])
+        force_all_finite = False if self.missing_values in ["NaN",
+                                                            np.nan] else True
+        X = check_array(X, accept_sparse=False, dtype=FLOAT_DTYPES,
+                        force_all_finite=force_all_finite, copy=self.copy)
+
+        # Check for +/- inf
+        if np.any(np.isinf(X)):
+            raise ValueError("+/- inf values are not allowed in data to be "
+                             "transformed.")
+
+        # Get fitted data and ensure correct dimension
+        n_rows_fit_X, n_cols_fit_X = self.fitted_X_.shape
+        n_rows_X, n_cols_X = X.shape
+
+        if n_cols_X != n_cols_fit_X:
+            raise ValueError("Incompatible dimension between the fitted "
+                             "dataset and the one to be transformed.")
+        mask = _get_mask(X, self.missing_values)
+
+        row_total_missing = mask.sum(axis=1)
+        if not np.any(row_total_missing):
+            return X
+
+        # Check for excessive missingness in rows
+        bad_rows = row_total_missing > (mask.shape[1] * self.row_max_missing)
+        if np.any(bad_rows):
+            warnings.warn(
+                "There are rows with more than {0}% missing values. The "
+                "missing features in these rows are imputed with column means."
+                    .format(self.row_max_missing * 100))
+            X_bad = X[bad_rows, :]
+            X = X[~bad_rows, :]
+            mask = mask[~bad_rows]
+            row_total_missing = mask.sum(axis=1)
+        row_has_missing = row_total_missing.astype(np.bool)
+
+        if np.any(row_has_missing):
+
+            # Mask for fitted_X
+            mask_fx = _get_mask(self.fitted_X_, self.missing_values)
+
+            # Pairwise distances between receivers and fitted samples
+            dist = np.empty((len(X), len(self.fitted_X_)))
+            dist[row_has_missing] = pairwise_distances(
+                X[row_has_missing], self.fitted_X_, metric=self.metric,
+                squared=False, missing_values=self.missing_values)
+
+            # Find and impute missing
+            X = self._impute(dist, X, self.fitted_X_, mask, mask_fx)
+
+        # Merge bad rows to X and mean impute their missing values
+        if np.any(bad_rows):
+            bad_missing_index = np.where(_get_mask(X_bad, self.missing_values))
+            X_bad[bad_missing_index] = np.take(self.statistics_,
+                                               bad_missing_index[1])
+            X_merged = np.empty((n_rows_X, n_cols_X))
+            X_merged[bad_rows, :] = X_bad
+            X_merged[~bad_rows, :] = X
+            X = X_merged
+        return X
+
+    def fit_transform(self, X, y=None, **fit_params):
+        """Fit KNNImputer and impute all missing values in X.
+
+        Parameters
+        ----------
+        X : {array-like}, shape (n_samples, n_features)
+            Input data, where ``n_samples`` is the number of samples and
+            ``n_features`` is the number of features.
+
+        Returns
+        -------
+        X : {array-like}, shape (n_samples, n_features)
+            Returns imputed dataset.
+        """
+        return self.fit(X).transform(X)