cpgtools 2.0.5__py3-none-any.whl

impyute/cs/central_tendency.py

@@ -0,0 +1,84 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+
+@wrapper.wrappers
+@wrapper.checks
+def mean(data):
+    """ Substitute missing values with the mean of that column.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    for x_i, y_i in nan_xy:
+        row_wo_nan = data[:, [y_i]][~np.isnan(data[:, [y_i]])]
+        new_value = np.mean(row_wo_nan)
+        data[x_i][y_i] = new_value
+    return data
+
+@wrapper.wrappers
+@wrapper.checks
+def median(data):
+    """ Substitute missing values with the median of that column(middle).
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    cols_missing = set(nan_xy.T[1])
+    medians = {}
+    for y_i in cols_missing:
+        cols_wo_nan = data[:, [y_i]][~np.isnan(data[:, [y_i]])]
+        median_y = np.median(cols_wo_nan)
+        medians[str(y_i)] = median_y
+    for x_i, y_i in nan_xy:
+        data[x_i][y_i] = medians[str(y_i)]
+    return data
+
+@wrapper.wrappers
+@wrapper.checks
+def mode(data):
+    """ Substitute missing values with the mode of that column(most frequent).
+
+    In the case that there is a tie (there are multiple, most frequent values)
+    for a column randomly pick one of them.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    modes = []
+    for y_i in range(np.shape(data)[1]):
+        unique_counts = np.unique(data[:, [y_i]], return_counts=True)
+        max_count = np.max(unique_counts[1])
+        mode_y = [unique for unique, count in np.transpose(unique_counts)
+                  if count == max_count and not np.isnan(unique)]
+        modes.append(mode_y)  # Appends index of column and column modes
+    for x_i, y_i in nan_xy:
+        data[x_i][y_i] = np.random.choice(modes[y_i])
+    return data

impyute/cs/em.py

@@ -0,0 +1,52 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+
+@wrapper.wrappers
+@wrapper.checks
+def em(data, eps=0.1):
+    """ Imputes given data using expectation maximization.
+
+    E-step: Calculates the expected complete data log likelihood ratio.
+    M-step: Finds the parameters that maximize the log likelihood of the
+    complete data.
+
+    Parameters
+    ----------
+    data: numpy.nd.array
+        Data to impute.
+    eps: float
+        The amount of minimum change between iterations to break, if relative change < eps, converge.
+        relative change = abs(current - previous) / previous
+    inplace: boolean
+        If True, operate on the numpy array reference
+
+    Returns
+    -------
+    numpy.nd.array
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    for x_i, y_i in nan_xy:
+        col = data[:, int(y_i)]
+        mu = col[~np.isnan(col)].mean()
+        std = col[~np.isnan(col)].std()
+        col[x_i] = np.random.normal(loc=mu, scale=std)
+        previous, i = 1, 1
+        while True:
+            i += 1
+            # Expectation
+            mu = col[~np.isnan(col)].mean()
+            std = col[~np.isnan(col)].std()
+            # Maximization
+            col[x_i] = np.random.normal(loc=mu, scale=std)
+            # Break out of loop if likelihood doesn't change at least 10%
+            # and has run at least 5 times
+            delta = np.abs(col[x_i]-previous)/previous
+            if i > 5 and delta < eps:
+                data[x_i][y_i] = col[x_i]
+                break
+            data[x_i][y_i] = col[x_i]
+            previous = col[x_i]
+    return data

impyute/cs/fast_knn.py

@@ -0,0 +1,130 @@
+import numpy as np
+import pandas as pd
+from scipy.spatial import KDTree
+from impyute.ops import matrix
+from impyute.ops import wrapper
+from impyute.ops import inverse_distance_weighting as idw
+
+from . import mean
+# pylint: disable=too-many-arguments
+
+@wrapper.wrappers
+@wrapper.checks
+def fast_knn(data, k=3, eps=0, p=2, distance_upper_bound=np.inf, leafsize=10,
+        idw_fn=idw.shepards, init_impute_fn=mean):
+    """ Impute using a variant of the nearest neighbours approach
+
+    Basic idea: Impute array with a passed in initial impute fn (mean impute)
+    and then use the resulting complete array to construct a KDTree. Use this
+    KDTree to compute nearest neighbours.  After finding `k` nearest
+    neighbours, take the weighted average of them. Basically, find the nearest
+    row in terms of distance
+
+    This approach is much, much faster than the other implementation (fit+transform
+    for each subset) which is almost prohibitively expensive.
+
+    Parameters
+    ----------
+    data: ndarray
+        2D matrix to impute.
+
+    k: int, optional
+        Parameter used for method querying the KDTree class object. Number of
+        neighbours used in the KNN query. Refer to the docs for
+        [`scipy.spatial.KDTree.query`]
+        (https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.KDTree.query.html).
+
+    eps: nonnegative float, optional
+        Parameter used for method querying the KDTree class object. From the
+        SciPy docs: "Return approximate nearest neighbors; the kth returned
+        value is guaranteed to be no further than (1+eps) times the distance to
+        the real kth nearest neighbor". Refer to the docs for
+        [`scipy.spatial.KDTree.query`]
+        (https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.KDTree.query.html).
+
+    p : float, 1<=p<=infinity, optional
+        Parameter used for method querying the KDTree class object. Straight from the
+        SciPy docs: "Which Minkowski p-norm to use. 1 is the
+        sum-of-absolute-values Manhattan distance 2 is the usual Euclidean
+        distance infinity is the maximum-coordinate-difference distance". Refer to
+        the docs for
+        [`scipy.spatial.KDTree.query`]
+        (https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.KDTree.query.html).
+
+    distance_upper_bound : nonnegative float, optional
+        Parameter used for method querying the KDTree class object. Straight
+        from the SciPy docs: "Return only neighbors within this distance. This
+        is used to prune tree searches, so if you are doing a series of
+        nearest-neighbor queries, it may help to supply the distance to the
+        nearest neighbor of the most recent point." Refer to the docs for
+        [`scipy.spatial.KDTree.query`]
+        (https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.KDTree.query.html).
+
+    leafsize: int, optional
+        Parameter used for construction of the `KDTree` class object. Straight from
+        the SciPy docs: "The number of points at which the algorithm switches
+        over to brute-force. Has to be positive". Refer to the docs for
+        [`scipy.spatial.KDTree`](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.spatial.KDTree.html)
+        for more information.
+
+    idw_fn: fn, optional
+        Function that takes one argument, a list of distances, and returns weighted percentages. You can define a custom
+        one or bootstrap from functions defined in `impy.util.inverse_distance_weighting` which can be using
+        functools.partial, for example: `functools.partial(impy.util.inverse_distance_weighting.shepards, power=1)`
+
+    init_impute_fn: fn, optional
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    Examples
+    --------
+
+        >>> data = np.arange(25).reshape((5, 5)).astype(np.float)
+        >>> data[0][2] =  np.nan
+        >>> data
+        array([[ 0.,  1., nan,  3.,  4.],
+               [ 5.,  6.,  7.,  8.,  9.],
+               [10., 11., 12., 13., 14.],
+               [15., 16., 17., 18., 19.],
+               [20., 21., 22., 23., 24.]])
+        >> fast_knn(data, k=1) # Weighted average (by distance) of nearest 1 neighbour
+        array([[ 0.,  1.,  7.,  3.,  4.],
+               [ 5.,  6.,  7.,  8.,  9.],
+               [10., 11., 12., 13., 14.],
+               [15., 16., 17., 18., 19.],
+               [20., 21., 22., 23., 24.]])
+        >> fast_knn(data, k=2) # Weighted average of nearest 2 neighbours
+        array([[ 0.        ,  1.        , 10.08608891,  3.        ,  4.        ],
+               [ 5.        ,  6.        ,  7.        ,  8.        ,  9.        ],
+               [10.        , 11.        , 12.        , 13.        , 14.        ],
+               [15.        , 16.        , 17.        , 18.        , 19.        ],
+               [20.        , 21.        , 22.        , 23.        , 24.        ]])
+        >> fast_knn(data, k=3)
+        array([[ 0.        ,  1.        , 13.40249283,  3.        ,  4.        ],
+               [ 5.        ,  6.        ,  7.        ,  8.        ,  9.        ],
+               [10.        , 11.        , 12.        , 13.        , 14.        ],
+               [15.        , 16.        , 17.        , 18.        , 19.        ],
+               [20.        , 21.        , 22.        , 23.        , 24.        ]])
+        >> fast_knn(data, k=5) # There are at most only 4 neighbours. Raises error
+        ...
+        IndexError: index 5 is out of bounds for axis 0 with size 5
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    data_c = init_impute_fn(data)
+    kdtree = KDTree(data_c, leafsize=leafsize)
+
+    for x_i, y_i in nan_xy:
+        distances, indices = kdtree.query(data_c[x_i], k=k+1, eps=eps,
+                                          p=p, distance_upper_bound=distance_upper_bound)
+        # Will always return itself in the first index. Delete it.
+        distances, indices = distances[1:], indices[1:]
+        # Add small constant to distances to avoid division by 0
+        distances += 1e-3
+        weights = idw_fn(distances)
+        # Assign missing value the weighted average of `k` nearest neighbours
+        data[x_i][y_i] = np.dot(weights, [data_c[ind][y_i] for ind in indices])
+    return data

impyute/cs/random.py

@@ -0,0 +1,27 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+
+@wrapper.wrappers
+@wrapper.checks
+def random_impute(data):
+    """ Fill missing values in with a randomly selected value from the same
+    column.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    for x, y in nan_xy:
+        uniques = np.unique(data[:, y])
+        uniques = uniques[~np.isnan(uniques)]
+        data[x][y] = np.random.choice(uniques)
+    return data

impyute/dataset/__init__.py

@@ -0,0 +1,6 @@
+""" Real-world/mock datasets and missingness corruptors to experiment with.  """
+from .base import randu
+from .base import randn
+from .base import mnist
+
+__all__ = ["randu", "randn", "mnist"]

impyute/dataset/base.py

@@ -0,0 +1,137 @@
+""" Shared functions to load/generate data """
+import itertools
+import math
+import random
+import string
+import numpy as np
+from impyute.dataset.corrupt import Corruptor
+from impyute.ops import error
+
+def randu(bound=(0, 10), shape=(5, 5), missingness="mcar", thr=0.2, dtype="int"):
+    """ Return randomly generated dataset of numbers with uniformly
+    distributed values between bound[0] and bound[1]
+
+    Parameters
+    ----------
+    bound:tuple (start,stop)
+        Determines the range of values in the matrix. Index 0 for start
+        value and index 1 for stop value. Start is inclusive, stop is
+        exclusive.
+    shape:tuple(optional)
+        Size of the randomly generated data
+    missingness: ('mcar', 'mar', 'mnar')
+        Type of missingness you want in your dataset
+    thr: float between [0,1]
+        Percentage of missing data in generated data
+    dtype: ('int','float')
+        Type of data
+
+    Returns
+    -------
+    numpy.ndarray
+    """
+    if dtype == "int":
+        data = np.random.randint(bound[0], bound[1], size=shape).astype(float)
+    elif dtype == "float":
+        data = np.random.uniform(bound[0], bound[1], size=shape)
+    corruptor = Corruptor(data, thr=thr)
+    raw_data = getattr(corruptor, missingness)()
+    return raw_data
+
+
+def randn(theta=(0, 1), shape=(5, 5), missingness="mcar", thr=0.2, dtype="float"):
+    """ Return randomly generated dataset of numbers with normally
+    distributed values with given and sigma.
+
+    Parameters
+    ----------
+    theta: tuple (mu, sigma)
+        Determines the range of values in the matrix
+    shape:tuple(optional)
+        Size of the randomly generated data
+    missingness: ('mcar', 'mar', 'mnar')
+        Type of missingness you want in your dataset
+    thr: float between [0,1]
+        Percentage of missing data in generated data
+    dtype: ('int','float')
+        Type of data
+
+    Returns
+    -------
+    numpy.ndarray
+    """
+    mean, sigma = theta
+    data = np.random.normal(mean, sigma, size=shape)
+    if dtype == "int":
+        data = np.round(data)
+    elif dtype == "float":
+        pass
+    corruptor = Corruptor(data, thr=thr)
+    raw_data = getattr(corruptor, missingness)()
+    return raw_data
+
+def randc(nlevels=5, shape=(5, 5), missingness="mcar", thr=0.2):
+    """ Return randomly generated dataset with uniformly distributed categorical data (alphabetic character)
+
+    Parameters
+    ----------
+    nlevels: int
+        Specify the number of different categories in the dataset
+    shape: tuple(optional)
+        Size of the randomly generated data
+    missingness: string in ('mcar', 'mar', 'mnar')
+        Type of missingness you want in your dataset
+    thr: float between [0,1]
+        Percentage of missing data in generated data
+
+    Returns
+    -------
+    numpy.ndarray
+    """
+    if shape[0]*shape[1] < nlevels:
+        raise error.BadInputError("nlevel exceeds the size of desired dataset. Please decrease the nlevel or increase the shape")
+
+    length = len(string.ascii_lowercase)
+    n_fold = int(math.floor(math.log(nlevels, length)))
+    cat_pool = list(string.ascii_lowercase)
+
+    # when nlevel > 26, the alphabetical character is used up, need to generate extra strings as categorical data
+    if n_fold > 0:
+        for i in range(2, n_fold+2):
+            pool_candidate = list(itertools.product(string.ascii_lowercase, repeat=i))
+            cat_pool.extend([''.join(w) for w in pool_candidate])
+            if len(cat_pool) > nlevels:
+                break
+
+    cat = random.sample(cat_pool, nlevels)
+    data = np.random.choice(cat, shape, replace=True)
+
+    # make sure the data frame has nlevel different categories
+    while len(np.unique(data)) != nlevels:
+        data = np.random.choice(cat, shape, replace=True)
+
+    corruptor = Corruptor(data, thr=thr, dtype=np.str)
+    raw_data = getattr(corruptor, missingness)()
+    return raw_data
+
+
+
+def mnist(missingness="mcar", thr=0.2):
+    """ Loads corrupted MNIST
+
+    Parameters
+    ----------
+    missingness: ('mcar', 'mar', 'mnar')
+        Type of missigness you want in your dataset
+    th: float between [0,1]
+        Percentage of missing data in generated data
+
+    Returns
+    -------
+    numpy.ndarray
+    """
+    from sklearn.datasets import fetch_mldata
+    dataset = fetch_mldata('MNIST original')
+    corruptor = Corruptor(dataset.data, thr=thr)
+    data = getattr(corruptor, missingness)()
+    return {"X": data, "Y": dataset.target}

impyute/dataset/corrupt.py

@@ -0,0 +1,55 @@
+""" impyute.dataset.corrupt """
+import numpy as np
+
+
+class Corruptor:
+    """ Adds missing values to a complete dataset.
+
+    Attributes
+    ----------
+    data: np.ndarray
+        Matrix of values with no NaN's that you want to add NaN's to.
+    thr: float (optional)
+        The percentage of null values you want in your dataset, a number
+        between 0 and 1.
+
+    Methods
+    -------
+    mcar()
+        Overwrite values with MCAR placed NaN's.
+    mar()
+        Overwrite values with MAR placed NaN's.
+    mnar()
+        Overwrite values with MNAR placed NaN's.
+
+    """
+    def __init__(self, data, thr=0.2, dtype=np.float):
+        self.dtype = data.dtype
+        self.shape = np.shape(data)
+        self.data = data.astype(dtype)
+        self.thr = thr
+
+    def mcar(self):
+        """ Overwrites values with MCAR placed NaN's """
+        data_1d = self.data.flatten()
+        n_total = len(data_1d)
+        nan_x = np.random.choice(range(n_total),
+                                  size=int(self.thr*n_total),
+                                  replace=False)
+        for x_i in nan_x:
+            data_1d[x_i] = np.nan
+        output = data_1d.reshape(self.shape)
+        return output
+
+    def mar(self):
+        """ Overwrites values with MAR placed NaN's """
+        pass
+
+    def mnar(self):
+        """ Overwrites values with MNAR placed NaN's """
+        pass
+
+    def complete(self):
+        """ Do nothing to the data """
+        output = self.data
+        return output

impyute/deletion/__init__.py

@@ -0,0 +1,5 @@
+""" Missing data approaches that delete values.  """
+
+from .complete_case import complete_case
+
+__all__ = ["complete_case"]

impyute/deletion/complete_case.py

@@ -0,0 +1,21 @@
+""" impyute.deletion.complete_case """
+import numpy as np
+from impyute.ops import wrapper
+
+@wrapper.wrappers
+@wrapper.checks
+def complete_case(data):
+    """ Return only data rows with all columns
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    return data[~np.isnan(data).any(axis=1)]

impyute/ops/__init__.py

@@ -0,0 +1,12 @@
+""" Unorganized set of utility functions """
+
+from . import error
+from . import inverse_distance_weighting
+from . import matrix
+from . import util
+from . import wrapper
+
+__all__ = [
+        "error", "inverse_distance_weighting", "matrix",
+        "util", "wrapper"
+        ]

impyute/ops/error.py

@@ -0,0 +1,9 @@
+""" Impyute specific error messages """
+
+class BadInputError(Exception):
+    "Error thrown when input args don't match spec"
+    pass
+
+class BadOutputError(Exception):
+    "Error thrown when outputs don't match spec"
+    pass

impyute/ops/inverse_distance_weighting.py

@@ -0,0 +1,31 @@
+""" Assign weights to distances in a way such that farther values are weighed less """
+import numpy as np
+
+def shepards(distances, power=2):
+    """ Basic inverse distance weighting function
+
+    Parameters
+    ----------
+    distances: list/numpy.ndarray
+        1D list of numbers (ex. distance results from call to KDTree.query)
+
+    power: int
+        Default of 2 used since the referenced paper stated an exponent of 2 "gives seemingly
+        satisfactory results"
+
+    Returns
+    -------
+    numpy.ndarray
+        1D list of numbers that sum to 1, represents weights of provided distances, in order.
+
+    References
+    ----------
+
+    Shepard, Donald (1968). "A two-dimensional interpolation function for irregularly-spaced data".
+    Proceedings of the 1968 ACM National Conference. pp. 517-524. doi:10.1145/800186.810616
+    """
+    return to_percentage(1/np.power(distances, power))
+
+def to_percentage(vec):
+    """ Converts list of real numbers into a list of percentages """
+    return vec/np.sum(vec)

impyute/ops/matrix.py

@@ -0,0 +1,47 @@
+""" Common operations on matrices
+
+*Look into whether it's worth writing these in raw c*
+"""
+import numpy as np
+
+def nan_indices(data):
+    """ Finds the indices of all missing values.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+
+    Returns
+    -------
+    List of tuples
+        Indices of all missing values in tuple format; (i, j)
+    """
+    return np.argwhere(np.isnan(data))
+
+def map_nd(fn, arr):
+    """ Map fn that takes a value over entire n-dim array
+
+    Parameters
+    ----------
+    arr: numpy.ndarray
+
+    Returns
+    -------
+    numpy.ndarray
+
+    """
+    return np.vectorize(fn)(arr)
+
+def every_nd(fn, arr):
+    """ Returns bool, true if fn is true for all elements of arr
+
+    Parameters
+    ----------
+    arr: numpy.ndarray
+
+    Returns
+    -------
+    bool
+
+    """
+    return all(map(fn, arr.flatten()))

impyute/ops/testing.py

@@ -0,0 +1,20 @@
+""" Utilities used for unit tests """
+import numpy as np
+
+
+def return_na_check(data):
+    """Helper function for tests to check if the data returned is a
+       numpy array and that the imputed data has no NaN's.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+    
+    Returns
+    -------
+    None
+
+    """
+    assert isinstance(data, np.ndarray)
+    assert not np.isnan(data).any()