cpgtools 2.0.0__py3-none-any.whl → 2.0.2__py3-none-any.whl

impyute/__init__.py

@@ -0,0 +1,3 @@
+""" Imputations for cross-sectional and time-series data.  """
+
+__all__ = ["cs", "ts"]

impyute/contrib/__init__.py

@@ -0,0 +1,7 @@
+""" Volatile code. Expect stuff in this to change. """
+
+from .describe import describe
+from .count_missing import count_missing
+from .compare import compare
+
+__all__ = ["describe", "count_missing", "compare"]

impyute/contrib/compare.py

@@ -0,0 +1,69 @@
+"""impyute.contrib.compare.py"""
+import importlib
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import accuracy_score
+# pylint: disable=too-many-locals, dangerous-default-value
+
+def compare(imputed, classifiers=["sklearn.svm.SVC"], log_path=None):
+    """
+    Given an imputed dataset with labels and a list of supervised machine
+    learning model, find accuracy score of all model/imputation pairs.
+
+    Parameters
+    ----------
+    imputed: [(str, np.ndarray), (str, np.ndarray)...]
+        List of tuples containing (imputation_name, imputed_data) where
+        `imputation_name` is a string and `imputed_data` is a tuple where
+        `imputed_data`[0] is the data, X and `imputed_data`[1] is the label, y
+    classifiers: [str, str...str] (optional)
+        Provide a list of classifiers to run imputed data sets on. Right now,
+        it ONLY works with sklearn, the format should be like so:
+        `sklearn.SUBMODULE.FUNCTION`. More generally its
+        'MODULE.SUBMODULE.FUNCTION'. If providing a custom classifier, make
+        sure to add the file location to sys.path first and the classifier
+        should also be structured like sklearn (with a `fit` and `predict`
+        method).
+    log_path: str (optional)
+        To write results to a file, provide a relative path
+
+    Returns
+    -------
+    results.txt
+        Classification results on imputed data
+
+    """
+    clfs = []
+    for clf_name in classifiers:
+        mod_name, smod_name, fn_name = clf_name.split(".")
+        try:
+            mod = importlib.import_module("{}.{}".format(mod_name, smod_name))
+            fn = getattr(mod, fn_name)
+            clfs.append([fn_name, fn])
+        except ModuleNotFoundError:
+            print("Cannot import '{}' from '{}.{}'".format(fn_name,
+                                                           mod_name,
+                                                           smod_name))
+
+    results = {imputation_name: [] for imputation_name, _ in imputed}
+
+    for imputation_name, data in imputed:
+        X, y = data
+        X_train, X_test, y_train, y_test = train_test_split(X, y,
+                                                            test_size=0.33,
+                                                            random_state=0)
+        print("Imputation {} =========".format(imputation_name))
+        for clf_name, clf in clfs:
+            clf = clf()
+            clf.fit(X_train, y_train)
+            y_pred = clf.predict(X_test)
+            accuracy = accuracy_score(y_test, y_pred)
+            results[imputation_name].append((clf_name, accuracy))
+            print("...{}".format(clf_name))
+
+    # If not None, write to path
+    if log_path:
+        with open(log_path, 'w') as f:
+            f.write(str(results))
+        print("Results saved to {}".format(log_path))
+
+    return results

impyute/contrib/count_missing.py

@@ -0,0 +1,30 @@
+""" impyute.contrib.count_missing.py """
+import numpy as np
+from impyute.ops import matrix
+
+def count_missing(data):
+    """ Calculate the total percentage of missing values and also the
+    percentage in each column.
+
+    Parameters
+    ----------
+    data: np.array
+        Data to impute.
+
+    Returns
+    -------
+    dict
+        Percentage of missing values in total and in each column.
+
+    """
+    size = len(data.flatten())
+    nan_xy = matrix.nan_indices(data)
+    np.unique(nan_xy)
+    counter = {y: 0. for y in np.unique(nan_xy.T[1])}
+    change_in_percentage = 1./size
+    for _, y in nan_xy:
+        counter[y] += change_in_percentage
+    total_missing = len(nan_xy)/size
+    counter["total"] = total_missing
+
+    return counter

impyute/contrib/describe.py

@@ -0,0 +1,63 @@
+""" impyute.contrib.describe """
+from impyute.ops import matrix
+
+def describe(data): # verbose=True):
+    """ Print input/output multiple times
+
+    Eventually will be used instead of matrix.nan_indices everywhere
+
+    Parameters
+    ----------
+    data: numpy.nd.array
+        The data you want to get a description from
+    verbose: boolean(optional)
+        Decides whether the description is short or long form
+
+    Returns
+    -------
+    dict
+        missingness: list
+            Confidence interval of data being MCAR, MAR or MNAR - in that order
+        nan_xy: list of tuples
+            Indices of all null points
+        nan_n: list
+            Total number of null values for each column
+        pmissing_n: float
+            Percentage of missing values in dataset
+        nan_rows: list
+            Indices of all rows that are completely null
+        nan_cols: list
+            Indices of all columns that are completely null
+        mean_rows: list
+            Mean value of each row
+        mean_cols: list
+            Mean value of each column
+        std_dev: list
+            std dev for each row/column
+        min_max: list
+            Finds the minimum and maximum for each row
+
+    """
+#    missingness = [0.33, 0.33, 0.33]  # find_missingness(data)
+    nan_xy = matrix.nan_indices(data)
+    nan_n = len(nan_xy)
+    pmissing_n = float(nan_n/len(data.flatten))
+#    pmissing_rows = ""
+#    pmissing_cols = ""
+#    nan_rows = ""
+#    nan_cols = ""
+#    mean_rows = ""
+#    mean_cols = ""
+#    std_dev = ""
+#                   "missingness": missingness,
+    description = {"nan_xy": nan_xy,
+                   "nan_n": nan_n,
+                   "pmissing_n": pmissing_n}
+#                   "pmissing_rows": pmissing_rows,
+#                   "pmissing_cols": pmissing_cols,
+#                   "nan_rows": nan_rows,
+#                   "nan_cols": nan_cols,
+#                   "mean_rows": mean_rows,
+#                   "mean_cols": mean_cols,
+#                   "std_dev": std_dev}
+    return description

impyute/cs/__init__.py

@@ -0,0 +1,11 @@
+""" Imputations for cross-sectional data.  """
+
+from .random import random_impute
+from .central_tendency import mean
+from .central_tendency import mode
+from .central_tendency import median
+from .buck_iterative import buck_iterative
+from .em import em
+from .fast_knn import fast_knn
+
+__all__ = ["random_impute", "mean", "mode", "median", "buck_iterative", "em", "fast_knn"]

impyute/cs/buck_iterative.py

@@ -0,0 +1,82 @@
+import numpy as np
+from sklearn.linear_model import LinearRegression
+from impyute.ops import matrix
+from impyute.ops import wrapper
+# pylint: disable=too-many-locals
+
+@wrapper.wrappers
+@wrapper.checks
+def buck_iterative(data):
+    """ Iterative variant of buck's method
+
+    - Variable to regress on is chosen at random.
+    - EM type infinite regression loop stops after change in prediction from
+      previous prediction < 10% for all columns with missing values
+
+    A Method of Estimation of Missing Values in Multivariate Data Suitable for
+    use with an Electronic Computer S. F. Buck Journal of the Royal Statistical
+    Society. Series B (Methodological) Vol. 22, No. 2 (1960), pp. 302-306
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+
+    # Add a column of zeros to the index values
+    nan_xyz = np.append(nan_xy, np.zeros((np.shape(nan_xy)[0], 1)), axis=1)
+
+    nan_xyz = [[int(x), int(y), v] for x, y, v in nan_xyz]
+    temp = []
+    cols_missing = {y for _, y, _ in nan_xyz}
+
+    # Step 1: Simple Imputation, these are just placeholders
+    for x_i, y_i, value in nan_xyz:
+        # Column containing nan value without the nan value
+        col = data[:, [y_i]][~np.isnan(data[:, [y_i]])]
+
+        new_value = np.mean(col)
+        data[x_i][y_i] = new_value
+        temp.append([x_i, y_i, new_value])
+    nan_xyz = temp
+
+    # Step 5: Repeat step 2 - 4 until convergence (the 100 is arbitrary)
+
+    converged = [False] * len(nan_xyz)
+    while not all(converged):
+        # Step 2: Placeholders are set back to missing for one variable/column
+        dependent_col = int(np.random.choice(list(cols_missing)))
+        missing_xs = [int(x) for x, y, value in nan_xyz if y == dependent_col]
+
+        # Step 3: Perform linear regression using the other variables
+        x_train, y_train = [], []
+        for x_i in (x_i for x_i in range(len(data)) if x_i not in missing_xs):
+            x_train.append(np.delete(data[x_i], dependent_col))
+            y_train.append(data[x_i][dependent_col])
+        model = LinearRegression()
+        model.fit(x_train, y_train)
+
+        # Step 4: Missing values for the missing variable/column are replaced
+        # with predictions from our new linear regression model
+        # For null indices with the dependent column that was randomly chosen
+        for i, z in enumerate(nan_xyz):
+            x_i = z[0]
+            y_i = z[1]
+            value = data[x_i, y_i]
+            if y_i == dependent_col:
+                # Row 'x' without the nan value
+                new_value = model.predict([np.delete(data[x_i], dependent_col)])
+                data[x_i][y_i] = new_value.reshape(1, -1)
+                if value == 0.0:
+                    delta = (new_value-value)/0.01
+                else:
+                    delta = (new_value-value)/value
+                converged[i] = abs(delta) < 0.1
+    return data

impyute/cs/central_tendency.py

@@ -0,0 +1,84 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+
+@wrapper.wrappers
+@wrapper.checks
+def mean(data):
+    """ Substitute missing values with the mean of that column.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    for x_i, y_i in nan_xy:
+        row_wo_nan = data[:, [y_i]][~np.isnan(data[:, [y_i]])]
+        new_value = np.mean(row_wo_nan)
+        data[x_i][y_i] = new_value
+    return data
+
+@wrapper.wrappers
+@wrapper.checks
+def median(data):
+    """ Substitute missing values with the median of that column(middle).
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    cols_missing = set(nan_xy.T[1])
+    medians = {}
+    for y_i in cols_missing:
+        cols_wo_nan = data[:, [y_i]][~np.isnan(data[:, [y_i]])]
+        median_y = np.median(cols_wo_nan)
+        medians[str(y_i)] = median_y
+    for x_i, y_i in nan_xy:
+        data[x_i][y_i] = medians[str(y_i)]
+    return data
+
+@wrapper.wrappers
+@wrapper.checks
+def mode(data):
+    """ Substitute missing values with the mode of that column(most frequent).
+
+    In the case that there is a tie (there are multiple, most frequent values)
+    for a column randomly pick one of them.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    modes = []
+    for y_i in range(np.shape(data)[1]):
+        unique_counts = np.unique(data[:, [y_i]], return_counts=True)
+        max_count = np.max(unique_counts[1])
+        mode_y = [unique for unique, count in np.transpose(unique_counts)
+                  if count == max_count and not np.isnan(unique)]
+        modes.append(mode_y)  # Appends index of column and column modes
+    for x_i, y_i in nan_xy:
+        data[x_i][y_i] = np.random.choice(modes[y_i])
+    return data

impyute/cs/em.py

@@ -0,0 +1,52 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+
+@wrapper.wrappers
+@wrapper.checks
+def em(data, eps=0.1):
+    """ Imputes given data using expectation maximization.
+
+    E-step: Calculates the expected complete data log likelihood ratio.
+    M-step: Finds the parameters that maximize the log likelihood of the
+    complete data.
+
+    Parameters
+    ----------
+    data: numpy.nd.array
+        Data to impute.
+    eps: float
+        The amount of minimum change between iterations to break, if relative change < eps, converge.
+        relative change = abs(current - previous) / previous
+    inplace: boolean
+        If True, operate on the numpy array reference
+
+    Returns
+    -------
+    numpy.nd.array
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    for x_i, y_i in nan_xy:
+        col = data[:, int(y_i)]
+        mu = col[~np.isnan(col)].mean()
+        std = col[~np.isnan(col)].std()
+        col[x_i] = np.random.normal(loc=mu, scale=std)
+        previous, i = 1, 1
+        while True:
+            i += 1
+            # Expectation
+            mu = col[~np.isnan(col)].mean()
+            std = col[~np.isnan(col)].std()
+            # Maximization
+            col[x_i] = np.random.normal(loc=mu, scale=std)
+            # Break out of loop if likelihood doesn't change at least 10%
+            # and has run at least 5 times
+            delta = np.abs(col[x_i]-previous)/previous
+            if i > 5 and delta < eps:
+                data[x_i][y_i] = col[x_i]
+                break
+            data[x_i][y_i] = col[x_i]
+            previous = col[x_i]
+    return data

impyute/cs/fast_knn.py

@@ -0,0 +1,130 @@
+import numpy as np
+import pandas as pd
+from scipy.spatial import KDTree
+from impyute.ops import matrix
+from impyute.ops import wrapper
+from impyute.ops import inverse_distance_weighting as idw
+
+from . import mean
+# pylint: disable=too-many-arguments
+
+@wrapper.wrappers
+@wrapper.checks
+def fast_knn(data, k=3, eps=0, p=2, distance_upper_bound=np.inf, leafsize=10,
+        idw_fn=idw.shepards, init_impute_fn=mean):
+    """ Impute using a variant of the nearest neighbours approach
+
+    Basic idea: Impute array with a passed in initial impute fn (mean impute)
+    and then use the resulting complete array to construct a KDTree. Use this
+    KDTree to compute nearest neighbours.  After finding `k` nearest
+    neighbours, take the weighted average of them. Basically, find the nearest
+    row in terms of distance
+
+    This approach is much, much faster than the other implementation (fit+transform
+    for each subset) which is almost prohibitively expensive.
+
+    Parameters
+    ----------
+    data: ndarray
+        2D matrix to impute.
+
+    k: int, optional
+        Parameter used for method querying the KDTree class object. Number of
+        neighbours used in the KNN query. Refer to the docs for
+        [`scipy.spatial.KDTree.query`]
+        (https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.KDTree.query.html).
+
+    eps: nonnegative float, optional
+        Parameter used for method querying the KDTree class object. From the
+        SciPy docs: "Return approximate nearest neighbors; the kth returned
+        value is guaranteed to be no further than (1+eps) times the distance to
+        the real kth nearest neighbor". Refer to the docs for
+        [`scipy.spatial.KDTree.query`]
+        (https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.KDTree.query.html).
+
+    p : float, 1<=p<=infinity, optional
+        Parameter used for method querying the KDTree class object. Straight from the
+        SciPy docs: "Which Minkowski p-norm to use. 1 is the
+        sum-of-absolute-values Manhattan distance 2 is the usual Euclidean
+        distance infinity is the maximum-coordinate-difference distance". Refer to
+        the docs for
+        [`scipy.spatial.KDTree.query`]
+        (https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.KDTree.query.html).
+
+    distance_upper_bound : nonnegative float, optional
+        Parameter used for method querying the KDTree class object. Straight
+        from the SciPy docs: "Return only neighbors within this distance. This
+        is used to prune tree searches, so if you are doing a series of
+        nearest-neighbor queries, it may help to supply the distance to the
+        nearest neighbor of the most recent point." Refer to the docs for
+        [`scipy.spatial.KDTree.query`]
+        (https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.KDTree.query.html).
+
+    leafsize: int, optional
+        Parameter used for construction of the `KDTree` class object. Straight from
+        the SciPy docs: "The number of points at which the algorithm switches
+        over to brute-force. Has to be positive". Refer to the docs for
+        [`scipy.spatial.KDTree`](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.spatial.KDTree.html)
+        for more information.
+
+    idw_fn: fn, optional
+        Function that takes one argument, a list of distances, and returns weighted percentages. You can define a custom
+        one or bootstrap from functions defined in `impy.util.inverse_distance_weighting` which can be using
+        functools.partial, for example: `functools.partial(impy.util.inverse_distance_weighting.shepards, power=1)`
+
+    init_impute_fn: fn, optional
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    Examples
+    --------
+
+        >>> data = np.arange(25).reshape((5, 5)).astype(np.float)
+        >>> data[0][2] =  np.nan
+        >>> data
+        array([[ 0.,  1., nan,  3.,  4.],
+               [ 5.,  6.,  7.,  8.,  9.],
+               [10., 11., 12., 13., 14.],
+               [15., 16., 17., 18., 19.],
+               [20., 21., 22., 23., 24.]])
+        >> fast_knn(data, k=1) # Weighted average (by distance) of nearest 1 neighbour
+        array([[ 0.,  1.,  7.,  3.,  4.],
+               [ 5.,  6.,  7.,  8.,  9.],
+               [10., 11., 12., 13., 14.],
+               [15., 16., 17., 18., 19.],
+               [20., 21., 22., 23., 24.]])
+        >> fast_knn(data, k=2) # Weighted average of nearest 2 neighbours
+        array([[ 0.        ,  1.        , 10.08608891,  3.        ,  4.        ],
+               [ 5.        ,  6.        ,  7.        ,  8.        ,  9.        ],
+               [10.        , 11.        , 12.        , 13.        , 14.        ],
+               [15.        , 16.        , 17.        , 18.        , 19.        ],
+               [20.        , 21.        , 22.        , 23.        , 24.        ]])
+        >> fast_knn(data, k=3)
+        array([[ 0.        ,  1.        , 13.40249283,  3.        ,  4.        ],
+               [ 5.        ,  6.        ,  7.        ,  8.        ,  9.        ],
+               [10.        , 11.        , 12.        , 13.        , 14.        ],
+               [15.        , 16.        , 17.        , 18.        , 19.        ],
+               [20.        , 21.        , 22.        , 23.        , 24.        ]])
+        >> fast_knn(data, k=5) # There are at most only 4 neighbours. Raises error
+        ...
+        IndexError: index 5 is out of bounds for axis 0 with size 5
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    data_c = init_impute_fn(data)
+    kdtree = KDTree(data_c, leafsize=leafsize)
+
+    for x_i, y_i in nan_xy:
+        distances, indices = kdtree.query(data_c[x_i], k=k+1, eps=eps,
+                                          p=p, distance_upper_bound=distance_upper_bound)
+        # Will always return itself in the first index. Delete it.
+        distances, indices = distances[1:], indices[1:]
+        # Add small constant to distances to avoid division by 0
+        distances += 1e-3
+        weights = idw_fn(distances)
+        # Assign missing value the weighted average of `k` nearest neighbours
+        data[x_i][y_i] = np.dot(weights, [data_c[ind][y_i] for ind in indices])
+    return data

impyute/cs/random.py

@@ -0,0 +1,27 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+
+@wrapper.wrappers
+@wrapper.checks
+def random_impute(data):
+    """ Fill missing values in with a randomly selected value from the same
+    column.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    nan_xy = matrix.nan_indices(data)
+    for x, y in nan_xy:
+        uniques = np.unique(data[:, y])
+        uniques = uniques[~np.isnan(uniques)]
+        data[x][y] = np.random.choice(uniques)
+    return data

impyute/dataset/__init__.py

@@ -0,0 +1,6 @@
+""" Real-world/mock datasets and missingness corruptors to experiment with.  """
+from .base import randu
+from .base import randn
+from .base import mnist
+
+__all__ = ["randu", "randn", "mnist"]