cpgtools 2.0.5__py3-none-any.whl

impyute/ops/util.py

@@ -0,0 +1,96 @@
+""" Random utility functions """
+from functools import wraps
+import numpy as np
+import pandas as pd
+
+# Things that get exposed from * import
+__all__ = [
+    "constantly", "complement", "identity", "thread",
+    "execute_fn_with_args_and_or_kwargs", "toy_df",
+    "insert_na",
+    ]
+
+def thread(arg, *fns):
+    if len(fns) > 0:
+        return thread(fns[0](arg), *fns[1:])
+    else:
+        return arg
+
+def identity(x):
+    return x
+
+def constantly(x):
+    """ Returns a function that takes any args and returns x """
+    def func(*args, **kwargs):
+        return x
+    return func
+
+def complement(fn):
+    """ Return fn that outputs the opposite truth values of the
+    input function
+    """
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        return not fn(*args, **kwargs)
+    return wrapper
+
+def execute_fn_with_args_and_or_kwargs(fn, args, kwargs):
+    """ If args + kwargs aren't accepted only args are passed in"""
+    try:
+        return fn(*args, **kwargs)
+    except TypeError:
+        return fn(*args)
+
+def toy_df(n_rows=20, n_cols=5, missingness=0.2, min_val=0, max_val=1,
+              missing_value=np.nan, rand_seed=1234, sample_prefix=None):
+    """Generate an array or DataFrame with NaNs"""
+    np.random.seed(rand_seed)
+    X = np.random.uniform(
+        low = min_val, high = max_val, size = n_rows * n_cols).reshape(n_rows, n_cols).astype(
+        float)
+    # check missingness
+    if missingness > 0:
+        # If missingness >= 1 then use it as approximate (see below) count
+        if missingness >= 1:
+            n_missing = int(missingness)
+        else:
+            n_missing = int(missingness * n_rows * n_cols)
+            print(n_missing)
+    
+    # Introduce NaNs until n_miss "NAs" are inserted.
+    missing_count = 0
+    for i,j in zip(np.random.choice(n_rows, n_missing), np.random.choice(n_cols, n_missing)):
+        if np.isnan(X[i][j]):
+            continue
+        else:
+            X[i][j] = missing_value
+            missing_count += 1
+        if missing_count >= n_missing:
+            break
+
+    # check sample_prefix
+    if sample_prefix is None:
+        return X
+    else:
+        colNames = [sample_prefix + '_' + str(i) for i in range(0, n_cols)]
+        return pd.DataFrame(X, columns=colNames)
+    
+
+def insert_na(df, n_miss, seed):
+    np.random.seed(seed)
+    nrow,ncol = df.shape
+    na_count = 0
+    if n_miss >= nrow*ncol:
+        out_df = df.replace(df.values, np.nan)
+    else:
+        tmp = df.to_numpy()
+        while(1):
+            if na_count >= n_miss:
+                break
+            x_ind = np.random.choice(nrow)
+            y_ind = np.random.choice(ncol)
+            if not np.isnan(tmp[x_ind][y_ind]):
+                tmp[x_ind][y_ind] = np.nan
+                na_count += 1
+        out_df = pd.DataFrame(tmp, index=df.index, columns=df.columns)
+    return out_df

impyute/ops/wrapper.py

@@ -0,0 +1,179 @@
+""" Decorator functions to wrap around entry and exit
+
+... to easily apply to a function, functions that check/process inputs
+and outputs
+"""
+from functools import wraps
+import numpy as np
+import pandas as pd
+
+from . import error
+from . import matrix
+from . import util as u
+
+## Hacky way to handle python2 not having `ModuleNotFoundError`
+# pylint: disable=redefined-builtin, missing-docstring
+try:
+    raise ModuleNotFoundError
+except NameError:
+    class ModuleNotFoundError(Exception):
+        pass
+except ModuleNotFoundError:
+    pass
+# pylint: enable=redefined-builtin, missing-docstring
+
+
+def handle_df(fn):
+    """ Decorator to handle pandas Dataframe object as input
+
+    If the first arg is a pandas dataframe, convert it to a numpy array
+    otherwise don't do anything. Cast back to a pandas Dataframe after
+    the imputation function has run
+    """
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        is_df = False
+        ## convert tuple to list so args can be modified
+        args = list(args)
+        ## Either make a copy or use a pointer to the original
+        if kwargs.get('inplace'):
+            args[0] = args[0]
+        else:
+            args[0] = args[0].copy()
+
+        ## If input data is a dataframe then cast the input to an np.array
+        ## and set an indicator flag before continuing
+        if isinstance(args[0], pd.DataFrame):
+            is_df = True
+            in_ind = args[0].index
+            in_columns = args[0].columns
+            args[0] = args[0].to_numpy()
+
+        ## function invokation
+        results = u.execute_fn_with_args_and_or_kwargs(fn, args, kwargs)
+
+        ## cast the output back to a DataFrame.
+        if is_df:
+            results = pd.DataFrame(results, index=in_ind, columns=in_columns)
+        return results
+    return wrapper
+
+def add_inplace_option(fn):
+    """ Decorator for inplace option
+
+    Functions wrapped by this can have an `inplace` kwarg to use either a copy of
+    data or reference """
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        """ Run input checks"""
+        ## convert tuple to list so args can be modified
+        args = list(args)
+        ## Either make a copy or use a pointer to the original
+        if kwargs.get('inplace'):
+            args[0] = args[0]
+        else:
+            args[0] = args[0].copy()
+
+        ## function invokation
+        return u.execute_fn_with_args_and_or_kwargs(fn, args, kwargs)
+    return wrapper
+
+def conform_output(fn):
+    """ Decorator to handle impossible values
+
+    Adds two optional kwargs, `coerce_fn` and `valid_fn`.
+
+    `valid_fn` function stub
+
+        def my_coerce_fn(some_literal) -> boolean
+
+    `coerce_fn` function stub
+
+        def my_coerce_fn(arr, x_i, y_i) -> some_literal
+
+    Valid function is something run on each element of the, this is
+    the function that we use to indicate whether the value is valid
+    or not
+
+    Coerce function has three arguments, the original matrix and
+    the two indices of the invalid value x_i and y_i. This function
+    will be run on all invalid values.
+    """
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        def raise_error(arr, x_i, y_i):
+            raise error.BadOutputError("{} does not conform".format(arr[x_i, y_i]))
+        ## convert tuple to list so args can be modified
+        args = list(args)
+        # function that checks if the value is valid
+        valid_fn = kwargs.get("valid_fn", u.constantly(True))
+        # function that modifies the invalid value to something valid
+        coerce_fn = kwargs.get("coerce_fn", raise_error)
+
+        ## function invokation
+        results = u.execute_fn_with_args_and_or_kwargs(fn, args, kwargs)
+
+        # check each value to see if it's valid
+        bool_arr = matrix.map_nd(u.complement(valid_fn), results)
+        # get indices of invalid values
+        invalid_indices = np.argwhere(bool_arr)
+        # run the coerce fn on each invalid indice
+        for x_i, y_i in invalid_indices:
+            results[x_i, y_i] = coerce_fn(results, x_i, y_i)
+
+        return results
+    return wrapper
+
+def wrappers(fn):
+    """ Helper decorator, all wrapper functions applied to modify input (matrix
+    with missing values) and output (matrix with imputed values)
+
+    NOTE: `handle_df` has to be last as it needs to be in the outer loop (first
+    entry point) since every other function assumes you're getting an np.array
+    as input
+    """
+    return u.thread(
+        fn,                 # function that's getting wrapped
+        add_inplace_option, # allow choosing reference/copy
+        conform_output,     # allow enforcing of some spec on returned outputs
+        handle_df,          # if df type, cast to np.array on in and df on out
+    )
+
+def _shape_2d(data):
+    """ True if array is 2D"""
+    return len(np.shape(data)) == 2
+
+def _shape_3d(data):
+    """ True if array is 3D"""
+    return len(np.shape(data)) == 3
+
+def _is_ndarray(data):
+    """ True if the array is an instance of numpy's ndarray"""
+    return isinstance(data, np.ndarray)
+
+def _dtype_float(data):
+    """ True if the values in the array are floating point"""
+    return data.dtype == float
+
+def _nan_exists(data):
+    """ True if there is at least one np.nan in the array"""
+    nan_xy = matrix.nan_indices(data)
+    return len(nan_xy) > 0
+
+def checks(fn):
+    """ Throw exception if error runs"""
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        data = args[0]
+        if len(np.shape(data)) != 2:
+            raise error.BadInputError("No support for arrays that aren't 2D yet.")
+        elif not _shape_2d(data):
+            raise error.BadInputError("Not a 2D array.")
+        elif not _is_ndarray(data):
+            raise error.BadInputError("Not a np.ndarray.")
+        elif not _dtype_float(data):
+            raise error.BadInputError("Data is not float.")
+        elif not _nan_exists(data):
+            raise error.BadInputError("No NaN's in given data")
+        return u.execute_fn_with_args_and_or_kwargs(fn, args, kwargs)
+    return wrapper

impyute/ts/__init__.py

@@ -0,0 +1,6 @@
+""" Imputations for time-series data.  """
+
+from .locf import locf
+from .moving_window import moving_window
+
+__all__ = ["locf", "moving_window"]

impyute/ts/locf.py

@@ -0,0 +1,57 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+from impyute.ops import error
+
+@wrapper.wrappers
+@wrapper.checks
+def locf(data, axis=0):
+    """ Last Observation Carried Forward
+
+    For each set of missing indices, use the value of one row before(same
+    column). In the case that the missing value is the first row, look one
+    row ahead instead. If this next row is also NaN, look to the next row.
+    Repeat until you find a row in this column that's not NaN. All the rows
+    before will be filled with this value.
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        Data to impute.
+    axis: boolean (optional)
+        0 if time series is in row format (Ex. data[0][:] is 1st data point).
+        1 if time series is in col format (Ex. data[:][0] is 1st data point).
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    if axis == 0:
+        data = np.transpose(data)
+    elif axis == 1:
+        pass
+    else:
+        raise error.BadInputError("Error: Axis value is invalid, please use either 0 (row format) or 1 (column format)")
+
+    nan_xy = matrix.nan_indices(data)
+    for x_i, y_i in nan_xy:
+        # Simplest scenario, look one row back
+        if x_i-1 > -1:
+            data[x_i][y_i] = data[x_i-1][y_i]
+        # Look n rows forward
+        else:
+            x_residuals = np.shape(data)[0]-x_i-1  # n datapoints left
+            val_found = False
+            for i in range(1, x_residuals):
+                if not np.isnan(data[x_i+i][y_i]):
+                    val_found = True
+                    break
+            if val_found:
+                # pylint: disable=undefined-loop-variable
+                for x_nan in range(i):
+                    data[x_i+x_nan][y_i] = data[x_i+i][y_i]
+            else:
+                raise Exception("Error: Entire Column is NaN")
+    return data

impyute/ts/moving_window.py

@@ -0,0 +1,128 @@
+import numpy as np
+from impyute.ops import matrix
+from impyute.ops import wrapper
+# pylint: disable=invalid-name, too-many-arguments, too-many-locals, too-many-branches, broad-except, len-as-condition
+
+@wrapper.wrappers
+@wrapper.checks
+def moving_window(data, nindex=None, wsize=5, errors="coerce", func=np.mean,
+        inplace=False):
+    """ Interpolate the missing values based on nearby values.
+
+    For example, with an array like this:
+
+        array([[-1.24940, -1.38673, -0.03214945,  0.08255145, -0.007415],
+               [ 2.14662,  0.32758 , -0.82601414,  1.78124027,  0.873998],
+               [-0.41400, -0.977629,         nan, -1.39255344,  1.680435],
+               [ 0.40975,  1.067599,  0.29152388, -1.70160145, -0.565226],
+               [-0.54592, -1.126187,  2.04004377,  0.16664863, -0.010677]])
+
+    Using a `k` or window size of 3. The one missing value would be set
+    to -1.18509122. The window operates on the horizontal axis.
+
+    Usage
+    -----
+
+    The parameters default the function to a moving mean. You may want to change
+    the default window size:
+
+        moving_window(data, wsize=10)
+
+    To only look at past data (null value is at the rightmost index in the window):
+
+        moving_window(data, nindex=-1)
+
+    To use a custom function:
+
+        moving_window(data, func=np.median)
+
+    You can also do something like take 1.5x the max of previous values in the window:
+
+        moving_window(data, func=lambda arr: max(arr) * 1.50, nindex=-1)
+
+    Parameters
+    ----------
+    data: numpy.ndarray
+        2D matrix to impute.
+    nindex: int
+        Null index. Index of the null value inside the moving average window.
+        Use cases: Say you wanted to make value skewed toward the left or right
+        side. 0 would only take the average of values from the right and -1
+        would only take the average of values from the left
+    wsize: int
+        Window size. Size of the moving average window/area of values being used
+        for each local imputation. This number includes the missing value.
+    errors: {"raise", "coerce", "ignore"}
+        Errors will occur with the indexing of the windows - for example if there
+        is a nan at data[x][0] and `nindex` is set to -1 or there is a nan at
+        data[x][-1] and `nindex` is set to 0. `"raise"` will raise an error,
+        `"coerce"` will try again using an nindex set to the middle and `"ignore"`
+        will just leave it as a nan.
+    inplace: {True, False}
+        Whether to return a copy or run on the passed-in array
+
+    Returns
+    -------
+    numpy.ndarray
+        Imputed data.
+
+    """
+    if errors == "ignore":
+        raise Exception("`errors` value `ignore` not implemented yet. Sorry!")
+
+    if not inplace:
+        data = data.copy()
+
+    if nindex is None: # If using equal window side lengths
+        assert wsize % 2 == 1, "The parameter `wsize` should not be even "\
+        "if the value `nindex` is not set since it defaults to the midpoint "\
+        "and an even `wsize` makes the midpoint ambiguous"
+        wside_left = wsize // 2
+        wside_right = wsize // 2
+    else: # If using custom window side lengths
+        assert nindex < wsize, "The null index must be smaller than the window size"
+        if nindex == -1:
+            wside_left = wsize - 1
+            wside_right = 0
+        else:
+            wside_left = nindex
+            wside_right = wsize - nindex - 1
+
+    while True:
+        nan_xy = matrix.nan_indices(data)
+        n_nan_prev = len(nan_xy)
+        for x_i, y_i in nan_xy:
+            left_i = max(0, y_i-wside_left)
+            right_i = min(len(data), y_i+wside_right+1)
+            window = data[x_i, left_i: right_i]
+            window_not_null = window[~np.isnan(window)]
+
+            if len(window_not_null) > 0:
+                try:
+                    data[x_i][y_i] = func(window_not_null)
+                    continue
+                except Exception as e:
+                    if errors == "raise":
+                        raise e
+
+            if errors == "coerce":
+                # If either the window has a length of 0 or the aggregate function fails somehow,
+                # do a fallback of just trying the best we can by using it as the middle and trying
+                # to recalculate. Use temporary wside_left/wside_right, for only the calculation of
+                # this specific problamatic value
+                wside_left_tmp = wsize // 2
+                wside_right_tmp = wside_left_tmp
+
+                left_i_tmp = max(0, y_i-wside_left_tmp)
+                right_i_tmp = min(len(data), y_i+wside_right_tmp+1)
+
+                window = data[x_i, left_i_tmp:right_i_tmp]
+                window_not_null = window[~np.isnan(window)]
+                try:
+                    data[x_i][y_i] = func(window_not_null)
+                except Exception as e:
+                    print("Exception:", e)
+        if n_nan_prev == len(matrix.nan_indices(data)):
+            break
+
+    return data