pyrolite 0.0.14__zip

ProgramData/Anaconda3_64/Lib/site-packages/pyrolite/comp/aggregate.py

@@ -0,0 +1,391 @@
+import numpy as np
+import pandas as pd
+import warnings
+from .codata import alr, inv_alr
+from .impute import *
+
+import logging
+
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+logger = logging.getLogger(__name__)
+
+
+def get_full_column(X: np.ndarray):
+    """
+    Returns the index of the first array column which contains only finite
+    numbers (i.e. no missing data, nan, inf).
+
+    Parameters
+    ---------------
+    X: np.ndarray
+        Array for which to find the first full column within.
+    """
+    if len(X.shape) == 1:
+        X = X.reshape((1, *X.shape))
+    inds = np.arange(X.shape[1])
+    wherenonnan = np.isfinite(X).all(axis=0)
+    ind = inds[wherenonnan][0]
+    return ind
+
+
+def weights_from_array(X: np.ndarray):
+    """
+    Returns a set of equal weights with size equal to that of the first axis of
+    an array.
+
+    Parameters
+    ---------------
+    X: np.ndarray
+        Array of compositions to produce weights for.
+    """
+    wts = np.ones((X.shape[0]))
+    return wts / np.sum(wts)
+
+
+def nan_weighted_mean(X: np.ndarray, weights=None):
+    """
+    Returns a weighted mean of compositions, where weights are renormalised
+    to account for missing data.
+
+    Parameters
+    ---------------
+    X: np.ndarray
+        Array of compositions to take a weighted mean of.
+    weights: np.ndarray
+        Array of weights.
+    """
+    if weights is None:
+        weights = weights_from_array(X)
+    weights = np.array(weights) / np.nansum(weights)
+
+    mask = (np.isnan(X) + np.isinf(X)) > 0
+    if not mask.any():
+        return np.average(X, weights=weights, axis=0)
+    else:
+        return np.ma.average(np.ma.array(X, mask=mask), weights=weights, axis=0)
+
+
+def compositional_mean(df, weights=[], **kwargs):
+    """
+    Implements an aggregation using a compositional weighted mean.
+
+    Parameters
+    ---------------
+    df: pd.DataFrame
+        Dataframe of compositions to aggregate.
+    weights: np.ndarray
+        Array of weights.
+    """
+    non_nan_cols = df.dropna(axis=1, how="all").columns
+    assert not df.loc[:, non_nan_cols].isna().values.any()
+    mean = df.iloc[0, :].copy()
+    if not weights:
+        weights = np.ones(len(df.index.values))
+    weights = np.array(weights) / np.nansum(weights)
+
+    logmean = alr(df.loc[:, non_nan_cols].values).T @ weights[:, np.newaxis]
+    # this renormalises by default
+    mean.loc[non_nan_cols] = inv_alr(logmean.T.squeeze())
+    return mean
+
+
+def nan_weighted_compositional_mean(
+    X: np.ndarray, weights=None, ind=None, renorm=True, **kwargs
+):
+    """
+    Implements an aggregation using a weighted mean, but accounts
+    for nans. Requires at least one non-nan column for alr mean.
+
+    When used for internal standardisation, there should be only a single
+    common element - this would be used by default as the divisor here. When
+    used for multiple-standardisation, the [specified] or first common
+    element will be used.
+
+    Parameters
+    ---------------
+    X: np.ndarray
+        Array of compositions to aggregate.
+    weights: np.ndarray
+        Array of weights.
+    ind: int
+        Index of the column to use as the alr divisor.
+    renorm: bool, True
+        Whether to renormalise the output compositional mean to unity.
+
+    Returns
+    -------
+    np.ndarray
+        An array with the mean composition.
+    """
+    if X.ndim == 1:  # if it's a single row
+        return X
+    else:
+        if weights is None:
+            weights = weights_from_array(X)
+        else:
+            weights = np.array(weights) / np.sum(weights, axis=-1)
+
+        if ind is None:  # take the first column which has no nans
+            ind = get_full_column(X)
+
+        if X.ndim < 3 and X.shape[0] == 1:
+            div = X[:, ind].squeeze()  # check this
+        else:
+            div = X[:, ind].squeeze()[:, np.newaxis]
+
+        logvals = np.log(np.divide(X, div))
+        mean = np.nan * np.ones(X.shape[1:])
+
+        ixs = np.arange(logvals.shape[1])
+        if X.ndim == 2:
+            indexes = ixs
+        elif X.ndim == 3:
+            iys = np.arange(logvals.shape[2])
+            indexes = np.ixs_(ixs, iys)
+
+        mean[indexes] = nan_weighted_mean(logvals[:, indexes], weights=weights)
+
+        mean = np.exp(mean.squeeze())
+        if renorm:
+            mean /= np.nansum(mean)
+        return mean
+
+
+def cross_ratios(df: pd.DataFrame):
+    """
+    Takes ratios of values across a dataframe, such that columns are
+    denominators and the row indexes the numerators, to create a square array.
+    Returns one array per record.
+
+    Parameters
+    ---------------
+    df: pd.DataFrame
+        Dataframe of compositions to create ratios of.
+
+    Returns
+    -------
+    np.ndarray
+        A 3D array of ratios.
+    """
+    ratios = np.ones((len(df.index), len(df.columns), len(df.columns)))
+    for idx in range(df.index.size):
+        row_vals = df.iloc[idx, :].values
+        r1 = row_vals.T[:, np.newaxis] @ np.ones_like(row_vals)[np.newaxis, :]
+        ratios[idx] = r1 / r1.T
+    return ratios
+
+
+def np_cross_ratios(X: np.ndarray, debug=False):
+    """
+    Takes ratios of values across an array, such that columns are
+    denominators and the row indexes the numerators, to create a square array.
+    Returns one array per record.
+
+    Parameters
+    ---------------
+    X: np.ndarray
+        Array of compositions to create ratios of.
+
+    Returns
+    -------
+    np.ndarray
+        A 3D array of ratios.
+    """
+    X = X.copy()
+    with warnings.catch_warnings():
+        # can get invalid values which raise RuntimeWarnings
+        # consider changing to np.errstate
+        warnings.simplefilter("ignore", category=RuntimeWarning)
+        X[X <= 0] = np.nan
+    if X.ndim == 1:
+        index_length = 1
+        X = X.reshape((1, *X.shape))
+    else:
+        index_length = X.shape[0]
+    dims = X.shape[-1]
+    ratios = np.ones((index_length, dims, dims))
+    for idx in range(index_length):
+        row_vals = X[idx, :]
+        r1 = row_vals.T[:, np.newaxis] @ np.ones_like(row_vals)[np.newaxis, :]
+        ratios[idx] = r1.T / r1
+
+    if debug:
+        try:
+            diags = ratios[:, np.arange(dims), np.arange(dims)]
+            # check all diags are 1.
+            assert np.allclose(diags, 1.0)
+        except:
+            # check all diags are 1. or nan
+            assert np.allclose(diags[~np.isnan(diags)], 1.0)
+
+    return ratios
+
+
+def standardise_aggregate(
+    df: pd.DataFrame, int_std=None, fixed_record_idx=0, renorm=True, **kwargs
+):
+    """
+    Performs internal standardisation and aggregates dissimilar geochemical
+    records. Note: this changes the closure parameter, and is generally
+    intended to integrate major and trace element records.
+
+    Parameters
+    ---------------
+    df: pd.DataFrame
+        Dataframe of compositions to aggregate of.
+    int_std: str
+        Name of the internal standard column.
+    fixed_record_idx: int
+        Numeric index of a specific record's for which to retain the internal
+        standard value (e.g for standardising trace element data).
+    renorm: bool, True
+        Whether to renormalise to unity.
+
+    Returns
+    -------
+    pd.Series
+        A series representing the internally standardised record.
+    """
+    if df.index.size == 1:  # catch single records
+        return df
+    else:
+        if int_std is None:
+            # Get the 'internal standard column'
+            potential_int_stds = df.count()[df.count() == df.count().max()].index.values
+            assert len(potential_int_stds) > 0
+            # Use an internal standard
+            int_std = potential_int_stds[0]
+            if len(potential_int_stds) > 1:
+                logging.info("Multiple int. stds possible. Using " + str(int_std))
+
+        non_nan_cols = df.dropna(axis=1, how="all").columns
+        assert len(non_nan_cols)
+        mean = nan_weighted_compositional_mean(
+            df.values, ind=df.columns.get_loc(int_std), renorm=False
+        )
+        ser = pd.Series(mean, index=df.columns)
+        multiplier = (
+            df.iloc[fixed_record_idx, df.columns.get_loc(int_std)] / ser[int_std]
+        )
+        ser *= multiplier
+        if renorm:
+            ser /= np.nansum(ser.values)
+        return ser
+
+
+def complex_standardise_aggregate(
+    df, int_std=None, fixed_record_idx=0, renorm=True  # fallback parameters
+):
+    """
+    Aggregation function which attempts to incorporate all ratio information
+    but creating a ratio matrix for each record which is then imputed for
+    missing data where possible. Falls back to internal standardistion where
+    int_std is specified.
+
+    Parameters
+    ---------------
+    df: pd.DataFrame
+        Dataframe of compositions to aggregate of.
+    int_std: str
+        Name of the internal standard column.
+    fixed_record_idx: int
+        Numeric index of a specific record's for which to retain the internal
+        standard value (e.g for standardising trace element data).
+    renorm: bool, True
+        Whether to renormalise to unity.
+
+    Returns
+    -------
+    pd.Series
+        A series representing the internally standardised record.
+    """
+    if int_std is None:
+        # create a n x d x d matrix for aggregating ratios
+        non_nan_cols = df.dropna(axis=1, how="all").columns
+        ratios = cross_ratios(df.loc[:, non_nan_cols])
+        # Average across record matricies
+        with warnings.catch_warnings():
+            # can get empty arrays which raise RuntimeWarnings
+            # consider changing to np.errstate
+            warnings.simplefilter("ignore", category=RuntimeWarning)
+            mean_ratios = pd.DataFrame(
+                np.exp(np.nanmean(np.log(ratios), axis=0)),
+                columns=non_nan_cols,
+                index=non_nan_cols,
+            )
+        # Filling in the null values in a ratio matrix
+        imputed_ratios = impute_ratios(mean_ratios)
+        # We simply pick the first non-nan column.
+        IS = non_nan_cols[0]
+        mean = np.exp(
+            np.mean(np.log(imputed_ratios / imputed_ratios.loc[IS, :]), axis=1)
+        )
+        # This needs to be renormalised to make logical sense
+        mean /= np.nansum(mean.values)
+
+        out = np.ones((1, len(df.columns))) * np.nan
+        out[:, [list(df.columns).index(c) for c in non_nan_cols]] = mean
+        return pd.Series(out.squeeze(), index=df.columns)
+    else:
+        # fallback to internal standardisation
+        return standardise_aggregate(
+            df, int_std=int_std, fixed_record_idx=fixed_record_idx, renorm=renorm
+        )
+
+
+def np_complex_standardise_aggregate(
+    df, int_std=None, renorm=True, fixed_record_idx=0  # fallback parameters
+):
+    """
+    Aggregation function which attempts to incorporate all ratio information
+    but creating a ratio matrix for each record which is then imputed for
+    missing data where possible. Falls back to internal standardistion where
+    int_std is specified. Uses numpy more extensively.
+
+    Parameters
+    ---------------
+    df: pd.DataFrame
+        Dataframe of compositions to aggregate of.
+    int_std: str
+        Name of the internal standard column.
+    fixed_record_idx: int
+        Numeric index of a specific record's for which to retain the internal
+        standard value (e.g for standardising trace element data).
+    renorm: bool, True
+        Whether to renormalise to unity.
+
+    Returns
+    -------
+    pd.Series
+        A series representing the internally standardised record.
+    """
+
+    if int_std is None:
+        # create a n x d x d matrix for aggregating ratios
+        non_nan_cols = df.dropna(axis=1, how="all").columns
+        assert len(non_nan_cols) > 0
+        ratios = np_cross_ratios(df.loc[:, non_nan_cols].values)
+        # Take the mean across the cross-ratio matricies
+        with warnings.catch_warnings():
+            # can get empty arrays which raise RuntimeWarnings
+            # consider changing to np.errstate
+            warnings.simplefilter("ignore", category=RuntimeWarning)
+            mean_logratios = np.nanmean(np.log(ratios), axis=0)
+        # Filling in the null values in a ratio matrix
+        imputed_log_ratios = np_impute_ratios(mean_logratios)
+        # We simply pick the first non-nan column.
+        # IS = 0
+        IS = np.argmax(np.count_nonzero(~np.isnan(imputed_log_ratios), axis=0))
+        # Convert to a composition by subtracting a row and taking negative
+        div_log_ratios = -(imputed_log_ratios - imputed_log_ratios[IS, :])
+        comp_abund = np.exp(np.nanmean(div_log_ratios, axis=1))
+        comp_abund /= np.nansum(comp_abund)
+        out = np.ones((1, len(df.columns))) * np.nan
+        inds = np.array([list(df.columns).index(c) for c in non_nan_cols])
+        out[:, inds] = comp_abund
+        return pd.Series(out.squeeze(), index=df.columns)
+    else:
+        # fallback to internal standardisation
+        return standardise_aggregate(
+            df, int_std=int_std, fixed_record_idx=fixed_record_idx, renorm=renorm
+        )

ProgramData/Anaconda3_64/Lib/site-packages/pyrolite/comp/codata.py

@@ -0,0 +1,266 @@
+import numpy as np
+import pandas as pd
+import pandas_flavor as pf
+import scipy.stats as scpstats
+import scipy.special as scpspec
+#from .renorm import renormalise, close
+from ..util.math import orthagonal_basis
+import logging
+
+logging.getLogger(__name__).addHandler(logging.NullHandler())
+logger = logging.getLogger(__name__)
+
+
+def close(X: np.ndarray):
+    if X.ndim == 2:
+        return np.divide(X, np.sum(X, axis=1)[:, np.newaxis])
+    else:
+        return np.divide(X, np.sum(X, axis=0))
+
+@pf.register_series_method
+@pf.register_dataframe_method
+def renormalise(df: pd.DataFrame, components: list = [], scale=100.0):
+    """
+    Renormalises compositional data to ensure closure.
+
+    Parameters
+    ------------
+    df: pd.DataFrame
+        Dataframe to renomalise.
+    components: list
+        Option subcompositon to renormalise to 100. Useful for the use case
+        where compostional data and non-compositional data are stored in the
+        same dataframe.
+    scale: float, 100.
+        Closure parameter. Typically either 100 or 1.
+    """
+    dfc = df.copy(deep=True)
+    if components:
+        cmpnts = [c for c in components if c in dfc.columns]
+        dfc.loc[:, cmpnts] = scale * dfc.loc[:, cmpnts].divide(
+            dfc.loc[:, cmpnts].sum(axis=1).replace(0, np.nan), axis=0
+        )
+        return dfc
+    else:
+        dfc = dfc.divide(dfc.sum(axis=1).replace(0, 100.0), axis=0) * scale
+        return dfc
+
+
+def additive_log_ratio(X: np.ndarray, ind: int = -1, null_col=False):
+    """
+    Inverse Additive Log Ratio transformation.
+
+    Parameters
+    ---------------
+    X: np.ndarray
+        Array on which to perform the inverse transformation.
+    ind: int
+        Index of column used as denominator.
+    """
+
+    Y = X.copy()
+    assert Y.ndim in [1, 2]
+    dimensions = Y.shape[Y.ndim - 1]
+    if ind < 0:
+        ind += dimensions
+
+    if Y.ndim == 2:
+        Y = np.divide(Y, Y[:, ind][:, np.newaxis])
+        if not null_col:
+            Y = Y[:, [i for i in range(dimensions) if not i == ind]]
+    else:
+        Y = np.divide(X, X[ind])
+        if not null_col:
+            Y = Y[[i for i in range(dimensions) if not i == ind]]
+
+    return np.log(Y)
+
+
+def inverse_additive_log_ratio(Y: np.ndarray, ind=-1, null_col=False):
+    """
+    Inverse Centred Log Ratio transformation.
+
+    Parameters
+    ---------------
+    X: np.ndarray
+        Array on which to perform the inverse transformation.
+    ind: int
+        Index of column used as denominator.
+    """
+    assert Y.ndim in [1, 2]
+
+    X = Y.copy()
+    dimensions = X.shape[X.ndim - 1]
+    if not null_col:
+        idx = np.arange(0, dimensions + 1)
+
+        if ind != -1:
+            idx = np.array(list(idx[idx < ind]) + [-1] + list(idx[idx >= ind + 1] - 1))
+
+        # Add a zero-column and reorder columns
+        if Y.ndim == 2:
+            X = np.concatenate((X, np.zeros((X.shape[0], 1))), axis=1)
+            X = X[:, idx]
+        else:
+            X = np.append(X, np.array([0]))
+            X = X[idx]
+
+    # Inverse log and closure operations
+    X = np.exp(X)
+    X = close(X)
+    return X
+
+
+def alr(*args, **kwargs):
+    """
+    Short form of Additive Log Ratio transformation.
+
+    Parameters
+    ---------------
+    Y: np.ndarray
+        Array on which to perform the inverse transformation.
+    ind: int
+        Index of column used as denominator.
+    """
+    return additive_log_ratio(*args, **kwargs)
+
+
+def inv_alr(*args, **kwargs):
+    """
+    Short form of Inverse Additive Log Ratio transformation.
+
+    Parameters
+    ---------------
+    Y: np.ndarray
+        Array on which to perform the inverse transformation.
+    ind: int
+        Index of column used as denominator.
+    """
+    return inverse_additive_log_ratio(*args, **kwargs)
+
+
+def clr(X: np.ndarray):
+    """
+    Centred Log Ratio transformation.
+
+    Parameters
+    ---------------
+    X: np.ndarray
+        Array on which to perform the transformation.
+    """
+    X = np.divide(X, np.sum(X, axis=1)[:, np.newaxis])  # Closure operation
+    Y = np.log(X)  # Log operation
+    Y -= 1 / X.shape[1] * np.nansum(Y, axis=1)[:, np.newaxis]
+    return Y
+
+
+def inv_clr(Y: np.ndarray):
+    """
+    Inverse Centred Log Ratio transformation.
+
+    Parameters
+    ---------------
+    Y: np.ndarray
+        Array on which to perform the inverse transformation.
+    """
+    # Inverse of log operation
+    X = np.exp(Y)
+    # Closure operation
+    X = np.divide(X, np.nansum(X, axis=1)[:, np.newaxis])
+    return X
+
+
+def ilr(X: np.ndarray):
+    """
+    Isotmetric Log Ratio transformation.
+
+    Parameters
+    ---------------
+    X: np.ndarray
+        Array on which to perform the transformation.
+    """
+    d = X.shape[1]
+    Y = clr(X)
+    psi = orthagonal_basis(X)  # Get a basis
+    psi = orthagonal_basis(clr(X))  # trying to get right algorithm
+    assert np.allclose(psi @ psi.T, np.eye(d - 1))
+    return Y @ psi.T
+
+
+def inv_ilr(Y: np.ndarray, X: np.ndarray = None):
+    """
+    Inverse Isometric Log Ratio transformation.
+
+    Parameters
+    ---------------
+    Y: np.ndarray
+        Array on which to perform the inverse transformation.
+    """
+    psi = orthagonal_basis(X)
+    C = Y @ psi
+    X = inv_clr(C)  # Inverse log operation
+    return X
+
+
+def boxcox(
+    X: np.ndarray,
+    lmbda=None,
+    lmbda_search_space=(-1, 5),
+    search_steps=100,
+    return_lmbda=False,
+):
+    """
+    Box-Cox transformation.
+
+    Parameters
+    ---------------
+    Y: np.ndarray
+        Array on which to perform the transformation.
+    lmbda: {None, np.float}
+        Lambda value used to forward-transform values. If none, it will be calculated
+        using the mean
+    """
+    if isinstance(X, pd.DataFrame) or isinstance(X, pd.Series):
+        _X = X.values
+    else:
+        _X = X.copy()
+
+    if lmbda is None:
+        l_search = np.linspace(*lmbda_search_space, search_steps)
+        llf = np.apply_along_axis(scpstats.boxcox_llf, 0, np.array([l_search]), _X.T)
+        if llf.shape[0] == 1:
+            mean_llf = llf[0]
+        else:
+            mean_llf = np.nansum(llf, axis=0)
+
+        lmbda = l_search[mean_llf == np.nanmax(mean_llf)]
+    if _X.ndim < 2:
+        out = scpstats.boxcox(_X, lmbda)
+    elif _X.shape[0] == 1:
+        out = scpstats.boxcox(np.squeeze(_X), lmbda)
+    else:
+        out = np.apply_along_axis(scpstats.boxcox, 0, _X, lmbda)
+
+    if isinstance(_X, pd.DataFrame) or isinstance(_X, pd.Series):
+        _out = X.copy()
+        _out.loc[:, :] = out
+        out = _out
+
+    if return_lmbda:
+        return out, lmbda
+    else:
+        return out
+
+
+def inv_boxcox(Y: np.ndarray, lmbda):
+    """
+    Inverse Box-Cox transformation.
+
+    Parameters
+    ---------------
+    Y: np.ndarray
+        Array on which to perform the transformation.
+    lmbda: np.float
+        Lambda value used to forward-transform values.
+    """
+    return scpspec.inv_boxcox(Y, lmbda)