scitex 2.0.0__py2.py3-none-any.whl

scitex/stats/desc/_describe.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-12-05 09:20:53 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/stats/desc/_describe.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/stats/desc/_describe.py"
+
+"""
+Functionality:
+    - Computes descriptive statistics on PyTorch tensors
+Input:
+    - PyTorch tensor or numpy array
+Output:
+    - Descriptive statistics (mean, std, quantiles, etc.)
+Prerequisites:
+    - PyTorch, NumPy
+"""
+
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+from ...decorators import batch_fn, torch_fn
+from ._nan import (
+    nancount,
+    nankurtosis,
+    nanmax,
+    nanmean,
+    nanmin,
+    nanq25,
+    nanq50,
+    nanq75,
+    nanskewness,
+    nanstd,
+    nanvar,
+)
+from ._real import kurtosis, mean, q25, q50, q75, skewness, std
+
+
+def verify_non_leakage(
+    x: torch.Tensor,
+    dim: Optional[Union[int, Tuple[int, ...]]] = None,
+):
+    """
+    Verifies that statistics computation doesn't leak information across samples.
+
+    Parameters
+    ----------
+    x : torch.Tensor
+        Input tensor
+    dim : Optional[Union[int, Tuple[int, ...]]]
+        Dimension(s) used for computation
+
+    Returns
+    -------
+    bool
+        True if verification passes
+
+    Raises
+    ------
+    AssertionError
+        If statistics leak information across samples
+    """
+    # Full calculation
+    described, _ = describe(x, dim=(1, 2))
+
+    # Compute statistics on first sample
+    x_first = x[:1]
+    described_first, _ = describe(x_first, dim=dim)
+
+    # Verify shapes match
+    assert (
+        described_first.shape == described[:1].shape
+    ), f"Shape mismatch: {described_first.shape} != {described[:1].shape}"
+
+    # Verify values match
+    torch.testing.assert_close(
+        described_first,
+        described[:1],
+        rtol=1e-5,
+        atol=1e-8,
+        msg="Statistics leak information across samples",
+    )
+
+    return True
+
+
+@batch_fn
+@torch_fn
+def describe(
+    x: torch.Tensor,
+    axis: int = -1,
+    dim: Optional[Union[int, Tuple[int, ...]]] = None,
+    keepdims: bool = False,
+    funcs: Union[List[str], str] = [
+        "nanmean",
+        "nanstd",
+        "nankurtosis",
+        "nanskewness",
+        "nanq25",
+        "nanq50",
+        "nanq75",
+    ],
+    device: Optional[torch.device] = None,
+    batch_size: int = -1,
+) -> Tuple[torch.Tensor, List[str]]:
+    """
+    Computes various descriptive statistics.
+
+    Parameters
+    ----------
+    x : torch.Tensor
+        Input tensor
+    axis : int, default=-1
+        Deprecated. Use dim instead
+    dim : int or tuple of ints, optional
+        Dimension(s) along which to compute statistics
+    keepdims : bool, default=True
+        Whether to keep reduced dimensions
+    funcs : list of str or "all"
+        Statistical functions to compute
+    device : torch.device, optional
+        Device to use for computation
+
+    Returns
+    -------
+    Tuple[torch.Tensor, List[str]]
+        Computed statistics and their names
+    """
+    dim = axis if dim is None else dim
+    dim = (dim,) if isinstance(dim, int) else tuple(dim)
+
+    func_names = funcs
+    func_candidates = {
+        "mean": mean,
+        "std": std,
+        "kurtosis": kurtosis,
+        "skewness": skewness,
+        "q25": q25,
+        "q50": q50,
+        "q75": q75,
+        "nanmean": nanmean,
+        "nanstd": nanstd,
+        "nanvar": nanvar,
+        "nankurtosis": nankurtosis,
+        "nanskewness": nanskewness,
+        "nanq25": nanq25,
+        "nanq50": nanq50,
+        "nanq75": nanq75,
+        "nanmax": nanmax,
+        "nanmin": nanmin,
+        "nancount": nancount,
+        # "nanprod": nanprod,
+        # "nanargmin": nanargmin,
+        # "nanargmax": nanargmax,
+    }
+
+    if funcs == "all":
+        _funcs = list(func_candidates.values())
+        func_names = list(func_candidates.keys())
+    else:
+        _funcs = [func_candidates[ff] for ff in func_names]
+
+    calculated = [ff(x, dim=dim, keepdims=keepdims) for ff in _funcs]
+    return torch.stack(calculated, dim=-1), func_names
+
+
+if __name__ == "__main__":
+    from scitex.stats.desc._describe import describe, verify_non_leakage
+
+    # x = np.random.rand(4, 3, 2)
+    # x = np.random.rand(390, 250, 16, 100, 100)
+    # print(scitex.stats.desc.nankurtosis(x, dim=(1,2)).shape)
+
+    x = np.random.rand(10, 250, 16, 100, 100)
+
+    described, _ = describe(x[:10], dim=(-2, -1), batch_size=1)
+    # verify_non_leakage(x, dim=(1, 2))
+    # # print(describe(x, dim=(1, 2), keepdims=False)[0].shape)
+    # # print(describe(x, funcs="all", dim=(1, 2), keepdims=False)[0].shape)
+
+
+"""
+python ./scitex_repo/src/scitex/stats/desc/_describe.py
+python -m src.scitex.stats.desc._describe
+"""
+
+# EOF

scitex/stats/desc/_nan.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 20:51:05 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/stats/desc/_nan.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/stats/desc/_nan.py"
+
+from scitex.decorators import torch_fn, batch_fn
+import torch
+
+
+@torch_fn
+@batch_fn
+def nanmax(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    min_value = torch.finfo(x.dtype).min
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        for d in sorted(dim, reverse=True):
+            x = x.nan_to_num(min_value).max(dim=d, keepdims=keepdims)[0]
+    else:
+        x = x.nan_to_num(min_value).max(dim=dim, keepdims=keepdims)[0]
+    return x
+
+
+@torch_fn
+@batch_fn
+def nanmin(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    max_value = torch.finfo(x.dtype).max
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        for d in sorted(dim, reverse=True):
+            x = x.nan_to_num(max_value).min(dim=d, keepdims=keepdims)[0]
+    else:
+        x = x.nan_to_num(max_value).min(dim=dim, keepdims=keepdims)[0]
+    return x
+
+
+@torch_fn
+@batch_fn
+def nansum(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    return torch.nansum(x, dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+@batch_fn
+def nanmean(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    return torch.nanmean(x, dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+@batch_fn
+def nanvar(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    tensor_mean = nanmean(x, dim=dim, keepdims=True)
+    return (x - tensor_mean).square().nanmean(dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+@batch_fn
+def nanstd(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    return torch.sqrt(nanvar(x, dim=dim, keepdims=keepdims))
+
+
+# @torch_fn
+# def nanzscore(x, axis=-1, dim=None, batch_size=None, keepdims=True):
+#     _mean = nanmean(x, dim=dim, keepdims=True)
+#     _std = nanstd(x, dim=dim, keepdims=True)
+#     zscores = (x - _mean) / _std
+#     return zscores if keepdims else zscores.squeeze(dim)
+@torch_fn
+@batch_fn
+def nanzscore(x, axis=-1, dim=None, batch_size=None, keepdims=True):
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        _mean = nanmean(x, dim=dim, keepdims=True)
+        _std = nanstd(x, dim=dim, keepdims=True)
+    else:
+        _mean = nanmean(x, dim=dim, keepdims=True)
+        _std = nanstd(x, dim=dim, keepdims=True)
+    zscores = (x - _mean) / _std
+    return zscores if keepdims else zscores.squeeze(dim)
+
+
+# @torch_fn
+# def nankurtosis(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+#     zscores = nanzscore(x, axis=axis, keepdims=True)
+#     n = (~torch.isnan(x)).sum(dim=dim, keepdim=True).to(x.dtype)  # Changed this line
+#     k = torch.nanmean(torch.pow(zscores, 4.0), dim=dim, keepdims=keepdims)
+#     correction = (n * (n + 1)) / ((n - 1) * (n - 2) * (n - 3))
+#     return correction * k - 3 * (n - 1)**2 / ((n - 2) * (n - 3))
+
+
+# @torch_fn
+# def nankurtosis(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+#     dim = axis if dim is None else dim
+#     if isinstance(dim, (tuple, list)):
+#         zscores = nanzscore(x, dim=dim, keepdims=True)
+#         n = (~torch.isnan(x)).sum(dim=dim, keepdim=True).to(x.dtype)
+#         k = torch.nanmean(torch.pow(zscores, 4.0), dim=dim, keepdims=keepdims)
+#         correction = (n * (n + 1)) / ((n - 1) * (n - 2) * (n - 3))
+#         result = correction * k - 3 * (n - 1)**2 / ((n - 2) * (n - 3))
+#         return result.squeeze() if not keepdims else result
+#     else:
+#         # Original code for single dimension
+#         zscores = nanzscore(x, dim=dim, keepdims=True)
+#         n = (~torch.isnan(x)).sum(dim=dim, keepdim=True).to(x.dtype)
+#         k = torch.nanmean(torch.pow(zscores, 4.0), dim=dim, keepdims=keepdims)
+#         correction = (n * (n + 1)) / ((n - 1) * (n - 2) * (n - 3))
+#         result = correction * k - 3 * (n - 1)**2 / ((n - 2) * (n - 3))
+#         return result.squeeze() if not keepdims else result
+
+# @torch_fn
+# def nanskewness(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+#     zscores = nanzscore(x, axis=axis, keepdims=True)
+#     n = (~torch.isnan(x)).sum(dim=dim, keepdim=True).to(x.dtype)  # Changed this line
+#     s = torch.nanmean(torch.pow(zscores, 3.0), dim=dim, keepdims=keepdims)
+#     correction = n**2 / ((n - 1) * (n - 2))
+#     return correction * s
+
+
+@torch_fn
+@batch_fn
+def nankurtosis(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    zscores = nanzscore(x, axis=axis, keepdims=True)
+    return torch.nanmean(torch.pow(zscores, 4.0), dim=dim, keepdims=keepdims) - 3.0
+
+
+@torch_fn
+@batch_fn
+def nanskewness(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    zscores = nanzscore(x, axis=axis, keepdims=True)
+    return torch.nanmean(torch.pow(zscores, 3.0), dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+@batch_fn
+def nanprod(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        for d in sorted(dim, reverse=True):
+            x = x.nan_to_num(1).prod(dim=d, keepdims=keepdims)
+    else:
+        x = x.nan_to_num(1).prod(dim=dim, keepdims=keepdims)
+    return x
+
+
+@torch_fn
+@batch_fn
+def nancumprod(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        raise ValueError("cumprod does not support multiple dimensions")
+    return x.nan_to_num(1).cumprod(dim=dim)
+
+
+@torch_fn
+@batch_fn
+def nancumsum(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        raise ValueError("cumsum does not support multiple dimensions")
+    return x.nan_to_num(0).cumsum(dim=dim)
+
+
+@torch_fn
+@batch_fn
+def nanargmin(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    max_value = torch.finfo(x.dtype).max
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        for d in sorted(dim, reverse=True):
+            x = x.nan_to_num(max_value).argmin(dim=d, keepdims=keepdims)
+    else:
+        x = x.nan_to_num(max_value).argmin(dim=dim, keepdims=keepdims)
+    return x
+
+
+@torch_fn
+@batch_fn
+def nanargmax(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    min_value = torch.finfo(x.dtype).min
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        for d in sorted(dim, reverse=True):
+            x = x.nan_to_num(min_value).argmax(dim=d, keepdims=keepdims)
+    else:
+        x = x.nan_to_num(min_value).argmax(dim=dim, keepdims=keepdims)
+    return x
+
+
+@torch_fn
+@batch_fn
+def nanquantile(x, q, axis=-1, dim=None, batch_size=None, keepdims=False):
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        # For multiple dimensions, flatten them first
+        # Save original shape for potential keepdims
+        original_shape = x.shape
+        
+        # Calculate new shape: keep dimensions not in dim, flatten those in dim
+        dim_list = list(dim) if isinstance(dim, tuple) else dim
+        # Normalize negative dimensions
+        dim_list = [d if d >= 0 else len(original_shape) + d for d in dim_list]
+        
+        # Determine which dimensions to keep
+        keep_dims = [i for i in range(len(original_shape)) if i not in dim_list]
+        
+        # Permute tensor to move dims to reduce to the end
+        perm_dims = keep_dims + dim_list
+        x_perm = x.permute(perm_dims)
+        
+        # Reshape to flatten the dimensions to reduce
+        new_shape = [original_shape[i] for i in keep_dims] + [-1]
+        x_flat = x_perm.reshape(new_shape)
+        
+        # Apply nanquantile on the flattened dimension
+        mask = ~torch.isnan(x_flat)
+        x_filtered = torch.where(mask, x_flat, torch.tensor(float("inf")))
+        result = torch.quantile(x_filtered, q / 100, dim=-1, keepdim=keepdims)
+        
+        # If keepdims, reshape back with singleton dimensions
+        if keepdims:
+            final_shape = list(original_shape)
+            for d in dim_list:
+                final_shape[d] = 1
+            result = result.reshape(final_shape)
+        
+        return result
+    else:
+        mask = ~torch.isnan(x)
+        x_filtered = torch.where(mask, x, torch.tensor(float("inf")))
+        x = torch.quantile(x_filtered, q / 100, dim=dim, keepdim=keepdims)
+    return x
+
+
+def nanq25(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    kwargs = {"axis": axis, "dim": dim, "keepdims": keepdims}
+    if batch_size is not None:
+        kwargs["batch_size"] = batch_size
+    return nanquantile(x, 25, **kwargs)
+
+
+def nanq50(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    kwargs = {"axis": axis, "dim": dim, "keepdims": keepdims}
+    if batch_size is not None:
+        kwargs["batch_size"] = batch_size
+    return nanquantile(x, 50, **kwargs)
+
+
+def nanq75(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    kwargs = {"axis": axis, "dim": dim, "keepdims": keepdims}
+    if batch_size is not None:
+        kwargs["batch_size"] = batch_size
+    return nanquantile(x, 75, **kwargs)
+
+
+@torch_fn
+@batch_fn
+def nancount(x, axis=-1, dim=None, batch_size=None, keepdims=False):
+    """Count number of non-NaN values along specified dimensions.
+
+    Parameters
+    ----------
+    x : torch.Tensor
+        Input tensor
+    axis : int, default=-1
+        Deprecated. Use dim instead
+    dim : int or tuple of ints, optional
+        Dimension(s) along which to count
+    keepdims : bool, default=True
+        Whether to keep reduced dimensions
+
+    Returns
+    -------
+    torch.Tensor
+        Count of non-NaN values
+    """
+    dim = axis if dim is None else dim
+    mask = ~torch.isnan(x)
+
+    if isinstance(dim, (tuple, list)):
+        for d in sorted(dim, reverse=True):
+            mask = mask.sum(dim=d, keepdims=keepdims)
+    else:
+        mask = mask.sum(dim=dim, keepdims=keepdims)
+
+    return mask
+
+
+# EOF

scitex/stats/desc/_real.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-17 21:17:13 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/stats/desc/_real.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/stats/desc/_real.py"
+
+"""
+Functionality:
+    - Computes descriptive statistics on PyTorch tensors
+Input:
+    - PyTorch tensor or numpy array
+Output:
+    - Descriptive statistics (mean, std, quantiles, etc.)
+Prerequisites:
+    - PyTorch, NumPy
+"""
+
+
+import numpy as np
+import torch
+
+from ...decorators import torch_fn
+
+
+@torch_fn
+def mean(x, axis=-1, dim=None, keepdims=False):
+    return x.mean(dim, keepdims=keepdims)
+
+
+@torch_fn
+def std(x, axis=-1, dim=None, keepdims=False):
+    return x.std(dim, keepdims=keepdims)
+
+
+@torch_fn
+def var(x, axis=-1, dim=None, keepdims=False):
+    return x.var(dim, keepdims=keepdims)
+
+
+@torch_fn
+def zscore(x, axis=-1, dim=None, keepdims=True):
+    _mean = mean(x, dim=dim, keepdims=True)
+    _std = std(x, dim=dim, keepdims=True)
+    zscores = (x - _mean) / _std
+    return zscores if keepdims else zscores.squeeze(dim)
+
+
+@torch_fn
+def skewness(x, axis=-1, dim=None, keepdims=False):
+    zscores = zscore(x, axis=axis, keepdims=True)
+    return torch.mean(torch.pow(zscores, 3.0), dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+def kurtosis(x, axis=-1, dim=None, keepdims=False):
+    zscores = zscore(x, axis=axis, keepdims=True)
+    return torch.mean(torch.pow(zscores, 4.0), dim=dim, keepdims=keepdims) - 3.0
+
+
+@torch_fn
+def quantile(x, q, axis=-1, dim=None, keepdims=False):
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        for d in sorted(dim, reverse=True):
+            x = torch.quantile(x, q / 100, dim=d, keepdims=keepdims)
+    else:
+        x = torch.quantile(x, q / 100, dim=dim, keepdims=keepdims)
+    return x
+
+
+@torch_fn
+def q25(x, axis=-1, dim=None, keepdims=False):
+    return quantile(x, 25, axis=axis, dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+def q50(x, axis=-1, dim=None, keepdims=False):
+    return quantile(x, 50, axis=axis, dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+def q75(x, axis=-1, dim=None, keepdims=False):
+    return quantile(x, 75, axis=axis, dim=dim, keepdims=keepdims)
+
+
+if __name__ == "__main__":
+    # from scitex.stats.desc import *
+
+    x = np.random.rand(4, 3, 2)
+    print(describe(x, dim=(1, 2), keepdims=False)[0].shape)
+    print(describe(x, funcs="all", dim=(1, 2), keepdims=False)[0].shape)
+
+# EOF

scitex/stats/multiple/__init__.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""Scitex multiple module."""
+
+from ._bonferroni_correction import bonferroni_correction, bonferroni_correction_torch
+from ._fdr_correction import ArrayLike, fdr_correction
+from ._multicompair import multicompair
+
+__all__ = [
+    "ArrayLike",
+    "bonferroni_correction",
+    "bonferroni_correction_torch",
+    "fdr_correction",
+    "multicompair",
+]

scitex/stats/multiple/_bonferroni_correction.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+# Time-stamp: "2021-09-25 15:39:51 (ylab)"
+
+import numpy as np
+import torch
+import scitex
+
+
+def bonferroni_correction(pval, alpha=0.05):
+    # https://github.com/mne-tools/mne-python/blob/main/mne/stats/multi_comp.py
+    """P-value correction with Bonferroni method.
+
+    Parameters
+    ----------
+    pval : array_like
+        Set of p-values of the individual tests.
+    alpha : float
+        Error rate.
+
+    Returns
+    -------
+    reject : array, bool
+        True if a hypothesis is rejected, False if not.
+    pval_corrected : array
+        P-values adjusted for multiple hypothesis testing to limit FDR.
+    """
+    pval = np.asarray(pval)
+    pval_corrected = pval * float(pval.size)
+    # p-values must not be larger than 1.
+    pval_corrected = pval_corrected.clip(max=1.0)
+    reject = pval_corrected < alpha
+    return reject, pval_corrected
+
+
+def bonferroni_correction_torch(pvals, alpha=0.05):
+    """P-value correction with Bonferroni method.
+
+    Parameters
+    ----------
+    pvals : array_like
+        Set of p-values of the individual tests.
+    alpha : float
+        Error rate.
+
+    Returns
+    -------
+    reject : array, bool
+        True if a hypothesis is rejected, False if not.
+    pvals_corrected : array
+        P-values adjusted for multiple hypothesis testing to limit FDR.
+    """
+    pvals = torch.tensor(pvals)
+    pvals_corrected = pvals * torch.tensor(pvals.size()).float()
+    # p-values must not be larger than 1.
+    pvals_corrected = pvals_corrected.clip(max=1.0)
+    reject = pvals_corrected < alpha
+    return reject, pvals_corrected
+
+
+if __name__ == "__main__":
+    pvals_npy = np.array([0.02, 0.03, 0.05])
+    pvals_torch = torch.tensor(np.array([0.02, 0.03, 0.05]))
+
+    reject, pvals_corrected = bonferroni_correction(pvals_npy, alpha=0.05)
+
+    reject_torch, pvals_corrected_torch = bonferroni_correction_torch(
+        pvals_torch, alpha=0.05
+    )
+
+    arr = pvals_corrected.astype(float)
+    tor = pvals_corrected_torch.numpy().astype(float)
+    print(scitex.gen.isclose(arr, tor))