scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/stats_v01/desc/_real.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-09-20 14:56:35 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/stats/desc/_real_dev.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = __file__
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+"""
+Functionalities:
+- Computes descriptive statistics on PyTorch tensors
+- Provides mean, standard deviation, variance calculations
+- Calculates z-scores, skewness, kurtosis
+- Computes quantiles (25th, 50th, 75th percentiles)
+- Demonstrates statistical computations with synthetic data
+
+Dependencies:
+- packages:
+  - torch
+  - numpy
+  - scitex
+
+IO:
+- input-files:
+  - PyTorch tensor or numpy array
+- output-files:
+  - Descriptive statistics results
+"""
+
+"""Imports"""
+import argparse
+
+import numpy as np
+import scitex as stx
+import torch
+from scitex import logging
+
+from ...decorators import torch_fn
+
+logger = logging.getLogger(__name__)
+
+"""Functions & Classes"""
+@torch_fn
+def mean(x, axis=-1, dim=None, keepdims=False):
+    return x.mean(dim, keepdims=keepdims)
+
+
+@torch_fn
+def std(x, axis=-1, dim=None, keepdims=False):
+    return x.std(dim, keepdims=keepdims)
+
+
+@torch_fn
+def var(x, axis=-1, dim=None, keepdims=False):
+    return x.var(dim, keepdims=keepdims)
+
+
+@torch_fn
+def zscore(x, axis=-1, dim=None, keepdims=True):
+    _mean = mean(x, dim=dim, keepdims=True)
+    _std = std(x, dim=dim, keepdims=True)
+    zscores = (x - _mean) / _std
+    return zscores if keepdims else zscores.squeeze(dim)
+
+
+@torch_fn
+def skewness(x, axis=-1, dim=None, keepdims=False):
+    zscores = zscore(x, axis=axis, keepdims=True)
+    return torch.mean(torch.pow(zscores, 3.0), dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+def kurtosis(x, axis=-1, dim=None, keepdims=False):
+    zscores = zscore(x, axis=axis, keepdims=True)
+    return (
+        torch.mean(torch.pow(zscores, 4.0), dim=dim, keepdims=keepdims) - 3.0
+    )
+
+
+@torch_fn
+def quantile(x, q, axis=-1, dim=None, keepdims=False):
+    dim = axis if dim is None else dim
+    if isinstance(dim, (tuple, list)):
+        for d in sorted(dim, reverse=True):
+            x = torch.quantile(x, q / 100, dim=d, keepdims=keepdims)
+    else:
+        x = torch.quantile(x, q / 100, dim=dim, keepdims=keepdims)
+    return x
+
+
+@torch_fn
+def q25(x, axis=-1, dim=None, keepdims=False):
+    return quantile(x, 25, axis=axis, dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+def q50(x, axis=-1, dim=None, keepdims=False):
+    return quantile(x, 50, axis=axis, dim=dim, keepdims=keepdims)
+
+
+@torch_fn
+def q75(x, axis=-1, dim=None, keepdims=False):
+    return quantile(x, 75, axis=axis, dim=dim, keepdims=keepdims)
+
+
+def main(args) -> int:
+    """Demonstrate descriptive statistics functions with synthetic data."""
+    x = np.random.rand(4, 3, 2)
+
+    # Compute statistics
+    x_mean = mean(x)
+    x_std = std(x)
+    x_var = var(x)
+    x_skew = skewness(x)
+    x_kurt = kurtosis(x)
+    x_q25 = q25(x)
+    x_q50 = q50(x)
+    x_q75 = q75(x)
+
+    # Store results
+    results = {
+        "input": x,
+        "mean": x_mean,
+        "std": x_std,
+        "variance": x_var,
+        "skewness": x_skew,
+        "kurtosis": x_kurt,
+        "q25": x_q25,
+        "q50": x_q50,
+        "q75": x_q75,
+    }
+
+    for k, v in results.items():
+        if isinstance(v, (np.ndarray, torch.Tensor)):
+            print(f"\n{k}, Type: {type(v)}, Shape: {v.shape}, Values: {v}")
+        elif isinstance(v, list):
+            print(f"\n{k}, Type: {type(v)}, Length: {len(v)}, Values: {v}")
+        else:
+            print(f"\n{k}, Type: {type(v)}, Values: {v}")
+
+    return 0
+
+
+def parse_args() -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Demonstrate descriptive statistics functions"
+    )
+    args = parser.parse_args()
+    return args
+
+
+def run_main() -> None:
+    """Initialize scitex framework, run main function, and cleanup."""
+    global CONFIG, CC, sys, plt, rng
+    import sys
+
+    import matplotlib.pyplot as plt
+    import scitex as stx
+
+    args = parse_args()
+    CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
+        sys,
+        plt,
+        args=args,
+        file=__FILE__,
+        sdir_suffix=None,
+        verbose=False,
+        agg=True,
+    )
+
+    exit_status = main(args)
+
+    stx.session.close(
+        CONFIG,
+        verbose=False,
+        notify=False,
+        message="",
+        exit_status=exit_status,
+    )
+
+
+if __name__ == "__main__":
+    run_main()
+
+# EOF

scitex/stats_v01/tests/__corr_test_optimized.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2025-07-25 05:00:00"
+# File: __corr_test_optimized.py
+
+"""
+Optimized correlation test implementation using vectorized operations and parallel processing.
+"""
+
+from bisect import bisect_right
+import numpy as np
+from scipy import stats
+from typing import Any, Literal, Dict, Callable
+from multiprocessing import Pool, cpu_count
+from functools import partial
+import warnings
+
+
+def _compute_correlation_batch(permuted_indices: np.ndarray, data1: np.ndarray, data2: np.ndarray, method: str) -> np.ndarray:
+    """
+    Compute correlations for a batch of permutations.
+    
+    Parameters
+    ----------
+    permuted_indices : np.ndarray
+        Array of shape (n_perms, n_samples) containing permutation indices
+    data1 : np.ndarray
+        First dataset
+    data2 : np.ndarray  
+        Second dataset
+    method : str
+        'pearson' or 'spearman'
+        
+    Returns
+    -------
+    np.ndarray
+        Array of correlation values
+    """
+    n_perms = permuted_indices.shape[0]
+    correlations = np.zeros(n_perms)
+    
+    if method == 'pearson':
+        # Vectorized Pearson correlation
+        for i in range(n_perms):
+            data2_perm = data2[permuted_indices[i]]
+            # Use numpy's corrcoef which is faster than scipy.stats.pearsonr
+            correlations[i] = np.corrcoef(data1, data2_perm)[0, 1]
+    else:  # spearman
+        # Convert to ranks once for data1
+        data1_ranks = stats.rankdata(data1)
+        for i in range(n_perms):
+            data2_perm = data2[permuted_indices[i]]
+            data2_ranks = stats.rankdata(data2_perm)
+            correlations[i] = np.corrcoef(data1_ranks, data2_ranks)[0, 1]
+    
+    return correlations
+
+
+def _corr_test_base_optimized(
+    data1: np.ndarray,
+    data2: np.ndarray,
+    only_significant: bool,
+    n_perm: int,
+    seed: int,
+    corr_func: Callable,
+    test_name: str,
+    n_jobs: int = -1,
+) -> Dict[str, Any]:
+    """
+    Optimized correlation test using vectorized operations and parallel processing.
+    
+    Parameters
+    ----------
+    data1 : np.ndarray
+        First dataset
+    data2 : np.ndarray
+        Second dataset  
+    only_significant : bool
+        If True, only return significant results
+    n_perm : int
+        Number of permutations
+    seed : int
+        Random seed
+    corr_func : Callable
+        Correlation function (scipy.stats.pearsonr or spearmanr)
+    test_name : str
+        Name of the test ('Pearson' or 'Spearman')
+    n_jobs : int
+        Number of parallel jobs (-1 for all CPUs)
+        
+    Returns
+    -------
+    Dict[str, Any]
+        Test results
+    """
+    np.random.seed(seed)
+    
+    # Remove NaN values
+    non_nan_indices = ~(np.isnan(data1) | np.isnan(data2))
+    data1_clean = data1[non_nan_indices]
+    data2_clean = data2[non_nan_indices]
+    n_samples = len(data1_clean)
+    
+    # Compute observed correlation
+    if test_name == "Pearson":
+        # Use numpy's corrcoef for speed
+        corr_obs = np.corrcoef(data1_clean, data2_clean)[0, 1]
+        _, p_analytical = stats.pearsonr(data1_clean, data2_clean)
+    else:
+        corr_obs, p_analytical = corr_func(data1_clean, data2_clean)
+    
+    # For small sample sizes or few permutations, use original method
+    if n_samples < 100 or n_perm < 1000:
+        # Original method for small datasets
+        surrogate = np.array([
+            corr_func(data1_clean, np.random.permutation(data2_clean))[0]
+            for _ in range(n_perm)
+        ])
+    else:
+        # Optimized parallel method for larger datasets
+        
+        # Determine number of workers
+        if n_jobs == -1:
+            n_workers = min(cpu_count(), 8)  # Cap at 8 workers
+        else:
+            n_workers = min(n_jobs, cpu_count())
+        
+        # Generate all permutation indices at once
+        perm_indices = np.array([
+            np.random.permutation(n_samples) for _ in range(n_perm)
+        ])
+        
+        # Split permutations into chunks for parallel processing
+        chunk_size = max(n_perm // n_workers, 100)
+        chunks = [perm_indices[i:i+chunk_size] for i in range(0, n_perm, chunk_size)]
+        
+        # Parallel computation
+        method = test_name.lower()
+        compute_func = partial(_compute_correlation_batch, 
+                              data1=data1_clean, 
+                              data2=data2_clean,
+                              method=method)
+        
+        if n_workers > 1 and len(chunks) > 1:
+            with Pool(n_workers) as pool:
+                results = pool.map(compute_func, chunks)
+            surrogate = np.concatenate(results)
+        else:
+            # Single-threaded for small jobs
+            surrogate = compute_func(perm_indices)
+    
+    # Calculate p-value using permutation test
+    rank = bisect_right(sorted(surrogate), corr_obs)
+    p_perm = min(rank, n_perm - rank) / n_perm * 2
+    
+    # Use the more conservative p-value
+    p_value = max(p_perm, p_analytical) if p_analytical is not None else p_perm
+    
+    # Import p2stars locally to avoid circular imports
+    try:
+        from .. import p2stars
+        stars = p2stars(p_value)
+    except:
+        # Fallback star calculation
+        if p_value < 0.001:
+            stars = "***"
+        elif p_value < 0.01:
+            stars = "**"
+        elif p_value < 0.05:
+            stars = "*"
+        else:
+            stars = ""
+    
+    effect_size = np.abs(corr_obs)
+    
+    result_string = (
+        f"{test_name} Corr. = {corr_obs:.3f}; p-value = {p_value:.3f} "
+        f"(n={n_samples:,}, eff={effect_size:.3f}) {stars}"
+    )
+    
+    if not only_significant or (only_significant and p_value < 0.05):
+        print(result_string)
+    
+    return {
+        "corr": round(corr_obs, 3),
+        "p_value": round(p_value, 3),
+        "p_perm": round(p_perm, 3),
+        "p_analytical": round(p_analytical, 3) if p_analytical is not None else None,
+        "stars": stars,
+        "effsize": round(effect_size, 3),
+        "surrogate": surrogate,
+        "n": n_samples,
+        "test_name": f"Permutation-based {test_name} correlation",
+        "statistic": round(corr_obs, 3),
+        "H0": f"There is no {test_name.lower()} correlation between the two variables",
+    }
+
+
+# Monkey patch to replace the original implementation
+def patch_correlation_functions():
+    """Replace original correlation functions with optimized versions."""
+    import scitex.stats.tests.__corr_test as corr_test_module
+    
+    # Store original for fallback
+    corr_test_module._corr_test_base_original = corr_test_module._corr_test_base
+    
+    # Replace with optimized version
+    corr_test_module._corr_test_base = _corr_test_base_optimized
+    
+    # Also update the wrapper if it exists
+    try:
+        import scitex.stats._corr_test_wrapper as wrapper_module
+        wrapper_module._corr_test_base = _corr_test_base_optimized
+    except:
+        pass
+
+
+# Auto-patch on import if enabled via environment variable
+import os
+if os.getenv('SCITEX_OPTIMIZE_STATS', 'true').lower() == 'true':
+    patch_correlation_functions()

scitex/stats_v01/tests/_corr_test_optimized.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2025-07-25 05:10:00"
+# File: _corr_test_optimized.py
+
+"""
+Optimized correlation test implementation using vectorized operations.
+"""
+
+from bisect import bisect_right
+import numpy as np
+from scipy import stats
+from typing import Any, Dict, Callable
+
+
+def _corr_test_base_vectorized(
+    data1: np.ndarray,
+    data2: np.ndarray,
+    only_significant: bool,
+    num_permutations: int,
+    seed: int,
+    corr_func: Callable,
+    test_name: str,
+) -> Dict[str, Any]:
+    """
+    Optimized correlation test using vectorized operations.
+    
+    This version focuses on vectorization rather than parallelization
+    for more consistent performance across different data sizes.
+    """
+    np.random.seed(seed)
+    
+    # Remove NaN values
+    non_nan_indices = ~(np.isnan(data1) | np.isnan(data2))
+    data1_clean = data1[non_nan_indices]
+    data2_clean = data2[non_nan_indices]
+    n_samples = len(data1_clean)
+    
+    # Compute observed correlation
+    if test_name == "Pearson":
+        # Use numpy's corrcoef for Pearson (faster than scipy)
+        corr_matrix = np.corrcoef(data1_clean, data2_clean)
+        corr_obs = corr_matrix[0, 1]
+        
+        # Still need scipy for p-value
+        _, p_analytical = stats.pearsonr(data1_clean, data2_clean)
+    else:  # Spearman
+        corr_obs, p_analytical = corr_func(data1_clean, data2_clean)
+    
+    # Vectorized permutation test
+    if test_name == "Pearson" and num_permutations >= 100:
+        # Optimized vectorized approach for Pearson correlation
+        
+        # Pre-compute means and stds
+        mean1 = np.mean(data1_clean)
+        mean2 = np.mean(data2_clean)
+        std1 = np.std(data1_clean, ddof=1)
+        std2 = np.std(data2_clean, ddof=1)
+        
+        # Standardize data
+        data1_std = (data1_clean - mean1) / std1
+        data2_std = (data2_clean - mean2) / std2
+        
+        # Generate all permutations at once
+        surrogate = np.zeros(num_permutations)
+        
+        # Batch process permutations
+        batch_size = min(100, num_permutations)
+        for i in range(0, num_permutations, batch_size):
+            end_idx = min(i + batch_size, num_permutations)
+            batch_perms = end_idx - i
+            
+            # Generate random permutations for this batch
+            perm_indices = np.array([
+                np.random.permutation(n_samples) 
+                for _ in range(batch_perms)
+            ])
+            
+            # Compute correlations for batch
+            for j in range(batch_perms):
+                data2_perm = data2_std[perm_indices[j]]
+                # Correlation of standardized data
+                surrogate[i + j] = np.mean(data1_std * data2_perm)
+                
+    else:
+        # Fall back to original method for Spearman or small permutations
+        surrogate = np.array([
+            corr_func(data1_clean, np.random.permutation(data2_clean))[0]
+            for _ in range(num_permutations)
+        ])
+    
+    # Calculate p-value using permutation test
+    rank = bisect_right(sorted(surrogate), corr_obs)
+    p_perm = min(rank, num_permutations - rank) / num_permutations * 2
+    
+    # Use the more conservative p-value
+    p_value = max(p_perm, p_analytical) if p_analytical is not None else p_perm
+    
+    # Get stars for significance
+    if p_value < 0.001:
+        stars = "***"
+    elif p_value < 0.01:
+        stars = "**"
+    elif p_value < 0.05:
+        stars = "*"
+    else:
+        stars = ""
+    
+    effect_size = np.abs(corr_obs)
+    
+    result_string = (
+        f"{test_name} Corr. = {corr_obs:.3f}; p-value = {p_value:.3f} "
+        f"(n={n_samples:,}, eff={effect_size:.3f}) {stars}"
+    )
+    
+    if not only_significant or (only_significant and p_value < 0.05):
+        print(result_string)
+    
+    return {
+        "p_value": round(p_value, 3),
+        "stars": stars,
+        "effsize": round(effect_size, 3),
+        "corr": round(corr_obs, 3),
+        "surrogate": surrogate,
+        "n": n_samples,
+        "test_name": f"Permutation-based {test_name} correlation",
+        "statistic": round(corr_obs, 3),
+        "H0": f"There is no {test_name.lower()} correlation between the two variables",
+    }
+
+
+def corr_test_optimized(
+    data1: np.ndarray,
+    data2: np.ndarray,
+    test: str = "pearson",
+    only_significant: bool = False,
+    num_permutations: int = 1_000,
+    seed: int = 42,
+) -> Dict[str, Any]:
+    """
+    Optimized correlation test.
+    
+    Parameters
+    ----------
+    data1 : np.ndarray
+        First dataset
+    data2 : np.ndarray
+        Second dataset
+    test : str
+        'pearson' or 'spearman'
+    only_significant : bool
+        Only return significant results
+    num_permutations : int
+        Number of permutations
+    seed : int
+        Random seed
+        
+    Returns
+    -------
+    Dict[str, Any]
+        Test results
+    """
+    if test.lower() == "pearson":
+        corr_func = stats.pearsonr
+        test_name = "Pearson"
+    elif test.lower() == "spearman":
+        corr_func = stats.spearmanr
+        test_name = "Spearman"
+    else:
+        raise ValueError("test must be 'pearson' or 'spearman'")
+    
+    return _corr_test_base_vectorized(
+        data1, data2,
+        only_significant,
+        num_permutations,
+        seed,
+        corr_func,
+        test_name
+    )

scitex/str/__init__.py

@@ -4,9 +4,7 @@
 # File: /ssh:sp:/home/ywatanabe/proj/SciTeX-Code/src/scitex/str/__init__.py
 # ----------------------------------------
 import os
-__FILE__ = (
-    "./src/scitex/str/__init__.py"
-)
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 """Scitex str module."""

scitex/str/_clean_path.py

@@ -49,10 +49,14 @@ def clean_path(path_string: str) -> str:
         Normalized path string
     """
     try:
-        is_directory = path_string.endswith("/")
-
+        # Convert Path objects to strings to avoid AttributeError
+        if hasattr(path_string, '__fspath__'):  # Check if it's a path-like object
+            path_string = str(path_string)
+        
         if not isinstance(path_string, str):
             raise TypeError("Input must be a string")
+            
+        is_directory = path_string.endswith("/")
 
         if path_string.startswith('f"'):
             path_string = path_string.replace('f"', "")[:-1]