scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/gen/_less.py

@@ -5,7 +5,7 @@
 #!./env/bin/python3
 # -*- coding: utf-8 -*-
 # Time-stamp: "2024-04-21 12:05:35"
-# Author: Yusuke Watanabe (ywata1989@gmail.com)
+# Author: Yusuke Watanabe (ywatanabe@scitex.ai)
 
 """
 This script does XYZ.

scitex/gen/_list_packages.py

@@ -118,7 +118,7 @@ if __name__ == "__main__":
     import matplotlib.pyplot as plt
     import scitex
 
-    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.gen.start(
+    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start(
         sys,
         plt,
         verbose=False,
@@ -127,7 +127,7 @@ if __name__ == "__main__":
 
     exit_status = main()
 
-    scitex.gen.close(
+    scitex.session.close(
         CONFIG,
         verbose=False,
         sys=sys,

scitex/gen/_norm.py

@@ -79,9 +79,20 @@ def to_01(x, axis=-1, dim=None, device="cuda"):
     torch.Tensor
         Min-max scaled tensor
     """
-    x_min = x.min(dim=dim, keepdim=True)[0]
-    x_max = x.max(dim=dim, keepdim=True)[0]
-    return (x - x_min) / (x_max - x_min)
+    # Use dim if provided, otherwise use axis
+    dimension = dim if dim is not None else axis
+    
+    if dimension is None:
+        # Scale entire tensor
+        x_min = x.min()
+        x_max = x.max()
+    else:
+        # Scale along specified dimension
+        x_min = x.min(dim=dimension, keepdim=True)[0]
+        x_max = x.max(dim=dimension, keepdim=True)[0]
+    
+    # Avoid division by zero
+    return (x - x_min) / (x_max - x_min + 1e-8)
 
 
 @torch_fn
@@ -104,9 +115,20 @@ def to_nan01(x, axis=-1, dim=None, device="cuda"):
     torch.Tensor
         Min-max scaled tensor with NaN handling
     """
-    x_min = torch.nanmin(x, dim=dim, keepdim=True)[0]
-    x_max = torch.nanmax(x, dim=dim, keepdim=True)[0]
-    return (x - x_min) / (x_max - x_min)
+    # Use dim if provided, otherwise use axis
+    dimension = dim if dim is not None else axis
+    
+    if dimension is None:
+        # Scale entire tensor
+        x_min = torch.nanmin(x)
+        x_max = torch.nanmax(x)
+    else:
+        # Scale along specified dimension
+        x_min = torch.nanmin(x, dim=dimension, keepdim=True)[0]
+        x_max = torch.nanmax(x, dim=dimension, keepdim=True)[0]
+    
+    # Avoid division by zero
+    return (x - x_min) / (x_max - x_min + 1e-8)
 
 
 @torch_fn
@@ -139,7 +161,7 @@ def unbias(x, axis=-1, dim=None, fn="mean", device="cuda"):
 
 
 @torch_fn
-def clip_perc(x, lower_perc=2.5, upper_perc=97.5, axis=-1, dim=None, device="cuda"):
+def clip_perc(x, lower_perc=2.5, upper_perc=97.5, low=None, high=None, axis=-1, dim=None, device="cuda"):
     """Clips tensor values between specified percentiles along dimension.
 
     Parameters
@@ -150,6 +172,10 @@ def clip_perc(x, lower_perc=2.5, upper_perc=97.5, axis=-1, dim=None, device="cud
         Lower percentile (0-100)
     upper_perc : float
         Upper percentile (0-100)
+    low : float, optional
+        Alternative name for lower_perc
+    high : float, optional
+        Alternative name for upper_perc
     dim : int
         Dimension along which to compute percentiles (preferred)
     axis : int
@@ -162,8 +188,17 @@ def clip_perc(x, lower_perc=2.5, upper_perc=97.5, axis=-1, dim=None, device="cud
     torch.Tensor
         Clipped tensor
     """
-    lower = torch.quantile(x, lower_perc / 100, dim=dim, keepdim=True)
-    upper = torch.quantile(x, upper_perc / 100, dim=dim, keepdim=True)
+    # Handle alternative parameter names
+    if low is not None:
+        lower_perc = low
+    if high is not None:
+        upper_perc = high
+        
+    # Use dim if provided, otherwise use axis
+    dimension = dim if dim is not None else axis
+    
+    lower = torch.quantile(x, lower_perc / 100, dim=dimension, keepdim=True)
+    upper = torch.quantile(x, upper_perc / 100, dim=dimension, keepdim=True)
     return torch.clamp(x, min=lower, max=upper)
 
 

scitex/gen/_norm_cache.py

@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2025-07-25 05:15:00"
+# File: _norm_cache.py
+
+"""
+Cached normalization functions for improved performance on repeated operations.
+"""
+
+import torch
+import numpy as np
+import weakref
+from functools import lru_cache
+from typing import Union, Tuple, Optional
+import hashlib
+
+from ..decorators import torch_fn
+
+
+# Cache for normalized data
+_norm_cache = weakref.WeakValueDictionary()
+_cache_metadata = {}
+_cache_config = {
+    'enabled': True,
+    'max_size': 64,
+    'verbose': False
+}
+
+
+def _get_array_key(x: Union[np.ndarray, torch.Tensor], axis: Optional[int] = None) -> str:
+    """
+    Generate a unique key for an array based on its content and parameters.
+    
+    Parameters
+    ----------
+    x : array-like
+        Input array
+    axis : int, optional
+        Axis parameter
+        
+    Returns
+    -------
+    str
+        Unique hash key
+    """
+    # Create a hash based on array properties
+    if isinstance(x, torch.Tensor):
+        # For tensors, use shape, dtype, device, and a sample of values
+        props = f"torch_{x.shape}_{x.dtype}_{x.device}_{axis}"
+        # Sample a few values for hash (avoid full array for performance)
+        if x.numel() > 100:
+            sample = x.flatten()[::max(1, x.numel() // 100)][:100]
+        else:
+            sample = x.flatten()
+        props += f"_{sample.sum().item():.6f}_{sample.std().item():.6f}"
+    else:
+        # For numpy arrays
+        props = f"numpy_{x.shape}_{x.dtype}_{axis}"
+        if x.size > 100:
+            sample = x.flatten()[::max(1, x.size // 100)][:100]
+        else:
+            sample = x.flatten()
+        props += f"_{np.sum(sample):.6f}_{np.std(sample):.6f}"
+    
+    return hashlib.md5(props.encode()).hexdigest()
+
+
+def _check_cache(key: str, x: Union[np.ndarray, torch.Tensor]) -> Optional[Union[np.ndarray, torch.Tensor]]:
+    """Check if cached result exists and is valid."""
+    if not _cache_config['enabled']:
+        return None
+        
+    if key in _cache_metadata:
+        # Verify array hasn't changed by checking a few properties
+        cached_info = _cache_metadata[key]
+        
+        if isinstance(x, torch.Tensor):
+            current_sum = x.sum().item()
+            current_mean = x.mean().item()
+        else:
+            current_sum = np.sum(x)
+            current_mean = np.mean(x)
+            
+        # Check if values match (within floating point tolerance)
+        if (abs(cached_info['sum'] - current_sum) < 1e-10 and 
+            abs(cached_info['mean'] - current_mean) < 1e-10):
+            
+            # Try to get from weak reference cache
+            if key in _norm_cache:
+                if _cache_config['verbose']:
+                    print(f"[Norm Cache HIT] {cached_info['op']}")
+                return _norm_cache[key]
+                
+    return None
+
+
+def _store_cache(key: str, x: Union[np.ndarray, torch.Tensor], 
+                result: Union[np.ndarray, torch.Tensor], op: str) -> None:
+    """Store result in cache."""
+    if not _cache_config['enabled']:
+        return
+        
+    # Store metadata
+    if isinstance(x, torch.Tensor):
+        _cache_metadata[key] = {
+            'sum': x.sum().item(),
+            'mean': x.mean().item(),
+            'shape': x.shape,
+            'op': op
+        }
+    else:
+        _cache_metadata[key] = {
+            'sum': np.sum(x),
+            'mean': np.mean(x),
+            'shape': x.shape,
+            'op': op
+        }
+    
+    # Try to store in weak reference cache
+    try:
+        _norm_cache[key] = result
+    except TypeError:
+        # Some types can't be weakly referenced
+        pass
+        
+    # Implement size limit
+    if len(_cache_metadata) > _cache_config['max_size']:
+        # Remove oldest entries
+        oldest = list(_cache_metadata.keys())[0]
+        del _cache_metadata[oldest]
+        if oldest in _norm_cache:
+            del _norm_cache[oldest]
+
+
+# Cached version of to_z
+@torch_fn
+def to_z_cached(x, axis=-1, dim=None, device="cuda"):
+    """
+    Cached version of z-score normalization.
+    
+    Caches results for repeated normalizations of the same data.
+    """
+    # Generate cache key
+    dimension = dim if dim is not None else axis
+    cache_key = _get_array_key(x, dimension) + "_z"
+    
+    # Check cache
+    cached = _check_cache(cache_key, x)
+    if cached is not None:
+        return cached
+    
+    # Compute normalization
+    if isinstance(x, torch.Tensor):
+        result = (x - x.mean(dim=dimension, keepdim=True)) / x.std(dim=dimension, keepdim=True)
+    else:
+        result = (x - np.mean(x, axis=dimension, keepdims=True)) / np.std(x, axis=dimension, keepdims=True)
+    
+    # Store in cache
+    _store_cache(cache_key, x, result, "z-score")
+    
+    return result
+
+
+# Cached version of to_01
+@torch_fn  
+def to_01_cached(x, axis=-1, dim=None, device="cuda"):
+    """
+    Cached version of min-max normalization.
+    
+    Caches results for repeated normalizations of the same data.
+    """
+    # Generate cache key
+    dimension = dim if dim is not None else axis
+    cache_key = _get_array_key(x, dimension) + "_01"
+    
+    # Check cache
+    cached = _check_cache(cache_key, x)
+    if cached is not None:
+        return cached
+    
+    # Compute normalization
+    if isinstance(x, torch.Tensor):
+        if dimension is None:
+            x_min = x.min()
+            x_max = x.max()
+        else:
+            x_min = x.min(dim=dimension, keepdim=True)[0]
+            x_max = x.max(dim=dimension, keepdim=True)[0]
+        result = (x - x_min) / (x_max - x_min + 1e-8)
+    else:
+        if dimension is None:
+            x_min = np.min(x)
+            x_max = np.max(x)
+        else:
+            x_min = np.min(x, axis=dimension, keepdims=True)
+            x_max = np.max(x, axis=dimension, keepdims=True)
+        result = (x - x_min) / (x_max - x_min + 1e-8)
+    
+    # Store in cache
+    _store_cache(cache_key, x, result, "min-max")
+    
+    return result
+
+
+def configure_norm_cache(enabled: Optional[bool] = None,
+                        max_size: Optional[int] = None,
+                        verbose: Optional[bool] = None) -> None:
+    """
+    Configure normalization cache settings.
+    
+    Parameters
+    ----------
+    enabled : bool, optional
+        Enable or disable caching
+    max_size : int, optional
+        Maximum number of arrays to cache
+    verbose : bool, optional
+        Enable verbose output
+    """
+    if enabled is not None:
+        _cache_config['enabled'] = enabled
+    if max_size is not None:
+        _cache_config['max_size'] = max_size
+    if verbose is not None:
+        _cache_config['verbose'] = verbose
+
+
+def clear_norm_cache() -> None:
+    """Clear all cached normalization results."""
+    _norm_cache.clear()
+    _cache_metadata.clear()
+
+
+def get_norm_cache_info() -> dict:
+    """Get information about the normalization cache."""
+    return {
+        'enabled': _cache_config['enabled'],
+        'max_size': _cache_config['max_size'],
+        'current_size': len(_cache_metadata),
+        'operations': [v['op'] for v in _cache_metadata.values()]
+    }
+
+
+# Monkey patch the original functions if enabled
+def patch_normalization_functions():
+    """Replace original normalization functions with cached versions."""
+    import scitex.gen._norm as norm_module
+    
+    # Store originals
+    norm_module.to_z_original = norm_module.to_z
+    norm_module.to_01_original = norm_module.to_01
+    
+    # Replace with cached versions
+    norm_module.to_z = to_z_cached
+    norm_module.to_01 = to_01_cached
+    
+    # Also patch in the gen module namespace
+    try:
+        import scitex.gen as gen_module
+        gen_module.to_z = to_z_cached
+        gen_module.to_01 = to_01_cached
+    except:
+        pass
+
+
+# Auto-patch if enabled
+import os
+if os.getenv('SCITEX_CACHE_NORM', 'true').lower() == 'true':
+    patch_normalization_functions()

scitex/gen/_src.py

@@ -4,9 +4,7 @@
 # File: /ssh:sp:/home/ywatanabe/proj/SciTeX-Code/src/scitex/gen/_src.py
 # ----------------------------------------
 import os
-__FILE__ = (
-    "./src/scitex/gen/_src.py"
-)
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
@@ -95,14 +93,14 @@ if __name__ == "__main__":
     import matplotlib.pyplot as plt
 
     # Start
-    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.gen.start(
+    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start(
         sys, plt, verbose=False
     )
     import sys
 
     # (YOUR AWESOME CODE)
     # Close
-    scitex.gen.close(CONFIG, verbose=False, notify=False)
+    scitex.session.close(CONFIG, verbose=False, notify=False)
 
 """
 /ssh:ywatanabe@444:/home/ywatanabe/proj/entrance/scitex/gen/_def.py

scitex/gen/_title_case.py

@@ -1,7 +1,7 @@
 #!./env/bin/python3
 # -*- coding: utf-8 -*-
 # Time-stamp: "2024-04-24 15:05:34"
-# Author: Yusuke Watanabe (ywata1989@gmail.com)
+# Author: Yusuke Watanabe (ywatanabe@scitex.ai)
 
 
 """
@@ -81,8 +81,8 @@ def main():
 
 
 if __name__ == "__main__":
-    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.gen.start(sys, plt, verbose=False)
+    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.session.start(sys, plt, verbose=False)
     main()
-    scitex.gen.close(CONFIG, verbose=False, notify=False)
+    scitex.session.close(CONFIG, verbose=False, notify=False)
 
 # EOF

scitex/io/__init__.py

@@ -1,12 +1,10 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Timestamp: "2025-06-14 16:00:50 (ywatanabe)"
-# File: /ssh:sp:/home/ywatanabe/proj/SciTeX-Code/src/scitex/io/__init__.py
+# Timestamp: "2025-07-12 04:23:59 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/__init__.py
 # ----------------------------------------
 import os
-__FILE__ = (
-    "./src/scitex/io/__init__.py"
-)
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 """Scitex IO module with lazy imports to avoid circular dependencies."""
@@ -19,7 +17,15 @@ from ._glob import glob, parse_glob
 from ._reload import reload
 from ._flush import flush
 from ._cache import cache
-from ._H5Explorer import H5Explorer, explore_h5, has_h5_key
+from ._load_modules._H5Explorer import H5Explorer, explore_h5, has_h5_key
+from ._load_modules._ZarrExplorer import ZarrExplorer, explore_zarr, has_zarr_key
+
+# Import load cache control functions
+from ._load_cache import (
+    get_cache_info,
+    configure_cache,
+    clear_cache as clear_load_cache
+)
 
 # Import save module functions
 try:
@@ -56,6 +62,13 @@ try:
 except ImportError:
     json2md = None
 
+# Import utils
+try:
+    from .utils import migrate_h5_to_zarr, migrate_h5_to_zarr_batch
+except ImportError:
+    migrate_h5_to_zarr = None
+    migrate_h5_to_zarr_batch = None
+
 __all__ = [
     "save",
     "load",
@@ -77,6 +90,15 @@ __all__ = [
     "save_listed_dfs_as_csv",
     "save_listed_scalars_as_csv",
     "save_optuna_study_as_csv_and_pngs",
+    "ZarrExplorer",
+    "explore_zarr",
+    "has_zarr_key",
+    "migrate_h5_to_zarr",
+    "migrate_h5_to_zarr_batch",
+    # Load cache control functions
+    "get_cache_info",
+    "configure_cache",
+    "clear_load_cache",
 ]
 
 # EOF

scitex/io/_glob.py

@@ -7,11 +7,13 @@ THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/io/_glob.py"
 
 import re as _re
 from glob import glob as _glob
+from pathlib import Path
+from typing import Union
 from ..str._parse import parse as _parse
 from natsort import natsorted as _natsorted
 
 
-def glob(expression, parse=False, ensure_one=False):
+def glob(expression: Union[str, Path], parse=False, ensure_one=False):
     """
     Perform a glob operation with natural sorting and extended pattern support.
 
@@ -20,9 +22,9 @@ def glob(expression, parse=False, ensure_one=False):
 
     Parameters:
     -----------
-    expression : str
-        The glob pattern to match against file paths. Supports standard glob syntax
-        and curly brace expansion (e.g., 'dir/{a,b}/*.txt').
+    expression : Union[str, Path]
+        The glob pattern to match against file paths. Can be a string or pathlib.Path object. 
+        Supports standard glob syntax and curly brace expansion (e.g., 'dir/{a,b}/*.txt').
     parse : bool, optional
         Whether to parse the matched paths. Default is False.
     ensure_one : bool, optional
@@ -51,6 +53,10 @@ def glob(expression, parse=False, ensure_one=False):
     >>> paths, parsed = glob('data/subj_{id}/run_{run}.txt', parse=True, ensure_one=True)
     AssertionError  # if more than one file matches
     """
+    # Convert Path objects to strings for consistency
+    if isinstance(expression, Path):
+        expression = str(expression)
+        
     glob_pattern = _re.sub(r"{[^}]*}", "*", expression)
     # Enable recursive globbing for ** patterns
     recursive = "**" in glob_pattern
@@ -70,14 +76,14 @@ def glob(expression, parse=False, ensure_one=False):
         return found_paths
 
 
-def parse_glob(expression, ensure_one=False):
+def parse_glob(expression: Union[str, Path], ensure_one=False):
     """
     Convenience function for glob with parsing enabled.
 
     Parameters:
     -----------
-    expression : str
-        The glob pattern to match against file paths.
+    expression : Union[str, Path]
+        The glob pattern to match against file paths. Can be a string or pathlib.Path object.
     ensure_one : bool, optional
         Ensure exactly one match is found. Default is False.