scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/io/_load.py

@@ -1,22 +1,31 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Timestamp: "2025-04-10 08:05:53 (ywatanabe)"
-# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load.py
+# Timestamp: "2025-08-11 05:54:51 (ywatanabe)"
+# File: /home/ywatanabe/proj/SciTeX-Code/src/scitex/io/_load.py
 # ----------------------------------------
+from __future__ import annotations
 import os
-
-__FILE__ = "/ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load.py"
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
-from typing import Any
 import glob
+from pathlib import Path
+from typing import Any, Union
+
 from ..decorators import preserve_doc
 from ..str._clean_path import clean_path
+from ._load_cache import (
+    cache_data,
+    configure_cache,
+    get_cache_info,
+    get_cached_data,
+    load_npy_cached,
+)
+from ._load_modules._bibtex import _load_bibtex
 
 # from ._load_modules._catboost import _load_catboost
 from ._load_modules._con import _load_con
-from ._load_modules._db import _load_sqlite3db
 from ._load_modules._docx import _load_docx
 from ._load_modules._eeg import _load_eeg_data
 from ._load_modules._hdf5 import _load_hdf5
@@ -24,32 +33,46 @@ from ._load_modules._image import _load_image
 from ._load_modules._joblib import _load_joblib
 from ._load_modules._json import _load_json
 from ._load_modules._markdown import _load_markdown
-from ._load_modules._numpy import _load_npy
 from ._load_modules._matlab import _load_matlab
+from ._load_modules._numpy import _load_npy
 from ._load_modules._pandas import _load_csv, _load_excel, _load_tsv
 from ._load_modules._pdf import _load_pdf
 from ._load_modules._pickle import _load_pickle
+from ._load_modules._sqlite3 import _load_db_sqlite3
 from ._load_modules._torch import _load_torch
 from ._load_modules._txt import _load_txt
 from ._load_modules._xml import _load_xml
 from ._load_modules._yaml import _load_yaml
-from ._load_modules._matlab import _load_matlab
+from ._load_modules._zarr import _load_zarr
 
 
-def load(lpath: str, show: bool = False, verbose: bool = False, **kwargs) -> Any:
+def load(
+    lpath: Union[str, Path],
+    ext: str = None,
+    show: bool = False,
+    verbose: bool = False,
+    cache: bool = True,
+    **kwargs,
+) -> Any:
     """
     Load data from various file formats.
 
-    This function supports loading data from multiple file formats.
+    This function supports loading data from multiple file formats with optional caching.
 
     Parameters
     ----------
-    lpath : str
-        The path to the file to be loaded.
+    lpath : Union[str, Path]
+        The path to the file to be loaded. Can be a string or pathlib.Path object.
+    ext : str, optional
+        File extension to use for loading. If None, automatically detects from filename.
+        Useful for files without extensions (e.g., UUID-named files).
+        Examples: 'pdf', 'json', 'csv'
     show : bool, optional
         If True, display additional information during loading. Default is False.
     verbose : bool, optional
         If True, print verbose output during loading. Default is False.
+    cache : bool, optional
+        If True, enable caching for faster repeated loads. Default is True.
     **kwargs : dict
         Additional keyword arguments to be passed to the specific loading function.
 
@@ -80,23 +103,66 @@ def load(lpath: str, show: bool = False, verbose: bool = False, **kwargs) -> Any
     >>> data = load('data.csv')
     >>> image = load('image.png')
     >>> model = load('model.pth')
+    >>> # Load file without extension (e.g., UUID PDF)
+    >>> pdf = load('f2694ccb-1b6f-4994-add8-5111fd4d52f1', ext='pdf')
     """
-    lpath = clean_path(lpath)
+
+    # Don't use clean_path as it breaks relative paths like ./file.txt
+    # lpath = clean_path(lpath)
+
+    # Convert Path objects to strings for consistency
+    if isinstance(lpath, Path):
+        lpath = str(lpath)
+        if verbose:
+            print(f"[DEBUG] After Path conversion: {lpath}")
 
     # Check if it's a glob pattern
     if "*" in lpath or "?" in lpath or "[" in lpath:
         # Handle glob pattern
         matched_files = sorted(glob.glob(lpath))
         if not matched_files:
-            raise FileNotFoundError(f"No files found matching pattern: {lpath}")
+            raise FileNotFoundError(
+                f"No files found matching pattern: {lpath}"
+            )
         # Load all matched files
         results = []
         for file_path in matched_files:
-            results.append(load(file_path, show=show, verbose=verbose, **kwargs))
+            results.append(
+                load(file_path, show=show, verbose=verbose, **kwargs)
+            )
         return results
 
+    # Handle broken symlinks - os.path.exists() returns False for broken symlinks
     if not os.path.exists(lpath):
-        raise FileNotFoundError(f"{lpath} not found.")
+        if os.path.islink(lpath):
+            # For symlinks, resolve the target path relative to symlink's directory
+            symlink_dir = os.path.dirname(os.path.abspath(lpath))
+            target = os.readlink(lpath)
+            resolved_target = os.path.join(symlink_dir, target)
+            resolved_target = os.path.abspath(resolved_target)
+            
+            if os.path.exists(resolved_target):
+                lpath = resolved_target
+            else:
+                raise FileNotFoundError(f"Symlink target not found: {resolved_target}")
+        else:
+            # Try general path resolution
+            try:
+                resolved_path = os.path.realpath(lpath)
+                if os.path.exists(resolved_path):
+                    lpath = resolved_path
+                else:
+                    raise FileNotFoundError(f"File not found: {lpath}")
+            except Exception:
+                raise FileNotFoundError(f"File not found: {lpath}")
+
+    # Try to get from cache first
+    if cache:
+        cached_data = get_cached_data(lpath)
+        if cached_data is not None:
+            if verbose:
+                print(f"[Cache HIT] Loaded from cache: {lpath}")
+            return cached_data
 
     loaders_dict = {
         # Default
@@ -106,6 +172,8 @@ def load(lpath: str, show: bool = False, verbose: bool = False, **kwargs) -> Any
         "yml": _load_yaml,
         "json": _load_json,
         "xml": _load_xml,
+        # Bibliography
+        "bib": _load_bibtex,
         # ML/DL Models
         "pth": _load_torch,
         "pt": _load_torch,
@@ -121,13 +189,14 @@ def load(lpath: str, show: bool = False, verbose: bool = False, **kwargs) -> Any
         "xlsx": _load_excel,
         "xlsm": _load_excel,
         "xlsb": _load_excel,
-        "db": _load_sqlite3db,
+        "db": _load_db_sqlite3,
         # Scientific Data
         "npy": _load_npy,
         "npz": _load_npy,
         "mat": _load_matlab,
         "hdf5": _load_hdf5,
         "h5": _load_hdf5,
+        "zarr": _load_zarr,
         "con": _load_con,
         # Documents
         "txt": _load_txt,
@@ -156,13 +225,31 @@ def load(lpath: str, show: bool = False, verbose: bool = False, **kwargs) -> Any
         "set": _load_eeg_data,
     }
 
-    ext = lpath.split(".")[-1] if "." in lpath else ""
-    loader = preserve_doc(loaders_dict.get(ext, _load_txt))
+    # Determine extension: use explicit ext parameter or detect from filename
+    if ext is not None:
+        # Use explicitly provided extension (strip leading dot if present)
+        detected_ext = ext.lstrip('.')
+    else:
+        # Auto-detect from filename
+        detected_ext = lpath.split(".")[-1] if "." in lpath else ""
+
+    # Special handling for numpy files with caching
+    if cache and detected_ext in ["npy", "npz"]:
+        return load_npy_cached(lpath, **kwargs)
+
+    loader = preserve_doc(loaders_dict.get(detected_ext, _load_txt))
 
     try:
-        return loader(lpath, **kwargs)
+        result = loader(lpath, **kwargs)
+
+        # Cache the result if caching is enabled
+        if cache:
+            cache_data(lpath, result)
+            if verbose:
+                print(f"[Cache STORED] Cached data for: {lpath}")
+
+        return result
     except (ValueError, FileNotFoundError) as e:
         raise ValueError(f"Error loading file {lpath}: {str(e)}")
 
-
 # EOF

scitex/io/_load_cache.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2025-07-25 04:40:00"
+# File: _load_cache.py
+
+"""
+File load caching infrastructure for scitex.io module.
+Provides intelligent caching with file modification detection.
+"""
+
+import os
+import hashlib
+import weakref
+from functools import lru_cache
+from typing import Any, Tuple, Optional, Dict
+from scitex import logging
+
+# Cache for file metadata (path -> (mtime, size, hash))
+_file_metadata_cache: Dict[str, Tuple[float, int, str]] = {}
+
+# Weak reference cache for actual data
+_file_data_cache = weakref.WeakValueDictionary()
+
+# Cache statistics
+_cache_stats = {
+    'hits': 0,
+    'misses': 0,
+    'evictions': 0
+}
+
+# Configuration
+_cache_config = {
+    'enabled': True,
+    'max_size': 32,  # Maximum number of files to track
+    'verbose': False
+}
+
+
+def get_file_key(file_path: str) -> Tuple[str, float, int]:
+    """
+    Get cache key based on file path, modification time, and size.
+    
+    Parameters
+    ----------
+    file_path : str
+        Path to the file
+        
+    Returns
+    -------
+    Tuple[str, float, int]
+        Tuple of (absolute_path, mtime, size)
+    """
+    abs_path = os.path.abspath(file_path)
+    stat = os.stat(abs_path)
+    return (abs_path, stat.st_mtime, stat.st_size)
+
+
+def get_file_hash(file_path: str) -> str:
+    """
+    Get a hash based on file metadata.
+    
+    Parameters
+    ----------
+    file_path : str
+        Path to the file
+        
+    Returns
+    -------
+    str
+        MD5 hash of file metadata
+    """
+    key = get_file_key(file_path)
+    hash_input = f"{key[0]}:{key[1]}:{key[2]}"
+    return hashlib.md5(hash_input.encode()).hexdigest()
+
+
+def is_cache_valid(file_path: str) -> bool:
+    """
+    Check if cached data for a file is still valid.
+    
+    Parameters
+    ----------
+    file_path : str
+        Path to the file
+        
+    Returns
+    -------
+    bool
+        True if cache is valid, False otherwise
+    """
+    abs_path = os.path.abspath(file_path)
+    
+    if abs_path not in _file_metadata_cache:
+        return False
+    
+    # Get current file metadata
+    try:
+        current_key = get_file_key(file_path)
+        cached_mtime, cached_size, cached_hash = _file_metadata_cache[abs_path]
+        
+        # Check if file has been modified
+        return (current_key[1] == cached_mtime and 
+                current_key[2] == cached_size)
+    except (OSError, KeyError):
+        return False
+
+
+def get_cached_data(file_path: str) -> Optional[Any]:
+    """
+    Retrieve cached data if available and valid.
+    
+    Parameters
+    ----------
+    file_path : str
+        Path to the file
+        
+    Returns
+    -------
+    Optional[Any]
+        Cached data if available, None otherwise
+    """
+    if not _cache_config['enabled']:
+        return None
+    
+    abs_path = os.path.abspath(file_path)
+    
+    # Check if cache is valid
+    if not is_cache_valid(file_path):
+        _cache_stats['misses'] += 1
+        return None
+    
+    # Try to get data from weak reference cache
+    if abs_path in _file_data_cache:
+        _cache_stats['hits'] += 1
+        if _cache_config['verbose']:
+            logging.info(f"[Cache HIT] {file_path}")
+        return _file_data_cache[abs_path]
+    
+    _cache_stats['misses'] += 1
+    return None
+
+
+def cache_data(file_path: str, data: Any) -> None:
+    """
+    Cache data for a file.
+    
+    Parameters
+    ----------
+    file_path : str
+        Path to the file
+    data : Any
+        Data to cache
+    """
+    if not _cache_config['enabled']:
+        return
+    
+    abs_path = os.path.abspath(file_path)
+    
+    # Update metadata cache
+    key = get_file_key(file_path)
+    _file_metadata_cache[abs_path] = (key[1], key[2], get_file_hash(file_path))
+    
+    # Try to cache the data (may fail for non-weakref-able types)
+    try:
+        # For numpy arrays and other special types, we need to wrap them
+        if hasattr(data, '__array__'):
+            # Numpy arrays can't be weakly referenced directly
+            # We'll skip caching for now (could implement a wrapper)
+            pass
+        else:
+            _file_data_cache[abs_path] = data
+    except TypeError:
+        # Some objects can't be weakly referenced
+        if _cache_config['verbose']:
+            logging.debug(f"Cannot cache {type(data).__name__} objects")
+    
+    # Implement cache size limit
+    if len(_file_metadata_cache) > _cache_config['max_size']:
+        # Remove oldest entries (simple FIFO for metadata)
+        oldest = list(_file_metadata_cache.keys())[0]
+        del _file_metadata_cache[oldest]
+        _cache_stats['evictions'] += 1
+
+
+def clear_cache() -> None:
+    """Clear all cached data."""
+    _file_metadata_cache.clear()
+    _file_data_cache.clear()
+    _numpy_cache_keys.clear()
+    _cached_load_npy.cache_clear()  # Clear LRU cache
+    _cache_stats['hits'] = 0
+    _cache_stats['misses'] = 0
+    _cache_stats['evictions'] = 0
+
+
+def get_cache_info() -> Dict[str, Any]:
+    """
+    Get cache statistics and configuration.
+    
+    Returns
+    -------
+    Dict[str, Any]
+        Cache information including stats and config
+    """
+    return {
+        'stats': _cache_stats.copy(),
+        'config': _cache_config.copy(),
+        'metadata_size': len(_file_metadata_cache),
+        'data_size': len(_file_data_cache)
+    }
+
+
+def configure_cache(enabled: Optional[bool] = None,
+                   max_size: Optional[int] = None,
+                   verbose: Optional[bool] = None) -> None:
+    """
+    Configure cache settings.
+    
+    Parameters
+    ----------
+    enabled : Optional[bool]
+        Enable or disable caching
+    max_size : Optional[int]
+        Maximum number of files to cache
+    verbose : Optional[bool]
+        Enable verbose logging
+    """
+    if enabled is not None:
+        _cache_config['enabled'] = enabled
+    if max_size is not None:
+        _cache_config['max_size'] = max_size
+    if verbose is not None:
+        _cache_config['verbose'] = verbose
+
+
+# Track numpy cache separately  
+_numpy_cache_keys = set()
+
+# Specialized cache for numpy arrays (using LRU cache)
+@lru_cache(maxsize=16)
+def _cached_load_npy(file_key: Tuple[str, float, int]) -> Any:
+    """
+    Cached numpy loader using LRU cache.
+    
+    Parameters
+    ----------
+    file_key : Tuple[str, float, int]
+        Cache key (path, mtime, size)
+        
+    Returns
+    -------
+    Any
+        Loaded numpy array
+    """
+    file_path = file_key[0]
+    # Import here to avoid circular imports
+    from ._load_modules._numpy import _load_npy
+    result = _load_npy(file_path)
+    
+    # Track that we've cached this key
+    _numpy_cache_keys.add(file_key)
+    
+    return result
+
+
+def load_npy_cached(file_path: str, **kwargs) -> Any:
+    """
+    Load numpy file with caching.
+    
+    Parameters
+    ----------
+    file_path : str
+        Path to numpy file
+    **kwargs
+        Additional arguments for numpy.load
+        
+    Returns
+    -------
+    Any
+        Loaded numpy array
+    """
+    if not _cache_config['enabled']:
+        from ._load_modules._numpy import _load_npy
+        return _load_npy(file_path, **kwargs)
+    
+    # Check if we have a cache hit
+    file_key = get_file_key(file_path)
+    
+    # Check if this key is already cached
+    if file_key in _numpy_cache_keys:
+        _cache_stats['hits'] += 1
+        if _cache_config['verbose']:
+            print(f"[Cache HIT] Loaded from cache: {file_path}")
+    else:
+        _cache_stats['misses'] += 1
+        if _cache_config['verbose']:
+            print(f"[Cache MISS] Loading from disk: {file_path}")
+    
+    # Use LRU cache for numpy files
+    return _cached_load_npy(file_key)
+
+
+# EOF

scitex/io/_load_configs.py

@@ -1,27 +1,31 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Timestamp: "2025-04-22 07:19:03 (ywatanabe)"
-# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_configs.py
+# Timestamp: "2025-10-11 23:54:07 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_configs.py
 # ----------------------------------------
+from __future__ import annotations
 import os
-
-__FILE__ = "/ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_configs.py"
+__FILE__ = (
+    "./src/scitex/io/_load_configs.py"
+)
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
-THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_configs.py"
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2025-02-27 11:09:00 (ywatanabe)"
-# File: ./scitex_repo/src/scitex/io/_load_configs.py
+from pathlib import Path
+from typing import Optional, Union
 
 from ..dict import DotDict
-from ._load import load
 from ._glob import glob
+from ._load import load
 
 
-def load_configs(IS_DEBUG=None, show=False, verbose=False):
-    """Load YAML configuration files from ./config directory.
+def load_configs(
+    IS_DEBUG=None,
+    show=False,
+    verbose=False,
+    config_dir: Optional[Union[str, Path]] = None,
+):
+    """Load YAML configuration files from specified directory.
 
     Parameters
     ----------
@@ -31,6 +35,9 @@ def load_configs(IS_DEBUG=None, show=False, verbose=False):
         Show configuration changes
     verbose : bool
         Print detailed information
+    config_dir : Union[str, Path], optional
+        Directory containing configuration files. Can be a string or pathlib.Path object.
+        Defaults to "./config" if None
 
     Returns
     -------
@@ -54,22 +61,40 @@ def load_configs(IS_DEBUG=None, show=False, verbose=False):
         return config
 
     try:
+        # Handle config directory parameter
+        if config_dir is None:
+            config_dir = "./config"
+        elif isinstance(config_dir, Path):
+            config_dir = str(config_dir)
+
         # Set debug mode
+        debug_config_path = f"{config_dir}/IS_DEBUG.yaml"
         IS_DEBUG = (
             IS_DEBUG
             or os.getenv("CI") == "True"
             or (
-                os.path.exists("./config/IS_DEBUG.yaml")
-                and load("./config/IS_DEBUG.yaml").get("IS_DEBUG")
+                os.path.exists(debug_config_path)
+                and load(debug_config_path).get("IS_DEBUG")
             )
         )
 
         # Load and merge configs
         CONFIGS = {}
-        for lpath in glob("./config/*.yaml"):
+
+        # Load from main config directory
+        config_pattern = f"{config_dir}/*.yaml"
+        for lpath in glob(config_pattern):
             if config := load(lpath):
                 CONFIGS.update(apply_debug_values(config, IS_DEBUG))
 
+        # Load from categories subdirectory if it exists
+        categories_dir = f"{config_dir}/categories"
+        if os.path.exists(categories_dir):
+            categories_pattern = f"{categories_dir}/*.yaml"
+            for lpath in glob(categories_pattern):
+                if config := load(lpath):
+                    CONFIGS.update(apply_debug_values(config, IS_DEBUG))
+
         return DotDict(CONFIGS)
 
     except Exception as e: