scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/io/{_H5Explorer.py → _load_modules/_H5Explorer.py}

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Timestamp: "2025-06-14 18:05:18 (ywatanabe)"
-# File: /ssh:sp:/home/ywatanabe/proj/SciTeX-Code/src/scitex/io/_H5Explorer.py
+# Timestamp: "2025-07-05 12:46:09 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_H5Explorer.py
 # ----------------------------------------
 import os
-__FILE__ = (
-    "./src/scitex/io/_H5Explorer.py"
-)
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
+
+import random
+import time
 import warnings
 
 # Time-stamp: "2025-06-13 21:00:00 (ywatanabe)"
@@ -132,6 +133,7 @@ class H5Explorer:
             # Handle pickled objects (stored as np.void)
             elif isinstance(data, np.void):
                 import pickle
+
                 return pickle.loads(data.tobytes())
             return data
         elif isinstance(item, h5py.Group):
@@ -268,25 +270,86 @@ def explore_h5(filepath: str) -> None:
         warnings.warn(f"Warning: File does not exist: {filepath}")
 
 
-def has_h5_key(h5_path, key):
-    """Check if key exists in HDF5 file."""
+def has_h5_key(h5_path, key, max_retries=3, action_on_corrupted="delete"):
+    """
+    Robust version of has_h5_key that handles corrupted files and lock conflicts.
+    """
     h5_path = os.path.realpath(h5_path)
 
     if not os.path.exists(h5_path):
         return False
 
-    try:
-        with h5py.File(h5_path, "r") as h5_file:
-            parts = key.split("/")
-            current = h5_file
-            for part in parts:
-                if part in current:
-                    current = current[part]
+    for attempt in range(max_retries):
+        try:
+            with h5py.File(h5_path, "r") as h5_file:
+                parts = [p for p in key.split("/") if p]
+                current = h5_file
+
+                for part in parts:
+                    if part in current:
+                        current = current[part]
+                    else:
+                        return False
+                return True
+
+        except (KeyError, FileNotFoundError):
+            return False
+
+        except (OSError, RuntimeError, ValueError) as e:
+            error_msg = str(e).lower()
+
+            lock_indicators = [
+                "resource temporarily unavailable",
+                "file is already open",
+                "unable to lock file",
+                "file locking failed",
+            ]
+
+            corruption_indicators = [
+                "unable to synchronously",
+                "bad symbol table",
+                "free block size is zero",
+                "truncated file",
+                "unable to read signature",
+                "corrupted",
+                "invalid file signature",
+                "unable to check link existence",
+                "bad heap free list",
+            ]
+
+            if any(indicator in error_msg for indicator in lock_indicators):
+                if attempt < max_retries - 1:
+                    base_wait = 0.1 * (2**attempt)
+                    jitter = random.uniform(0, base_wait * 0.5)
+                    wait_time = base_wait + jitter
+                    time.sleep(wait_time)
+                    continue
                 else:
                     return False
-            return True
 
-    except (KeyError, FileNotFoundError, OSError):
-        return False
+            elif any(
+                indicator in error_msg for indicator in corruption_indicators
+            ):
+                if action_on_corrupted == "delete":
+                    if _delete_corrupted_entry(h5_path, key):
+                        return False
+                return False
+            else:
+                raise e
+
+    return False
+
+
+def _delete_corrupted_entry(h5_path, key):
+    """Delete corrupted entry from HDF5 file."""
+    try:
+        with h5py.File(h5_path, "r+") as h5_file:
+            if key in h5_file:
+                del h5_file[key]
+                print(f"Deleted corrupted entry: {key}")
+                return True
+    except:
+        pass
+    return False
 
 # EOF

scitex/io/_load_modules/_ZarrExplorer.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-07-11 15:45:49 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_ZarrExplorer.py
+# ----------------------------------------
+import os
+__FILE__ = __file__
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+from typing import Any, List, Optional
+
+import zarr
+from ._zarr import _load_zarr_dataset
+
+
+class ZarrExplorer:
+    """Interactive Zarr store explorer."""
+
+    def __init__(self, storepath: str, mode: str = "r"):
+        self.storepath = storepath
+        self.mode = mode
+        self.store = zarr.open(storepath, mode=mode)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        pass  # Zarr doesn't need explicit closing
+
+    def explore(self, path: str = "/", max_depth: Optional[int] = None):
+        """Explore Zarr store structure."""
+        self.show(path, max_depth)
+
+    def show(
+        self,
+        path: str = "/",
+        max_depth: Optional[int] = None,
+        indent: str = "",
+        _current_depth: int = 0,
+    ):
+        """Display Zarr store structure."""
+        if max_depth is not None and _current_depth > max_depth:
+            return
+
+        if path == "/":
+            target = self.store
+        else:
+            target = self.store[path.lstrip("/")]
+
+        if hasattr(target, "keys"):  # Group
+            if path != "/":
+                print(f"{indent}[{path.split('/')[-1]}]")
+
+            for key in sorted(target.keys()):
+                subpath = f"{path}/{key}".replace("//", "/")
+                self.show(
+                    subpath, max_depth, indent + "  ", _current_depth + 1
+                )
+
+        else:  # Array
+            name = path.split("/")[-1]
+            shape = target.shape
+            dtype = target.dtype
+            size = target.size
+            compressor = getattr(target, "compressor", None)
+            compressed_size = getattr(target, "nbytes_stored", "unknown")
+
+            print(
+                f"{indent}{name}: shape={shape}, dtype={dtype}, "
+                f"size={size}, compressor={compressor}, "
+                f"compressed_size={compressed_size}"
+            )
+
+    def keys(self, path: str = "/") -> List[str]:
+        """Get keys at specified path."""
+        if path == "/":
+            target = self.store
+        else:
+            target = self.store[path.lstrip("/")]
+
+        if hasattr(target, "keys"):
+            return list(target.keys())
+        return []
+
+    def load(self, path: str) -> Any:
+        """Load data from specified path."""
+        return _load_zarr_dataset(self.store[path.lstrip("/")])
+
+    def has_key(self, path: str) -> bool:
+        """Check if key exists (no locking issues!)."""
+        try:
+            _ = self.store[path.lstrip("/")]
+            return True
+        except KeyError:
+            return False
+
+
+def explore_zarr(storepath: str) -> None:
+    """Explore Zarr store structure."""
+    explorer = ZarrExplorer(storepath)
+    explorer.explore()
+
+
+def has_zarr_key(zarr_path: str, key: str) -> bool:
+    """Check if key exists in Zarr store (no locking issues!)."""
+    try:
+        store = zarr.open(zarr_path, mode="r")
+        _ = store[key.lstrip("/")]
+        return True
+    except (KeyError, ValueError):
+        return False
+
+# EOF

scitex/io/_load_modules/_bibtex.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-01-19 14:20:00 (ywatanabe)"
+# File: ./src/scitex/io/_load_modules/_bibtex.py
+# ----------------------------------------
+import os
+__FILE__ = __file__
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+"""
+BibTeX file loading module for SciTeX IO.
+
+This module provides functionality to load and parse .bib files,
+returning structured data that can be used by other SciTeX modules.
+"""
+
+import re
+from scitex import logging
+from typing import List, Dict, Any, Optional
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+def _load_bibtex(lpath: str, **kwargs) -> List[Dict[str, Any]]:
+    """
+    Load BibTeX file and parse entries.
+    
+    Args:
+        lpath: Path to the .bib file
+        **kwargs: Additional arguments
+            - parse_mode: 'full' (default) or 'basic'
+            - encoding: File encoding (default: utf-8)
+    
+    Returns:
+        List of dictionaries, each representing a BibTeX entry
+    """
+    parse_mode = kwargs.get('parse_mode', 'full')
+    encoding = kwargs.get('encoding', 'utf-8')
+    
+    try:
+        # Read file content
+        with open(lpath, 'r', encoding=encoding) as f:
+            content = f.read()
+        
+        # Parse BibTeX entries
+        entries = _parse_bibtex_content(content, parse_mode)
+        
+        logger.info(f"Loaded {len(entries)} BibTeX entries from {lpath}")
+        return entries
+        
+    except Exception as e:
+        raise ValueError(f"Error loading BibTeX file {lpath}: {str(e)}")
+
+
+def _parse_bibtex_content(content: str, parse_mode: str = 'full') -> List[Dict[str, Any]]:
+    """
+    Parse BibTeX content into structured entries.
+    
+    Args:
+        content: BibTeX file content
+        parse_mode: 'full' for complete parsing, 'basic' for minimal parsing
+    
+    Returns:
+        List of parsed entries
+    """
+    entries = []
+    
+    # Pattern to match BibTeX entries
+    entry_pattern = r'@(\w+)\s*\{\s*([^,]+)\s*,(.*?)\n\s*\}'
+    matches = re.finditer(entry_pattern, content, re.DOTALL | re.IGNORECASE)
+    
+    for match in matches:
+        entry_type = match.group(1).lower()
+        entry_key = match.group(2).strip()
+        entry_body = match.group(3)
+        
+        # Parse entry fields
+        entry = {
+            'entry_type': entry_type,
+            'key': entry_key,
+            'fields': {}
+        }
+        
+        if parse_mode == 'full':
+            entry['fields'] = _parse_bibtex_fields(entry_body)
+        else:
+            # Basic parsing - just extract key-value pairs
+            entry['fields'] = _parse_bibtex_fields_basic(entry_body)
+        
+        entries.append(entry)
+    
+    return entries
+
+
+def _parse_bibtex_fields(body: str) -> Dict[str, str]:
+    """
+    Parse BibTeX entry fields with proper handling of nested braces.
+    
+    Args:
+        body: The body of a BibTeX entry
+    
+    Returns:
+        Dictionary of field names to values
+    """
+    fields = {}
+    
+    # Pattern to match field = value pairs
+    # Handles both {braced} and "quoted" values
+    field_pattern = r'(\w+)\s*=\s*(?:\{([^{}]*(?:\{[^{}]*\}[^{}]*)*)\}|"([^"]*)")'
+    
+    for match in re.finditer(field_pattern, body):
+        field_name = match.group(1).lower()
+        # Get value from either braced or quoted group
+        field_value = match.group(2) if match.group(2) is not None else match.group(3)
+        
+        # Clean up the value
+        field_value = field_value.strip()
+        
+        # Handle special characters
+        field_value = _unescape_bibtex(field_value)
+        
+        fields[field_name] = field_value
+    
+    return fields
+
+
+def _parse_bibtex_fields_basic(body: str) -> Dict[str, str]:
+    """
+    Basic parsing of BibTeX fields without special character handling.
+    
+    Args:
+        body: The body of a BibTeX entry
+    
+    Returns:
+        Dictionary of field names to values
+    """
+    fields = {}
+    
+    # Simple pattern for basic parsing
+    lines = body.strip().split(',')
+    
+    for line in lines:
+        if '=' in line:
+            parts = line.split('=', 1)
+            if len(parts) == 2:
+                field_name = parts[0].strip().lower()
+                field_value = parts[1].strip()
+                
+                # Remove surrounding braces or quotes
+                if field_value.startswith('{') and field_value.endswith('}'):
+                    field_value = field_value[1:-1]
+                elif field_value.startswith('"') and field_value.endswith('"'):
+                    field_value = field_value[1:-1]
+                
+                fields[field_name] = field_value.strip()
+    
+    return fields
+
+
+def _unescape_bibtex(text: str) -> str:
+    """
+    Unescape BibTeX special characters.
+    
+    Args:
+        text: BibTeX text with escaped characters
+    
+    Returns:
+        Unescaped text
+    """
+    # Common BibTeX escapes
+    replacements = {
+        r'\&': '&',
+        r'\%': '%',
+        r'\$': '$',
+        r'\#': '#',
+        r'\_': '_',
+        r'\{': '{',
+        r'\}': '}',
+        r'~': ' ',  # Non-breaking space
+        r'--': '–',  # En dash
+        r'---': '—',  # Em dash
+    }
+    
+    for escaped, unescaped in replacements.items():
+        text = text.replace(escaped, unescaped)
+    
+    return text
+
+
+# Convenience function for loading BibTeX
+def load_bibtex(filepath: str, **kwargs) -> List[Dict[str, Any]]:
+    """
+    Load and parse a BibTeX file.
+    
+    Args:
+        filepath: Path to the .bib file
+        **kwargs: Additional arguments passed to _load_bibtex
+    
+    Returns:
+        List of parsed BibTeX entries
+    """
+    return _load_bibtex(filepath, **kwargs)
+
+
+# EOF

scitex/io/_load_modules/_hdf5.py

@@ -1,196 +1,71 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Timestamp: "2025-06-16 17:40:55 (ywatanabe)"
-# File: /ssh:sp:/home/ywatanabe/proj/SciTeX-Code/src/scitex/io/_load_modules/_hdf5.py
+# Timestamp: "2025-07-12 07:04:14 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_hdf5.py
 # ----------------------------------------
 import os
-__FILE__ = (
-    "./src/scitex/io/_load_modules/_hdf5.py"
-)
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
-import warnings
-from typing import Any
+import time
 
 import h5py
 import numpy as np
 
-# Time-stamp: "2024-11-14 07:55:37 (ywatanabe)"
+from .._save_modules._hdf5 import SWMRFile
 
 
+def _load_hdf5(lpath, key=None, swmr=True, max_retries=10, **kwargs):
+    """Load HDF5 file with SWMR support."""
+    for attempt in range(max_retries):
+        try:
+            with SWMRFile(lpath, "r", swmr=swmr) as h5_file:
+                if key:
+                    if key not in h5_file:
+                        return None
+                    target = h5_file[key]
+                else:
+                    target = h5_file
 
+                # Load data recursively
+                return _load_h5_object(target)
 
-def _load_group(group):
-    """Recursively load an HDF5 group."""
-    obj = {}
-    for key in group.keys():
-        if isinstance(group[key], h5py.Group):
-            # Recursively load subgroups
-            obj[key] = _load_group(group[key])
-        else:
-            # Load dataset
-            dataset = group[key]
-            # Check if it's a scalar dataset
-            if dataset.shape == ():
-                data = dataset[()]
-            else:
-                data = dataset[:]
-
-            # Decode bytes to string if needed
-            if isinstance(data, bytes):
-                obj[key] = data.decode("utf-8")
-            elif isinstance(data, np.void):
-                # Handle pickled data
-                import pickle
-
-                obj[key] = pickle.loads(data.tobytes())
-            else:
-                obj[key] = data
-    # Load attributes
-    for key in group.attrs.keys():
-        obj[key] = group.attrs[key]
-    return obj
-
-
-def _load_hdf5(lpath: str, key: str = None, **kwargs) -> Any:
-    """Load HDF5 file with automatic group/root switching."""
-    try:
-        with h5py.File(lpath, "r") as hf:
-            if key:
-                if key not in hf:
-                    return None
-                target = hf[key]
+        except (OSError, IOError) as e:
+            if attempt < max_retries - 1:
+                time.sleep(0.1 * (2**attempt))
             else:
-                target = hf
-
-            obj = {}
-            for key_name in target.keys():
-                try:
-                    if isinstance(target[key_name], h5py.Group):
-                        obj[key_name] = _load_group(target[key_name])
-                    else:
-                        dataset = target[key_name]
-                        if dataset.shape == ():
-                            data = dataset[()]
-                        else:
-                            data = dataset[:]
-
-                        if isinstance(data, bytes):
-                            obj[key_name] = data.decode("utf-8")
-                        elif isinstance(data, np.void):
-                            import pickle
-
-                            obj[key_name] = pickle.loads(data.tobytes())
-                        else:
-                            obj[key_name] = data
-                except (RuntimeError, OSError) as e:
-                    print(f"Warning: Could not load key '{key_name}': {e}")
-                    continue
-
-            for attr_name in target.attrs.keys():
-                obj[attr_name] = target.attrs[attr_name]
-            return obj
-
-    except (RuntimeError, OSError) as e:
-        key_warning_str = f" with {key}" if key else ""
-        warnings.warn(f"Error loading {lpath}{key_warning_str}:\n{e}")
-        return None
-
-
-# def _load_hdf5(lpath: str, key: str = None, **kwargs) -> Any:
-#     """Load HDF5 file with automatic group/root switching."""
-#     with h5py.File(lpath, "r") as hf:
-#         if key:
-#             if key not in hf:
-#                 return None
-#             target = hf[key]
-#         else:
-#             target = hf
-
-#         obj = {}
-#         for key in target.keys():
-#             if isinstance(target[key], h5py.Group):
-#                 # Recursively load groups
-#                 obj[key] = _load_group(target[key])
-#             else:
-#                 # Load dataset
-#                 dataset = target[key]
-#                 # Check if it's a scalar dataset
-#                 if dataset.shape == ():
-#                     data = dataset[()]
-#                 else:
-#                     data = dataset[:]
-
-#                 # Decode bytes to string if needed
-#                 if isinstance(data, bytes):
-#                     obj[key] = data.decode("utf-8")
-#                 elif isinstance(data, np.void):
-#                     # Handle pickled data
-#                     import pickle
-
-#                     obj[key] = pickle.loads(data.tobytes())
-#                 else:
-#                     obj[key] = data
-#         for key in target.attrs.keys():
-#             obj[key] = target.attrs[key]
-#         return obj
-
-# def _load_hdf5(lpath: str, **kwargs) -> Any:
-#     """Load HDF5 file."""
-#     if not (lpath.endswith(".hdf5") or lpath.endswith(".h5")):
-#         raise ValueError("File must have .hdf5 or .h5 extension")
-
-#     def load_item(item):
-#         """Recursively load items from HDF5."""
-#         if isinstance(item, h5py.Group):
-#             # Load groups as nested dicts
-#             result = {}
-#             for key in item.keys():
-#                 result[key] = load_item(item[key])
-#             return result
-#         elif isinstance(item, h5py.Dataset):
-#             # Load datasets
-#             data = item[()]
-#             # Check if it's a pickled object
-#             if isinstance(data, np.void):
-#                 import pickle
-
-#                 return pickle.loads(data.tobytes())
-#             # Convert bytes to string
-#             elif isinstance(data, bytes):
-#                 return data.decode("utf-8")
-#             # Convert numpy scalars to Python types
-#             elif isinstance(data, np.integer):
-#                 return int(data)
-#             elif isinstance(data, np.floating):
-#                 return float(data)
-#             elif isinstance(data, np.bool_):
-#                 return bool(data)
-#             else:
-#                 return data
-#         else:
-#             return item
-
-#     with h5py.File(lpath, "r") as hf:
-#         return load_item(hf)
-
-# def _load_hdf5_group(lpath: str, key: str = None, **kwargs) -> Any:
-#     """Load specific group from HDF5 file."""
-#     with h5py.File(lpath, "r") as hf:
-#         if key:
-#             if key not in hf:
-#                 return None
-#             group = hf[key]
-#             result = {}
-#             for key in group.keys():
-#                 result[key] = group[key][:]
-#             for key in group.attrs.keys():
-#                 result[key] = group.attrs[key]
-#             return result
-#         else:
-#             return _load_hdf5(lpath, **kwargs)
-
-# # EOF
+                raise
+
+    return None
+
+
+def _load_h5_object(h5_obj):
+    """Recursively load HDF5 object."""
+    if isinstance(h5_obj, h5py.Group):
+        result = {}
+        # Load datasets
+        for key in h5_obj.keys():
+            result[key] = _load_h5_object(h5_obj[key])
+        # Load attributes
+        for key in h5_obj.attrs.keys():
+            result[f"_attr_{key}"] = h5_obj.attrs[key]
+        return result
+
+    elif isinstance(h5_obj, h5py.Dataset):
+        data = h5_obj[()]
+
+        # Handle different data types
+        if isinstance(data, bytes):
+            return data.decode("utf-8")
+        elif isinstance(data, np.void):
+            # Unpickle data
+            import pickle
+
+            return pickle.loads(data.tobytes())
+        else:
+            return data
+    else:
+        return h5_obj
 
 # EOF