scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/io/_save_modules/_hdf5.py

@@ -1,249 +1,274 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Timestamp: "2025-06-14 18:26:02 (ywatanabe)"
-# File: /ssh:sp:/home/ywatanabe/proj/SciTeX-Code/src/scitex/io/_save_modules/_hdf5.py
+# Timestamp: "2025-07-14 05:28:34 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_save_modules/_hdf5.py
 # ----------------------------------------
 import os
-__FILE__ = (
-    "./src/scitex/io/_save_modules/_hdf5.py"
-)
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 
-def _save_hdf5(obj, spath, key=None, override=False, **kwargs):
-    import os
-    import pickle
+import fcntl
+import pickle
+import shutil
+import tempfile
+import time
 
-    import h5py
-    import numpy as np
+import h5py
+import numpy as np
 
-    if not isinstance(obj, dict):
-        obj = {"data": obj}
 
-    if "compression" not in kwargs:
-        kwargs["compression"] = "gzip"
+class SWMRFile:
+    """Context manager for SWMR-enabled HDF5 files."""
 
-    mode = "a"
+    def __init__(self, filepath, mode="r", swmr=True, timeout=300):
+        self.filepath = filepath
+        self.mode = mode
+        self.swmr = swmr
+        self.timeout = timeout
+        self.file = None
+        self.temp_file = None
 
-    with h5py.File(spath, mode) as h5_file:
-        if key:
-            # Check if the full path exists
-            key_exists = False
+    def __enter__(self):
+        start_time = time.time()
+
+        while time.time() - start_time < self.timeout:
             try:
-                parts = key.split("/")
-                current = h5_file
-                for part in parts:
-                    if part in current:
-                        current = current[part]
+                if self.mode in ["w", "w-"]:
+                    # For write mode, create new file
+                    self.file = h5py.File(
+                        self.filepath, self.mode, libver="latest"
+                    )
+                    if self.swmr:
+                        self.file.swmr_mode = True
+                    return self.file
+
+                elif self.mode == "a":
+                    # For append mode, handle SWMR carefully
+                    if os.path.exists(self.filepath):
+                        # Try to open in read mode first to check SWMR status
+                        try:
+                            with h5py.File(self.filepath, "r") as test_file:
+                                is_swmr = test_file.swmr_mode
+                        except:
+                            is_swmr = False
+
+                        if is_swmr:
+                            # File is in SWMR mode, we need to copy and modify
+                            self.temp_file = tempfile.NamedTemporaryFile(
+                                delete=False, suffix=".h5"
+                            )
+                            shutil.copy2(self.filepath, self.temp_file.name)
+                            self.file = h5py.File(
+                                self.temp_file.name, "r+", libver="latest"
+                            )
+                        else:
+                            # Normal append mode
+                            self.file = h5py.File(
+                                self.filepath, "r+", libver="latest"
+                            )
+                            if self.swmr:
+                                self.file.swmr_mode = True
                     else:
-                        break
+                        # Create new file
+                        self.file = h5py.File(
+                            self.filepath, "w", libver="latest"
+                        )
+                        if self.swmr:
+                            self.file.swmr_mode = True
+                    return self.file
+
+                elif self.mode == "r":
+                    # Read mode with SWMR
+                    self.file = h5py.File(
+                        self.filepath, "r", libver="latest", swmr=True
+                    )
+                    return self.file
+
+            except (OSError, IOError) as e:
+                if "Resource temporarily unavailable" in str(e):
+                    time.sleep(
+                        0.1 * (2 ** min(5, (time.time() - start_time) / 10))
+                    )
                 else:
-                    key_exists = True
-            except:
-                key_exists = False
-            
-            if not override and key_exists:
-                return
-
-            if override and key_exists:
-                del h5_file[key]
-
-            key_parts = key.split("/")
-            current_group = h5_file
-
-            for part in key_parts[:-1]:
-                if part:
-                    current_group = current_group.require_group(part)
-
-            final_key = key_parts[-1] if key_parts[-1] else ""
-            target_group = (
-                current_group.create_group(final_key)
-                if final_key
-                else current_group
-            )
-        else:
-            target_group = h5_file
+                    raise
+
+        raise TimeoutError(
+            f"Could not open {self.filepath} within {self.timeout} seconds"
+        )
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.file:
+            self.file.close()
+
+        # If we used a temp file, move it back
+        if self.temp_file and exc_type is None:
+            shutil.move(self.temp_file.name, self.filepath)
+        elif self.temp_file:
+            os.unlink(self.temp_file.name)
+
+
+def _save_hdf5(
+    obj,
+    spath,
+    key=None,
+    override=False,
+    swmr=True,
+    compression="gzip",
+    compression_opts=4,
+    max_retries=10,
+    **kwargs,
+):
+    """
+    HDF5 save function with SWMR support for HPC environments.
+
+    Parameters:
+    -----------
+    obj : dict or any
+        Object to save
+    spath : str
+        Path to HDF5 file
+    key : str, optional
+        Key/group path within HDF5 file
+    override : bool
+        Whether to override existing keys
+    swmr : bool
+        Enable SWMR mode (default: True)
+    compression : str
+        Compression algorithm (default: 'gzip')
+    compression_opts : int
+        Compression level (default: 4)
+    max_retries : int
+        Maximum number of retry attempts
+    """
+    if not isinstance(obj, dict):
+        obj = {"data": obj}
+
+    # Ensure directory exists
+    os.makedirs(os.path.dirname(spath) or ".", exist_ok=True)
+
+    # Lock file for coordination
+    lock_file = f"{spath}.lock"
+
+    for attempt in range(max_retries):
+        try:
+            # Use file locking for write coordination
+            with open(lock_file, "w") as lock_fd:
+                fcntl.flock(lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+
+                try:
+                    mode = "a" if os.path.exists(spath) else "w"
+
+                    with SWMRFile(spath, mode, swmr=swmr) as h5_file:
+                        if key:
+                            # Check if key exists
+                            if key in h5_file and not override:
+                                return
+
+                            if key in h5_file and override:
+                                del h5_file[key]
 
-        for dataset_name, data in obj.items():
-            if isinstance(data, str):
-                target_group.create_dataset(
-                    dataset_name, data=data, dtype=h5py.string_dtype()
-                )
+                            # Create group structure
+                            parts = [p for p in key.split("/") if p]
+                            current_group = h5_file
+
+                            for part in parts[:-1]:
+                                if part:
+                                    current_group = (
+                                        current_group.require_group(part)
+                                    )
+
+                            final_key = parts[-1] if parts else ""
+                            if final_key:
+                                target_group = current_group.create_group(
+                                    final_key
+                                )
+                            else:
+                                target_group = current_group
+                        else:
+                            target_group = h5_file
+
+                        # Save datasets with chunking for large arrays
+                        for dataset_name, data in obj.items():
+                            _save_dataset(
+                                target_group,
+                                dataset_name,
+                                data,
+                                compression,
+                                compression_opts,
+                                **kwargs,
+                            )
+
+                        # Flush to ensure data is written
+                        h5_file.flush()
+
+                finally:
+                    fcntl.flock(lock_fd, fcntl.LOCK_UN)
+
+                return  # Success
+
+        except (OSError, IOError) as e:
+            if attempt < max_retries - 1:
+                time.sleep(0.1 * (2**attempt))
             else:
-                data_array = np.asarray(data)
+                raise
 
-                if data_array.dtype == np.object_:
-                    pickled_data = pickle.dumps(data)
-                    target_group.create_dataset(
-                        dataset_name, data=np.void(pickled_data)
-                    )
-                elif data_array.ndim == 0:
-                    target_group.create_dataset(dataset_name, data=data)
+    raise RuntimeError(
+        f"Failed to save to {spath} after {max_retries} attempts"
+    )
+
+
+def _save_dataset(group, name, data, compression, compression_opts, **kwargs):
+    """Save a single dataset with appropriate settings."""
+    try:
+        if isinstance(data, str):
+            group.create_dataset(name, data=data, dtype=h5py.string_dtype())
+
+        elif hasattr(data, "__array__"):
+            # NumPy arrays and array-like objects
+            data_array = np.asarray(data)
+
+            # Determine chunk size for large arrays
+            chunks = None
+            if data_array.size > 1000:
+                chunks = True  # Let h5py auto-determine chunks
+
+            # Check if we should use compression
+            use_compression = compression if data_array.size > 100 else None
+
+            group.create_dataset(
+                name,
+                data=data_array,
+                compression=use_compression,
+                compression_opts=compression_opts if use_compression else None,
+                chunks=chunks,
+                **kwargs,
+            )
+
+        elif isinstance(data, (list, tuple)) and len(data) > 0:
+            # Try to convert to array
+            try:
+                data_array = np.asarray(data)
+                if data_array.dtype != np.object_:
+                    group.create_dataset(name, data=data_array)
                 else:
-                    target_group.create_dataset(
-                        dataset_name, data=data, **kwargs
-                    )
+                    # Pickle complex objects
+                    pickled_data = pickle.dumps(data)
+                    group.create_dataset(name, data=np.void(pickled_data))
+            except:
+                # Pickle if conversion fails
+                pickled_data = pickle.dumps(data)
+                group.create_dataset(name, data=np.void(pickled_data))
 
+        else:
+            # For all other types, try direct save or pickle
+            try:
+                group.create_dataset(name, data=data)
+            except:
+                pickled_data = pickle.dumps(data)
+                group.create_dataset(name, data=np.void(pickled_data))
 
-# def _save_hdf5(obj, spath, key=None, **kwargs):
-#     """
-#     Save a dictionary of arrays or a single array to an HDF5 file.
-
-#     Parameters
-#     ----------
-#     obj : dict or array-like
-#         Dictionary of arrays to save (keys will be dataset names) or
-#         a single array/data to save (will be saved as 'data').
-#     spath : str
-#         Path where the HDF5 file will be saved.
-#     key : str, optional
-#         Path/key within the HDF5 file where data will be saved.
-#         If None, saves to root.
-#     **kwargs : dict
-#         Additional keyword arguments passed to create_dataset.
-#         Common: compression='gzip', compression_opts=9, shuffle=True
-
-#     Returns
-#     -------
-#     None
-#     """
-#     import numpy as np
-
-#     # Convert single array to dict
-#     if not isinstance(obj, dict):
-#         obj = {'data': obj}
-
-#     # Set default compression if not specified
-#     if "compression" not in kwargs:
-#         kwargs["compression"] = "gzip"
-
-#     mode = "a" if key else "w"
-
-#     with h5py.File(spath, mode) as hf:
-#         if key:
-#             if key in hf:
-#                 del hf[key]
-#             group = hf.create_group(key)
-#         else:
-#             group = hf
-
-#         for name, data in obj.items():
-#             try:
-#                 # Check if data is scalar and remove incompatible options
-#                 if np.isscalar(data) or (isinstance(data, np.ndarray) and data.ndim == 0):
-#                     # Remove chunk/compression options for scalars
-#                     scalar_kwargs = {k: v for k, v in kwargs.items()
-#                                    if k not in ['chunks', 'compression', 'compression_opts',
-#                                                'shuffle', 'fletcher32', 'maxshape']}
-#                     group.create_dataset(name, data=data, **scalar_kwargs)
-#                 else:
-#                     group.create_dataset(name, data=data, **kwargs)
-#             except TypeError as e:
-#                 if "Object dtype" in str(
-#                     e
-#                 ) or "has no native HDF5 equivalent" in str(e):
-#                     if isinstance(data, np.ndarray) and data.dtype == object:
-#                         if all(isinstance(item, str) for item in data.flat):
-#                             dt = h5py.string_dtype(encoding="utf-8")
-#                             group.create_dataset(
-#                                 name, data=data.astype(dt), **kwargs
-#                             )
-#                         else:
-#                             str_data = np.array(
-#                                 [str(item) for item in data.flat]
-#                             ).reshape(data.shape)
-#                             dt = h5py.string_dtype(encoding="utf-8")
-#                             group.create_dataset(
-#                                 name, data=str_data, dtype=dt, **kwargs
-#                             )
-#                     else:
-#                         group.attrs[name] = str(data)
-#                 else:
-#                     raise
-
-# #!/usr/bin/env python3
-# # -*- coding: utf-8 -*-
-# # Timestamp: "2025-05-16 12:24:04 (ywatanabe)"
-# # File: /data/gpfs/projects/punim2354/ywatanabe/scitex_repo/src/scitex/io/_save_modules/_hdf5.py
-
-# import h5py
-
-# def _save_hdf5(obj, spath, key=None, **kwargs):
-#     """
-#     Save a dictionary of arrays to an HDF5 file.
-
-#     Parameters
-#     ----------
-#     obj : dict
-#         Dictionary of arrays to save. Keys will be dataset names.
-#     spath : str
-#         Path where the HDF5 file will be saved.
-#     key : str, optional
-#         Path to the group within the HDF5 file. If None, saves to root.
-#     **kwargs : dict
-#         Additional keyword arguments passed to create_dataset.
-
-#     Returns
-#     -------
-#     None
-#     """
-#     import numpy as np
-
-#     name_list, obj_list = [], []
-#     for k, v in obj.items():
-#         name_list.append(k)
-#         obj_list.append(v)
-
-#     with h5py.File(spath, "w") as hf:
-#         # If key is specified, create/get the group
-#         if key:
-#             group = hf.require_group(key)
-#         else:
-#             group = hf
-
-#         for name, data in zip(name_list, obj_list):
-#             try:
-#                 # Try to save directly
-#                 group.create_dataset(name, data=data, **kwargs)
-#             except TypeError as e:
-#                 if "Object dtype" in str(
-#                     e
-#                 ) or "has no native HDF5 equivalent" in str(e):
-#                     # Handle object dtype by converting to appropriate format
-#                     if isinstance(data, np.ndarray) and data.dtype == object:
-#                         # Try to convert to string array
-#                         try:
-#                             # Check if all elements are strings
-#                             if all(
-#                                 isinstance(item, str) for item in data.flat
-#                             ):
-#                                 # Save as variable-length string
-#                                 dt = h5py.string_dtype(encoding="utf-8")
-#                                 group.create_dataset(
-#                                     name, data=data.astype(dt), **kwargs
-#                                 )
-#                             else:
-#                                 # Convert to string representation
-#                                 str_data = np.array(
-#                                     [str(item) for item in data.flat]
-#                                 ).reshape(data.shape)
-#                                 dt = h5py.string_dtype(encoding="utf-8")
-#                                 group.create_dataset(
-#                                     name, data=str_data, dtype=dt, **kwargs
-#                                 )
-#                         except Exception:
-#                             # If conversion fails, save as attributes or skip
-#                             group.attrs[name] = str(data)
-#                             print(
-#                                 f"Warning: Dataset '{name}' saved as attribute due to incompatible dtype"
-#                             )
-#                     else:
-#                         # For non-array objects, save as attribute
-#                         group.attrs[name] = str(data)
-#                         print(
-#                             f"Warning: Dataset '{name}' saved as attribute due to incompatible type"
-#                         )
-#                 else:
-#                     raise  # Re-raise if it's a different error
+    except Exception as e:
+        print(f"Warning: Could not save dataset '{name}': {e}")
 
 # EOF

scitex/io/_save_modules/_image.py

@@ -4,9 +4,7 @@
 # File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_save_modules/_save_image.py
 # ----------------------------------------
 import os
-__FILE__ = (
-    "./src/scitex/io/_save_modules/_save_image.py"
-)
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 

scitex/io/_save_modules/_json.py

@@ -4,22 +4,67 @@
 # File: /data/gpfs/projects/punim2354/ywatanabe/scitex_repo/src/scitex/io/_save_modules/_json.py
 
 import json
+from typing import Any
+
+
+def _convert_to_serializable(obj: Any) -> Any:
+    """
+    Convert non-JSON-serializable objects to serializable format.
+
+    Handles:
+    - DotDict -> dict
+    - DataFrame -> dict of lists
+    - numpy arrays -> lists
+    - etc.
+    """
+    # Handle DotDict
+    if hasattr(obj, '__class__') and obj.__class__.__name__ == 'DotDict':
+        return _convert_to_serializable(dict(obj))
+
+    # Handle pandas DataFrame
+    if hasattr(obj, '__class__') and obj.__class__.__name__ == 'DataFrame':
+        return obj.to_dict('list')
+
+    # Handle numpy arrays
+    if hasattr(obj, 'tolist'):
+        return obj.tolist()
+
+    # Handle dict recursively
+    if isinstance(obj, dict):
+        return {key: _convert_to_serializable(value) for key, value in obj.items()}
+
+    # Handle list recursively
+    if isinstance(obj, list):
+        return [_convert_to_serializable(item) for item in obj]
+
+    # Handle tuple
+    if isinstance(obj, tuple):
+        return list(obj)
+
+    # Default: return as-is and let json.dump handle or error
+    return obj
 
 
 def _save_json(obj, spath):
     """
     Save a Python object as a JSON file.
-    
+
+    Automatically converts DotDict objects and pandas DataFrames
+    to JSON-serializable formats.
+
     Parameters
     ----------
-    obj : dict or list
+    obj : dict, list, DotDict, or other JSON-serializable object
         The object to serialize to JSON.
     spath : str
         Path where the JSON file will be saved.
-        
+
     Returns
     -------
     None
     """
+    # Convert to serializable format
+    serializable_obj = _convert_to_serializable(obj)
+
     with open(spath, "w") as f:
-        json.dump(obj, f, indent=4)
+        json.dump(serializable_obj, f, indent=4)

scitex/io/_save_modules/_listed_dfs_as_csv.py

@@ -4,9 +4,7 @@
 # File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_save_modules/_save_listed_dfs_as_csv.py
 # ----------------------------------------
 import os
-__FILE__ = (
-    "./src/scitex/io/_save_modules/_save_listed_dfs_as_csv.py"
-)
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 # Time-stamp: "2024-11-02 21:28:56 (ywatanabe)"

scitex/io/_save_modules/_listed_scalars_as_csv.py

@@ -4,9 +4,7 @@
 # File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_save_modules/_save_listed_scalars_as_csv.py
 # ----------------------------------------
 import os
-__FILE__ = (
-    "./src/scitex/io/_save_modules/_save_listed_scalars_as_csv.py"
-)
+__FILE__ = __file__
 __DIR__ = os.path.dirname(__FILE__)
 # ----------------------------------------
 # Time-stamp: "2024-11-02 21:26:48 (ywatanabe)"