scitex 2.0.0__py2.py3-none-any.whl

scitex/io/_load_modules/_catboost.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-12-12 06:50:19 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_catboost.py
+
+from typing import Union
+
+try:
+    from catboost import CatBoostClassifier, CatBoostRegressor
+    CATBOOST_AVAILABLE = True
+except ImportError:
+    CATBOOST_AVAILABLE = False
+    # Create placeholder classes for testing
+    class CatBoostClassifier:
+        pass
+    class CatBoostRegressor:
+        pass
+
+
+def _load_catboost(
+    lpath: str, **kwargs
+) -> Union["CatBoostClassifier", "CatBoostRegressor"]:
+    """
+    Loads a CatBoost model from a file.
+
+    Parameters
+    ----------
+    lpath : str
+        Path to the CatBoost model file (.cbm extension)
+    **kwargs : dict
+        Additional keyword arguments passed to load_model method
+
+    Returns
+    -------
+    Union[CatBoostClassifier, CatBoostRegressor]
+        Loaded CatBoost model object
+
+    Raises
+    ------
+    ValueError
+        If file extension is not .cbm
+    FileNotFoundError
+        If model file does not exist
+    ImportError
+        If CatBoost is not installed
+
+    Examples
+    --------
+    >>> model = _load_catboost('model.cbm')
+    >>> predictions = model.predict(X_test)
+    """
+    if not CATBOOST_AVAILABLE:
+        raise ImportError("CatBoost is not installed. Please install with: pip install catboost")
+    
+    if not (lpath.endswith(".cbm") or lpath.endswith(".CBM")):
+        raise ValueError("File must have .cbm extension")
+
+    try:
+        model = CatBoostClassifier().load_model(lpath, **kwargs)
+    except:
+        model = CatBoostRegressor().load_model(lpath, **kwargs)
+
+    return model
+
+
+# EOF

scitex/io/_load_modules/_con.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:51:45 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_con.py
+
+from typing import Any
+
+import mne
+
+
+def _load_con(lpath: str, **kwargs) -> Any:
+    if not lpath.endswith(".con"):
+        raise ValueError("File must have .con extension")
+    obj = mne.io.read_raw_fif(lpath, preload=True, **kwargs)
+    obj = obj.to_data_frame()
+    obj["samp_rate"] = obj.info["sfreq"]
+    return obj
+
+
+# EOF

scitex/io/_load_modules/_db.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 11:50:05 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_db.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_db.py"
+
+from typing import Any
+
+from ...db._SQLite3 import SQLite3
+
+
+def _load_sqlite3db(lpath: str, use_temp=False) -> Any:
+    if not lpath.endswith(".db"):
+        raise ValueError("File must have .db extension")
+    try:
+        obj = SQLite3(lpath, use_temp=use_temp)
+
+        return obj
+    except Exception as e:
+        raise ValueError(str(e))
+
+
+# EOF

scitex/io/_load_modules/_docx.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:55:35 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_docx.py
+
+from typing import Any
+
+
+def _load_docx(lpath: str, **kwargs) -> Any:
+    """
+    Load and extract text content from a .docx file.
+
+    Parameters:
+    -----------
+    lpath : str
+        The path to the .docx file.
+
+    Returns:
+    --------
+    str
+        The extracted text content from the .docx file.
+
+    Raises:
+    -------
+    FileNotFoundError
+        If the specified file does not exist.
+    docx.opc.exceptions.PackageNotFoundError
+        If the file is not a valid .docx file.
+    """
+    if not lpath.endswith(".docx"):
+        raise ValueError("File must have .docx extension")
+
+    from docx import Document
+
+    doc = Document(lpath)
+    full_text = []
+    for para in doc.paragraphs:
+        full_text.append(para.text)
+    return "".join(full_text)
+
+
+# EOF

scitex/io/_load_modules/_eeg.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:56:27 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_eeg.py
+
+import os
+import warnings
+from typing import Any
+
+import mne
+
+
+def _load_eeg_data(path: str, **kwargs) -> Any:
+    """
+    Load EEG data based on file extension and associated files using MNE-Python.
+
+    This function supports various EEG file formats including BrainVision, EDF, BDF, GDF, CNT, EGI, and SET.
+    It also handles special cases for .eeg files (BrainVision and Nihon Koden).
+
+    Parameters:
+    -----------
+    lpath : str
+        The path to the EEG file to be loaded.
+    **kwargs : dict
+        Additional keyword arguments to be passed to the specific MNE loading function.
+
+    Returns:
+    --------
+    raw : mne.io.Raw
+        The loaded raw EEG data.
+
+    Raises:
+    -------
+    ValueError
+        If the file extension is not supported.
+
+    Notes:
+    ------
+    This function uses MNE-Python to load the EEG data. It automatically detects the file format
+    based on the file extension and uses the appropriate MNE function to load the data.
+    """
+    # Get the file extension
+    extension = lpath.split(".")[-1]
+
+    allowed_extensions = [
+        ".vhdr",
+        ".vmrk",
+        ".edf",
+        ".bdf",
+        ".gdf",
+        ".cnt",
+        ".egi",
+        ".eeg",
+        ".set",
+    ]
+
+    if extension not in allowed_extensions:
+        raise ValueError(
+            f"File must have one of these extensions: {', '.join(allowed_extensions)}"
+        )
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", RuntimeWarning)
+
+        # Load the data based on the file extension
+        if extension in ["vhdr", "vmrk"]:
+            # Load BrainVision data
+            raw = mne.io.read_raw_brainvision(lpath, preload=True, **kwargs)
+        elif extension == "edf":
+            # Load European data format
+            raw = mne.io.read_raw_edf(lpath, preload=True, **kwargs)
+        elif extension == "bdf":
+            # Load BioSemi data format
+            raw = mne.io.read_raw_bdf(lpath, preload=True, **kwargs)
+        elif extension == "gdf":
+            # Load Gen data format
+            raw = mne.io.read_raw_gdf(lpath, preload=True, **kwargs)
+        elif extension == "cnt":
+            # Load Neuroscan CNT data
+            raw = mne.io.read_raw_cnt(lpath, preload=True, **kwargs)
+        elif extension == "egi":
+            # Load EGI simple binary data
+            raw = mne.io.read_raw_egi(lpath, preload=True, **kwargs)
+        elif extension == "set":
+            # ???
+            raw = mne.io.read_raw(lpath, preload=True, **kwargs)
+        elif extension == "eeg":
+            is_BrainVision = any(
+                os.path.isfile(lpath.replace(".eeg", ext)) for ext in [".vhdr", ".vmrk"]
+            )
+            is_NihonKoden = any(
+                os.path.isfile(lpath.replace(".eeg", ext))
+                for ext in [".21e", ".pnt", ".log"]
+            )
+
+            # Brain Vision
+            if is_BrainVision:
+                lpath_v = lpath.replace(".eeg", ".vhdr")
+                raw = mne.io.read_raw_brainvision(lpath_v, preload=True, **kwargs)
+            # Nihon Koden
+            if is_NihonKoden:
+                # raw = mne.io.read_raw_nihon(lpath, preload=True, **kwargs)
+                raw = mne.io.read_raw(lpath, preload=True, **kwargs)
+        else:
+            raise ValueError(f"Unsupported file extension: {extension}")
+
+        return raw
+
+
+# EOF

scitex/io/_load_modules/_hdf5.py

@@ -0,0 +1,196 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-06-16 17:40:55 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/SciTeX-Code/src/scitex/io/_load_modules/_hdf5.py
+# ----------------------------------------
+import os
+__FILE__ = (
+    "./src/scitex/io/_load_modules/_hdf5.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+import warnings
+from typing import Any
+
+import h5py
+import numpy as np
+
+# Time-stamp: "2024-11-14 07:55:37 (ywatanabe)"
+
+
+
+
+def _load_group(group):
+    """Recursively load an HDF5 group."""
+    obj = {}
+    for key in group.keys():
+        if isinstance(group[key], h5py.Group):
+            # Recursively load subgroups
+            obj[key] = _load_group(group[key])
+        else:
+            # Load dataset
+            dataset = group[key]
+            # Check if it's a scalar dataset
+            if dataset.shape == ():
+                data = dataset[()]
+            else:
+                data = dataset[:]
+
+            # Decode bytes to string if needed
+            if isinstance(data, bytes):
+                obj[key] = data.decode("utf-8")
+            elif isinstance(data, np.void):
+                # Handle pickled data
+                import pickle
+
+                obj[key] = pickle.loads(data.tobytes())
+            else:
+                obj[key] = data
+    # Load attributes
+    for key in group.attrs.keys():
+        obj[key] = group.attrs[key]
+    return obj
+
+
+def _load_hdf5(lpath: str, key: str = None, **kwargs) -> Any:
+    """Load HDF5 file with automatic group/root switching."""
+    try:
+        with h5py.File(lpath, "r") as hf:
+            if key:
+                if key not in hf:
+                    return None
+                target = hf[key]
+            else:
+                target = hf
+
+            obj = {}
+            for key_name in target.keys():
+                try:
+                    if isinstance(target[key_name], h5py.Group):
+                        obj[key_name] = _load_group(target[key_name])
+                    else:
+                        dataset = target[key_name]
+                        if dataset.shape == ():
+                            data = dataset[()]
+                        else:
+                            data = dataset[:]
+
+                        if isinstance(data, bytes):
+                            obj[key_name] = data.decode("utf-8")
+                        elif isinstance(data, np.void):
+                            import pickle
+
+                            obj[key_name] = pickle.loads(data.tobytes())
+                        else:
+                            obj[key_name] = data
+                except (RuntimeError, OSError) as e:
+                    print(f"Warning: Could not load key '{key_name}': {e}")
+                    continue
+
+            for attr_name in target.attrs.keys():
+                obj[attr_name] = target.attrs[attr_name]
+            return obj
+
+    except (RuntimeError, OSError) as e:
+        key_warning_str = f" with {key}" if key else ""
+        warnings.warn(f"Error loading {lpath}{key_warning_str}:\n{e}")
+        return None
+
+
+# def _load_hdf5(lpath: str, key: str = None, **kwargs) -> Any:
+#     """Load HDF5 file with automatic group/root switching."""
+#     with h5py.File(lpath, "r") as hf:
+#         if key:
+#             if key not in hf:
+#                 return None
+#             target = hf[key]
+#         else:
+#             target = hf
+
+#         obj = {}
+#         for key in target.keys():
+#             if isinstance(target[key], h5py.Group):
+#                 # Recursively load groups
+#                 obj[key] = _load_group(target[key])
+#             else:
+#                 # Load dataset
+#                 dataset = target[key]
+#                 # Check if it's a scalar dataset
+#                 if dataset.shape == ():
+#                     data = dataset[()]
+#                 else:
+#                     data = dataset[:]
+
+#                 # Decode bytes to string if needed
+#                 if isinstance(data, bytes):
+#                     obj[key] = data.decode("utf-8")
+#                 elif isinstance(data, np.void):
+#                     # Handle pickled data
+#                     import pickle
+
+#                     obj[key] = pickle.loads(data.tobytes())
+#                 else:
+#                     obj[key] = data
+#         for key in target.attrs.keys():
+#             obj[key] = target.attrs[key]
+#         return obj
+
+# def _load_hdf5(lpath: str, **kwargs) -> Any:
+#     """Load HDF5 file."""
+#     if not (lpath.endswith(".hdf5") or lpath.endswith(".h5")):
+#         raise ValueError("File must have .hdf5 or .h5 extension")
+
+#     def load_item(item):
+#         """Recursively load items from HDF5."""
+#         if isinstance(item, h5py.Group):
+#             # Load groups as nested dicts
+#             result = {}
+#             for key in item.keys():
+#                 result[key] = load_item(item[key])
+#             return result
+#         elif isinstance(item, h5py.Dataset):
+#             # Load datasets
+#             data = item[()]
+#             # Check if it's a pickled object
+#             if isinstance(data, np.void):
+#                 import pickle
+
+#                 return pickle.loads(data.tobytes())
+#             # Convert bytes to string
+#             elif isinstance(data, bytes):
+#                 return data.decode("utf-8")
+#             # Convert numpy scalars to Python types
+#             elif isinstance(data, np.integer):
+#                 return int(data)
+#             elif isinstance(data, np.floating):
+#                 return float(data)
+#             elif isinstance(data, np.bool_):
+#                 return bool(data)
+#             else:
+#                 return data
+#         else:
+#             return item
+
+#     with h5py.File(lpath, "r") as hf:
+#         return load_item(hf)
+
+# def _load_hdf5_group(lpath: str, key: str = None, **kwargs) -> Any:
+#     """Load specific group from HDF5 file."""
+#     with h5py.File(lpath, "r") as hf:
+#         if key:
+#             if key not in hf:
+#                 return None
+#             group = hf[key]
+#             result = {}
+#             for key in group.keys():
+#                 result[key] = group[key][:]
+#             for key in group.attrs.keys():
+#                 result[key] = group.attrs[key]
+#             return result
+#         else:
+#             return _load_hdf5(lpath, **kwargs)
+
+# # EOF
+
+# EOF

scitex/io/_load_modules/_image.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:55:38 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_image.py
+
+from typing import Any
+
+from PIL import Image
+
+
+def _load_image(lpath: str, **kwargs) -> Any:
+    """Load image file."""
+    supported_exts = [".jpg", ".jpeg", ".png", ".tiff", ".tif"]
+    if not any(lpath.lower().endswith(ext) for ext in supported_exts):
+        raise ValueError("Unsupported image format")
+    return Image.open(lpath, **kwargs)
+
+
+# EOF

scitex/io/_load_modules/_joblib.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:55:39 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_joblib.py
+
+from typing import Any
+
+import joblib
+
+
+def _load_joblib(lpath: str, **kwargs) -> Any:
+    """Load joblib file."""
+    if not lpath.endswith(".joblib"):
+        raise ValueError("File must have .joblib extension")
+    with open(lpath, "rb") as f:
+        return joblib.load(f, **kwargs)
+
+
+# EOF

scitex/io/_load_modules/_json.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:55:40 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_json.py
+
+import json
+from typing import Any
+
+
+def _load_json(lpath: str, **kwargs) -> Any:
+    """Load JSON file."""
+    if not lpath.endswith(".json"):
+        raise ValueError("File must have .json extension")
+    with open(lpath, "r") as f:
+        return json.load(f)
+
+
+# EOF

scitex/io/_load_modules/_markdown.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:55:42 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_markdown.py
+
+
+def _load_markdown(lpath_md, style="plain_text", **kwargs):
+    """
+    Load and convert Markdown content from a file.
+
+    This function reads a Markdown file and converts it to either HTML or plain text format.
+
+    Parameters:
+    -----------
+    lpath_md : str
+        The path to the Markdown file to be loaded.
+    style : str, optional
+        The output style of the converted content.
+        Options are "html" or "plain_text" (default).
+
+    Returns:
+    --------
+    str
+        The converted content of the Markdown file, either as HTML or plain text.
+
+    Raises:
+    -------
+    FileNotFoundError
+        If the specified file does not exist.
+    IOError
+        If there's an error reading the file.
+    ValueError
+        If an invalid style option is provided.
+
+    Notes:
+    ------
+    This function uses the 'markdown' library to convert Markdown to HTML,
+    and 'html2text' to convert HTML to plain text when necessary.
+    """
+    import html2text
+    import markdown
+
+    # Load Markdown content from a file
+    with open(lpath_md, "r") as file:
+        markdown_content = file.read()
+
+    # Convert Markdown to HTML
+    html_content = markdown.markdown(markdown_content)
+    if style == "html":
+        return html_content
+    elif style == "plain_text":
+        text_maker = html2text.HTML2Text()
+        text_maker.ignore_links = True
+        text_maker.bypass_tables = False
+        plain_text = text_maker.handle(html_content)
+        return plain_text
+    else:
+        raise ValueError("Invalid style option. Choose 'html' or 'plain_text'.")
+
+
+def load_markdown(lpath_md, style="plain_text"):
+    """
+    Load and convert a Markdown file to either HTML or plain text.
+
+    Parameters:
+    -----------
+    lpath_md : str
+        The path to the Markdown file.
+    style : str, optional
+        The output style, either "html" or "plain_text" (default).
+
+    Returns:
+    --------
+    str
+        The converted content of the Markdown file.
+    """
+    import html2text
+    import markdown
+
+    # Load Markdown content from a file
+    with open(lpath_md, "r") as file:
+        markdown_content = file.read()
+
+    # Convert Markdown to HTML
+    html_content = markdown.markdown(markdown_content)
+    if style == "html":
+        return html_content
+
+    elif style == "plain_text":
+        text_maker = html2text.HTML2Text()
+        text_maker.ignore_links = True
+        text_maker.bypass_tables = False
+        plain_text = text_maker.handle(html_content)
+
+        return plain_text
+
+
+# def _load_markdown(lpath):
+#     md_text = StringIO(lpath.read().decode("utf-8"))
+#     html = markdown.markdown(md_text.read())
+#     return html
+
+# EOF

scitex/io/_load_modules/_matlab.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-04-10 08:07:03 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_matlab.py
+# ----------------------------------------
+import os
+
+__FILE__ = "/ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_matlab.py"
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+from typing import Any
+
+
+def _load_matlab(lpath: str, **kwargs) -> Any:
+    """Load MATLAB file."""
+    if not lpath.endswith(".mat"):
+        raise ValueError("File must have .mat extension")
+
+    # Try using scipy.io first for binary .mat files
+    try:
+        # For MATLAB v7.3 files (HDF5 format)
+        from scipy.io import loadmat
+
+        return loadmat(lpath, **kwargs)
+    except Exception as e1:
+        # If scipy fails, try pymatreader  or older MAT files
+        try:
+            from pymatreader import read_mat
+
+            return read_mat(lpath, **kwargs)
+        except Exception as e2:
+            # Both methods failed
+            raise ValueError(f"Error loading file {lpath}: {str(e1)}\nAnd: {str(e2)}")
+
+
+# EOF