scitex 2.0.0__py2.py3-none-any.whl

scitex/io/_cache.py

@@ -0,0 +1,101 @@
+#!./env/bin/python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-08-20 19:42:38 (ywatanabe)"
+# ./src/scitex/io/_cache.py
+
+
+import os
+import pickle
+import sys
+from pathlib import Path
+
+
+def cache(id, *args):
+    """
+    Store or fetch data using a pickle file.
+
+    This function provides a simple caching mechanism for storing and retrieving
+    Python objects. It uses pickle to serialize the data and stores it in a file
+    with a unique identifier. If the data is already cached, it can be retrieved
+    without recomputation.
+
+    Parameters:
+    -----------
+    id : str
+        A unique identifier for the cache file.
+    *args : str
+        Variable names to be cached or loaded.
+
+    Returns:
+    --------
+    tuple
+        A tuple of cached values corresponding to the input variable names.
+
+    Raises:
+    -------
+    ValueError
+        If the cache file is not found and not all variables are defined.
+
+    Example:
+    --------
+    >>> import scitex
+    >>> import numpy as np
+    >>>
+    >>> # Variables to cache
+    >>> var1 = "x"
+    >>> var2 = 1
+    >>> var3 = np.ones(10)
+    >>>
+    >>> # Saving
+    >>> var1, var2, var3 = scitex.io.cache("my_id", "var1", "var2", "var3")
+    >>> print(var1, var2, var3)
+    >>>
+    >>> # Loading when not all variables are defined and the id exists
+    >>> del var1, var2, var3
+    >>> var1, var2, var3 = scitex.io.cache("my_id", "var1", "var2", "var3")
+    >>> print(var1, var2, var3)
+    """
+    cache_dir = Path.home() / ".cache" / "your_app_name"
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    cache_file = cache_dir / f"{id}.pkl"
+
+    does_cache_file_exist = cache_file.exists()
+
+    # Get the caller's local variables
+    caller_locals = sys._getframe(1).f_locals
+    are_all_variables_defined = all(arg in caller_locals for arg in args)
+
+    if are_all_variables_defined:
+        # If all variables are defined, save them to cache and return as-is
+        data_to_cache = {arg: caller_locals[arg] for arg in args}
+        with cache_file.open("wb") as f:
+            pickle.dump(data_to_cache, f)
+        return tuple(data_to_cache.values())
+    else:
+        if does_cache_file_exist:
+            # If cache exists, load and return the values
+            with cache_file.open("rb") as f:
+                loaded_data = pickle.load(f)
+            return tuple(loaded_data[arg] for arg in args)
+        else:
+            raise ValueError("Cache file not found and not all variables are defined.")
+
+
+# Usage example
+if __name__ == "__main__":
+    import scitex
+    import numpy as np
+
+    # Variables to cache
+    var1 = "x"
+    var2 = 1
+    var3 = np.ones(10)
+
+    # Saving
+    var1, var2, var3 = scitex.io.cache("my_id", "var1", "var2", "var3")
+    print(var1, var2, var3)
+
+    # Loading when not all variables are defined and the id exists
+    del var1, var2, var3
+    var1, var2, var3 = scitex.io.cache("my_id", "var1", "var2", "var3")
+    print(var1, var2, var3)

scitex/io/_flush.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-02 03:23:44 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_flush.py
+
+import os
+import sys
+import warnings
+
+
+def flush(sys=sys):
+    """
+    Flushes the system's stdout and stderr, and syncs the file system.
+    This ensures all pending write operations are completed.
+    """
+    if sys is None:
+        warnings.warn("flush needs sys. Skipping.")
+    else:
+        sys.stdout.flush()
+        sys.stderr.flush()
+        os.sync()
+
+
+# EOF

scitex/io/_glob.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 00:31:08 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_glob.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/io/_glob.py"
+
+import re as _re
+from glob import glob as _glob
+from ..str._parse import parse as _parse
+from natsort import natsorted as _natsorted
+
+
+def glob(expression, parse=False, ensure_one=False):
+    """
+    Perform a glob operation with natural sorting and extended pattern support.
+
+    This function extends the standard glob functionality by adding natural sorting
+    and support for curly brace expansion in the glob pattern.
+
+    Parameters:
+    -----------
+    expression : str
+        The glob pattern to match against file paths. Supports standard glob syntax
+        and curly brace expansion (e.g., 'dir/{a,b}/*.txt').
+    parse : bool, optional
+        Whether to parse the matched paths. Default is False.
+    ensure_one : bool, optional
+        Ensure exactly one match is found. Default is False.
+
+    Returns:
+    --------
+    Union[List[str], Tuple[List[str], List[dict]]]
+        If parse=False: A naturally sorted list of file paths
+        If parse=True: Tuple of (paths, parsed results)
+
+    Examples:
+    ---------
+    >>> glob('data/*.txt')
+    ['data/file1.txt', 'data/file2.txt', 'data/file10.txt']
+
+    >>> glob('data/{a,b}/*.txt')
+    ['data/a/file1.txt', 'data/a/file2.txt', 'data/b/file1.txt']
+
+    >>> paths, parsed = glob('data/subj_{id}/run_{run}.txt', parse=True)
+    >>> paths
+    ['data/subj_001/run_01.txt', 'data/subj_001/run_02.txt']
+    >>> parsed
+    [{'id': '001', 'run': '01'}, {'id': '001', 'run': '02'}]
+
+    >>> paths, parsed = glob('data/subj_{id}/run_{run}.txt', parse=True, ensure_one=True)
+    AssertionError  # if more than one file matches
+    """
+    glob_pattern = _re.sub(r"{[^}]*}", "*", expression)
+    # Enable recursive globbing for ** patterns
+    recursive = "**" in glob_pattern
+    try:
+        found_paths = _natsorted(_glob(eval(glob_pattern), recursive=recursive))
+    except:
+        found_paths = _natsorted(_glob(glob_pattern, recursive=recursive))
+
+    if ensure_one:
+        assert len(found_paths) == 1
+
+    if parse:
+        parsed = [_parse(found_path, expression) for found_path in found_paths]
+        return found_paths, parsed
+
+    else:
+        return found_paths
+
+
+def parse_glob(expression, ensure_one=False):
+    """
+    Convenience function for glob with parsing enabled.
+
+    Parameters:
+    -----------
+    expression : str
+        The glob pattern to match against file paths.
+    ensure_one : bool, optional
+        Ensure exactly one match is found. Default is False.
+
+    Returns:
+    --------
+    Tuple[List[str], List[dict]]
+        Matched paths and parsed results.
+
+    Examples:
+    ---------
+    >>> paths, parsed = pglob('data/subj_{id}/run_{run}.txt')
+    >>> paths
+    ['data/subj_001/run_01.txt', 'data/subj_001/run_02.txt']
+    >>> parsed
+    [{'id': '001', 'run': '01'}, {'id': '001', 'run': '02'}]
+
+    >>> paths, parsed = pglob('data/subj_{id}/run_{run}.txt', ensure_one=True)
+    AssertionError  # if more than one file matches
+    """
+    return glob(expression, parse=True, ensure_one=ensure_one)
+
+
+# EOF

scitex/io/_json2md.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-12-19 15:37:46 (ywatanabe)"
+# File: ./Ninja/workspace/formats/json2md.py
+
+THIS_FILE = "/home/ywatanabe/.emacs.d/lisp/Ninja/workspace/formats/json2md.py"
+
+import json
+import sys
+import argparse
+
+
+def json2md(obj, level=1):
+    output = []
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if output:  # Add extra newline between sections
+                output.append("")
+            output.append("#" * level + " " + str(key))
+            if isinstance(value, (dict, list)):
+                output.append(json2md(value, level + 1))
+            else:
+                output.append(str(value) + "\n")
+    elif isinstance(obj, list):
+        for item in obj:
+            if isinstance(item, (dict, list)):
+                output.append(json2md(item, level))
+            else:
+                output.append("* " + str(item))
+    return "\n".join(filter(None, output))
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Convert JSON to Markdown")
+    parser.add_argument("input", help="Input JSON file")
+    parser.add_argument("-o", "--output", help="Output file (default: stdout)")
+    args = parser.parse_args()
+
+    try:
+        with open(args.input, "r") as f:
+            data = json.load(f)
+
+        result = json2md(data)
+
+        if args.output:
+            with open(args.output, "w") as f:
+                f.write(result)
+        else:
+            print(result)
+
+    except FileNotFoundError:
+        print(f"Error: File {args.input} not found", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
+
+"""
+python ./Ninja/workspace/formats/json2md.py
+python -m workspace.formats.json2md
+"""
+# EOF
+
+# #!/usr/bin/env python3
+# # -*- coding: utf-8 -*-
+# # Time-stamp: "2024-12-19 15:29:28 (ywatanabe)"
+# # File: ./Ninja/workspace/formats/json2md.py
+
+# THIS_FILE = "/home/ywatanabe/.emacs.d/lisp/Ninja/workspace/formats/json2md.py"
+
+# import json
+# import sys
+
+# def json2md(obj, level=1):
+#     output = []
+#     if isinstance(obj, dict):
+#         for key, value in obj.items():
+#             if output:  # Add extra newline between sections
+#                 output.append("")
+#             output.append("#" * level + " " + str(key))
+#             if isinstance(value, (dict, list)):
+#                 output.append(json2md(value, level + 1))
+#             else:
+#                 output.append(str(value) + "\n")
+#     elif isinstance(obj, list):
+#         for item in obj:
+#             if isinstance(item, (dict, list)):
+#                 output.append(json2md(item, level))
+#             else:
+#                 output.append("* " + str(item))
+#     return "\n".join(filter(None, output))
+
+# def main():
+#     if len(sys.argv) != 2:
+#         print("Usage: json2md.py <input.json>")
+#         sys.exit(1)
+
+#     lpath = sys.argv[1].replace("/./", "/")
+#     with open(lpath, "r") as f:
+#         data = json.load(f)
+
+
+# if __name__ == "__main__":
+#     main()
+
+
+# """
+# python ./Ninja/workspace/formats/json2md.py
+# python -m workspace.formats.json2md
+# """
+
+# # EOF

scitex/io/_load.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-04-10 08:05:53 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load.py
+# ----------------------------------------
+import os
+
+__FILE__ = "/ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load.py"
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+from typing import Any
+import glob
+from ..decorators import preserve_doc
+from ..str._clean_path import clean_path
+
+# from ._load_modules._catboost import _load_catboost
+from ._load_modules._con import _load_con
+from ._load_modules._db import _load_sqlite3db
+from ._load_modules._docx import _load_docx
+from ._load_modules._eeg import _load_eeg_data
+from ._load_modules._hdf5 import _load_hdf5
+from ._load_modules._image import _load_image
+from ._load_modules._joblib import _load_joblib
+from ._load_modules._json import _load_json
+from ._load_modules._markdown import _load_markdown
+from ._load_modules._numpy import _load_npy
+from ._load_modules._matlab import _load_matlab
+from ._load_modules._pandas import _load_csv, _load_excel, _load_tsv
+from ._load_modules._pdf import _load_pdf
+from ._load_modules._pickle import _load_pickle
+from ._load_modules._torch import _load_torch
+from ._load_modules._txt import _load_txt
+from ._load_modules._xml import _load_xml
+from ._load_modules._yaml import _load_yaml
+from ._load_modules._matlab import _load_matlab
+
+
+def load(lpath: str, show: bool = False, verbose: bool = False, **kwargs) -> Any:
+    """
+    Load data from various file formats.
+
+    This function supports loading data from multiple file formats.
+
+    Parameters
+    ----------
+    lpath : str
+        The path to the file to be loaded.
+    show : bool, optional
+        If True, display additional information during loading. Default is False.
+    verbose : bool, optional
+        If True, print verbose output during loading. Default is False.
+    **kwargs : dict
+        Additional keyword arguments to be passed to the specific loading function.
+
+    Returns
+    -------
+    object
+        The loaded data object, which can be of various types depending on the input file format.
+
+    Raises
+    ------
+    ValueError
+        If the file extension is not supported.
+    FileNotFoundError
+        If the specified file does not exist.
+
+    Supported Extensions
+    -------------------
+    - Data formats: .csv, .tsv, .xls, .xlsx, .xlsm, .xlsb, .json, .yaml, .yml
+    - Scientific: .npy, .npz, .mat, .hdf5, .con
+    - ML/DL: .pth, .pt, .cbm, .joblib, .pkl
+    - Documents: .txt, .log, .event, .md, .docx, .pdf, .xml
+    - Images: .jpg, .png, .tiff, .tif
+    - EEG data: .vhdr, .vmrk, .edf, .bdf, .gdf, .cnt, .egi, .eeg, .set
+    - Database: .db
+
+    Examples
+    --------
+    >>> data = load('data.csv')
+    >>> image = load('image.png')
+    >>> model = load('model.pth')
+    """
+    lpath = clean_path(lpath)
+
+    # Check if it's a glob pattern
+    if "*" in lpath or "?" in lpath or "[" in lpath:
+        # Handle glob pattern
+        matched_files = sorted(glob.glob(lpath))
+        if not matched_files:
+            raise FileNotFoundError(f"No files found matching pattern: {lpath}")
+        # Load all matched files
+        results = []
+        for file_path in matched_files:
+            results.append(load(file_path, show=show, verbose=verbose, **kwargs))
+        return results
+
+    if not os.path.exists(lpath):
+        raise FileNotFoundError(f"{lpath} not found.")
+
+    loaders_dict = {
+        # Default
+        "": _load_txt,
+        # Config/Settings
+        "yaml": _load_yaml,
+        "yml": _load_yaml,
+        "json": _load_json,
+        "xml": _load_xml,
+        # ML/DL Models
+        "pth": _load_torch,
+        "pt": _load_torch,
+        # "cbm": _load_catboost,
+        "joblib": _load_joblib,
+        "pkl": _load_pickle,
+        "pickle": _load_pickle,
+        "gz": _load_pickle,  # For .pkl.gz files
+        # Tabular Data
+        "csv": _load_csv,
+        "tsv": _load_tsv,
+        "xls": _load_excel,
+        "xlsx": _load_excel,
+        "xlsm": _load_excel,
+        "xlsb": _load_excel,
+        "db": _load_sqlite3db,
+        # Scientific Data
+        "npy": _load_npy,
+        "npz": _load_npy,
+        "mat": _load_matlab,
+        "hdf5": _load_hdf5,
+        "h5": _load_hdf5,
+        "con": _load_con,
+        # Documents
+        "txt": _load_txt,
+        "tex": _load_txt,
+        "log": _load_txt,
+        "event": _load_txt,
+        "py": _load_txt,
+        "sh": _load_txt,
+        "md": _load_markdown,
+        "docx": _load_docx,
+        "pdf": _load_pdf,
+        # Images
+        "jpg": _load_image,
+        "png": _load_image,
+        "tiff": _load_image,
+        "tif": _load_image,
+        # EEG Data
+        "vhdr": _load_eeg_data,
+        "vmrk": _load_eeg_data,
+        "edf": _load_eeg_data,
+        "bdf": _load_eeg_data,
+        "gdf": _load_eeg_data,
+        "cnt": _load_eeg_data,
+        "egi": _load_eeg_data,
+        "eeg": _load_eeg_data,
+        "set": _load_eeg_data,
+    }
+
+    ext = lpath.split(".")[-1] if "." in lpath else ""
+    loader = preserve_doc(loaders_dict.get(ext, _load_txt))
+
+    try:
+        return loader(lpath, **kwargs)
+    except (ValueError, FileNotFoundError) as e:
+        raise ValueError(f"Error loading file {lpath}: {str(e)}")
+
+
+# EOF

scitex/io/_load_configs.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-04-22 07:19:03 (ywatanabe)"
+# File: /ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_configs.py
+# ----------------------------------------
+import os
+
+__FILE__ = "/ssh:sp:/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_configs.py"
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_configs.py"
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2025-02-27 11:09:00 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_configs.py
+
+from ..dict import DotDict
+from ._load import load
+from ._glob import glob
+
+
+def load_configs(IS_DEBUG=None, show=False, verbose=False):
+    """Load YAML configuration files from ./config directory.
+
+    Parameters
+    ----------
+    IS_DEBUG : bool, optional
+        Debug mode flag. If None, reads from IS_DEBUG.yaml
+    show : bool
+        Show configuration changes
+    verbose : bool
+        Print detailed information
+
+    Returns
+    -------
+    DotDict
+        Merged configuration dictionary
+    """
+
+    def apply_debug_values(config, IS_DEBUG):
+        """Apply debug values if IS_DEBUG is True."""
+        if not IS_DEBUG or not isinstance(config, (dict, DotDict)):
+            return config
+
+        for key, value in list(config.items()):
+            if key.startswith(("DEBUG_", "debug_")):
+                dk_wo_debug_prefix = key.split("_", 1)[1]
+                config[dk_wo_debug_prefix] = value
+                if show or verbose:
+                    print(f"{key} -> {dk_wo_debug_prefix}")
+            elif isinstance(value, (dict, DotDict)):
+                config[key] = apply_debug_values(value, IS_DEBUG)
+        return config
+
+    try:
+        # Set debug mode
+        IS_DEBUG = (
+            IS_DEBUG
+            or os.getenv("CI") == "True"
+            or (
+                os.path.exists("./config/IS_DEBUG.yaml")
+                and load("./config/IS_DEBUG.yaml").get("IS_DEBUG")
+            )
+        )
+
+        # Load and merge configs
+        CONFIGS = {}
+        for lpath in glob("./config/*.yaml"):
+            if config := load(lpath):
+                CONFIGS.update(apply_debug_values(config, IS_DEBUG))
+
+        return DotDict(CONFIGS)
+
+    except Exception as e:
+        print(f"Error loading configs: {e}")
+        return DotDict({})
+
+
+# def load_configs(IS_DEBUG=None, show=False, verbose=False):
+#     """
+#     Load configuration files from the ./config directory.
+
+#     Parameters:
+#     -----------
+#     IS_DEBUG : bool, optional
+#         If True, use debug configurations. If None, check ./config/IS_DEBUG.yaml.
+#     show : bool, optional
+#         If True, display additional information during loading.
+#     verbose : bool, optional
+#         If True, print verbose output during loading.
+
+#     Returns:
+#     --------
+#     DotDict
+#         A dictionary-like object containing the loaded configurations.
+#     """
+
+#     def apply_debug_values(config, IS_DEBUG):
+#         if IS_DEBUG:
+#             if isinstance(config, (dict, DotDict)):
+#                 for key, value in list(config.items()):
+#                     try:
+#                         if key.startswith(("DEBUG_", "debug_")):
+#                             dk_wo_debug_prefix = key.split("_", 1)[1]
+#                             config[dk_wo_debug_prefix] = value
+#                             if show or verbose:
+#                                 print(f"\n{key} -> {dk_wo_debug_prefix}\n")
+#                         elif isinstance(value, (dict, DotDict)):
+#                             config[key] = apply_debug_values(value, IS_DEBUG)
+#                     except Exception as e:
+#                         print(e)
+#         return config
+
+#     if os.getenv("CI") == "True":
+#         IS_DEBUG = True
+
+#     try:
+#         # Check ./config/IS_DEBUG.yaml file if IS_DEBUG argument is not passed
+#         if IS_DEBUG is None:
+#             IS_DEBUG_PATH = "./config/IS_DEBUG.yaml"
+#             if os.path.exists(IS_DEBUG_PATH):
+#                 IS_DEBUG = load("./config/IS_DEBUG.yaml").get("IS_DEBUG")
+#             else:
+#                 IS_DEBUG = False
+
+#         # Main
+#         CONFIGS = {}
+#         for lpath in glob("./config/*.yaml"):
+#             config = load(lpath)
+#             if config:
+#                 CONFIG = apply_debug_values(config, IS_DEBUG)
+#                 CONFIGS.update(CONFIG)
+
+#         CONFIGS = DotDict(CONFIGS)
+
+#     except Exception as e:
+#         print(e)
+#         CONFIGS = DotDict({})
+
+#     return CONFIGS
+
+
+#
+
+# EOF

scitex/io/_load_modules/__init__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:41:25 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/__init__.py
+
+import importlib as __importlib
+import inspect as __inspect
+import os as __os
+
+# Get the current directory
+current_dir = __os.path.dirname(__file__)
+
+# Iterate through all Python files in the current directory
+for filename in __os.listdir(current_dir):
+    if filename.endswith(".py") and not filename.startswith("__"):
+        module_name = filename[:-3]  # Remove .py extension
+        module = __importlib.import_module(f".{module_name}", package=__name__)
+
+        # Import only functions and classes from the module
+        for name, obj in __inspect.getmembers(module):
+            if __inspect.isfunction(obj) or __inspect.isclass(obj):
+                if not name.startswith("_"):
+                    globals()[name] = obj
+
+# Clean up temporary variables
+del (
+    __os,
+    __importlib,
+    __inspect,
+    current_dir,
+    filename,
+    module_name,
+    module,
+    name,
+    obj,
+)
+
+# EOF