scitex 2.0.0__py2.py3-none-any.whl

scitex/io/_load_modules/_numpy.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:55:44 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_numpy.py
+
+from typing import Any
+
+import numpy as np
+
+
+def _load_npy(lpath: str, **kwargs) -> Any:
+    """Load NPY or NPZ file."""
+    if lpath.endswith(".npy"):
+        return __load_npy(lpath, **kwargs)
+    elif lpath.endswith(".npz"):
+        return __load_npz(lpath, **kwargs)
+    raise ValueError("File must have .npy or .npz extension")
+
+
+def __load_npy(lpath: str, **kwargs) -> Any:
+    """Load NPY file."""
+    return np.load(lpath, allow_pickle=True, **kwargs)
+
+
+def __load_npz(lpath: str, **kwargs) -> Any:
+    """Load NPZ file."""
+    obj = np.load(lpath, allow_pickle=True)
+
+    # Check if it's a single array saved with default key
+    if len(obj.files) == 1 and obj.files[0] == "arr_0":
+        # Return the single array directly for backward compatibility
+        return obj["arr_0"]
+
+    # Return the NpzFile object so users can access arrays by key
+    # This preserves the dictionary-like interface
+    return obj
+
+
+# EOF

scitex/io/_load_modules/_optuna.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:55:45 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_optuna.py
+
+from ._yaml import _load_yaml
+
+
+def load_yaml_as_an_optuna_dict(fpath_yaml, trial):
+    """
+    Load a YAML file and convert it to an Optuna-compatible dictionary.
+
+    This function reads a YAML file containing hyperparameter configurations
+    and converts it to a dictionary suitable for use with Optuna trials.
+
+    Parameters:
+    -----------
+    fpath_yaml : str
+        The file path to the YAML configuration file.
+    trial : optuna.trial.Trial
+        The Optuna trial object to use for suggesting hyperparameters.
+
+    Returns:
+    --------
+    dict
+        A dictionary containing the hyperparameters with values suggested by Optuna.
+
+    Raises:
+    -------
+    FileNotFoundError
+        If the specified YAML file does not exist.
+    ValueError
+        If the YAML file contains invalid configuration for Optuna.
+    """
+    _d = _load_yaml(fpath_yaml)
+
+    for k, v in _d.items():
+        dist = v["distribution"]
+
+        if dist == "categorical":
+            _d[k] = trial.suggest_categorical(k, v["values"])
+
+        elif dist == "uniform":
+            _d[k] = trial.suggest_int(k, float(v["min"]), float(v["max"]))
+
+        elif dist == "loguniform":
+            _d[k] = trial.suggest_loguniform(k, float(v["min"]), float(v["max"]))
+
+        elif dist == "intloguniform":
+            _d[k] = trial.suggest_int(k, float(v["min"]), float(v["max"]), log=True)
+
+    return _d
+
+
+def load_study_rdb(study_name, rdb_raw_bytes_url):
+    """
+    Load an Optuna study from a RDB (Relational Database) file.
+
+    This function loads an Optuna study from a given RDB file URL.
+
+    Parameters:
+    -----------
+    study_name : str
+        The name of the Optuna study to load.
+    rdb_raw_bytes_url : str
+        The URL of the RDB file, typically in the format "sqlite:///*.db".
+
+    Returns:
+    --------
+    optuna.study.Study
+        The loaded Optuna study object.
+
+    Raises:
+    -------
+    optuna.exceptions.StorageInvalidUsageError
+        If there's an error loading the study from the storage.
+
+    Example:
+    --------
+    >>> study = load_study_rdb(
+    ...     study_name="YOUR_STUDY_NAME",
+    ...     rdb_raw_bytes_url="sqlite:///path/to/your/study.db"
+    ... )
+    """
+    import optuna
+
+    storage = optuna.storages.RDBStorage(url=rdb_raw_bytes_url)
+    study = optuna.load_study(study_name=study_name, storage=storage)
+    print(f"Loaded: {rdb_raw_bytes_url}")
+    return study
+
+
+def load_yaml_as_an_optuna_dict(fpath_yaml, trial):
+    """
+    Load a YAML file and convert it to an Optuna-compatible dictionary.
+
+    Parameters:
+    -----------
+    fpath_yaml : str
+        The path to the YAML file.
+    trial : optuna.trial.Trial
+        The Optuna trial object.
+
+    Returns:
+    --------
+    dict
+        A dictionary with Optuna-compatible parameter suggestions.
+    """
+    _d = _load_yaml(fpath_yaml)
+
+    for k, v in _d.items():
+
+        dist = v["distribution"]
+
+        if dist == "categorical":
+            _d[k] = trial.suggest_categorical(k, v["values"])
+
+        elif dist == "uniform":
+            _d[k] = trial.suggest_int(k, float(v["min"]), float(v["max"]))
+
+        elif dist == "loguniform":
+            _d[k] = trial.suggest_loguniform(k, float(v["min"]), float(v["max"]))
+
+        elif dist == "intloguniform":
+            _d[k] = trial.suggest_int(k, float(v["min"]), float(v["max"]), log=True)
+
+    return _d
+
+
+def load_study_rdb(study_name, rdb_raw_bytes_url):
+    """
+    Load an Optuna study from a RDB storage.
+
+    Parameters:
+    -----------
+    study_name : str
+        The name of the Optuna study.
+    rdb_raw_bytes_url : str
+        The URL of the RDB storage.
+
+    Returns:
+    --------
+    optuna.study.Study
+        The loaded Optuna study object.
+    """
+    import optuna
+
+    # rdb_raw_bytes_url = "sqlite:////tmp/fake/ywatanabe/_MicroNN_WindowSize-1.0-sec_MaxEpochs_100_2021-1216-1844/optuna_study_test_file#0.db"
+    storage = optuna.storages.RDBStorage(url=rdb_raw_bytes_url)
+    study = optuna.load_study(study_name=study_name, storage=storage)
+    print(f"\nLoaded: {rdb_raw_bytes_url}\n")
+    return study
+
+
+# EOF

scitex/io/_load_modules/_pandas.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:41:31 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_pandas.py
+
+import pandas as pd
+
+
+def _load_csv(lpath, **kwargs):
+    """Load CSV files."""
+    if not lpath.endswith(".csv"):
+        raise ValueError("File must have .csv extension")
+
+    # Handle index column - default to None to match default index=False in saving
+    index_col = kwargs.pop("index_col", None)
+    obj = pd.read_csv(lpath, index_col=index_col, **kwargs)
+
+    # Remove unnamed columns only if they exist
+    unnamed_cols = obj.columns.str.contains("^Unnamed")
+    if unnamed_cols.any():
+        obj = obj.loc[:, ~unnamed_cols]
+
+    return obj
+
+
+# def _load_csv(lpath, **kwargs):
+#     """Load CSV files."""
+#     if not lpath.endswith('.csv'):
+#         raise ValueError("File must have .csv extension")
+#     index_col = kwargs.get("index_col", 0)
+#     obj = pd.read_csv(lpath, **kwargs)
+#     return obj.loc[:, ~obj.columns.str.contains("^Unnamed")]
+
+
+def _load_tsv(lpath, **kwargs):
+    """Load TSV files."""
+    if not lpath.endswith(".tsv"):
+        raise ValueError("File must have .tsv extension")
+    return pd.read_csv(lpath, sep="\t", **kwargs)
+
+
+def _load_excel(lpath, **kwargs):
+    """Load Excel files."""
+    if not lpath.endswith((".xls", ".xlsx", ".xlsm", ".xlsb")):
+        raise ValueError("File must have Excel extension")
+    return pd.read_excel(lpath, **kwargs)
+
+
+def _load_parquet(lpath, **kwargs):
+    """Load Parquet files."""
+    if not lpath.endswith(".parquet"):
+        raise ValueError("File must have .parquet extension")
+    return pd.read_parquet(lpath, **kwargs)
+
+
+# def _load_excel(lpath):
+#     workbook = openpyxl.load_workbook(lpath)
+#     all_text = []
+#     for sheet in workbook:
+#         for row in sheet.iter_rows(values_only=True):
+#             all_text.append(
+#                 " ".join(
+#                     [str(cell) if cell is not None else "" for cell in row]
+#                 )
+#             )
+#     return "\n".join(all_text)
+
+
+# EOF

scitex/io/_load_modules/_pdf.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:55:46 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_pdf.py
+
+try:
+    import PyPDF2
+except ImportError:
+    PyPDF2 = None
+
+
+def _load_pdf(lpath, **kwargs):
+    """Load PDF file and return extracted text."""
+    if PyPDF2 is None:
+        raise ImportError("PyPDF2 is required for PDF loading. Install with: pip install PyPDF2")
+        
+    try:
+        if not lpath.endswith(".pdf"):
+            raise ValueError("File must have .pdf extension")
+
+        reader = PyPDF2.PdfReader(lpath)
+        full_text = []
+        for page_num in range(len(reader.pages)):
+            page = reader.pages[page_num]
+            full_text.append(page.extract_text())
+        return "\n".join(full_text)
+    except (ValueError, FileNotFoundError, PyPDF2.PdfReadError) as e:
+        raise ValueError(f"Error loading PDF {lpath}: {str(e)}")
+
+
+# EOF

scitex/io/_load_modules/_pickle.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:41:33 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_pickle.py
+
+import pickle
+import gzip
+
+
+def _load_pickle(lpath, **kwargs):
+    """Load pickle file (compressed or uncompressed)."""
+    if lpath.endswith(".pkl.gz"):
+        # Handle gzip compressed pickle
+        with gzip.open(lpath, "rb") as f:
+            return pickle.load(f, **kwargs)
+    elif lpath.endswith(".pkl") or lpath.endswith(".pickle"):
+        # Handle regular pickle
+        with open(lpath, "rb") as f:
+            return pickle.load(f, **kwargs)
+    else:
+        raise ValueError("File must have .pkl, .pickle, or .pkl.gz extension")
+
+
+# EOF

scitex/io/_load_modules/_torch.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:41:34 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_torch.py
+
+import torch
+
+
+def _load_torch(lpath, **kwargs):
+    """Load PyTorch model/checkpoint file."""
+    if not lpath.endswith((".pth", ".pt")):
+        raise ValueError("File must have .pth or .pt extension")
+    return torch.load(lpath, **kwargs)
+
+
+# EOF

scitex/io/_load_modules/_txt.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-05-03 11:58:11 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/io/_load_modules/_txt.py
+# ----------------------------------------
+import os
+
+__FILE__ = "./src/scitex/io/_load_modules/_txt.py"
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+import warnings
+
+# # UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8a in position 30173: invalid start byte
+# def _load_txt(lpath, **kwargs):
+#     """Load text file and return non-empty lines."""
+#     SUPPORTED_EXTENSIONS = (".txt", ".log", ".event", ".py", ".sh", "")
+#     try:
+#         if not lpath.endswith(SUPPORTED_EXTENSIONS):
+#             warnings.warn(
+#                 f"File must have supported extensions: {SUPPORTED_EXTENSIONS}"
+#             )
+
+#         # Try UTF-8 first (most common)
+#         try:
+#             with open(lpath, "r", encoding="utf-8") as f:
+#                 return [
+#                     line.strip()
+#                     for line in f.read().splitlines()
+#                     if line.strip()
+#                 ]
+#         except UnicodeDecodeError:
+#             # Fallback to system default encoding
+#             with open(lpath, "r") as f:
+#                 return [
+#                     line.strip()
+#                     for line in f.read().splitlines()
+#                     if line.strip()
+#                 ]
+
+
+#     except (ValueError, FileNotFoundError) as e:
+#         raise ValueError(f"Error loading file {lpath}: {str(e)}")
+# Removed duplicate function - see main _load_txt below
+
+
+def _load_txt(lpath, strip=False, as_lines=False):
+    """
+    Load text file and return its content.
+    - Warn if extension is unexpected.
+    - Try UTF-8 first, then default encoding.
+    - If strip=True, strip whitespace.
+    - If as_lines=True, return list of lines (backward compatibility).
+    """
+    if not lpath.endswith((".txt", ".log", ".event", ".py", ".sh")):
+        warnings.warn(f"Unexpected extension for file: {lpath}")
+
+    try:
+        with open(lpath, "r", encoding="utf-8") as file:
+            content = file.read()
+    except UnicodeDecodeError:
+        # Fallback: try to detect correct encoding
+        encoding = _check_encoding(lpath)
+        with open(lpath, "r", encoding=encoding) as file:
+            content = file.read()
+
+    # For backward compatibility, check if as_lines parameter or legacy behavior needed
+    if as_lines:
+        raw_lines = content.splitlines()
+        if strip:
+            return [line.strip() for line in raw_lines if line.strip()]
+        return [line for line in raw_lines if line.strip()]
+
+    # Default: return full content (possibly stripped)
+    if strip:
+        return content.strip()
+
+    return content
+
+
+def _check_encoding(file_path):
+    """Check file encoding by trying common encodings."""
+    encodings = ["utf-8", "latin1", "cp1252", "iso-8859-1", "ascii"]
+
+    for encoding in encodings:
+        try:
+            with open(file_path, "r", encoding=encoding) as f:
+                f.read()
+            return encoding
+        except UnicodeDecodeError:
+            continue
+
+    raise ValueError(f"Unable to determine encoding for {file_path}")
+
+
+# def _check_encoding(file_path):
+#     """
+#     Check the encoding of a given file.
+
+#     This function attempts to read the file with different encodings
+#     to determine the correct one.
+
+#     Parameters:
+#     -----------
+#     file_path : str
+#         The path to the file to check.
+
+#     Returns:
+#     --------
+#     str
+#         The detected encoding of the file.
+
+#     Raises:
+#     -------
+#     IOError
+#         If the file cannot be read or the encoding cannot be determined.
+#     """
+#     import chardet
+
+#     with open(file_path, "rb") as file:
+#         raw_data = file.read()
+
+#     result = chardet.detect(raw_data)
+#     return result["encoding"]
+
+# EOF

scitex/io/_load_modules/_xml.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:55:49 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_xml.py
+
+
+def _load_xml(lpath, **kwargs):
+    """Load XML file and convert to dict."""
+    if not lpath.endswith(".xml"):
+        raise ValueError("File must have .xml extension")
+    
+    # Import xml2dict locally to avoid circular imports
+    from xml.etree import cElementTree as ElementTree
+    
+    # Inline the xml2dict functionality to avoid circular import
+    tree = ElementTree.parse(lpath)
+    root = tree.getroot()
+    
+    # Simplified XML to dict conversion - basic implementation
+    def xml_element_to_dict(element):
+        result = {}
+        
+        # Add attributes
+        if element.attrib:
+            result.update(element.attrib)
+        
+        # Handle child elements
+        for child in element:
+            if child.tag in result:
+                # Convert to list if multiple elements with same tag
+                if not isinstance(result[child.tag], list):
+                    result[child.tag] = [result[child.tag]]
+                result[child.tag].append(xml_element_to_dict(child))
+            else:
+                result[child.tag] = xml_element_to_dict(child)
+        
+        # Handle text content
+        if element.text and element.text.strip():
+            if result:
+                result['text'] = element.text.strip()
+            else:
+                return element.text.strip()
+        
+        return result
+    
+    return xml_element_to_dict(root)
+
+
+# EOF

scitex/io/_load_modules/_yaml.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-14 07:41:37 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_load_modules/_yaml.py
+
+import yaml
+
+
+def _load_yaml(lpath, **kwargs):
+    """Load YAML file with optional key lowercasing."""
+    if not lpath.endswith((".yaml", ".yml")):
+        raise ValueError("File must have .yaml or .yml extension")
+
+    lower = kwargs.pop("lower", False)
+    with open(lpath) as f:
+        obj = yaml.safe_load(f, **kwargs)
+
+    if lower:
+        obj = {k.lower(): v for k, v in obj.items()}
+    return obj
+
+
+# EOF

scitex/io/_mv_to_tmp.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-02 21:25:50 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/io/_mv_to_tmp.py
+
+from shutil import move
+
+
+def _mv_to_tmp(fpath, L=2):
+    try:
+        tgt_fname = "-".join(fpath.split("/")[-L:])
+        tgt_fpath = "/tmp/{}".format(tgt_fname)
+        move(fpath, tgt_fpath)
+        print("Moved to: {}".format(tgt_fpath))
+    except:
+        pass
+
+
+# EOF