scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/db/_SQLite3Mixins/_BlobMixin.py

@@ -1,229 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2024-12-01 05:13:49 (ywatanabe)"
-# File: ./scitex_repo/src/scitex/db/_SQLite3Mixins/_BlobMixin.py
-
-THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_SQLite3Mixins/_BlobMixin.py"
-
-import sqlite3
-from typing import Any as _Any
-from typing import Dict, List, Optional, Tuple, Union
-
-import numpy as np
-from .._BaseMixins._BaseBlobMixin import _BaseBlobMixin
-
-
-class _BlobMixin:
-    """BLOB data handling functionality"""
-
-    def save_array(
-        self,
-        table_name: str,
-        data: np.ndarray,
-        column: str = "data",
-        ids: Optional[Union[int, List[int]]] = None,
-        where: str = None,
-        additional_columns: Dict[str, _Any] = None,
-        batch_size: int = 1000,
-    ) -> None:
-        with self.lock:
-            if not isinstance(data, (np.ndarray, list)):
-                raise ValueError("Input must be a NumPy array or list of arrays")
-
-            try:
-                if ids is not None:
-                    if isinstance(ids, int):
-                        ids = [ids]
-                        data = [data]
-                    if len(ids) != len(data):
-                        raise ValueError("Length of ids must match number of arrays")
-
-                    for id_, arr in zip(ids, data):
-                        if not isinstance(arr, np.ndarray):
-                            raise ValueError(
-                                f"Element for id {id_} must be a NumPy array"
-                            )
-
-                        binary = arr.tobytes()
-                        columns = [
-                            column,
-                            f"{column}_dtype",
-                            f"{column}_shape",
-                        ]
-                        values = [binary, str(arr.dtype), str(arr.shape)]
-
-                        if additional_columns:
-                            columns = list(additional_columns.keys()) + columns
-                            values = list(additional_columns.values()) + values
-
-                        update_cols = [f"{col}=?" for col in columns]
-                        query = f"UPDATE {table_name} SET {','.join(update_cols)} WHERE id=?"
-                        values.append(id_)
-                        self.execute(query, tuple(values))
-
-                else:
-                    if not isinstance(data, np.ndarray):
-                        raise ValueError("Single input must be a NumPy array")
-
-                    binary = data.tobytes()
-                    columns = [column, f"{column}_dtype", f"{column}_shape"]
-                    values = [binary, str(data.dtype), str(data.shape)]
-
-                    if additional_columns:
-                        columns = list(additional_columns.keys()) + columns
-                        values = list(additional_columns.values()) + values
-
-                    if where is not None:
-                        update_cols = [f"{col}=?" for col in columns]
-                        query = f"UPDATE {table_name} SET {','.join(update_cols)} WHERE {where}"
-                        self.execute(query, tuple(values))
-                    else:
-                        placeholders = ",".join(["?" for _ in columns])
-                        columns_str = ",".join(columns)
-                        query = f"INSERT INTO {table_name} ({columns_str}) VALUES ({placeholders})"
-                        self.execute(query, tuple(values))
-
-            except Exception as err:
-                raise ValueError(f"Failed to save array: {err}")
-
-    def load_array(
-        self,
-        table_name: str,
-        column: str,
-        ids: Union[int, List[int], str] = "all",
-        where: str = None,
-        order_by: str = None,
-        batch_size: int = 128,
-        dtype: np.dtype = None,
-        shape: Optional[Tuple] = None,
-    ) -> Optional[np.ndarray]:
-        try:
-            if ids == "all":
-                query = f"SELECT id FROM {table_name}"
-                if where:
-                    query += f" WHERE {where}"
-                self.cursor.execute(query)
-                ids = [row[0] for row in self.cursor.fetchall()]
-            elif isinstance(ids, int):
-                ids = [ids]
-
-            id_to_data = {}
-            unique_ids = list(set(ids))
-
-            for idx in range(0, len(unique_ids), batch_size):
-                batch_ids = unique_ids[idx : idx + batch_size]
-                placeholders = ",".join("?" for _ in batch_ids)
-
-                try:
-                    query = f"""
-                        SELECT id, {column},
-                               {column}_dtype,
-                               {column}_shape
-                        FROM {table_name}
-                        WHERE id IN ({placeholders})
-                    """
-                    self.cursor.execute(query, tuple(batch_ids))
-                    has_metadata = True
-                except sqlite3.OperationalError:
-                    query = f"SELECT id, {column} FROM {table_name} WHERE id IN ({placeholders})"
-                    self.cursor.execute(query, tuple(batch_ids))
-                    has_metadata = False
-
-                if where:
-                    query += f" AND {where}"
-                if order_by:
-                    query += f" ORDER BY {order_by}"
-
-                results = self.cursor.fetchall()
-                if results:
-                    for result in results:
-                        if has_metadata:
-                            id_val, blob, dtype_str, shape_str = result
-                            data = np.frombuffer(
-                                blob, dtype=np.dtype(dtype_str)
-                            ).reshape(eval(shape_str))
-                        else:
-                            id_val, blob = result
-                            data = (
-                                np.frombuffer(blob, dtype=dtype)
-                                if dtype
-                                else np.frombuffer(blob)
-                            )
-                            if shape:
-                                data = data.reshape(shape)
-                        id_to_data[id_val] = data
-
-            all_data = [id_to_data[id_val] for id_val in ids if id_val in id_to_data]
-            return np.stack(all_data, axis=0) if all_data else None
-
-        except Exception as err:
-            raise ValueError(f"Failed to load array: {err}")
-
-    def binary_to_array(
-        self,
-        binary_data,
-        dtype_str=None,
-        shape_str=None,
-        dtype=None,
-        shape=None,
-    ):
-        if binary_data is None:
-            return None
-
-        if dtype_str and shape_str:
-            return np.frombuffer(binary_data, dtype=np.dtype(dtype_str)).reshape(
-                eval(shape_str)
-            )
-        elif dtype and shape:
-            return np.frombuffer(binary_data, dtype=dtype).reshape(shape)
-        return binary_data
-
-    def get_array_dict(self, df, columns=None, dtype=None, shape=None):
-        result = {}
-        if columns is None:
-            columns = [
-                col
-                for col in df.columns
-                if not (col.endswith("_dtype") or col.endswith("_shape"))
-            ]
-
-        for col in columns:
-            if f"{col}_dtype" in df.columns and f"{col}_shape" in df.columns:
-                arrays = [
-                    self.binary_to_array(
-                        row[col], row[f"{col}_dtype"], row[f"{col}_shape"]
-                    )
-                    for _, row in df.iterrows()
-                ]
-            elif dtype and shape:
-                arrays = [
-                    self.binary_to_array(x, dtype=dtype, shape=shape) for x in df[col]
-                ]
-            result[col] = np.stack(arrays)
-
-        return result
-
-    def decode_array_columns(self, df, columns=None, dtype=None, shape=None):
-        if columns is None:
-            columns = [
-                col
-                for col in df.columns
-                if not (col.endswith("_dtype") or col.endswith("_shape"))
-            ]
-
-        for col in columns:
-            if f"{col}_dtype" in df.columns and f"{col}_shape" in df.columns:
-                df[col] = df.apply(
-                    lambda row: self.binary_to_array(
-                        row[col], row[f"{col}_dtype"], row[f"{col}_shape"]
-                    ),
-                    axis=1,
-                )
-            elif dtype and shape:
-                df[col] = df[col].apply(
-                    lambda x: self.binary_to_array(x, dtype=dtype, shape=shape)
-                )
-        return df
-
-
-# EOF

scitex/gen/_close.py

@@ -1,222 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-02-14 21:12:25 (ywatanabe)"
-# File: ./src/scitex/gen/_close.py
-
-THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/gen/_close.py"
-
-import os
-import re
-import shutil
-import time
-from datetime import datetime
-from glob import glob as _glob
-
-from ..io import flush as scitex_io_flush
-from ..io import save as scitex_io_save
-from ..str._printc import printc
-from ..utils._notify import notify as scitex_utils_notify
-
-
-def _format_diff_time(diff_time):
-    # Get total seconds from the timedelta object
-    total_seconds = int(diff_time.total_seconds())
-
-    # Calculate hours, minutes and seconds
-    hours = total_seconds // 3600
-    minutes = (total_seconds % 3600) // 60
-    seconds = total_seconds % 60
-
-    # Format the time difference as a string
-    diff_time_str = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
-    return diff_time_str
-
-
-def _process_timestamp(CONFIG, verbose=True):
-    try:
-        CONFIG["END_TIME"] = datetime.now()
-        CONFIG["RUN_TIME"] = _format_diff_time(
-            CONFIG["END_TIME"] - CONFIG["START_TIME"]
-        )
-        if verbose:
-            print()
-            print(f"START TIME: {CONFIG['START_TIME']}")
-            print(f"END TIME: {CONFIG['END_TIME']}")
-            print(f"RUN TIME: {CONFIG['RUN_TIME']}")
-            print()
-
-    except Exception as e:
-        print(e)
-
-    return CONFIG
-
-
-def _save_configs(CONFIG):
-    scitex_io_save(CONFIG, CONFIG["SDIR"] + "CONFIGS/CONFIG.pkl", verbose=False)
-    scitex_io_save(CONFIG, CONFIG["SDIR"] + "CONFIGS/CONFIG.yaml", verbose=False)
-
-
-def _escape_ansi_from_log_files(log_files):
-    """Remove ANSI escape sequences from log files.
-    
-    Parameters
-    ----------
-    log_files : list
-        List of log file paths to clean
-    """
-    ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
-
-    # ANSI code escape
-    for f in log_files:
-        with open(f, "r", encoding="utf-8") as file:
-            content = file.read()
-        cleaned_content = ansi_escape.sub("", content)
-        with open(f, "w", encoding="utf-8") as file:
-            file.write(cleaned_content)
-
-
-def _args_to_str(args_dict):
-    """Convert args dictionary to formatted string."""
-    if args_dict:
-        max_key_length = max(len(str(k)) for k in args_dict.keys())
-        return "\n".join(
-            f"{str(k):<{max_key_length}} : {str(v)}"
-            for k, v in sorted(args_dict.items())
-        )
-    else:
-        return ""
-
-
-def close(CONFIG, message=":)", notify=False, verbose=True, exit_status=None):
-    sys = None  # Initialize sys outside try block
-    try:
-        CONFIG.EXIT_STATUS = exit_status
-        CONFIG = CONFIG.to_dict()
-        CONFIG = _process_timestamp(CONFIG, verbose=verbose)
-        sys = CONFIG.pop("sys")
-        _save_configs(CONFIG)
-        # scitex_io_flush(sys=sys)
-
-        # RUNNING to RUNNING2FINISHEDED
-        CONFIG = running2finished(CONFIG, exit_status=exit_status)
-        # scitex_io_flush(sys=sys)
-
-        # ANSI code escape
-        log_files = _glob(CONFIG["SDIR"] + "logs/*.log")
-        _escape_ansi_from_log_files(log_files)
-        # scitex_io_flush(sys=sys)
-
-        if CONFIG.get("ARGS"):
-            message += f"\n{_args_to_str(CONFIG.get('ARGS'))}"
-
-        if notify:
-            try:
-                message = (
-                    f"[DEBUG]\n" + str(message)
-                    if CONFIG.get("DEBUG", False)
-                    else str(message)
-                )
-                scitex_utils_notify(
-                    message=message,
-                    ID=CONFIG["ID"],
-                    file=CONFIG.get("FILE"),
-                    attachment_paths=log_files,
-                    verbose=verbose,
-                )
-                # scitex_io_flush(sys=sys)
-            except Exception as e:
-                print(e)
-
-    finally:
-        # Only close if they're custom file objects (Tee objects)
-        if sys:
-            try:
-                # First, flush all outputs
-                if hasattr(sys, "stdout") and hasattr(sys.stdout, "flush"):
-                    sys.stdout.flush()
-                if hasattr(sys, "stderr") and hasattr(sys.stderr, "flush"):
-                    sys.stderr.flush()
-
-                # Then close Tee objects
-                if hasattr(sys, "stdout") and hasattr(sys.stdout, "_log_file"):
-                    sys.stdout.close()
-                if hasattr(sys, "stderr") and hasattr(sys.stderr, "_log_file"):
-                    sys.stderr.close()
-            except Exception:
-                # Silent fail to ensure logs are saved even if there's an error
-                pass
-    # finally:
-    #     # Ensure file handles are closed
-    #     if hasattr(sys, 'stdout') and hasattr(sys.stdout, 'close'):
-    #         sys.stdout.close()
-    #     if hasattr(sys, 'stderr') and hasattr(sys.stderr, 'close'):
-    #         sys.stderr.close()
-    # # try:
-    # #     sys.stdout.close()
-    # #     sys.stderr.close()
-    # # except Exception as e:
-    # #     print(e)
-
-
-def running2finished(CONFIG, exit_status=None, remove_src_dir=True, max_wait=60):
-    if exit_status == 0:
-        dest_dir = CONFIG["SDIR"].replace("RUNNING/", "FINISHED_SUCCESS/")
-    elif exit_status == 1:
-        dest_dir = CONFIG["SDIR"].replace("RUNNING/", "FINISHED_ERROR/")
-    else:  # exit_status is None:
-        dest_dir = CONFIG["SDIR"].replace("RUNNING/", "FINISHED/")
-
-    src_dir = CONFIG["SDIR"]
-    # if dest_dir is None:
-    #     dest_dir = src_dir.replace("RUNNING/", "FINISHED/")
-
-    os.makedirs(dest_dir, exist_ok=True)
-    try:
-
-        # Copy files individually
-        for item in os.listdir(src_dir):
-            s = os.path.join(src_dir, item)
-            d = os.path.join(dest_dir, item)
-            if os.path.isdir(s):
-                shutil.copytree(s, d)
-            else:
-                shutil.copy2(s, d)
-
-        start_time = time.time()
-        while not os.path.exists(dest_dir) and time.time() - start_time < max_wait:
-            time.sleep(0.1)
-        if os.path.exists(dest_dir):
-            printc(
-                f"Congratulations! The script completed.\n\n{dest_dir}",
-                c="yellow",
-            )
-            if remove_src_dir:
-                shutil.rmtree(src_dir)
-        else:
-            print(f"Copy operation timed out after {max_wait} seconds")
-
-        CONFIG["SDIR"] = dest_dir
-    except Exception as e:
-        print(e)
-
-    finally:
-        return CONFIG
-
-
-if __name__ == "__main__":
-    import sys
-
-    import matplotlib.pyplot as plt
-    from icecream import ic
-
-    from .._start import start
-
-    CONFIG, sys.stdout, sys.stderr, plt, CC = start(sys, plt, verbose=False)
-
-    ic("aaa")
-    ic("bbb")
-    ic("ccc")
-
-    close(CONFIG)
-
-# EOF