scitex 2.0.0__py2.py3-none-any.whl

scitex/gen/__init__.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Scitex gen module."""
+
+from ._DimHandler import DimHandler
+from ._TimeStamper import TimeStamper
+from ._alternate_kwarg import alternate_kwarg
+from ._cache import cache
+from ._check_host import check_host, is_host, verify_host
+from ._ci import ci
+from ._close import close, running2finished
+from ._embed import embed
+from ._inspect_module import inspect_module
+from ._is_ipython import is_ipython, is_script
+from ._less import less
+from ._list_packages import list_packages, main
+from ._mat2py import dir2npy, keys2npa, mat2dict, mat2npa, mat2npy, public_keys, save_npa
+from ._norm import clip_perc, to_01, to_nan01, to_nanz, to_z, unbias
+from ._paste import paste
+from ._print_config import print_config, print_config_main
+from ._shell import run_shellcommand, run_shellscript
+from ._src import src
+from ._start import start
+from ._symlink import symlink
+from ._symlog import symlog
+from ._tee import Tee, main, tee
+from ._title2path import title2path
+from ._title_case import main, title_case
+from ._to_even import to_even
+from ._to_odd import to_odd
+from ._to_rank import to_rank
+from ._transpose import transpose
+from ._type import ArrayLike, var_info
+from ._var_info import ArrayLike, var_info
+from ._wrap import wrap
+from ._xml2dict import XmlDictConfig, XmlListConfig, xml2dict
+
+__all__ = [
+    "ArrayLike",
+    "ArrayLike",
+    "DimHandler",
+    "Tee",
+    "TimeStamper",
+    "XmlDictConfig",
+    "XmlListConfig",
+    "alternate_kwarg",
+    "cache",
+    "check_host",
+    "ci",
+    "clip_perc",
+    "close",
+    "dir2npy",
+    "embed",
+    "inspect_module",
+    "is_host",
+    "is_ipython",
+    "is_script",
+    "keys2npa",
+    "less",
+    "list_packages",
+    "main",
+    "main",
+    "main",
+    "mat2dict",
+    "mat2npa",
+    "mat2npy",
+    "paste",
+    "print_config",
+    "print_config_main",
+    "public_keys",
+    "run_shellcommand",
+    "run_shellscript",
+    "running2finished",
+    "save_npa",
+    "src",
+    "start",
+    "symlink",
+    "symlog",
+    "tee",
+    "title2path",
+    "title_case",
+    "to_01",
+    "to_even",
+    "to_nan01",
+    "to_nanz",
+    "to_odd",
+    "to_rank",
+    "to_z",
+    "transpose",
+    "unbias",
+    "var_info",
+    "var_info",
+    "verify_host",
+    "wrap",
+    "xml2dict",
+]

scitex/gen/_alternate_kwarg.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-02 13:30:41 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/gen/_alternate_kwarg.py
+
+
+def alternate_kwarg(kwargs, primary_key, alternate_key):
+    alternate_value = kwargs.pop(alternate_key, None)
+    kwargs[primary_key] = kwargs.get(primary_key) or alternate_value
+    return kwargs
+
+
+# EOF

scitex/gen/_cache.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-02 13:30:24 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/gen/_cache.py
+
+from functools import lru_cache
+
+cache = lru_cache(maxsize=None)
+
+
+# EOF

scitex/gen/_check_host.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-02 13:43:36 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/gen/_check_host.py
+
+
+from .._sh import sh
+import sys
+
+
+def check_host(keyword):
+    return keyword in sh("echo $(hostname)", verbose=False)
+
+
+is_host = check_host
+
+
+def verify_host(keyword):
+    if is_host(keyword):
+        print(f"Host verification successed for keyword: {keyword}")
+        return
+    else:
+        print(f"Host verification failed for keyword: {keyword}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    # check_host("ywata")
+    verify_host("titan")
+    verify_host("ywata")
+    verify_host("crest")
+
+
+# EOF

scitex/gen/_ci.py

@@ -0,0 +1,12 @@
+#!./env/bin/python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-06-04 06:55:56 (ywatanabe)"
+# /home/ywatanabe/proj/scitex/src/scitex/gen/_ci.py
+
+
+import numpy as np
+
+
+def ci(xx, axis=None):
+    indi = ~np.isnan(xx)
+    return 1.96 * (xx[indi]).std(axis=axis) / np.sqrt(indi.sum())

scitex/gen/_close.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-02-14 21:12:25 (ywatanabe)"
+# File: ./src/scitex/gen/_close.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/gen/_close.py"
+
+import os
+import re
+import shutil
+import time
+from datetime import datetime
+from glob import glob as _glob
+
+from ..io import flush as scitex_io_flush
+from ..io import save as scitex_io_save
+from ..str._printc import printc
+from ..utils._notify import notify as scitex_utils_notify
+
+
+def _format_diff_time(diff_time):
+    # Get total seconds from the timedelta object
+    total_seconds = int(diff_time.total_seconds())
+
+    # Calculate hours, minutes and seconds
+    hours = total_seconds // 3600
+    minutes = (total_seconds % 3600) // 60
+    seconds = total_seconds % 60
+
+    # Format the time difference as a string
+    diff_time_str = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
+    return diff_time_str
+
+
+def _process_timestamp(CONFIG, verbose=True):
+    try:
+        CONFIG["END_TIME"] = datetime.now()
+        CONFIG["RUN_TIME"] = _format_diff_time(
+            CONFIG["END_TIME"] - CONFIG["START_TIME"]
+        )
+        if verbose:
+            print()
+            print(f"START TIME: {CONFIG['START_TIME']}")
+            print(f"END TIME: {CONFIG['END_TIME']}")
+            print(f"RUN TIME: {CONFIG['RUN_TIME']}")
+            print()
+
+    except Exception as e:
+        print(e)
+
+    return CONFIG
+
+
+def _save_configs(CONFIG):
+    scitex_io_save(CONFIG, CONFIG["SDIR"] + "CONFIGS/CONFIG.pkl", verbose=False)
+    scitex_io_save(CONFIG, CONFIG["SDIR"] + "CONFIGS/CONFIG.yaml", verbose=False)
+
+
+def _escape_ansi_from_log_files(log_files):
+    """Remove ANSI escape sequences from log files.
+    
+    Parameters
+    ----------
+    log_files : list
+        List of log file paths to clean
+    """
+    ansi_escape = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
+
+    # ANSI code escape
+    for f in log_files:
+        with open(f, "r", encoding="utf-8") as file:
+            content = file.read()
+        cleaned_content = ansi_escape.sub("", content)
+        with open(f, "w", encoding="utf-8") as file:
+            file.write(cleaned_content)
+
+
+def _args_to_str(args_dict):
+    """Convert args dictionary to formatted string."""
+    if args_dict:
+        max_key_length = max(len(str(k)) for k in args_dict.keys())
+        return "\n".join(
+            f"{str(k):<{max_key_length}} : {str(v)}"
+            for k, v in sorted(args_dict.items())
+        )
+    else:
+        return ""
+
+
+def close(CONFIG, message=":)", notify=False, verbose=True, exit_status=None):
+    sys = None  # Initialize sys outside try block
+    try:
+        CONFIG.EXIT_STATUS = exit_status
+        CONFIG = CONFIG.to_dict()
+        CONFIG = _process_timestamp(CONFIG, verbose=verbose)
+        sys = CONFIG.pop("sys")
+        _save_configs(CONFIG)
+        # scitex_io_flush(sys=sys)
+
+        # RUNNING to RUNNING2FINISHEDED
+        CONFIG = running2finished(CONFIG, exit_status=exit_status)
+        # scitex_io_flush(sys=sys)
+
+        # ANSI code escape
+        log_files = _glob(CONFIG["SDIR"] + "logs/*.log")
+        _escape_ansi_from_log_files(log_files)
+        # scitex_io_flush(sys=sys)
+
+        if CONFIG.get("ARGS"):
+            message += f"\n{_args_to_str(CONFIG.get('ARGS'))}"
+
+        if notify:
+            try:
+                message = (
+                    f"[DEBUG]\n" + str(message)
+                    if CONFIG.get("DEBUG", False)
+                    else str(message)
+                )
+                scitex_utils_notify(
+                    message=message,
+                    ID=CONFIG["ID"],
+                    file=CONFIG.get("FILE"),
+                    attachment_paths=log_files,
+                    verbose=verbose,
+                )
+                # scitex_io_flush(sys=sys)
+            except Exception as e:
+                print(e)
+
+    finally:
+        # Only close if they're custom file objects (Tee objects)
+        if sys:
+            try:
+                # First, flush all outputs
+                if hasattr(sys, "stdout") and hasattr(sys.stdout, "flush"):
+                    sys.stdout.flush()
+                if hasattr(sys, "stderr") and hasattr(sys.stderr, "flush"):
+                    sys.stderr.flush()
+
+                # Then close Tee objects
+                if hasattr(sys, "stdout") and hasattr(sys.stdout, "_log_file"):
+                    sys.stdout.close()
+                if hasattr(sys, "stderr") and hasattr(sys.stderr, "_log_file"):
+                    sys.stderr.close()
+            except Exception:
+                # Silent fail to ensure logs are saved even if there's an error
+                pass
+    # finally:
+    #     # Ensure file handles are closed
+    #     if hasattr(sys, 'stdout') and hasattr(sys.stdout, 'close'):
+    #         sys.stdout.close()
+    #     if hasattr(sys, 'stderr') and hasattr(sys.stderr, 'close'):
+    #         sys.stderr.close()
+    # # try:
+    # #     sys.stdout.close()
+    # #     sys.stderr.close()
+    # # except Exception as e:
+    # #     print(e)
+
+
+def running2finished(CONFIG, exit_status=None, remove_src_dir=True, max_wait=60):
+    if exit_status == 0:
+        dest_dir = CONFIG["SDIR"].replace("RUNNING/", "FINISHED_SUCCESS/")
+    elif exit_status == 1:
+        dest_dir = CONFIG["SDIR"].replace("RUNNING/", "FINISHED_ERROR/")
+    else:  # exit_status is None:
+        dest_dir = CONFIG["SDIR"].replace("RUNNING/", "FINISHED/")
+
+    src_dir = CONFIG["SDIR"]
+    # if dest_dir is None:
+    #     dest_dir = src_dir.replace("RUNNING/", "FINISHED/")
+
+    os.makedirs(dest_dir, exist_ok=True)
+    try:
+
+        # Copy files individually
+        for item in os.listdir(src_dir):
+            s = os.path.join(src_dir, item)
+            d = os.path.join(dest_dir, item)
+            if os.path.isdir(s):
+                shutil.copytree(s, d)
+            else:
+                shutil.copy2(s, d)
+
+        start_time = time.time()
+        while not os.path.exists(dest_dir) and time.time() - start_time < max_wait:
+            time.sleep(0.1)
+        if os.path.exists(dest_dir):
+            printc(
+                f"Congratulations! The script completed.\n\n{dest_dir}",
+                c="yellow",
+            )
+            if remove_src_dir:
+                shutil.rmtree(src_dir)
+        else:
+            print(f"Copy operation timed out after {max_wait} seconds")
+
+        CONFIG["SDIR"] = dest_dir
+    except Exception as e:
+        print(e)
+
+    finally:
+        return CONFIG
+
+
+if __name__ == "__main__":
+    import sys
+
+    import matplotlib.pyplot as plt
+    from icecream import ic
+
+    from .._start import start
+
+    CONFIG, sys.stdout, sys.stderr, plt, CC = start(sys, plt, verbose=False)
+
+    ic("aaa")
+    ic("bbb")
+    ic("ccc")
+
+    close(CONFIG)
+
+# EOF

scitex/gen/_embed.py

@@ -0,0 +1,78 @@
+"""
+This script does XYZ.
+"""
+
+# import os
+# import sys
+
+# import matplotlib.pyplot as plt
+
+# # Imports
+#
+# import numpy as np
+# import pandas as pd
+# import torch
+# import torch.nn as nn
+# import torch.nn.functional as F
+
+# # Config
+# CONFIG = scitex.gen.load_configs()
+
+# Functions
+# from IPython import embed as _embed
+# import pyperclip
+
+# def embed_with_clipboard_exec():
+#     # Try to get text from the clipboard
+#     try:
+#         clipboard_content = pyperclip.paste()
+#     except pyperclip.PyperclipException as e:
+#         clipboard_content = ""
+#         print("Could not access the clipboard:", e)
+
+#     # Start IPython session with the clipboard content preloaded
+#     ipython_shell = embed(header='IPython is now running with the following clipboard content executed:', compile_flags=None)
+
+#     # Optionally, execute the clipboard content automatically
+#     if clipboard_content:
+#         # Execute the content as if it was typed in directly
+#         ipython_shell.run_cell(clipboard_content)
+
+
+def embed():
+    import pyperclip
+    from IPython import embed as _embed
+
+    try:
+        clipboard_content = pyperclip.paste()
+    except pyperclip.PyperclipException as e:
+        clipboard_content = ""
+        print("Could not access the clipboard:", e)
+
+    print("Clipboard content loaded. Do you want to execute it? [y/n]")
+    execute_clipboard = input().strip().lower() == "y"
+
+    # Start IPython shell
+    ipython_shell = _embed(
+        header="IPython is now running. Clipboard content will be executed if confirmed."
+    )
+
+    # Execute if confirmed
+    if clipboard_content and execute_clipboard:
+        ipython_shell.run_cell(clipboard_content)
+
+
+if __name__ == "__main__":
+    # Start
+    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.gen.start(sys, plt)
+
+    embed()
+
+    # Close
+    scitex.gen.close(CONFIG)
+
+# EOF
+
+"""
+/ssh:ywatanabe@444:/home/ywatanabe/proj/entrance/scitex/gen/_embed.py
+"""