scitex 2.0.0__py2.py3-none-any.whl

scitex/utils/_grid.py

@@ -0,0 +1,148 @@
+#!./env/bin/python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-04-22 23:54:02"
+# Author: Yusuke Watanabe (ywata1989@gmail.com)
+
+"""
+This script defines scitex.ml.utils.grid_search
+"""
+
+# Imports
+import itertools as _itertools
+import random as _random
+import sys as _sys
+
+import matplotlib.pyplot as _plt
+import scitex as _scitex
+
+
+# Functions
+def yield_grids(params_grid: dict, random=False):
+    """
+    Generator function that yields combinations of parameters from a grid.
+
+    Args:
+        params_grid (dict): A dictionary where keys are parameter names and values are lists of parameter values.
+        random (bool): If True, yields the parameter combinations in random order.
+
+    Yields:
+        dict: A dictionary of parameters for one set of conditions from the grid.
+
+    Example:
+        # Parameters
+        params_grid = {
+            "batch_size": [2**i for i in range(7)],
+            "n_chs": [2**i for i in range(7)],
+            "seq_len": [2**i for i in range(15)],
+            "fs": [2**i for i in range(8, 11)],
+            "n_segments": [2**i for i in range(6)],
+            "n_bands_pha": [2**i for i in range(7)],
+            "n_bands_amp": [2**i for i in range(7)],
+            "precision": ['fp16', 'fp32'],
+            "device": ['cpu', 'cuda'],
+            "package": ['tensorpac', 'scitex'],
+        }
+
+        # Example of using the generator
+        for param_dict in yield_grids(params_grid, random=True):
+            print(param_dict)
+    """
+    combinations = list(_itertools.product(*params_grid.values()))
+    if random:
+        _random.shuffle(combinations)  # [REVISED]
+    for values in combinations:
+        yield dict(zip(params_grid.keys(), values))
+
+
+# def yield_grids(params_grid: dict, random=False):
+#     """
+#     Generator function that yields combinations of parameters from a grid.
+
+#     Args:
+#         params_grid (dict): A dictionary where keys are parameter names and values are lists of parameter values.
+
+#     Yields:
+#         dict: A dictionary of parameters for one set of conditions from the grid.
+
+#     Example:
+#         # Parameters
+#         params_grid = {
+#             "batch_size": [2**i for i in range(7)],
+#             "n_chs": [2**i for i in range(7)],
+#             "seq_len": [2**i for i in range(15)],
+#             "fs": [2**i for i in range(8, 11)],
+#             "n_segments": [2**i for i in range(6)],
+#             "n_bands_pha": [2**i for i in range(7)],
+#             "n_bands_amp": [2**i for i in range(7)],
+#             "precision": ['fp16', 'fp32'],
+#             "device": ['cpu', 'cuda'],
+#             "package": ['tensorpac', 'scitex'],
+#         }
+
+#         # Example of using the generator
+#         for param_dict in yield_grids(params_grid):
+#             print(param_dict)
+#     """
+#     print(f"\nThe Number of Combinations: {count_grids(params_grid):,}")
+
+#     for values in _itertools.product(*params_grid.values()):
+#         yield dict(zip(params_grid.keys(), values))
+
+
+def count_grids(params_grid):
+    """
+    Calculate the total number of combinations possible from the given parameter grid.
+
+    Args:
+        params_grid (dict): A dictionary where keys are parameter names and values are lists of parameter values.
+
+    Returns:
+        int: The total number of combinations that can be generated from the parameter grid.
+    """
+    # Get the number of values for each parameter and multiply them
+    num_combinations = 1
+    for values in params_grid.values():
+        num_combinations *= len(values)
+    return num_combinations
+
+
+if __name__ == "__main__":
+    import pandas as pd
+
+    # Start
+    CONFIG, _sys.stdout, _sys.stderr, _plt, CC = _scitex.gen.start(
+        _sys, _plt, verbose=False
+    )
+
+    # Parameters
+    N = 15
+    print(pd.DataFrame(pd.Series({f"2^{ii}": 2**ii for ii in range(N)})))
+
+    params_grid = {
+        "batch_size": [2**i for i in [3, 4, 5, 6]],
+        "n_chs": [2**i for i in [3, 4, 5, 6]],
+        "seq_len": [2**i for i in range(8, 13)],
+        "fs": [2**i for i in range(7, 10)],
+        "n_segments": [2**i for i in range(5)],
+        "n_bands_pha": [2**i for i in range(7)],
+        "n_bands_amp": [2**i for i in range(7)],
+        "precision": ["fp16", "fp32"],
+        "device": ["cpu", "cuda"],
+        "package": ["tensorpac", "_scitex"],
+    }
+
+    print(params_grid)
+    print(f"{count_grids(params_grid):,}")
+
+    # Example of using the generator
+    for param_dict in yield_grids(params_grid):
+        print(param_dict)
+
+    # Close
+    _scitex.gen.close(CONFIG, verbose=False, notify=False)
+
+# EOF
+
+"""
+/home/ywatanabe/proj/entrance/_scitex/ml/utils/grid_search.py
+"""

scitex/utils/_notify.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-06-15 20:25:22 (ywatanabe)"
+# File: /ssh:ywatanabe@sp:/home/ywatanabe/proj/SciTeX-Code/src/scitex/utils/_notify.py
+# ----------------------------------------
+import os
+__FILE__ = (
+    "./src/scitex/utils/_notify.py"
+)
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+# Time-stamp: "2024-11-24 17:54:38 (ywatanabe)"
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/utils/_notify.py"
+
+# Time-stamp: "2024-11-07 05:38:14 (ywatanabe)"
+
+"""This script does XYZ."""
+
+import inspect
+import pwd
+import socket
+import subprocess
+import sys
+import warnings
+
+from ._email import send_gmail
+
+
+def get_username():
+    try:
+        return pwd.getpwuid(os.getuid()).pw_name
+    except:
+        return os.getenv("USER") or os.getenv("LOGNAME") or "unknown"
+
+
+def get_hostname():
+    return socket.gethostname()
+
+
+def get_git_branch(scitex):
+    try:
+        branch = (
+            subprocess.check_output(
+                ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+                cwd=scitex.__path__[0],
+                stderr=subprocess.DEVNULL,
+            )
+            .decode()
+            .strip()
+        )
+        return branch
+
+    except Exception as e:
+        print(e)
+        return "main"
+
+
+def gen_footer(sender, script_name, scitex, branch):
+    return f"""
+
+{'-'*30}
+Sent via
+- Host: {sender}
+- Script: {script_name}
+- Source: scitex v{scitex.__version__} (github.com/ywatanabe1989/scitex/blob/{branch}/src/scitex/gen/system_ops/_notify.py)
+{'-'*30}"""
+
+
+# This is an automated system notification. If received outside working hours, please disregard.
+
+
+def notify(
+    subject="",
+    message=":)",
+    file=None,
+    ID="auto",
+    sender_name=None,
+    recipient_email=None,
+    cc=None,
+    attachment_paths=None,
+    verbose=False,
+):
+    import scitex
+
+    try:
+        message = str(message)
+    except Exception as e:
+        warnings.warn(str(e))
+
+    FAKE_PYTHON_SCRIPT_NAME = "$ python -c ..."
+    sender_gmail = os.getenv("SCITEX_SENDER_GMAIL")
+    sender_password = os.getenv("SCITEX_SENDER_GMAIL_PASSWORD")
+    recipient_email = recipient_email or os.getenv("SCITEX_RECIPIENT_GMAIL")
+
+    if file is not None:
+        script_name = str(file)
+    else:
+        if sys.argv[0]:
+            script_name = os.path.basename(sys.argv[0])
+        else:
+            frames = inspect.stack()
+            script_name = (
+                os.path.basename(frames[-1].filename)
+                if frames
+                else "(Not found)"
+            )
+        if (script_name == "-c") or (not script_name.endswith(".py")):
+            script_name = FAKE_PYTHON_SCRIPT_NAME
+
+    sender = f"{get_username()}@{get_hostname()}"
+    branch = get_git_branch(scitex)
+    footer = gen_footer(sender, script_name, scitex, branch)
+
+    full_message = script_name + "\n\n" + message + "\n\n" + footer
+    full_subject = (
+        f"{script_name}—{subject}"
+        if subject and (script_name != FAKE_PYTHON_SCRIPT_NAME)
+        else f"{subject}"
+    )
+
+    if sender_gmail is None or sender_password is None:
+        print(
+            f"""
+        Please set environmental variables to use this function (f"{inspect.stack()[0][3]}"):\n\n
+        $ export SciTeX_SENDER_GMAIL="scitex.notification@gmail.com"
+        $ export SciTeX_SENDER_GMAIL_PASSWORD="YOUR_APP_PASSWORD"
+        $ export SciTeX_RECIPIENT_GMAIL="YOUR_GMAIL_ADDRESS"
+        """
+        )
+
+    send_gmail(
+        sender_gmail,
+        sender_password,
+        recipient_email,
+        full_subject,
+        full_message,
+        sender_name=sender_name,
+        cc=cc,
+        ID=ID,
+        attachment_paths=attachment_paths,
+        verbose=verbose,
+    )
+
+
+if __name__ == "__main__":
+    notify(verbose=True)
+
+    # python -c "; scitex.gen.notify()"
+
+
+# # Example in shell
+# #!/bin/bash
+# # /home/ywatanabe/.dotfiles/.bin/notify
+# # Author: ywatanabe (ywatanabe@alumni.u-tokyo.ac.jp)
+# # Date: $(date +"%Y-%m-%d-%H-%M")
+
+# # LOG_FILE="${0%.sh}.log"
+
+# usage() {
+#     echo "Usage: $0 [-s|--subject <subject>] [-m|--message <message>] [-r|--recipient-name <name>] [-t|--to <email>] [-c|--cc <email>] [-h|--help]"
+#     echo "Options:"
+#     echo "  -s, --subject   Subject of the notification (default: 'Subject')"
+#     echo "  -m, --message   Message body of the notification (default: 'Message')"
+#     echo "  -t, --to        The email address of the recipient"
+#     echo "  -c, --cc        CC email address(es) (can be used multiple times)"
+#     echo "  -h, --help      Display this help message"
+#     echo
+#     echo "Example:"
+#     echo "  $0 -s \"About the Project A\" -m \"Hi, ...\" -r \"John\" -t \"john@example.com\" -c \"cc1@example.com\" -c \"cc2@example.com\""
+#     echo "  $0 -s \"Notification\" -m \"This is a notification from ...\" -r \"Team\" -t \"team@example.com\""
+#     exit 1
+# }
+
+# main() {
+#     subject="Subject"
+#     message="Message"
+#     recipient_email=""
+#     cc_addresses=()
+
+#     while [[ $# -gt 0 ]]; do
+#         case $1 in
+#             -s|--subject)
+#                 shift
+#                 subject=""
+#                 while [[ $# -gt 0 && ! $1 =~ ^- ]]; do
+#                     subject+="$1 "
+#                     shift
+#                 done
+#                 subject=${subject% }
+#                 ;;
+#             -m|--message)
+#                 shift
+#                 message=""
+#                 while [[ $# -gt 0 && ! $1 =~ ^- ]]; do
+#                     message+="$1 "
+#                     shift
+#                 done
+#                 message=${message% }
+#                 ;;
+#             -t|--to)
+#                 shift
+#                 recipient_email="$1"
+#                 shift
+#                 ;;
+#             -c|--cc)
+#                 shift
+#                 cc_addresses+=("$1")
+#                 shift
+#                 ;;
+#             -h|--help)
+#                 usage
+#                 ;;
+#             *)
+#                 echo "Unknown option: $1"
+#                 usage
+#                 ;;
+#         esac
+#     done
+
+#     subject=$(echo "$subject" | sed "s/'/'\\\\''/g")
+#     message=$(echo "$message" | sed "s/'/'\\\\''/g")
+#     recipient_email=$(echo "$recipient_email" | sed "s/'/'\\\\''/g")
+#     cc_string=$(IFS=,; echo "${cc_addresses[*]}" | sed "s/'/'\\\\''/g")
+
+#     python -c "
+#
+
+# cc_list = [$(printf "'%s', " "${cc_addresses[@]}")]
+# cc_list = [addr.strip() for addr in cc_list if addr.strip()]
+
+# scitex.gen.notify(
+#     subject='$subject',
+#     message='$message',
+#     ID=None,
+#     recipient_email='$recipient_email',
+#     cc=cc_list
+# )
+# "
+# }
+
+# main "$@"
+# # { main "$@"; } 2>&1 | tee "$LOG_FILE"
+
+#
+
+# EOF

scitex/utils/_search.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-02 13:01:38 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/utils/_search.py
+
+import numpy as np
+import re
+from collections import abc
+
+try:
+    import pandas as pd
+except ImportError:
+    pd = None
+
+try:
+    import xarray as xr
+except ImportError:
+    xr = None
+
+try:
+    from natsort import natsorted
+except ImportError:
+    # Fallback to regular sorted if natsort not available
+    def natsorted(iterable):
+        return sorted(iterable)
+
+
+def search(
+    patterns, strings, only_perfect_match=False, as_bool=False, ensure_one=False
+):
+    """Search for patterns in strings using regular expressions.
+
+    Parameters
+    ----------
+    patterns : str or list of str
+        The pattern(s) to search for. Can be a single string or a list of strings.
+    strings : str or list of str
+        The string(s) to search in. Can be a single string or a list of strings.
+    only_perfect_match : bool, optional
+        If True, only exact matches are considered (default is False).
+    as_bool : bool, optional
+        If True, return a boolean array instead of indices (default is False).
+    ensure_one : bool, optional
+        If True, ensures only one match is found (default is False).
+
+    Returns
+    -------
+    tuple
+        A tuple containing two elements:
+        - If as_bool is False: (list of int, list of str)
+          The first element is a list of indices where matches were found.
+          The second element is a list of matched strings.
+        - If as_bool is True: (numpy.ndarray of bool, list of str)
+          The first element is a boolean array indicating matches.
+          The second element is a list of matched strings.
+
+    Example
+    -------
+    >>> patterns = ['orange', 'banana']
+    >>> strings = ['apple', 'orange', 'apple', 'apple_juice', 'banana', 'orange_juice']
+    >>> search(patterns, strings)
+    ([1, 4, 5], ['orange', 'banana', 'orange_juice'])
+
+    >>> patterns = 'orange'
+    >>> strings = ['apple', 'orange', 'apple', 'apple_juice', 'banana', 'orange_juice']
+    >>> search(patterns, strings)
+    ([1, 5], ['orange', 'orange_juice'])
+    """
+
+    def to_list(string_or_pattern):
+        # Check for numpy arrays first
+        if isinstance(string_or_pattern, np.ndarray):
+            return string_or_pattern.tolist()
+        
+        # Check for pandas types if pandas is available
+        if pd is not None:
+            if isinstance(string_or_pattern, (pd.Series, pd.Index)):
+                return string_or_pattern.tolist()
+        
+        # Check for xarray types if xarray is available
+        if xr is not None:
+            if isinstance(string_or_pattern, xr.DataArray):
+                return string_or_pattern.tolist()
+        
+        # Check for other iterables
+        if isinstance(string_or_pattern, abc.KeysView):
+            return list(string_or_pattern)
+        elif not isinstance(string_or_pattern, (list, tuple)):
+            return [string_or_pattern]
+        return string_or_pattern
+
+    patterns = to_list(patterns)
+    strings = to_list(strings)
+
+    indices_matched = []
+    for pattern in patterns:
+        for index_str, string in enumerate(strings):
+            if only_perfect_match:
+                if pattern == string:
+                    indices_matched.append(index_str)
+            else:
+                if re.search(pattern, string):
+                    indices_matched.append(index_str)
+
+    indices_matched = natsorted(indices_matched)
+    keys_matched = list(np.array(strings)[indices_matched])
+
+    if ensure_one:
+        assert (
+            len(indices_matched) == 1
+        ), "Expected exactly one match, but found {}".format(len(indices_matched))
+
+    if as_bool:
+        bool_matched = np.zeros(len(strings), dtype=bool)
+        bool_matched[np.unique(indices_matched)] = True
+        return bool_matched, keys_matched
+    else:
+        return indices_matched, keys_matched
+
+
+# EOF

scitex/web/__init__.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Web-related utilities module for scitex."""
+
+from ._search_pubmed import (
+    search_pubmed,
+    _search_pubmed,
+    _fetch_details,
+    _parse_abstract_xml,
+    _get_citation,
+    get_crossref_metrics,
+    save_bibtex,
+    format_bibtex,
+    fetch_async,
+    batch__fetch_details,
+    parse_args,
+    run_main,
+)
+from ._summarize_url import summarize_url, extract_main_content, crawl_url, crawl_to_json, summarize_all
+
+__all__ = [
+    "search_pubmed",
+    "_search_pubmed",
+    "_fetch_details",
+    "_parse_abstract_xml",
+    "_get_citation",
+    "get_crossref_metrics",
+    "save_bibtex",
+    "format_bibtex",
+    "fetch_async",
+    "batch__fetch_details",
+    "parse_args",
+    "run_main",
+    "summarize_url",
+    "extract_main_content",
+    "crawl_url",
+    "crawl_to_json",
+    "summarize_all",
+]