scitex 2.0.0__py2.py3-none-any.whl

scitex/ai/utils/__init__.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+"""Scitex utils module."""
+
+from ._check_params import check_params
+from ._default_dataset import DefaultDataset
+from ._format_samples_for_sktime import format_samples_for_sktime
+from ._label_encoder import LabelEncoder
+from ._merge_labels import merge_labels
+from ._sliding_window_data_augmentation import sliding_window_data_augmentation
+from ._under_sample import under_sample
+from ._verify_n_gpus import verify_n_gpus
+
+__all__ = [
+    "DefaultDataset",
+    "LabelEncoder",
+    "check_params",
+    "format_samples_for_sktime",
+    "merge_labels",
+    "sliding_window_data_augmentation",
+    "under_sample",
+    "verify_n_gpus",
+]

scitex/ai/utils/_check_params.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# Time-stamp: "2024-02-17 12:38:40 (ywatanabe)"
+
+from pprint import pprint as _pprint
+from time import sleep
+
+# def get_params(model, tgt_name=None, sleep_sec=2, show=False):
+
+#     name_shape_dict = {}
+#     for name, param in model.named_parameters():
+#         learnable = "Learnable" if param.requires_grad else "Freezed"
+
+#         if (tgt_name is not None) & (name == tgt_name):
+#             return param
+#         if tgt_name is None:
+#             # print(f"\n{param}\n{param.shape}\nname: {name}\n")
+#             if show is True:
+#                 print(
+#                     f"\n{param}: {param.shape}\nname: {name}\nStatus: {learnable}\n"
+#                 )
+#                 sleep(sleep_sec)
+#             name_shape_dict[name] = list(param.shape)
+
+#     if tgt_name is None:
+#         print()
+#         _pprint(name_shape_dict)
+#         print()
+
+
+def check_params(model, tgt_name=None, show=False):
+
+    out_dict = {}
+
+    for name, param in model.named_parameters():
+        learnable = "Learnable" if param.requires_grad else "Freezed"
+
+        if tgt_name is None:
+            out_dict[name] = (param.shape, learnable)
+
+        elif (tgt_name is not None) & (name == tgt_name):
+            out_dict[name] = (param.shape, learnable)
+
+        elif (tgt_name is not None) & (name != tgt_name):
+            continue
+
+    if show:
+        for k, v in out_dict.items():
+            print(f"\n{k}\n{v}")
+
+    return out_dict

scitex/ai/utils/_default_dataset.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+from torch.utils.data import Dataset
+import numpy as np
+
+
+class DefaultDataset(Dataset):
+    """
+    Apply transform for the first element of arrs_list
+
+    Example:
+        n = 1024
+        n_chs = 19
+        X = np.random.rand(n, n_chs, 1000)
+        T = np.random.randint(0, 4, size=(n, 1))
+        S = np.random.randint(0, 999, size=(n, 1))
+        Sr = np.random.randint(0, 4, size=(n, 1))
+
+        arrs_list = [X, T, S, Sr]
+        transform = None
+        ds = _DefaultDataset(arrs_list, transform=transform)
+        len(ds) # 1024
+    """
+
+    def __init__(self, arrs_list, transform=None):
+        self.arrs_list = arrs_list
+        self.arrs = arrs_list  # alias
+
+        assert np.all([len(arr) for arr in arrs_list])
+
+        self.length = len(arrs_list[0])
+        self.transform = transform
+
+    def __len__(self):
+        return self.length
+
+    def __getitem__(self, idx):
+        arrs_list_idx = [arr[idx] for arr in self.arrs_list]
+
+        # Here, you might want to transform, or apply DA on X as a numpy array
+        if self.transform:
+            dtype_orig = arrs_list_idx[0].dtype
+            arrs_list_idx[0] = self.transform(
+                arrs_list_idx[0].astype(np.float64)
+            ).astype(dtype_orig)
+        return arrs_list_idx

scitex/ai/utils/_format_samples_for_sktime.py

@@ -0,0 +1,26 @@
+import pandas as pd
+import torch
+import numpy as np
+
+
+def _format_a_sample_for_sktime(x):
+    """
+    x.shape: (n_chs, seq_len)
+    """
+    dims = pd.Series(
+        [pd.Series(x[d], name=f"dim_{d}") for d in range(len(x))],
+        index=[f"dim_{i}" for i in np.arange(len(x))],
+    )
+    return dims
+
+
+def format_samples_for_sktime(X):
+    """
+    X.shape: (n_samples, n_chs, seq_len)
+    """
+    if torch.is_tensor(X):
+        X = X.numpy()  # (64, 160, 1024)
+
+        X = X.astype(np.float64)
+
+    return pd.DataFrame([_format_a_sample_for_sktime(X[i]) for i in range(len(X))])

scitex/ai/utils/_label_encoder.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-03-02 09:52:28 (ywatanabe)"
+
+from warnings import warn
+
+import numpy as np
+import pandas as pd
+import torch
+from sklearn.preprocessing import LabelEncoder as SklearnLabelEncoder
+
+
+class LabelEncoder(SklearnLabelEncoder):
+    """
+    An extension of the sklearn.preprocessing.LabelEncoder that supports incremental learning.
+    This means it can handle new classes without forgetting the old ones.
+
+    Attributes:
+        classes_ (np.ndarray): Holds the label for each class.
+
+    Example usage:
+        encoder = IncrementalLabelEncoder()
+        encoder.fit(np.array(["apple", "banana"]))
+        encoded_labels = encoder.transform(["apple", "banana"])  # This will give you the encoded labels
+
+        encoder.fit(["cherry"])  # Incrementally add "cherry"
+        encoder.transform(["apple", "banana", "cherry"])  # Now it works, including "cherry"
+
+        # Now you can use inverse_transform with the encoded labels
+        print(encoder.classes_)
+        original_labels = encoder.inverse_transform(encoded_labels)
+        print(original_labels)  # This should print ['apple', 'banana']
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.classes_ = np.array([])
+
+    def _check_input(self, y):
+        """
+        Check and convert the input to a NumPy array if it is a list, tuple, pandas.Series, pandas.DataFrame, or torch.Tensor.
+
+        Arguments:
+            y (list, tuple, pd.Series, pd.DataFrame, torch.Tensor): The input labels.
+
+        Returns:
+            np.ndarray: The input labels converted to a NumPy array.
+        """
+        if isinstance(y, (list, tuple)):
+            y = np.array(y)
+        elif isinstance(y, pd.Series):
+            y = y.values
+        elif isinstance(y, torch.Tensor):
+            y = y.numpy()
+        return y
+
+    def fit(self, y):
+        """
+        Fit the label encoder with an array of labels, incrementally adding new classes.
+
+        Arguments:
+            y (list, tuple, np.ndarray, pd.Series, pd.DataFrame, torch.Tensor): The input labels.
+
+        Returns:
+            IncrementalLabelEncoder: The instance itself.
+        """
+        y = self._check_input(y)
+        new_unique_labels = np.unique(y)
+        unique_labels = np.unique(np.concatenate((self.classes_, new_unique_labels)))
+        self.classes_ = unique_labels
+        return self
+
+    def transform(self, y):
+        """
+        Transform labels to normalized encoding.
+
+        Arguments:
+            y (list, tuple, np.ndarray, pd.Series, pd.DataFrame, torch.Tensor): The input labels.
+
+        Returns:
+            np.ndarray: The encoded labels as a NumPy array.
+
+        Raises:
+            ValueError: If the input contains new labels that haven't been seen during `fit`.
+        """
+
+        y = self._check_input(y)
+        diff = set(y) - set(self.classes_)
+        if diff:
+            raise ValueError(f"y contains new labels: {diff}")
+        return super().transform(y)
+
+    def inverse_transform(self, y):
+        """
+        Transform labels back to original encoding.
+
+        Arguments:
+            y (np.ndarray): The encoded labels as a NumPy array.
+
+        Returns:
+            np.ndarray: The original labels as a NumPy array.
+        """
+
+        return super().inverse_transform(y)
+
+
+# # Obsolete warning for future compatibility
+# class LabelEncoder(IncrementalLabelEncoder):
+#     def __init__(self, *args, **kwargs):
+#         """
+#         Initialize the LabelEncoder with a deprecation warning.
+#         """
+#         warn(
+#             "LabelEncoder is now obsolete; use IncrementalLabelEncoder instead.",
+#             category=FutureWarning,
+#         )
+#         super().__init__(*args, **kwargs)
+
+
+if __name__ == "__main__":
+    # Example usage of IncrementalLabelEncoder
+    le = LabelEncoder()
+    le.fit(["A", "B"])
+    print(le.classes_)
+
+    le.fit(["C"])
+    print(le.classes_)
+
+    le.inverse_transform([0, 1, 2])
+
+    le.fit(["X"])
+    print(le.classes_)
+
+    le.inverse_transform([3])

scitex/ai/utils/_merge_labels.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import scitex
+import numpy as np
+
+# y1, y2 = T_tra, M_tra
+# def merge_labels(y1, y2):
+#     y = [str(z1) + "-" + str(z2) for z1, z2 in zip(y1, y2)]
+#     conv_d = {z: i for i, z in enumerate(np.unique(y))}
+#     y = [conv_d[z] for z in y]
+#     return y
+
+
+def merge_labels(*ys, to_int=False):
+    if not len(ys) > 1:  # Check if more than two arguments are passed
+        return ys[0]
+    else:
+        y = [scitex.gen.connect_nums(zs) for zs in zip(*ys)]
+        if to_int:
+            conv_d = {z: i for i, z in enumerate(np.unique(y))}
+            y = [conv_d[z] for z in y]
+        return np.array(y)

scitex/ai/utils/_sliding_window_data_augmentation.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-01-24 13:56:36 (ywatanabe)"
+
+import random
+
+
+def sliding_window_data_augmentation(x, window_size_pts):
+    start = random.randint(0, x.shape[-1] - window_size_pts)
+    end = start + window_size_pts
+    return x[..., start:end]

scitex/ai/utils/_under_sample.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+
+
+from collections import Counter
+
+import numpy as np
+
+
+def under_sample(y, replace=False):
+    """
+    Input:
+        Labels
+    Return:
+        Indices
+
+    Example:
+        t = ['a', 'b', 'c', 'b', 'c', 'a', 'c']
+        print(under_sample(t))
+        # [5 0 1 3 4 6]
+        print(under_sample(t))
+        # [5 0 1 3 6 2]
+    """
+
+    # find the minority and majority classes
+    class_counts = Counter(y)
+    # majority_class = max(class_counts, key=class_counts.get)
+    minority_class = min(class_counts, key=class_counts.get)
+
+    # compute the number of sample to draw from the majority class using
+    # a negative binomial distribution
+    n_minority_class = class_counts[minority_class]
+    n_majority_resampled = n_minority_class
+
+    # draw randomly with or without replacement
+    indices = np.hstack(
+        [
+            np.random.choice(
+                np.flatnonzero(y == k),
+                size=n_majority_resampled,
+                replace=replace,
+            )
+            for k in class_counts.keys()
+        ]
+    )
+
+    return indices
+
+
+if __name__ == "__main__":
+    t = np.array(["a", "b", "c", "b", "c", "a", "c"])
+    print(under_sample(t))

scitex/ai/utils/_verify_n_gpus.py

@@ -0,0 +1,16 @@
+import torch
+import warnings
+
+
+def verify_n_gpus(n_gpus):
+    if torch.cuda.device_count() < n_gpus:
+        warnings.warn(
+            f"N_GPUS ({n_gpus}) is larger "
+            f"than n_gpus torch can acesses (= {torch.cuda.device_count()})"
+            f"Please check $CUDA_VISIBLE_DEVICES and your setting in this script.",
+            UserWarning,
+        )
+        return torch.cuda.device_count()
+
+    else:
+        return n_gpus

scitex/ai/utils/grid_search.py

@@ -0,0 +1,148 @@
+#!./env/bin/python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-04-22 23:54:02"
+# Author: Yusuke Watanabe (ywata1989@gmail.com)
+
+"""
+This script defines scitex.ml.utils.grid_search
+"""
+
+# Imports
+import itertools as _itertools
+import random as _random
+import sys as _sys
+
+import matplotlib.pyplot as _plt
+import scitex as _scitex
+
+
+# Functions
+def yield_grids(params_grid: dict, random=False):
+    """
+    Generator function that yields combinations of parameters from a grid.
+
+    Args:
+        params_grid (dict): A dictionary where keys are parameter names and values are lists of parameter values.
+        random (bool): If True, yields the parameter combinations in random order.
+
+    Yields:
+        dict: A dictionary of parameters for one set of conditions from the grid.
+
+    Example:
+        # Parameters
+        params_grid = {
+            "batch_size": [2**i for i in range(7)],
+            "n_chs": [2**i for i in range(7)],
+            "seq_len": [2**i for i in range(15)],
+            "fs": [2**i for i in range(8, 11)],
+            "n_segments": [2**i for i in range(6)],
+            "n_bands_pha": [2**i for i in range(7)],
+            "n_bands_amp": [2**i for i in range(7)],
+            "precision": ['fp16', 'fp32'],
+            "device": ['cpu', 'cuda'],
+            "package": ['tensorpac', 'scitex'],
+        }
+
+        # Example of using the generator
+        for param_dict in yield_grids(params_grid, random=True):
+            print(param_dict)
+    """
+    combinations = list(_itertools.product(*params_grid.values()))
+    if random:
+        _random.shuffle(combinations)  # [REVISED]
+    for values in combinations:
+        yield dict(zip(params_grid.keys(), values))
+
+
+# def yield_grids(params_grid: dict, random=False):
+#     """
+#     Generator function that yields combinations of parameters from a grid.
+
+#     Args:
+#         params_grid (dict): A dictionary where keys are parameter names and values are lists of parameter values.
+
+#     Yields:
+#         dict: A dictionary of parameters for one set of conditions from the grid.
+
+#     Example:
+#         # Parameters
+#         params_grid = {
+#             "batch_size": [2**i for i in range(7)],
+#             "n_chs": [2**i for i in range(7)],
+#             "seq_len": [2**i for i in range(15)],
+#             "fs": [2**i for i in range(8, 11)],
+#             "n_segments": [2**i for i in range(6)],
+#             "n_bands_pha": [2**i for i in range(7)],
+#             "n_bands_amp": [2**i for i in range(7)],
+#             "precision": ['fp16', 'fp32'],
+#             "device": ['cpu', 'cuda'],
+#             "package": ['tensorpac', 'scitex'],
+#         }
+
+#         # Example of using the generator
+#         for param_dict in yield_grids(params_grid):
+#             print(param_dict)
+#     """
+#     print(f"\nThe Number of Combinations: {count_grids(params_grid):,}")
+
+#     for values in _itertools.product(*params_grid.values()):
+#         yield dict(zip(params_grid.keys(), values))
+
+
+def count_grids(params_grid):
+    """
+    Calculate the total number of combinations possible from the given parameter grid.
+
+    Args:
+        params_grid (dict): A dictionary where keys are parameter names and values are lists of parameter values.
+
+    Returns:
+        int: The total number of combinations that can be generated from the parameter grid.
+    """
+    # Get the number of values for each parameter and multiply them
+    num_combinations = 1
+    for values in params_grid.values():
+        num_combinations *= len(values)
+    return num_combinations
+
+
+if __name__ == "__main__":
+    import pandas as pd
+
+    # Start
+    CONFIG, _sys.stdout, _sys.stderr, _plt, CC = _scitex.gen.start(
+        _sys, _plt, verbose=False
+    )
+
+    # Parameters
+    N = 15
+    print(pd.DataFrame(pd.Series({f"2^{ii}": 2**ii for ii in range(N)})))
+
+    params_grid = {
+        "batch_size": [2**i for i in [3, 4, 5, 6]],
+        "n_chs": [2**i for i in [3, 4, 5, 6]],
+        "seq_len": [2**i for i in range(8, 13)],
+        "fs": [2**i for i in range(7, 10)],
+        "n_segments": [2**i for i in range(5)],
+        "n_bands_pha": [2**i for i in range(7)],
+        "n_bands_amp": [2**i for i in range(7)],
+        "precision": ["fp16", "fp32"],
+        "device": ["cpu", "cuda"],
+        "package": ["tensorpac", "_scitex"],
+    }
+
+    print(params_grid)
+    print(f"{count_grids(params_grid):,}")
+
+    # Example of using the generator
+    for param_dict in yield_grids(params_grid):
+        print(param_dict)
+
+    # Close
+    _scitex.gen.close(CONFIG, verbose=False, notify=False)
+
+# EOF
+
+"""
+/home/ywatanabe/proj/entrance/_scitex/ml/utils/grid_search.py
+"""

scitex/context/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+"""Scitex context module."""
+
+from ._suppress_output import quiet, suppress_output
+
+__all__ = [
+    "quiet",
+    "suppress_output",
+]

scitex/context/_suppress_output.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-05-01 09:11:04 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/context/_suppress_output.py
+# ----------------------------------------
+import os
+
+__FILE__ = "./src/scitex/context/_suppress_output.py"
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
+
+from contextlib import contextmanager, redirect_stderr, redirect_stdout
+
+
+@contextmanager
+def suppress_output(suppress=True):
+    """
+    A context manager that suppresses stdout and stderr.
+
+    Example:
+        with suppress_output():
+            print("This will not be printed to the console.")
+    """
+    if suppress:
+        # Open a file descriptor that points to os.devnull (a black hole for data)
+        with open(os.devnull, "w") as fnull:
+            # Temporarily redirect stdout and stderr to the file descriptor fnull
+            with redirect_stdout(fnull), redirect_stderr(fnull):
+                # Yield control back to the context block
+                yield
+    else:
+        # If suppress is False, just yield without redirecting output
+        yield
+
+
+quiet = suppress_output
+
+# EOF

scitex/db/_BaseMixins/_BaseBackupMixin.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Timestamp: "2025-02-27 22:16:38 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_dev/src/scitex/db/_BaseMixins/_BaseBackupMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_BaseMixins/_BaseBackupMixin.py"
+
+from typing import Optional
+
+
+class _BaseBackupMixin:
+    def backup_table(self, table: str, file_path: str):
+        raise NotImplementedError
+
+    def restore_table(self, table: str, file_path: str):
+        raise NotImplementedError
+
+    def backup_database(self, file_path: str):
+        raise NotImplementedError
+
+    def restore_database(self, file_path: str):
+        raise NotImplementedError
+
+    def copy_table(
+        self, source_table: str, target_table: str, where: Optional[str] = None
+    ):
+        raise NotImplementedError
+
+
+# EOF

scitex/db/_BaseMixins/_BaseBatchMixin.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-25 01:43:41 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/db/_BaseMixins/_BaseBatchMixin.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/db/_BaseMixins/_BaseBatchMixin.py"
+
+from typing import List, Any, Optional, Dict, Union
+import pandas as pd
+
+
+class _BaseBatchMixin:
+    def insert_many(
+        self,
+        table: str,
+        records: List[Dict[str, Any]],
+        batch_size: Optional[int] = None,
+    ):
+        raise NotImplementedError
+
+    def _prepare_insert_query(self, table: str, record: Dict[str, Any]) -> str:
+        raise NotImplementedError
+
+    def _prepare_batch_parameters(self, records: List[Dict[str, Any]]) -> tuple:
+        raise NotImplementedError
+
+    def dataframe_to_sql(self, df: pd.DataFrame, table: str, if_exists: str = "fail"):
+        raise NotImplementedError
+
+
+# EOF