scitex 2.0.0__py2.py3-none-any.whl

scitex/gen/_inspect_module.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-07 18:58:55 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/gen/_inspect_module.py
+
+import inspect
+import sys
+import warnings
+from typing import Any, List, Optional, Set, Union
+
+import scitex
+import pandas as pd
+
+
+def inspect_module(
+    module: Union[str, Any],
+    columns: List[str] = ["Type", "Name", "Docstring", "Depth"],
+    prefix: str = "",
+    max_depth: int = 5,
+    visited: Optional[Set[str]] = None,
+    docstring: bool = False,
+    tree: bool = True,
+    current_depth: int = 0,
+    print_output: bool = False,
+    skip_depwarnings: bool = True,
+    drop_duplicates: bool = True,
+    root_only: bool = False,
+) -> pd.DataFrame:
+    return _inspect_module(
+        module=module,
+        prefix=prefix,
+        max_depth=max_depth,
+        visited=visited,
+        docstring=docstring,
+        tree=tree,
+        current_depth=current_depth,
+        print_output=print_output,
+        skip_depwarnings=skip_depwarnings,
+        drop_duplicates=drop_duplicates,
+        root_only=root_only,
+    )[columns]
+
+
+def _inspect_module(
+    module: Union[str, Any],
+    columns: List[str] = ["Type", "Name", "Docstring", "Depth"],
+    prefix: str = "",
+    max_depth: int = 5,
+    visited: Optional[Set[str]] = None,
+    docstring: bool = False,
+    tree: bool = True,
+    current_depth: int = 0,
+    print_output: bool = False,
+    skip_depwarnings: bool = True,
+    drop_duplicates: bool = True,
+    root_only: bool = False,
+) -> pd.DataFrame:
+    """List the contents of a module recursively and return as a DataFrame.
+
+    Example
+    -------
+    >>>
+    >>> df = inspect_module(scitex)
+    >>> print(df)
+       Type           Name                    Docstring  Depth
+    0    M            scitex  Module description              0
+    1    F  scitex.some_function  Function description        1
+    2    C  scitex.SomeClass  Class description               1
+    ...
+
+    Parameters
+    ----------
+    module : Union[str, Any]
+        Module to inspect (string name or actual module)
+    columns : List[str]
+        Columns to include in output DataFrame
+    prefix : str
+        Prefix for nested modules
+    max_depth : int
+        Maximum recursion depth
+    visited : Optional[Set[str]]
+        Set of visited modules to prevent cycles
+    docstring : bool
+        Whether to include docstrings
+    tree : bool
+        Whether to display tree structure
+    current_depth : int
+        Current recursion depth
+    print_output : bool
+        Whether to print results
+    skip_depwarnings : bool
+        Whether to skip DeprecationWarnings
+    drop_duplicates : bool
+        Whether to remove duplicate module entries
+    root_only : bool
+        Whether to show only root-level modules
+
+    Returns
+    -------
+    pd.DataFrame
+        Module structure with specified columns
+    """
+    if skip_depwarnings:
+        warnings.filterwarnings("ignore", category=DeprecationWarning)
+        warnings.filterwarnings("ignore", category=UserWarning)
+
+    if isinstance(module, str):
+        try:
+            module = __import__(module)
+        except ImportError as err:
+            print(f"Error importing module {module}: {err}")
+            return pd.DataFrame(columns=columns)
+
+    if visited is None:
+        visited = set()
+
+    content_list = []
+
+    try:
+        module_name = getattr(module, "__name__", "")
+        if max_depth < 0 or module_name in visited:
+            return pd.DataFrame(content_list, columns=columns)
+
+        visited.add(module_name)
+        base_name = module_name.split(".")[-1]
+        full_path = f"{prefix}.{base_name}" if prefix else base_name
+
+        try:
+            module_version = (
+                f" (v{module.__version__})" if hasattr(module, "__version__") else ""
+            )
+            content_list.append(("M", full_path, module_version, current_depth))
+        except Exception:
+            pass
+
+        for name, obj in inspect.getmembers(module):
+            if name.startswith("_"):
+                continue
+
+            obj_name = f"{full_path}.{name}"
+
+            if inspect.ismodule(obj):
+                if obj.__name__ not in visited:
+                    content_list.append(
+                        (
+                            "M",
+                            obj_name,
+                            obj.__doc__ if docstring and obj.__doc__ else "",
+                            current_depth,
+                        )
+                    )
+                    try:
+                        sub_df = _inspect_module(
+                            obj,
+                            columns=columns,
+                            prefix=full_path,
+                            max_depth=max_depth - 1,
+                            visited=visited,
+                            docstring=docstring,
+                            tree=tree,
+                            current_depth=current_depth + 1,
+                            print_output=print_output,
+                            skip_depwarnings=skip_depwarnings,
+                            drop_duplicates=drop_duplicates,
+                            root_only=root_only,
+                        )
+                        if sub_df is not None and not sub_df.empty:
+                            content_list.extend(sub_df.values.tolist())
+                    except Exception as err:
+                        print(f"Error processing module {obj_name}: {err}")
+            elif inspect.isfunction(obj):
+                content_list.append(
+                    (
+                        "F",
+                        obj_name,
+                        obj.__doc__ if docstring and obj.__doc__ else "",
+                        current_depth,
+                    )
+                )
+            elif inspect.isclass(obj):
+                content_list.append(
+                    (
+                        "C",
+                        obj_name,
+                        obj.__doc__ if docstring and obj.__doc__ else "",
+                        current_depth,
+                    )
+                )
+
+    except Exception as err:
+        print(f"Error processing module structure: {err}")
+        return pd.DataFrame(columns=columns)
+
+    df = pd.DataFrame(content_list, columns=columns)
+
+    if drop_duplicates:
+        df = df.drop_duplicates(subset="Name", keep="first")
+
+    if root_only:
+        mask = df["Name"].str.count(r"\.") <= 1
+        df = df[mask]
+
+    if tree and current_depth == 0 and print_output:
+        _print_module_contents(df)
+
+    return df[columns]
+
+
+def _print_module_contents(df: pd.DataFrame) -> None:
+    """Prints module contents in tree structure.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        DataFrame containing module structure
+    """
+    df_sorted = df.sort_values(["Depth", "Name"])
+    depth_last = {}
+
+    for index, row in df_sorted.iterrows():
+        depth = row["Depth"]
+        is_last = (
+            index == len(df_sorted) - 1 or df_sorted.iloc[index + 1]["Depth"] <= depth
+        )
+
+        prefix = ""
+        for d in range(depth):
+            if d == depth - 1:
+                prefix += "└── " if is_last else "├── "
+            else:
+                prefix += "    " if depth_last.get(d, False) else "│   "
+
+        print(f"{prefix}({row['Type']}) {row['Name']}{row['Docstring']}")
+        depth_last[depth] = is_last
+
+
+if __name__ == "__main__":
+
+    sys.setrecursionlimit(10_000)
+    df = inspect_module(scitex, docstring=True, print_output=False, columns=["Name"])
+    print(scitex.pd.round(df))
+    #                                 Name
+    # 0                               scitex
+    # 1                            scitex.ai
+    # 3     scitex.ai.ClassificationReporter
+    # 4           scitex.ai.ClassifierServer
+    # 5              scitex.ai.EarlyStopping
+    # ...                              ...
+    # 5373                     scitex.typing
+    # 5375                 scitex.typing.Any
+    # 5376            scitex.typing.Iterable
+    # 5377                        scitex.web
+    # 5379          scitex.web.summarize_url
+
+    # [5361 rows x 1 columns]
+
+# EOF

scitex/gen/_is_ipython.py

@@ -0,0 +1,12 @@
+def is_ipython():
+    try:
+        __IPYTHON__
+        ipython_mode = True
+    except NameError:
+        ipython_mode = False
+
+    return ipython_mode
+
+
+def is_script():
+    return not is_ipython()

scitex/gen/_less.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-03 02:11:18 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/gen/_less.py
+#!./env/bin/python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-04-21 12:05:35"
+# Author: Yusuke Watanabe (ywata1989@gmail.com)
+
+"""
+This script does XYZ.
+"""
+
+import sys
+
+import matplotlib.pyplot as plt
+import scitex
+
+# Imports
+
+# # Config
+# CONFIG = scitex.gen.load_configs()
+
+
+# Functions
+def less(output):
+    """
+    Print the given output using `less` in an IPython or IPdb session.
+    """
+    import os
+    import tempfile
+
+    from IPython import get_ipython
+
+    # Create a temporary file to hold the output
+    with tempfile.NamedTemporaryFile(delete=False, mode="w+t") as tmpfile:
+        # Write the output to the temporary file
+        tmpfile.write(output)
+        tmpfile_name = tmpfile.name
+
+    # Use IPython's system command access to pipe the content of the temporary file to `less`
+    get_ipython().system(f"less {tmpfile_name}")
+
+    # Clean up the temporary file
+    os.remove(tmpfile_name)
+
+
+# EOF

scitex/gen/_list_packages.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-03 02:11:54 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/gen/_list_packages.py
+"""
+Functionality:
+    * Lists and analyzes installed Python packages and their modules
+Input:
+    * None (uses importlib.metadata to get installed packages)
+Output:
+    * DataFrame containing package module information
+Prerequisites:
+    * importlib.metadata (Python 3.8+) or importlib_metadata, pandas
+"""
+
+import sys
+from typing import Optional
+
+import pandas as pd
+
+try:
+    # Python 3.8+ standard library
+    from importlib.metadata import distributions
+except ImportError:
+    # Fallback for older Python versions
+    from importlib_metadata import distributions
+
+from ._inspect_module import inspect_module
+
+
+def list_packages(
+    max_depth: int = 1,
+    root_only: bool = True,
+    skip_errors: bool = True,
+    verbose: bool = False,
+) -> pd.DataFrame:
+    """Lists all installed packages and their modules."""
+    sys.setrecursionlimit(10_000)
+
+    # Skip known problematic packages
+    skip_patterns = [
+        "nvidia",
+        "cuda",
+        "pillow",
+        "fonttools",
+        "ipython",
+        "jsonschema",
+        "readme",
+        "importlib-metadata",
+    ]
+
+    # Get installed packages, excluding problematic ones
+    installed_packages = [
+        dist.name.replace("-", "_")
+        for dist in distributions()
+        if not any(pat in dist.name.lower() for pat in skip_patterns)
+    ]
+
+    # Focus on commonly used packages first
+    safelist = [
+        "numpy",
+        "pandas",
+        "scipy",
+        "matplotlib",
+        "sklearn",
+        "torch",
+        "tensorflow",
+        "keras",
+        "xarray",
+        "dask",
+        "pytest",
+        "requests",
+        "flask",
+        "django",
+        "seaborn",
+    ]
+
+    # Prioritize safelist packages
+    installed_packages = [pkg for pkg in installed_packages if pkg in safelist] + [
+        pkg for pkg in installed_packages if pkg not in safelist
+    ]
+
+    all_dfs = []
+    for package_name in installed_packages:
+        try:
+            df = inspect_module(
+                package_name,
+                docstring=False,  # Speed up by skipping docstrings
+                print_output=False,
+                columns=["Name"],
+                root_only=root_only,
+                max_depth=max_depth,
+                skip_depwarnings=True,
+            )
+            if not df.empty:
+                all_dfs.append(df)
+        except Exception as err:
+            if verbose:
+                print(f"Error processing {package_name}: {err}")
+            if not skip_errors:
+                raise
+
+    if not all_dfs:
+        return pd.DataFrame(columns=["Name"])
+
+    combined_df = pd.concat(all_dfs, ignore_index=True)
+    return combined_df.drop_duplicates().sort_values("Name")
+
+
+def main() -> Optional[int]:
+    """Main function for testing package listing functionality."""
+    df = list_packages(verbose=True)
+    __import__("ipdb").set_trace()
+    return 0
+
+
+if __name__ == "__main__":
+    import matplotlib.pyplot as plt
+    import scitex
+
+    CONFIG, sys.stdout, sys.stderr, plt, CC = scitex.gen.start(
+        sys,
+        plt,
+        verbose=False,
+        agg=True,
+    )
+
+    exit_status = main()
+
+    scitex.gen.close(
+        CONFIG,
+        verbose=False,
+        sys=sys,
+        notify=False,
+        message="",
+        exit_status=exit_status,
+    )
+
+# EOF

scitex/gen/_mat2py.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-03 18:57:14 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/gen/_mat2py.py
+
+"""Helper script for loading .mat files into python.
+For .mat with multiple variables use mat2dict to get return dictionary with .mat variables.
+For .mat with 1 matrix use mat2npa to return np.array
+For .mat with 1 matrix use mat2npy to save np.array to .npy
+For multiple .mat files with 1 matrix use dir2npy to save 1 np.array of each .mat to .npy
+
+
+Examples:
+mat2py.mat2npa(fname = '/vol/ccnlab-scratch1/julber/chill_nn_regression/data/chill_wav_time_16kHz.mat', typ = np.float32)
+mat2py.dir2npa(dir = '/vol/ccnlab-scratch1/julber/phoneme_decoding/data/', typ = np.float32, regex = '*xdata')
+mat2py.dir2npa(dir = '/vol/ccnlab-scratch1/julber/phoneme_decoding/data/', typ = np.int32, regex = '*ylabels')
+
+
+September 07, 2017
+JB"""
+
+import numpy as np
+import h5py
+from glob import glob as _glob
+import os
+from scipy.io import loadmat
+
+
+def mat2dict(fname):
+    """Function returns a dictionary with .mat variables"""
+    try:
+        D = h5py.File(fname)
+        d = {}
+        for key, value in D.items():
+            d[key] = value
+        d["__hdf__"] = True
+    except:
+        d = loadmat(fname)
+        d["__hdf__"] = False
+    return d
+
+
+def keys2npa(d, typ):
+    import pdb
+
+    pdb.set_trace()
+    d2 = {}
+    for key in public_keys(d):
+        x = np.array(d[key], dtype=typ)
+        if d["__hdf__"]:
+            x = np.squeeze(np.swapaxes(x, 0, -1))
+        assert type(x.flatten()[0]) == typ
+        d2[key] = x.copy()
+    return d2
+
+
+def public_keys(d):
+    return [k for k in d.keys() if not k.startswith("_")]
+
+
+def mat2npa(fname, typ):
+    """Function returns np array from 1st entry in .mat file"""
+    import pdb
+
+    pdb.set_trace()
+    d = keys2npa(mat2dict(fname), typ)
+    return d[d.keys()[0]]
+
+
+def save_npa(fname, x):
+    np.save(fname, x)
+
+
+def mat2npy(fname, typ):
+    """Function save np array from 1st entry in .mat file to .npy file"""
+    x = mat2npa(fname, typ)
+    save_npa(fname=fname.replace(".mat", ""), x=x)
+
+
+def dir2npy(dir, typ, regex="*"):
+    """Function saves np array from 1st entry in each regex + .mat file in dir"""
+    os.chdir(dir)
+    for fname in _glob(regex + ".mat"):
+        print("File " + fname + " to" + " .npa")
+        mat2npy(dir + fname, typ)
+
+
+# EOF

scitex/gen/_norm.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-19 01:09:55 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/gen/_norm.py
+
+THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/gen/_norm.py"
+
+import torch
+
+from ..decorators import torch_fn
+from ..torch import nanstd
+
+
+@torch_fn
+def to_z(x, axis=-1, dim=None, device="cuda"):
+    """Standardizes tensor to zero mean and unit variance along specified dimension.
+
+    Parameters
+    ----------
+    xx : torch.Tensor
+        Input tensor
+    dim : int, optional
+        Dimension along which to standardize (preferred)
+    axis : int, optional
+        Alternative to dim for numpy compatibility
+    device : str
+        Device to use for computation
+
+    Returns
+    -------
+    torch.Tensor
+        Z-scored tensor
+    """
+    return (x - x.mean(dim=dim, keepdim=True)) / x.std(dim=dim, keepdim=True)
+
+
+@torch_fn
+def to_nanz(x, axis=-1, dim=None, device="cuda"):
+    """Standardizes tensor handling NaN values along specified dimension.
+
+    Parameters
+    ----------
+    xx : torch.Tensor
+        Input tensor
+    dim : int, optional
+        Dimension along which to standardize (preferred)
+    axis : int, optional
+        Alternative to dim for numpy compatibility
+    device : str
+        Device to use for computation
+
+    Returns
+    -------
+    torch.Tensor
+        Z-scored tensor with NaN handling
+    """
+    nan_mean = torch.nanmean(x, dim=dim, keepdim=True)
+    nan_std = nanstd(x, dim=dim, keepdim=True)
+    return (x - nan_mean) / nan_std
+
+
+@torch_fn
+def to_01(x, axis=-1, dim=None, device="cuda"):
+    """Min-max scales tensor to [0, 1] range along specified dimension.
+
+    Parameters
+    ----------
+    xx : torch.Tensor
+        Input tensor
+    dim : int, optional
+        Dimension along which to scale (preferred)
+    axis : int, optional
+        Alternative to dim for numpy compatibility
+    device : str
+        Device to use for computation
+
+    Returns
+    -------
+    torch.Tensor
+        Min-max scaled tensor
+    """
+    x_min = x.min(dim=dim, keepdim=True)[0]
+    x_max = x.max(dim=dim, keepdim=True)[0]
+    return (x - x_min) / (x_max - x_min)
+
+
+@torch_fn
+def to_nan01(x, axis=-1, dim=None, device="cuda"):
+    """Min-max scales tensor handling NaN values along specified dimension.
+
+    Parameters
+    ----------
+    xx : torch.Tensor
+        Input tensor
+    dim : int, optional
+        Dimension along which to scale (preferred)
+    axis : int, optional
+        Alternative to dim for numpy compatibility
+    device : str
+        Device to use for computation
+
+    Returns
+    -------
+    torch.Tensor
+        Min-max scaled tensor with NaN handling
+    """
+    x_min = torch.nanmin(x, dim=dim, keepdim=True)[0]
+    x_max = torch.nanmax(x, dim=dim, keepdim=True)[0]
+    return (x - x_min) / (x_max - x_min)
+
+
+@torch_fn
+def unbias(x, axis=-1, dim=None, fn="mean", device="cuda"):
+    """Removes bias from tensor using specified method along dimension.
+
+    Parameters
+    ----------
+    xx : torch.Tensor
+        Input tensor
+    dim : int, optional
+        Dimension along which to unbias (preferred)
+    axis : int, optional
+        Alternative to dim for numpy compatibility
+    fn : str
+        Method to use for unbiasing ('mean' or 'min')
+    device : str
+        Device to use for computation
+
+    Returns
+    -------
+    torch.Tensor
+        Unbiased tensor
+    """
+    if fn == "mean":
+        return x - x.mean(dim=dim, keepdims=True)
+    if fn == "min":
+        return x - x.min(dim=dim, keepdims=True)[0]
+    raise ValueError(f"Unsupported unbiasing method: {fn}")
+
+
+@torch_fn
+def clip_perc(x, lower_perc=2.5, upper_perc=97.5, axis=-1, dim=None, device="cuda"):
+    """Clips tensor values between specified percentiles along dimension.
+
+    Parameters
+    ----------
+    x : torch.Tensor
+        Input tensor
+    lower_perc : float
+        Lower percentile (0-100)
+    upper_perc : float
+        Upper percentile (0-100)
+    dim : int
+        Dimension along which to compute percentiles (preferred)
+    axis : int
+        Alternative to dim for numpy compatibility
+    device : str
+        Device to use for computation
+
+    Returns
+    -------
+    torch.Tensor
+        Clipped tensor
+    """
+    lower = torch.quantile(x, lower_perc / 100, dim=dim, keepdim=True)
+    upper = torch.quantile(x, upper_perc / 100, dim=dim, keepdim=True)
+    return torch.clamp(x, min=lower, max=upper)
+
+
+# EOF