scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/ai/genai/response_handler.py

@@ -1,235 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2025-05-31 10:20:00"
-# Author: ywatanabe
-# File: ./src/scitex/ai/genai/response_handler.py
-
-"""
-Handles response processing for AI providers.
-
-This module provides response handling functionality including:
-- Static response processing
-- Stream response handling
-- Output formatting
-- Error response generation
-"""
-
-import sys
-from typing import Generator, List, Union, Optional, Any
-
-from .format_output_func import format_output_func
-
-
-class ResponseHandler:
-    """Handles processing of AI provider responses.
-
-    Example
-    -------
-    >>> handler = ResponseHandler()
-    >>> # Process static response
-    >>> result = handler.process_static("Hello, world!")
-    >>> print(result)
-    Hello, world!
-
-    >>> # Process stream
-    >>> stream = ["Hello", ", ", "world!"]
-    >>> for chunk in handler.process_stream(stream):
-    ...     print(chunk, end="")
-    Hello, world!
-    """
-
-    def __init__(self):
-        """Initialize response handler."""
-        self._accumulated_response = []
-
-    def process_static(self, response: str, format_output: bool = False) -> str:
-        """Process a static (non-streaming) response.
-
-        Parameters
-        ----------
-        response : str
-            The response text to process
-        format_output : bool
-            Whether to apply output formatting
-
-        Returns
-        -------
-        str
-            Processed response text
-        """
-        if format_output:
-            response = self.format_output(response)
-        return response
-
-    def process_stream(
-        self, stream: Generator[str, None, None], format_output: bool = False
-    ) -> Generator[str, None, None]:
-        """Process a streaming response.
-
-        Parameters
-        ----------
-        stream : Generator[str, None, None]
-            The stream of response chunks
-        format_output : bool
-            Whether to apply output formatting to final result
-
-        Yields
-        ------
-        str
-            Response chunks as they arrive
-        """
-        self._accumulated_response = []
-
-        for chunk in stream:
-            if chunk:
-                self._accumulated_response.append(chunk)
-                yield chunk
-
-        # Apply formatting to accumulated response if requested
-        if format_output and self._accumulated_response:
-            full_response = "".join(self._accumulated_response)
-            formatted = self.format_output(full_response)
-
-            # If formatting changed the response, yield the difference
-            if formatted != full_response:
-                # This is tricky - we've already yielded the unformatted chunks
-                # In practice, formatting is usually applied after streaming
-                pass
-
-    def format_output(self, text: str) -> str:
-        """Apply output formatting to text.
-
-        Parameters
-        ----------
-        text : str
-            Text to format
-
-        Returns
-        -------
-        str
-            Formatted text
-        """
-        return format_output_func(text)
-
-    def yield_stream_with_print(self, stream: Generator[str, None, None]) -> str:
-        """Yield stream chunks while printing to stdout.
-
-        Parameters
-        ----------
-        stream : Generator[str, None, None]
-            The stream to process
-
-        Returns
-        -------
-        str
-            Complete accumulated response
-        """
-        accumulated = []
-
-        for chunk in stream:
-            if chunk:
-                sys.stdout.write(chunk)
-                sys.stdout.flush()
-                accumulated.append(chunk)
-
-        return "".join(accumulated)
-
-    def create_error_response(
-        self, error_messages: List[str], as_stream: bool = False
-    ) -> Union[str, Generator[str, None, None]]:
-        """Create an error response.
-
-        Parameters
-        ----------
-        error_messages : List[str]
-            List of error messages
-        as_stream : bool
-            Whether to return as stream
-
-        Returns
-        -------
-        Union[str, Generator[str, None, None]]
-            Error response as string or stream
-        """
-        error_text = "".join(error_messages)
-
-        if not as_stream:
-            return error_text
-
-        return self._text_to_stream(error_text)
-
-    def _text_to_stream(self, text: str) -> Generator[str, None, None]:
-        """Convert text to a stream generator.
-
-        Parameters
-        ----------
-        text : str
-            Text to convert
-
-        Yields
-        ------
-        str
-            Text as stream chunks
-        """
-        # Yield text in reasonable chunks
-        chunk_size = 50
-        for i in range(0, len(text), chunk_size):
-            yield text[i : i + chunk_size]
-
-    def extract_content_from_response(self, response: Any) -> str:
-        """Extract text content from various response formats.
-
-        Parameters
-        ----------
-        response : Any
-            Response object from provider
-
-        Returns
-        -------
-        str
-            Extracted text content
-        """
-        # Handle string responses
-        if isinstance(response, str):
-            return response
-
-        # Handle dict responses
-        if isinstance(response, dict):
-            if "content" in response:
-                return str(response["content"])
-            if "text" in response:
-                return str(response["text"])
-            if "message" in response:
-                return str(response["message"])
-
-        # Handle object responses with attributes
-        if hasattr(response, "content"):
-            return str(response.content)
-        if hasattr(response, "text"):
-            return str(response.text)
-        if hasattr(response, "message"):
-            return str(response.message)
-
-        # Fallback to string conversion
-        return str(response)
-
-    def reset(self) -> None:
-        """Reset the handler state."""
-        self._accumulated_response = []
-
-    def get_accumulated_response(self) -> str:
-        """Get the accumulated response from streaming.
-
-        Returns
-        -------
-        str
-            Accumulated response text
-        """
-        return "".join(self._accumulated_response)
-
-    def __repr__(self) -> str:
-        """String representation of ResponseHandler."""
-        return f"ResponseHandler(accumulated_chunks={len(self._accumulated_response)})"
-
-
-# EOF

scitex/ai/layer/_Pass.py

@@ -1,21 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2024-11-20 00:29:47 (ywatanabe)"
-# File: ./scitex_repo/src/scitex/ai/layer/_Pass.py
-
-THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/ai/layer/_Pass.py"
-
-import torch.nn as nn
-
-
-class Pass(nn.Module):
-    def __init__(
-        self,
-    ):
-        super().__init__()
-
-    def forward(self, x):
-        return x
-
-
-# EOF

scitex/ai/layer/__init__.py

@@ -1,10 +0,0 @@
-#!/usr/bin/env python3
-"""Scitex layer module."""
-
-from ._Pass import Pass
-from ._switch import switch
-
-__all__ = [
-    "Pass",
-    "switch",
-]

scitex/ai/layer/_switch.py

@@ -1,8 +0,0 @@
-from ._Pass import Pass
-
-
-def switch(layer, is_used):
-    if is_used:
-        return layer
-    else:
-        return Pass()

scitex/ai/metrics/_bACC.py

@@ -1,51 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2024-02-26 16:32:42 (ywatanabe)"
-
-import warnings
-
-import numpy as np
-import torch
-from sklearn.metrics import balanced_accuracy_score
-
-
-def bACC(true_class, pred_class):
-    """
-    Calculates the balanced accuracy score between predicted and true class labels.
-
-    Parameters:
-    - true_class (array-like or torch.Tensor): True class labels.
-    - pred_class (array-like or torch.Tensor): Predicted class labels.
-
-    Returns:
-    - bACC (float): The balanced accuracy score rounded to three decimal places.
-    """
-    if isinstance(true_class, torch.Tensor):  # [REVISED]
-        true_class = true_class.detach().cpu().numpy()  # [REVISED]
-    if isinstance(pred_class, torch.Tensor):  # [REVISED]
-        pred_class = pred_class.detach().cpu().numpy()  # [REVISED]
-
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore")
-        bACC_score = balanced_accuracy_score(
-            true_class.reshape(-1),  # [REVISED]
-            pred_class.reshape(-1),  # [REVISED]
-        )
-    return round(bACC_score, 3)  # [REVISED]
-
-
-# Snake_case alias for consistency
-def balanced_accuracy(true_class, pred_class):
-    """
-    Calculates the balanced accuracy score between predicted and true class labels.
-    
-    This is an alias for bACC() with snake_case naming.
-    
-    Parameters:
-    - true_class (array-like or torch.Tensor): True class labels.
-    - pred_class (array-like or torch.Tensor): Predicted class labels.
-    
-    Returns:
-    - float: The balanced accuracy score rounded to three decimal places.
-    """
-    return bACC(true_class, pred_class)

scitex/ai/plt/_learning_curve.py

@@ -1,194 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2024-03-12 19:52:48 (ywatanabe)"
-
-import re
-
-import matplotlib
-import matplotlib.pyplot as plt
-import scitex
-import numpy as np
-import pandas as pd
-
-
-def process_i_global(metrics_df):
-    if metrics_df.index.name != "i_global":
-        try:
-            metrics_df = metrics_df.set_index("i_global")
-        except KeyError:
-            print(
-                "Error: The DataFrame does not contain a column named 'i_global'. Please check the column names."
-            )
-        except Exception as e:
-            print(f"An unexpected error occurred: {e}")
-    else:
-        print("The index is already set to 'i_global'.")
-    metrics_df["i_global"] = metrics_df.index  # alias
-    return metrics_df
-
-
-def set_yaxis_for_acc(ax, key_plt):
-    if re.search("[aA][cC][cC]", key_plt):  # acc, ylim, yticks
-        ax.set_ylim(0, 1)
-        ax.set_yticks([0, 0.5, 1.0])
-    return ax
-
-
-def plot_tra(ax, metrics_df, key_plt, lw=1, color="blue"):
-    indi_step = scitex.gen.search("^[Tt]rain(ing)?", metrics_df.step, as_bool=True)[0]
-    step_df = metrics_df[indi_step]
-
-    if len(step_df) != 0:
-        ax.plot(
-            step_df.index,  # i_global
-            step_df[key_plt],
-            label="Training",
-            color=color,
-            linewidth=lw,
-        )
-        ax.legend()
-
-    return ax
-
-
-def scatter_val(ax, metrics_df, key_plt, s=3, color="green"):
-    indi_step = scitex.gen.search("^[Vv]alid(ation)?", metrics_df.step, as_bool=True)[0]
-    step_df = metrics_df[indi_step]
-    if len(step_df) != 0:
-        ax.scatter(
-            step_df.index,
-            step_df[key_plt],
-            label="Validation",
-            color=color,
-            s=s,
-            alpha=0.9,
-        )
-        ax.legend()
-    return ax
-
-
-def scatter_tes(ax, metrics_df, key_plt, s=3, color="red"):
-    indi_step = scitex.gen.search("^[Tt]est", metrics_df.step, as_bool=True)[0]
-    step_df = metrics_df[indi_step]
-    if len(step_df) != 0:
-        ax.scatter(
-            step_df.index,
-            step_df[key_plt],
-            label="Test",
-            color=color,
-            s=s,
-            alpha=0.9,
-        )
-        ax.legend()
-    return ax
-
-
-def vline_at_epochs(ax, metrics_df, color="grey"):
-    # Determine the global iteration values where new epochs start
-    epoch_starts = metrics_df[metrics_df["i_batch"] == 0].index.values
-    epoch_labels = metrics_df[metrics_df["i_batch"] == 0].index.values
-    ax.vlines(
-        x=epoch_starts,
-        ymin=-1e4,  # ax.get_ylim()[0],
-        ymax=1e4,  # ax.get_ylim()[1],
-        linestyle="--",
-        color=color,
-    )
-    return ax
-
-
-def select_ticks(metrics_df, max_n_ticks=4):
-    # Calculate epoch starts and their corresponding labels for ticks
-    unique_epochs = metrics_df["i_epoch"].drop_duplicates().values
-    epoch_starts = (
-        metrics_df[metrics_df["i_batch"] == 0]["i_global"].drop_duplicates().values
-    )
-
-    # Given the performance issue, let's just select a few epoch starts for labeling
-    # We use MaxNLocator to pick ticks; however, it's used here to choose a reasonable number of epoch markers
-    if len(epoch_starts) > max_n_ticks:
-        selected_ticks = np.linspace(
-            epoch_starts[0], epoch_starts[-1], max_n_ticks, dtype=int
-        )
-        # Ensure selected ticks are within the epoch starts for accurate labeling
-        selected_labels = [
-            metrics_df[metrics_df["i_global"] == tick]["i_epoch"].iloc[0]
-            for tick in selected_ticks
-        ]
-    else:
-        selected_ticks = epoch_starts
-        selected_labels = unique_epochs
-    return selected_ticks, selected_labels
-
-
-def learning_curve(
-    metrics_df,
-    keys,
-    title="Title",
-    max_n_ticks=4,
-    scattersize=3,
-    linewidth=1,
-    yscale="linear",
-    spath=None,
-):
-    _plt, cc = scitex.plt.configure_mpl(plt, show=False)
-    """
-    Example:
-        print(metrics_df)
-        #                 step  i_global  i_epoch  i_batch      loss
-        # 0       Training         0        0        0  0.717023
-        # 1       Training         1        0        1  0.703844
-        # 2       Training         2        0        2  0.696279
-        # 3       Training         3        0        3  0.685384
-        # 4       Training         4        0        4  0.670675
-        # ...          ...       ...      ...      ...       ...
-        # 123266      Test     66900      299      866  0.000067
-        # 123267      Test     66900      299      867  0.000067
-        # 123268      Test     66900      299      868  0.000067
-        # 123269      Test     66900      299      869  0.000067
-        # 123270      Test     66900      299      870  0.000068
-
-        # [123271 rows x 5 columns]
-    """
-    metrics_df = process_i_global(metrics_df)
-    selected_ticks, selected_labels = select_ticks(metrics_df)
-
-    # fig, axes = plt.subplots(len(keys), 1, sharex=True, sharey=False)
-    fig, axes = scitex.plt.subplots(len(keys), 1, sharex=True, sharey=False)
-    axes = axes if len(keys) != 1 else [axes]
-
-    axes[-1].set_xlabel("Iteration #")
-    fig.text(0.5, 0.95, title, ha="center")
-
-    for i_plt, key_plt in enumerate(keys):
-        ax = axes[i_plt]
-        ax.set_yscale(yscale)
-        ax.set_ylabel(key_plt)
-
-        ax = set_yaxis_for_acc(ax, key_plt)
-        ax = plot_tra(ax, metrics_df, key_plt, lw=linewidth, color=cc["blue"])
-        ax = scatter_val(ax, metrics_df, key_plt, s=scattersize, color=cc["green"])
-        ax = scatter_tes(ax, metrics_df, key_plt, s=scattersize, color=cc["red"])
-
-        # # Custom tick marks
-        # ax = scitex.plt.ax.map_ticks(
-        #     ax, selected_ticks, selected_labels, axis="x"
-        # )
-
-    if spath is not None:
-        scitex.io.save(fig, spath)
-
-    return fig
-
-
-if __name__ == "__main__":
-
-    plt, cc = scitex.plt.configure_mpl(plt)
-    # lpath = "./scripts/ml/.old/pretrain_EEGPT_old/2024-01-29-12-04_eDflsnWv_v8/metrics.csv"
-    lpath = "./scripts/ml/pretrain_EEGPT/[DEBUG] 2024-02-11-06-45_4uUpdfpb/metrics.csv"
-
-    sdir, _, _ = scitex.gen.split_fpath(lpath)
-    metrics_df = scitex.io.load(lpath)
-    fig = learning_curve(metrics_df, title="Pretraining on db_v8", yscale="log")
-    # plt.show()
-    scitex.io.save(fig, sdir + "learning_curve.png")

scitex/ai/plt/_optuna_study.py

@@ -1,111 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2024-03-30 08:24:55 (ywatanabe)"
-import os
-
-
-def optuna_study(lpath, value_str, sort=False):
-    """
-    Loads an Optuna study and generates various visualizations for each target metric.
-
-    Parameters:
-    - lpath (str): Path to the Optuna study database.
-    - value_str (str): The name of the column to be used as the optimization target.
-
-    Returns:
-    - None
-    """
-    import matplotlib
-
-    matplotlib.use("Agg")
-    import matplotlib.pyplot as plt
-    import scitex
-    import optuna
-    import pandas as pd
-
-    plt, CC = scitex.plt.configure_mpl(plt, fig_scale=3)
-
-    lpath = lpath.replace("./", "/")
-
-    study = optuna.load_study(study_name=None, storage=lpath)
-
-    sdir = lpath.replace("sqlite:///", "./").replace(".db", "/")
-
-    # To get the best trial:
-    best_trial = study.best_trial
-    print(f"Best trial number: {best_trial.number}")
-    print(f"Best trial value: {best_trial.value}")
-    print(f"Best trial parameters: {best_trial.params}")
-    print(f"Best trial user attributes: {best_trial.user_attrs}")
-
-    # Merge the user attributes into the study history DataFrame
-    study_history = study.trials_dataframe().rename(columns={"value": value_str})
-
-    if sort:
-        ascending = "MINIMIZE" in str(study.directions[0])  # [REVISED]
-        study_history = study_history.sort_values([value_str], ascending=ascending)
-
-    # Add user attributes to the study history DataFrame
-    attrs_df = []
-    for trial in study.trials:
-        user_attrs = trial.user_attrs
-        user_attrs = {k: v for k, v in user_attrs.items()}
-        attrs_df.append({"number": trial.number, **user_attrs})
-    attrs_df = pd.DataFrame(attrs_df).set_index("number")
-
-    # Updates study history
-    study_history = study_history.merge(
-        attrs_df, left_index=True, right_index=True, how="left"
-    ).set_index("number")
-    try:
-        study_history = scitex.gen.mv_col(study_history, "SDIR", 1)
-        study_history["SDIR"] = study_history["SDIR"].apply(
-            lambda x: str(x).replace("RUNNING", "FINISHED")
-        )
-        best_trial_dir = study_history["SDIR"].iloc[0]
-        scitex.gen.symlink(best_trial_dir, sdir + "best_trial", force=True)
-    except Exception as e:
-        print(e)
-    scitex.io.save(study_history, sdir + "study_history.csv")
-    print(study_history)
-
-    # To visualize the optimization history:
-    fig = optuna.visualization.plot_optimization_history(study, target_name=value_str)
-    scitex.io.save(fig, sdir + "optimization_history.png")
-    scitex.io.save(fig, sdir + "optimization_history.html")
-    plt.close()
-
-    # To visualize the parameter importances:
-    fig = optuna.visualization.plot_param_importances(study, target_name=value_str)
-    scitex.io.save(fig, sdir + "param_importances.png")
-    scitex.io.save(fig, sdir + "param_importances.html")
-    plt.close()
-
-    # To visualize the slice of the study:
-    fig = optuna.visualization.plot_slice(study, target_name=value_str)
-    scitex.io.save(fig, sdir + "slice.png")
-    scitex.io.save(fig, sdir + "slice.html")
-    plt.close()
-
-    # To visualize the contour plot of the study:
-    fig = optuna.visualization.plot_contour(study, target_name=value_str)
-    scitex.io.save(fig, sdir + "contour.png")
-    scitex.io.save(fig, sdir + "contour.html")
-    plt.close()
-
-    # To visualize the parallel coordinate plot of the study:
-    fig = optuna.visualization.plot_parallel_coordinate(study, target_name=value_str)
-    scitex.io.save(fig, sdir + "parallel_coordinate.png")
-    scitex.io.save(fig, sdir + "parallel_coordinate.html")
-    plt.close()
-
-
-if __name__ == "__main__":
-    scitex.plt.configure_mpl(plt, fig_scale=3)
-    lpath = "sqlite:///scripts/ml/clf/sub_conv_transformer_optuna/optuna_studies/optuna_study_v001.db"
-    lpath = "sqlite:///scripts/ml/clf/rocket_optuna/optuna_studies/optuna_study_v001.db"
-    optuna_study(lpath, "Validation bACC")
-    # scripts/ml/clf/sub_conv_transformer/optuna_studies/optuna_study_v032
-
-    lpath = "sqlite:///scripts/ml/clf/sub_conv_transformer_optuna/optuna_studies/optuna_study_v020.db"
-    scitex.ml.plt.optuna_study(lpath, "val_loss", sort=True)

scitex/ai/plt/aucs/__init__.py

@@ -1,2 +0,0 @@
-#!/usr/bin/env python3
-# Time-stamp: "2021-09-16 01:25:35 (ywatanabe)"

scitex/ai/plt/aucs/example.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Time-stamp: "2024-11-07 18:56:57 (ywatanabe)"
-# File: ./scitex_repo/src/scitex/ai/plt/aucs/example.py
-
-import matplotlib.pyplot as plt
-import scitex
-import numpy as np
-from sklearn import datasets, svm
-from sklearn.model_selection import train_test_split
-from .roc_auc import roc_auc
-from .pre_rec_auc import pre_rec_auc
-
-################################################################################
-## MNIST
-################################################################################
-
-digits = datasets.load_digits()
-
-# flatten the images
-n_samples = len(digits.images)
-data = digits.images.reshape((n_samples, -1))
-
-# Create a classifier: a support vector classifier
-clf = svm.SVC(gamma=0.001, probability=True)
-
-# Split data into 50% train and 50% test subsets
-X_train, X_test, y_train, y_test = train_test_split(
-    data, digits.target, test_size=0.5, shuffle=False
-)
-
-# Learn the digits on the train subset
-clf.fit(X_train, y_train)
-
-# Predict the value of the digit on the test subset
-predicted_proba = clf.predict_proba(X_test)
-predicted = clf.predict(X_test)
-
-n_classes = len(np.unique(digits.target))
-labels = ["Class {}".format(i) for i in range(n_classes)]
-
-## Configures matplotlib
-plt.rcParams["font.size"] = 20
-plt.rcParams["legend.fontsize"] = "xx-small"
-scale = 0.75
-plt.rcParams["figure.figsize"] = (16 * scale, 9 * scale)
-
-################################################################################
-## Main
-################################################################################
-## ROC Curve
-fig_roc, metrics_roc = roc_auc(plt, y_test, predicted_proba, labels)
-fig_roc.show()
-## Precision-Recall Curve
-fig_pre_rec, metrics_pre_rec = pre_rec_auc(plt, y_test, predicted_proba, labels)
-fig_pre_rec.show()
-
-#
-
-# EOF