scitex 2.0.0__py2.py3-none-any.whl

scitex/ai/plt/_learning_curve.py

@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-03-12 19:52:48 (ywatanabe)"
+
+import re
+
+import matplotlib
+import matplotlib.pyplot as plt
+import scitex
+import numpy as np
+import pandas as pd
+
+
+def process_i_global(metrics_df):
+    if metrics_df.index.name != "i_global":
+        try:
+            metrics_df = metrics_df.set_index("i_global")
+        except KeyError:
+            print(
+                "Error: The DataFrame does not contain a column named 'i_global'. Please check the column names."
+            )
+        except Exception as e:
+            print(f"An unexpected error occurred: {e}")
+    else:
+        print("The index is already set to 'i_global'.")
+    metrics_df["i_global"] = metrics_df.index  # alias
+    return metrics_df
+
+
+def set_yaxis_for_acc(ax, key_plt):
+    if re.search("[aA][cC][cC]", key_plt):  # acc, ylim, yticks
+        ax.set_ylim(0, 1)
+        ax.set_yticks([0, 0.5, 1.0])
+    return ax
+
+
+def plot_tra(ax, metrics_df, key_plt, lw=1, color="blue"):
+    indi_step = scitex.gen.search("^[Tt]rain(ing)?", metrics_df.step, as_bool=True)[0]
+    step_df = metrics_df[indi_step]
+
+    if len(step_df) != 0:
+        ax.plot(
+            step_df.index,  # i_global
+            step_df[key_plt],
+            label="Training",
+            color=color,
+            linewidth=lw,
+        )
+        ax.legend()
+
+    return ax
+
+
+def scatter_val(ax, metrics_df, key_plt, s=3, color="green"):
+    indi_step = scitex.gen.search("^[Vv]alid(ation)?", metrics_df.step, as_bool=True)[0]
+    step_df = metrics_df[indi_step]
+    if len(step_df) != 0:
+        ax.scatter(
+            step_df.index,
+            step_df[key_plt],
+            label="Validation",
+            color=color,
+            s=s,
+            alpha=0.9,
+        )
+        ax.legend()
+    return ax
+
+
+def scatter_tes(ax, metrics_df, key_plt, s=3, color="red"):
+    indi_step = scitex.gen.search("^[Tt]est", metrics_df.step, as_bool=True)[0]
+    step_df = metrics_df[indi_step]
+    if len(step_df) != 0:
+        ax.scatter(
+            step_df.index,
+            step_df[key_plt],
+            label="Test",
+            color=color,
+            s=s,
+            alpha=0.9,
+        )
+        ax.legend()
+    return ax
+
+
+def vline_at_epochs(ax, metrics_df, color="grey"):
+    # Determine the global iteration values where new epochs start
+    epoch_starts = metrics_df[metrics_df["i_batch"] == 0].index.values
+    epoch_labels = metrics_df[metrics_df["i_batch"] == 0].index.values
+    ax.vlines(
+        x=epoch_starts,
+        ymin=-1e4,  # ax.get_ylim()[0],
+        ymax=1e4,  # ax.get_ylim()[1],
+        linestyle="--",
+        color=color,
+    )
+    return ax
+
+
+def select_ticks(metrics_df, max_n_ticks=4):
+    # Calculate epoch starts and their corresponding labels for ticks
+    unique_epochs = metrics_df["i_epoch"].drop_duplicates().values
+    epoch_starts = (
+        metrics_df[metrics_df["i_batch"] == 0]["i_global"].drop_duplicates().values
+    )
+
+    # Given the performance issue, let's just select a few epoch starts for labeling
+    # We use MaxNLocator to pick ticks; however, it's used here to choose a reasonable number of epoch markers
+    if len(epoch_starts) > max_n_ticks:
+        selected_ticks = np.linspace(
+            epoch_starts[0], epoch_starts[-1], max_n_ticks, dtype=int
+        )
+        # Ensure selected ticks are within the epoch starts for accurate labeling
+        selected_labels = [
+            metrics_df[metrics_df["i_global"] == tick]["i_epoch"].iloc[0]
+            for tick in selected_ticks
+        ]
+    else:
+        selected_ticks = epoch_starts
+        selected_labels = unique_epochs
+    return selected_ticks, selected_labels
+
+
+def learning_curve(
+    metrics_df,
+    keys,
+    title="Title",
+    max_n_ticks=4,
+    scattersize=3,
+    linewidth=1,
+    yscale="linear",
+    spath=None,
+):
+    _plt, cc = scitex.plt.configure_mpl(plt, show=False)
+    """
+    Example:
+        print(metrics_df)
+        #                 step  i_global  i_epoch  i_batch      loss
+        # 0       Training         0        0        0  0.717023
+        # 1       Training         1        0        1  0.703844
+        # 2       Training         2        0        2  0.696279
+        # 3       Training         3        0        3  0.685384
+        # 4       Training         4        0        4  0.670675
+        # ...          ...       ...      ...      ...       ...
+        # 123266      Test     66900      299      866  0.000067
+        # 123267      Test     66900      299      867  0.000067
+        # 123268      Test     66900      299      868  0.000067
+        # 123269      Test     66900      299      869  0.000067
+        # 123270      Test     66900      299      870  0.000068
+
+        # [123271 rows x 5 columns]
+    """
+    metrics_df = process_i_global(metrics_df)
+    selected_ticks, selected_labels = select_ticks(metrics_df)
+
+    # fig, axes = plt.subplots(len(keys), 1, sharex=True, sharey=False)
+    fig, axes = scitex.plt.subplots(len(keys), 1, sharex=True, sharey=False)
+    axes = axes if len(keys) != 1 else [axes]
+
+    axes[-1].set_xlabel("Iteration #")
+    fig.text(0.5, 0.95, title, ha="center")
+
+    for i_plt, key_plt in enumerate(keys):
+        ax = axes[i_plt]
+        ax.set_yscale(yscale)
+        ax.set_ylabel(key_plt)
+
+        ax = set_yaxis_for_acc(ax, key_plt)
+        ax = plot_tra(ax, metrics_df, key_plt, lw=linewidth, color=cc["blue"])
+        ax = scatter_val(ax, metrics_df, key_plt, s=scattersize, color=cc["green"])
+        ax = scatter_tes(ax, metrics_df, key_plt, s=scattersize, color=cc["red"])
+
+        # # Custom tick marks
+        # ax = scitex.plt.ax.map_ticks(
+        #     ax, selected_ticks, selected_labels, axis="x"
+        # )
+
+    if spath is not None:
+        scitex.io.save(fig, spath)
+
+    return fig
+
+
+if __name__ == "__main__":
+
+    plt, cc = scitex.plt.configure_mpl(plt)
+    # lpath = "./scripts/ml/.old/pretrain_EEGPT_old/2024-01-29-12-04_eDflsnWv_v8/metrics.csv"
+    lpath = "./scripts/ml/pretrain_EEGPT/[DEBUG] 2024-02-11-06-45_4uUpdfpb/metrics.csv"
+
+    sdir, _, _ = scitex.gen.split_fpath(lpath)
+    metrics_df = scitex.io.load(lpath)
+    fig = learning_curve(metrics_df, title="Pretraining on db_v8", yscale="log")
+    # plt.show()
+    scitex.io.save(fig, sdir + "learning_curve.png")

scitex/ai/plt/_optuna_study.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-03-30 08:24:55 (ywatanabe)"
+import os
+
+
+def optuna_study(lpath, value_str, sort=False):
+    """
+    Loads an Optuna study and generates various visualizations for each target metric.
+
+    Parameters:
+    - lpath (str): Path to the Optuna study database.
+    - value_str (str): The name of the column to be used as the optimization target.
+
+    Returns:
+    - None
+    """
+    import matplotlib
+
+    matplotlib.use("Agg")
+    import matplotlib.pyplot as plt
+    import scitex
+    import optuna
+    import pandas as pd
+
+    plt, CC = scitex.plt.configure_mpl(plt, fig_scale=3)
+
+    lpath = lpath.replace("./", "/")
+
+    study = optuna.load_study(study_name=None, storage=lpath)
+
+    sdir = lpath.replace("sqlite:///", "./").replace(".db", "/")
+
+    # To get the best trial:
+    best_trial = study.best_trial
+    print(f"Best trial number: {best_trial.number}")
+    print(f"Best trial value: {best_trial.value}")
+    print(f"Best trial parameters: {best_trial.params}")
+    print(f"Best trial user attributes: {best_trial.user_attrs}")
+
+    # Merge the user attributes into the study history DataFrame
+    study_history = study.trials_dataframe().rename(columns={"value": value_str})
+
+    if sort:
+        ascending = "MINIMIZE" in str(study.directions[0])  # [REVISED]
+        study_history = study_history.sort_values([value_str], ascending=ascending)
+
+    # Add user attributes to the study history DataFrame
+    attrs_df = []
+    for trial in study.trials:
+        user_attrs = trial.user_attrs
+        user_attrs = {k: v for k, v in user_attrs.items()}
+        attrs_df.append({"number": trial.number, **user_attrs})
+    attrs_df = pd.DataFrame(attrs_df).set_index("number")
+
+    # Updates study history
+    study_history = study_history.merge(
+        attrs_df, left_index=True, right_index=True, how="left"
+    ).set_index("number")
+    try:
+        study_history = scitex.gen.mv_col(study_history, "SDIR", 1)
+        study_history["SDIR"] = study_history["SDIR"].apply(
+            lambda x: str(x).replace("RUNNING", "FINISHED")
+        )
+        best_trial_dir = study_history["SDIR"].iloc[0]
+        scitex.gen.symlink(best_trial_dir, sdir + "best_trial", force=True)
+    except Exception as e:
+        print(e)
+    scitex.io.save(study_history, sdir + "study_history.csv")
+    print(study_history)
+
+    # To visualize the optimization history:
+    fig = optuna.visualization.plot_optimization_history(study, target_name=value_str)
+    scitex.io.save(fig, sdir + "optimization_history.png")
+    scitex.io.save(fig, sdir + "optimization_history.html")
+    plt.close()
+
+    # To visualize the parameter importances:
+    fig = optuna.visualization.plot_param_importances(study, target_name=value_str)
+    scitex.io.save(fig, sdir + "param_importances.png")
+    scitex.io.save(fig, sdir + "param_importances.html")
+    plt.close()
+
+    # To visualize the slice of the study:
+    fig = optuna.visualization.plot_slice(study, target_name=value_str)
+    scitex.io.save(fig, sdir + "slice.png")
+    scitex.io.save(fig, sdir + "slice.html")
+    plt.close()
+
+    # To visualize the contour plot of the study:
+    fig = optuna.visualization.plot_contour(study, target_name=value_str)
+    scitex.io.save(fig, sdir + "contour.png")
+    scitex.io.save(fig, sdir + "contour.html")
+    plt.close()
+
+    # To visualize the parallel coordinate plot of the study:
+    fig = optuna.visualization.plot_parallel_coordinate(study, target_name=value_str)
+    scitex.io.save(fig, sdir + "parallel_coordinate.png")
+    scitex.io.save(fig, sdir + "parallel_coordinate.html")
+    plt.close()
+
+
+if __name__ == "__main__":
+    scitex.plt.configure_mpl(plt, fig_scale=3)
+    lpath = "sqlite:///scripts/ml/clf/sub_conv_transformer_optuna/optuna_studies/optuna_study_v001.db"
+    lpath = "sqlite:///scripts/ml/clf/rocket_optuna/optuna_studies/optuna_study_v001.db"
+    optuna_study(lpath, "Validation bACC")
+    # scripts/ml/clf/sub_conv_transformer/optuna_studies/optuna_study_v032
+
+    lpath = "sqlite:///scripts/ml/clf/sub_conv_transformer_optuna/optuna_studies/optuna_study_v020.db"
+    scitex.ml.plt.optuna_study(lpath, "val_loss", sort=True)

scitex/ai/plt/aucs/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+# Time-stamp: "2021-09-16 01:25:35 (ywatanabe)"

scitex/ai/plt/aucs/example.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Time-stamp: "2024-11-07 18:56:57 (ywatanabe)"
+# File: ./scitex_repo/src/scitex/ai/plt/aucs/example.py
+
+import matplotlib.pyplot as plt
+import scitex
+import numpy as np
+from sklearn import datasets, svm
+from sklearn.model_selection import train_test_split
+from .roc_auc import roc_auc
+from .pre_rec_auc import pre_rec_auc
+
+################################################################################
+## MNIST
+################################################################################
+
+digits = datasets.load_digits()
+
+# flatten the images
+n_samples = len(digits.images)
+data = digits.images.reshape((n_samples, -1))
+
+# Create a classifier: a support vector classifier
+clf = svm.SVC(gamma=0.001, probability=True)
+
+# Split data into 50% train and 50% test subsets
+X_train, X_test, y_train, y_test = train_test_split(
+    data, digits.target, test_size=0.5, shuffle=False
+)
+
+# Learn the digits on the train subset
+clf.fit(X_train, y_train)
+
+# Predict the value of the digit on the test subset
+predicted_proba = clf.predict_proba(X_test)
+predicted = clf.predict(X_test)
+
+n_classes = len(np.unique(digits.target))
+labels = ["Class {}".format(i) for i in range(n_classes)]
+
+## Configures matplotlib
+plt.rcParams["font.size"] = 20
+plt.rcParams["legend.fontsize"] = "xx-small"
+scale = 0.75
+plt.rcParams["figure.figsize"] = (16 * scale, 9 * scale)
+
+################################################################################
+## Main
+################################################################################
+## ROC Curve
+fig_roc, metrics_roc = roc_auc(plt, y_test, predicted_proba, labels)
+fig_roc.show()
+## Precision-Recall Curve
+fig_pre_rec, metrics_pre_rec = pre_rec_auc(plt, y_test, predicted_proba, labels)
+fig_pre_rec.show()
+
+#
+
+# EOF

scitex/ai/plt/aucs/pre_rec_auc.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+
+import warnings
+from itertools import cycle
+
+import numpy as np
+from sklearn.metrics import average_precision_score, precision_recall_curve
+
+
+def solve_the_intersection_of_a_line_and_iso_f1_curve(f1, a, b):
+    """
+    Determines the intersection of the following lines:
+        1) a line: y = a * x + b
+        2) the iso-f1 curve: y = f1 * x / (2 * x - f1)
+    , where a, b, and f1 are the constant values.
+    """
+    _a = 2 * a
+    _b = -a * f1 + 2 * b - f1
+    _c = -b * f1
+
+    x_f = (-_b + np.sqrt(_b**2 - 4 * _a * _c)) / (2 * _a)
+    y_f = a * x_f + b
+
+    return (x_f, y_f)
+
+
+def to_onehot(labels, n_classes):
+    eye = np.eye(n_classes, dtype=int)
+    return eye[labels]
+
+
+def pre_rec_auc(plt, true_class, pred_proba, labels):
+    """
+    Calculates the precision recall curve.
+    """
+
+    # Use label_binarize to be multi-label like settings
+    n_classes = len(labels)
+    true_class_onehot = to_onehot(true_class, n_classes)
+
+    # For each class
+    precision = dict()
+    recall = dict()
+    threshold = dict()
+    pre_rec_auc = dict()
+    for i in range(n_classes):
+        true_class_i_onehot = true_class_onehot[:, i]
+        pred_proba_i = pred_proba[:, i]
+
+        try:
+            precision[i], recall[i], threshold[i] = precision_recall_curve(
+                true_class_i_onehot,
+                pred_proba_i,
+            )
+            pre_rec_auc[i] = average_precision_score(true_class_i_onehot, pred_proba_i)
+        except Exception as e:
+            print(e)
+            precision[i], recall[i], threshold[i], pre_rec_auc[i] = (
+                np.nan,
+                np.nan,
+                np.nan,
+                np.nan,
+            )
+
+    ## Average precision: micro and macro
+
+    # A "micro-average": quantifying score on all classes jointly
+    precision["micro"], recall["micro"], threshold["micro"] = precision_recall_curve(
+        true_class_onehot.ravel(), pred_proba.ravel()
+    )
+    pre_rec_auc["micro"] = average_precision_score(
+        true_class_onehot, pred_proba, average="micro"
+    )
+
+    # macro
+    _pre_rec_aucs = []
+    for i in range(n_classes):
+        try:
+            _pre_rec_aucs.append(
+                average_precision_score(
+                    true_class_onehot[:, i], pred_proba[:, i], average="macro"
+                )
+            )
+        except Exception as e:
+            print(
+                f'\nPRE-REC-AUC for "{labels[i]}" was not defined and NaN-filled '
+                "for a calculation purpose (for the macro avg.)\n"
+            )
+            _pre_rec_aucs.append(np.nan)
+    pre_rec_auc["macro"] = np.nanmean(_pre_rec_aucs)
+
+    # pre_rec_auc["macro"] = average_precision_score(
+    #     true_class_onehot, pred_proba, average="macro"
+    # )
+
+    # Plot Precision-Recall curve for each class and iso-f1 curves
+    colors = cycle(["navy", "turquoise", "darkorange", "cornflowerblue", "teal"])
+    fig, ax = plt.subplots()
+    ax.set_box_aspect(1)
+    lines = []
+    legends = []
+
+    # iso-F1: By definition, an iso-F1 curve contains all points
+    #         in the precision/recall space whose F1 scores are the same.
+    f_scores = np.linspace(0.2, 0.8, num=4)
+    # for f_score in f_scores:
+    for i_f, f_score in enumerate(f_scores):
+        x = np.linspace(0.01, 1)  # num=50
+        y = f_score * x / (2 * x - f_score)
+        (l,) = ax.plot(x[y >= 0], y[y >= 0], color="gray", alpha=0.2)
+
+        # ax.annotate("f1={0:0.1f}".format(f_score), xy=(0.9, y[45] + 0.02))
+        x_f, y_f = solve_the_intersection_of_a_line_and_iso_f1_curve(f_score, 0.5, 0.5)
+        ax.annotate("f1={0:0.1f}".format(f_score), xy=(x_f - 0.1, y_f - 0.1 * 0.5))
+        # ax.annotate("f1={0:0.1f}".format(f_score), xy=(y[35] - 0.02 * (3 - i_f), 0.85))
+
+    lines.append(l)
+    legends.append("iso-f1 curves")
+
+    """
+    ## In this project, average precision-recall curve is not drawn.
+    (l,) = ax.plot(recall["micro"], precision["micro"], color="gold", lw=2)
+    lines.append(l)
+    legends.append("micro-average\n(AUC = {0:0.2f})" "".format(pre_rec_auc["micro"]))
+    """
+
+    ## Each Class
+    for i, color in zip(range(n_classes), colors):
+        (l,) = ax.plot(recall[i], precision[i], color=color, lw=2)
+        lines.append(l)
+        legends.append("{0} (AUC = {1:0.2f})" "".format(labels[i], pre_rec_auc[i]))
+
+    # fig = plt.gcf()
+    fig.subplots_adjust(bottom=0.25)
+    ax.set_xlim([-0.01, 1.01])
+    ax.set_ylim([-0.01, 1.01])
+    ax.set_xticks([0.0, 0.5, 1.0])
+    ax.set_yticks([0.0, 0.5, 1.0])
+    ax.set_xlabel("Recall")
+    ax.set_ylabel("Precision")
+    ax.set_title("Precision-Recall Curve")
+    ax.legend(lines, legends, loc="lower left")
+
+    metrics = dict(
+        pre_rec_auc=pre_rec_auc,
+        precision=precision,
+        recall=recall,
+        threshold=threshold,
+    )
+
+    return fig, metrics
+
+
+if __name__ == "__main__":
+    import matplotlib.pyplot as plt
+    import numpy as np
+    from scipy.special import softmax
+    from sklearn import datasets, svm
+    from sklearn.model_selection import train_test_split
+
+    def mk_demo_data(n_classes=2, batch_size=16):
+        labels = ["cls{}".format(i_cls) for i_cls in range(n_classes)]
+        true_class = np.random.randint(0, n_classes, size=(batch_size,))
+        pred_proba = softmax(np.random.rand(batch_size, n_classes), axis=-1)
+        pred_class = np.argmax(pred_proba, axis=-1)
+        return labels, true_class, pred_proba, pred_class
+
+    ## Fix seed
+    np.random.seed(42)
+
+    """
+    ################################################################################
+    ## A Minimal Example
+    ################################################################################    
+    labels, true_class, pred_proba, pred_class = \
+        mk_demo_data(n_classes=10, batch_size=256)
+
+    pre_rec_auc, precision, recall, threshold = \
+        calc_pre_rec_auc(true_class, pred_proba, labels, plot=False)
+    """
+
+    ################################################################################
+    ## MNIST
+    ################################################################################
+    from sklearn import datasets, metrics, svm
+    from sklearn.model_selection import train_test_split
+
+    digits = datasets.load_digits()
+
+    # flatten the images
+    n_samples = len(digits.images)
+    data = digits.images.reshape((n_samples, -1))
+
+    # Create a classifier: a support vector classifier
+    clf = svm.SVC(gamma=0.001, probability=True)
+
+    # Split data into 50% train and 50% test subsets
+    X_train, X_test, y_train, y_test = train_test_split(
+        data, digits.target, test_size=0.5, shuffle=False
+    )
+
+    # Learn the digits on the train subset
+    clf.fit(X_train, y_train)
+
+    # Predict the value of the digit on the test subset
+    predicted_proba = clf.predict_proba(X_test)
+    predicted = clf.predict(X_test)
+
+    n_classes = len(np.unique(digits.target))
+    labels = ["Class {}".format(i) for i in range(n_classes)]
+
+    ## Configures matplotlib
+    plt.rcParams["font.size"] = 20
+    plt.rcParams["legend.fontsize"] = "xx-small"
+    plt.rcParams["figure.figsize"] = (16 * 1.2, 9 * 1.2)
+
+    ## Main
+    fig, metrics_dict = pre_rec_auc(plt, y_test, predicted_proba, labels)
+
+    fig.show()
+
+    print(metrics_dict.keys())
+    # dict_keys(['pre_rec_auc', 'precision', 'recall', 'threshold'])