scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/ai/classification_reporter.py

@@ -1,1161 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-# Timestamp: "2025-02-15 01:38:28 (ywatanabe)"
-# File: ./src/scitex/ai/ClassificationReporter.py
-
-THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/ai/ClassificationReporter.py"
-
-import os as _os
-import random as _random
-import sys as _sys
-from collections import defaultdict as _defaultdict
-from glob import glob as _glob
-from pprint import pprint as _pprint
-
-import matplotlib as _matplotlib
-import matplotlib.pyplot as _plt
-import scitex as _scitex
-import numpy as _np
-import pandas as _pd
-import torch as _torch
-from sklearn.metrics import (
-    balanced_accuracy_score as _balanced_accuracy_score,
-    classification_report as _classification_report,
-    confusion_matrix as _confusion_matrix,
-    matthews_corrcoef as _matthews_corrcoef,
-)
-
-from ..reproduce import fix_seeds as _fix_seeds
-
-
-class MultiClassificationReporter(object):
-    def __init__(self, sdir, tgts=None):
-        if tgts is None:
-            sdirs = [""]
-        else:
-            sdirs = [_os.path.join(sdir, tgt, "/") for tgt in tgts]
-        sdirs = [sdir + tgt + "/" for tgt in tgts]
-
-        self.tgt2id = {tgt: i_tgt for i_tgt, tgt in enumerate(tgts)}
-        self.reporters = [ClassificationReporter(sdir) for sdir in sdirs]
-
-    def add(self, obj_name, obj, tgt=None):
-        i_tgt = self.tgt2id[tgt]
-        self.reporters[i_tgt].add(obj_name, obj)
-
-    def calc_metrics(
-        self,
-        true_class,
-        pred_class,
-        pred_proba,
-        labels=None,
-        i_fold=None,
-        show=True,
-        auc_plt_config=dict(
-            figsize=(7, 7),
-            labelsize=8,
-            fontsize=7,
-            legendfontsize=6,
-            tick_size=0.8,
-            tick_width=0.2,
-        ),
-        tgt=None,
-    ):
-        i_tgt = self.tgt2id[tgt]
-        self.reporters[i_tgt].calc_metrics(
-            true_class,
-            pred_class,
-            pred_proba,
-            labels=labels,
-            i_fold=i_fold,
-            show=show,
-            auc_plt_config=auc_plt_config,
-        )
-
-    def summarize(
-        self,
-        n_round=3,
-        show=False,
-        tgt=None,
-    ):
-        i_tgt = self.tgt2id[tgt]
-        self.reporters[i_tgt].summarize(
-            n_round=n_round,
-            show=show,
-        )
-
-    def save(
-        self,
-        files_to_reproduce=None,
-        meta_dict=None,
-        tgt=None,
-    ):
-        i_tgt = self.tgt2id[tgt]
-        self.reporters[i_tgt].save(
-            files_to_reproduce=files_to_reproduce,
-            meta_dict=meta_dict,
-        )
-
-    def plot_and_save_conf_mats(
-        self,
-        plt,
-        extend_ratio=1.0,
-        colorbar=True,
-        confmat_plt_config=None,
-        sci_notation_kwargs=None,
-        tgt=None,
-    ):
-        i_tgt = self.tgt2id[tgt]
-        self.reporters[i_tgt].plot_and_save_conf_mats(
-            plt,
-            extend_ratio=extend_ratio,
-            colorbar=colorbar,
-            confmat_plt_config=confmat_plt_config,
-            sci_notation_kwargs=sci_notation_kwargs,
-        )
-
-
-class ClassificationReporter(object):
-    """Saves the following metrics under sdir.
-       - Balanced Accuracy
-       - MCC
-       - Confusion Matrix
-       - Classification Report
-       - ROC AUC score / curve
-       - PRE-REC AUC score / curve
-
-    Example is described in this file.
-    """
-
-    def __init__(self, sdir):
-        self.sdir = sdir
-        self.folds_dict = _defaultdict(list)
-        _fix_seeds(os=_os, random=_random, np=_np, torch=_torch, verbose=False)
-
-    def add(
-        self,
-        obj_name,
-        obj,
-    ):
-        """
-        ## fig
-        fig, ax = plt.subplots()
-        ax.plot(np.random.rand(10))
-        reporter.add("manu_figs", fig)
-
-        ## DataFrame
-        df = pd.DataFrame(np.random.rand(5, 3))
-        reporter.add("manu_dfs", df)
-
-        ## scalar
-        scalar = random.random()
-        reporter.add("manu_scalers", scalar)
-        """
-        assert isinstance(obj_name, str)
-        self.folds_dict[obj_name].append(obj)
-
-    @staticmethod
-    def calc_bACC(true_class, pred_class, i_fold, show=False):
-        """Balanced ACC"""
-        balanced_acc = _balanced_accuracy_score(true_class, pred_class)
-        if show:
-            print(f"\nBalanced ACC in fold#{i_fold} was {balanced_acc:.3f}\n")
-        return balanced_acc
-    
-    @staticmethod
-    def calc_balanced_accuracy(true_class, pred_class, i_fold, show=False):
-        """Balanced accuracy (snake_case alias for calc_bACC)"""
-        return ClassificationReporter.calc_bACC(true_class, pred_class, i_fold, show)
-
-    @staticmethod
-    def calc_mcc(true_class, pred_class, i_fold, show=False):
-        """MCC"""
-        mcc = float(_matthews_corrcoef(true_class, pred_class))
-        if show:
-            print(f"\nMCC in fold#{i_fold} was {mcc:.3f}\n")
-        return mcc
-
-    @staticmethod
-    def calc_conf_mat(true_class, pred_class, labels, i_fold, show=False):
-        """
-        Confusion Matrix
-        This method assumes unique classes of true_class and pred_class are the same.
-        """
-        conf_mat = _pd.DataFrame(
-            data=_confusion_matrix(
-                true_class, pred_class, labels=_np.arange(len(labels))
-            ),
-            columns=labels,
-        ).set_index(_pd.Series(list(labels)))
-
-        if show:
-            print(f"\nConfusion Matrix in fold#{i_fold}: \n")
-            _pprint(conf_mat)
-            print()
-
-        return conf_mat
-
-    @staticmethod
-    def calc_clf_report(
-        true_class, pred_class, labels, balanced_acc, i_fold, show=False
-    ):
-        """Classification Report"""
-        clf_report = _pd.DataFrame(
-            _classification_report(
-                true_class,
-                pred_class,
-                labels=_np.arange(len(labels)),
-                target_names=labels,
-                output_dict=True,
-            )
-        )
-
-        clf_report["accuracy"] = balanced_acc
-        clf_report = _pd.concat(
-            [
-                clf_report[labels],
-                clf_report[["accuracy", "macro avg", "weighted avg"]],
-            ],
-            axis=1,
-        )
-        clf_report = clf_report.rename(columns={"accuracy": "balanced accuracy"})
-        clf_report = clf_report.round(3)
-        clf_report["index"] = clf_report.index
-        clf_report.loc["support", "index"] = "sample size"
-        clf_report.set_index("index", drop=True, inplace=True)
-        clf_report.index.name = None
-        if show:
-            print(f"\nClassification Report for fold#{i_fold}:\n")
-            _pprint(clf_report)
-            print()
-        return clf_report
-
-    def calc_AUCs(
-        self,
-        true_class,
-        pred_proba,
-        labels,
-        i_fold,
-        show=True,
-        auc_plt_config=dict(
-            figsize=(7, 7),
-            labelsize=8,
-            fontsize=7,
-            legendfontsize=6,
-            tick_size=0.8,
-            tick_width=0.2,
-        ),
-    ):
-        """ROC AUC and PRE-REC AUC."""
-        n_classes = len(labels)
-        assert len(_np.unique(true_class)) == n_classes
-        if n_classes == 2:
-            roc_auc = self._calc_AUCs_binary(
-                true_class,
-                pred_proba,
-                i_fold,
-                show=show,
-                auc_plt_config=auc_plt_config,
-            )
-        else:
-            roc_auc = self._calc_AUCs_multiple(
-                true_class,
-                pred_proba,
-                labels,
-                i_fold,
-                show=show,
-                auc_plt_config=auc_plt_config,
-            )
-        return roc_auc
-    
-    def calc_aucs(self, true_class, pred_proba, labels, i_fold, show=True, auc_plt_config=None):
-        """Calculate AUCs (snake_case alias for calc_AUCs)"""
-        if auc_plt_config is None:
-            auc_plt_config = dict(
-                figsize=(7, 7),
-                labelsize=8,
-                fontsize=7,
-                legendfontsize=6,
-                tick_size=0.8,
-                tick_width=0.2,
-            )
-        return self.calc_AUCs(true_class, pred_proba, labels, i_fold, show, auc_plt_config)
-
-    def _calc_AUCs_binary(
-        self,
-        true_class,
-        pred_proba,
-        i_fold,
-        show=False,
-        auc_plt_config=dict(
-            figsize=(7, 7),
-            labelsize=8,
-            fontsize=7,
-            legendfontsize=6,
-            tick_size=0.8,
-            tick_width=0.2,
-        ),
-    ):
-        """Calculates metrics for binary classification."""
-        from sklearn.metrics import (
-            PrecisionRecallDisplay,
-            RocCurveDisplay,
-            auc,
-            precision_recall_curve,
-            roc_curve,
-        )
-
-        unique_classes = sorted(list(_np.unique(true_class)))
-        n_classes = len(unique_classes)
-        assert n_classes == 2, "This method is only for binary classification"
-
-        # ROC curve
-        fpr, tpr, _ = roc_curve(true_class, pred_proba)
-        roc_auc = auc(fpr, tpr)
-
-        fig_size = auc_plt_config["figsize"]
-        fontsize = auc_plt_config["fontsize"]
-        labelsize = auc_plt_config["labelsize"]
-        legendfontsize = auc_plt_config["legendfontsize"]
-        tick_size = auc_plt_config["tick_size"]
-        tick_width = auc_plt_config["tick_width"]
-
-        fig_roc, ax_roc = _plt.subplots(figsize=fig_size)
-        RocCurveDisplay(
-            fpr=fpr,
-            tpr=tpr,
-            roc_auc=roc_auc,
-        ).plot(ax=ax_roc)
-        ax_roc.plot([0, 1], [0, 1], "k:")
-        ax_roc.set_xlabel("False Positive Rate", fontsize=labelsize)
-        ax_roc.set_ylabel("True Positive Rate", fontsize=labelsize)
-        ax_roc.set_title("ROC Curve", fontsize=fontsize)
-        ax_roc.legend(fontsize=legendfontsize)
-        ax_roc.tick_params(
-            axis="both",
-            which="major",
-            labelsize=tick_size,
-            width=tick_width,
-        )
-        self.folds_dict["ROC_fig"].append(fig_roc)
-        if show:
-            print(f"\nROC AUC in fold#{i_fold} is {roc_auc:.3f}\n")
-
-        # PRE-REC curve
-        fig_prerec, ax_prerec = _plt.subplots(figsize=fig_size)
-        PrecisionRecallDisplay.from_predictions(
-            true_class,
-            pred_proba,
-            ax=ax_prerec,
-        )
-        ax_prerec.set_xlabel("Recall", fontsize=labelsize)
-        ax_prerec.set_ylabel("Precision", fontsize=labelsize)
-        ax_prerec.set_title("Precision-Recall Curve", fontsize=fontsize)
-        ax_prerec.legend(fontsize=legendfontsize)
-        ax_prerec.tick_params(
-            axis="both",
-            which="major",
-            labelsize=tick_size,
-            width=tick_width,
-        )
-        self.folds_dict["PRE_REC_fig"].append(fig_prerec)
-
-        return roc_auc
-    
-    def _calc_aucs_binary(self, true_class, pred_proba, i_fold, show=False, auc_plt_config=None):
-        """Calculates metrics for binary classification (snake_case alias)."""
-        if auc_plt_config is None:
-            auc_plt_config = dict(
-                figsize=(7, 7),
-                labelsize=8,
-                fontsize=7,
-                legendfontsize=6,
-                tick_size=0.8,
-                tick_width=0.2,
-            )
-        return self._calc_AUCs_binary(true_class, pred_proba, i_fold, show, auc_plt_config)
-
-
-# #!/usr/bin/env python3
-# # -*- coding: utf-8 -*-
-# # Time-stamp: "2024-11-20 00:15:08 (ywatanabe)"
-# # File: ./scitex_repo/src/scitex/ai/ClassificationReporter.py
-
-# THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/ai/ClassificationReporter.py"
-
-# #!/usr/bin/env python3
-# # -*- coding: utf-8 -*-
-# # Time-stamp: "2024-11-13 12:54:17 (ywatanabe)"
-# # File: ./scitex_repo/src/scitex/ai/ClassificationReporter.py
-
-# import os
-# import random
-# import sys
-# from collections import defaultdict as _defaultdict
-# from glob import glob as _glob
-# from pprint import pprint as _pprint
-
-# import matplotlib
-# import matplotlib.pyplot as plt
-# import scitex
-# import numpy as np
-# import pandas as pd
-# import torch
-# from sklearn.metrics import (
-#     balanced_accuracy_score,
-#     classification_report,
-#     confusion_matrix,
-#     matthews_corrcoef,
-# )
-
-# from ..reproduce import fix_seeds
-
-
-# class MultiClassificationReporter(object):
-#     def __init__(self, sdir, tgts=None):
-#         if tgts is None:
-#             sdirs = [""]
-#         else:
-#             sdirs = [os.path.join(sdir, tgt, "/") for tgt in tgts]
-#         sdirs = [sdir + tgt + "/" for tgt in tgts]
-
-#         self.tgt2id = {tgt: i_tgt for i_tgt, tgt in enumerate(tgts)}
-#         self.reporters = [ClassificationReporter(sdir) for sdir in sdirs]
-
-#     def add(self, obj_name, obj, tgt=None):
-#         i_tgt = self.tgt2id[tgt]
-#         self.reporters[i_tgt].add(obj_name, obj)
-
-#     def calc_metrics(
-#         self,
-#         true_class,
-#         pred_class,
-#         pred_proba,
-#         labels=None,
-#         i_fold=None,
-#         show=True,
-#         auc_plt_config=dict(
-#             figsize=(7, 7),
-#             labelsize=8,
-#             fontsize=7,
-#             legendfontsize=6,
-#             tick_size=0.8,
-#             tick_width=0.2,
-#         ),
-#         tgt=None,
-#     ):
-#         i_tgt = self.tgt2id[tgt]
-#         self.reporters[i_tgt].calc_metrics(
-#             true_class,
-#             pred_class,
-#             pred_proba,
-#             labels=labels,
-#             i_fold=i_fold,
-#             show=show,
-#             auc_plt_config=auc_plt_config,
-#         )
-
-#     def summarize(
-#         self,
-#         n_round=3,
-#         show=False,
-#         tgt=None,
-#     ):
-#         i_tgt = self.tgt2id[tgt]
-#         self.reporters[i_tgt].summarize(
-#             n_round=n_round,
-#             show=show,
-#         )
-
-#     def save(
-#         self,
-#         files_to_reproduce=None,
-#         meta_dict=None,
-#         tgt=None,
-#     ):
-#         i_tgt = self.tgt2id[tgt]
-#         self.reporters[i_tgt].save(
-#             files_to_reproduce=files_to_reproduce,
-#             meta_dict=meta_dict,
-#         )
-
-#     def plot_and_save_conf_mats(
-#         self,
-#         plt,
-#         extend_ratio=1.0,
-#         colorbar=True,
-#         confmat_plt_config=None,
-#         sci_notation_kwargs=None,
-#         tgt=None,
-#     ):
-#         i_tgt = self.tgt2id[tgt]
-#         self.reporters[i_tgt].plot_and_save_conf_mats(
-#             plt,
-#             extend_ratio=extend_ratio,
-#             colorbar=colorbar,
-#             confmat_plt_config=confmat_plt_config,
-#             sci_notation_kwargs=sci_notation_kwargs,
-#         )
-
-
-# class ClassificationReporter(object):
-#     """Saves the following metrics under sdir.
-#        - Balanced Accuracy
-#        - MCC
-#        - Confusion Matrix
-#        - Classification Report
-#        - ROC AUC score / curve
-#        - PRE-REC AUC score / curve
-
-#     Example is described in this file.
-#     """
-
-#     def __init__(self, sdir):
-#         self.sdir = sdir
-#         self.folds_dict = _defaultdict(list)
-#         fix_seeds(os=os, random=random, np=np, torch=torch, show=False)
-
-#     def add(
-#         self,
-#         obj_name,
-#         obj,
-#     ):
-#         """
-#         ## fig
-#         fig, ax = plt.subplots()
-#         ax.plot(np.random.rand(10))
-#         reporter.add("manu_figs", fig)
-
-#         ## DataFrame
-#         df = pd.DataFrame(np.random.rand(5, 3))
-#         reporter.add("manu_dfs", df)
-
-#         ## scalar
-#         scalar = random.random()
-#         reporter.add("manu_scalers", scalar)
-#         """
-#         assert isinstance(obj_name, str)
-#         self.folds_dict[obj_name].append(obj)
-
-#     @staticmethod
-#     def calc_bACC(true_class, pred_class, i_fold, show=False):
-#         """Balanced ACC"""
-#         balanced_acc = balanced_accuracy_score(true_class, pred_class)
-#         if show:
-#             print(f"\nBalanced ACC in fold#{i_fold} was {balanced_acc:.3f}\n")
-#         return balanced_acc
-
-#     @staticmethod
-#     def calc_mcc(true_class, pred_class, i_fold, show=False):
-#         """MCC"""
-#         mcc = float(matthews_corrcoef(true_class, pred_class))
-#         if show:
-#             print(f"\nMCC in fold#{i_fold} was {mcc:.3f}\n")
-#         return mcc
-
-#     @staticmethod
-#     def calc_conf_mat(true_class, pred_class, labels, i_fold, show=False):
-#         """
-#         Confusion Matrix
-#         This method assumes unique classes of true_class and pred_class are the same.
-#         """
-#         # conf_mat = pd.DataFrame(
-#         #     data=confusion_matrix(true_class, pred_class),
-#         #     columns=pred_labels,
-#         #     index=true_labels,
-#         # )
-
-#         conf_mat = pd.DataFrame(
-#             data=confusion_matrix(
-#                 true_class, pred_class, labels=np.arange(len(labels))
-#             ),
-#             columns=labels,
-#         ).set_index(pd.Series(list(labels)))
-
-#         if show:
-#             print(f"\nConfusion Matrix in fold#{i_fold}: \n")
-#             _pprint(conf_mat)
-#             print()
-
-#         return conf_mat
-
-#     @staticmethod
-#     def calc_clf_report(
-#         true_class, pred_class, labels, balanced_acc, i_fold, show=False
-#     ):
-#         """Classification Report"""
-#         clf_report = pd.DataFrame(
-#             classification_report(
-#                 true_class,
-#                 pred_class,
-#                 labels=np.arange(len(labels)),
-#                 target_names=labels,
-#                 output_dict=True,
-#             )
-#         )
-
-#         # ACC to bACC
-#         clf_report["accuracy"] = balanced_acc
-#         clf_report = pd.concat(
-#             [
-#                 clf_report[labels],
-#                 clf_report[["accuracy", "macro avg", "weighted avg"]],
-#             ],
-#             axis=1,
-#         )
-#         clf_report = clf_report.rename(
-#             columns={"accuracy": "balanced accuracy"}
-#         )
-#         clf_report = clf_report.round(3)
-#         # Renames 'support' to 'sample size'
-#         clf_report["index"] = clf_report.index
-#         clf_report.loc["support", "index"] = "sample size"
-#         clf_report.set_index("index", drop=True, inplace=True)
-#         clf_report.index.name = None
-#         if show:
-#             print(f"\nClassification Report for fold#{i_fold}:\n")
-#             _pprint(clf_report)
-#             print()
-#         return clf_report
-
-#     @staticmethod
-#     def calc_and_plot_roc_curve(
-#         true_class, pred_proba, labels, sdir_for_csv=None
-#     ):
-#         # ROC-AUC
-#         fig_roc, metrics_roc_auc_dict = scitex.ml.plt.roc_auc(
-#             plt,
-#             true_class,
-#             pred_proba,
-#             labels,
-#             sdir_for_csv=sdir_for_csv,
-#         )
-#         plt.close()
-#         return fig_roc, metrics_roc_auc_dict
-
-#     @staticmethod
-#     def calc_and_plot_pre_rec_curve(true_class, pred_proba, labels):
-#         # PRE-REC AUC
-#         fig_pre_rec, metrics_pre_rec_auc_dict = scitex.ml.plt.pre_rec_auc(
-#             plt, true_class, pred_proba, labels
-#         )
-#         plt.close()
-#         return fig_pre_rec, metrics_pre_rec_auc_dict
-
-#     def calc_metrics(
-#         self,
-#         true_class,
-#         pred_class,
-#         pred_proba,
-#         labels=None,
-#         i_fold=None,
-#         show=True,
-#         auc_plt_config=dict(
-#             figsize=(7, 7),
-#             labelsize=8,
-#             fontsize=7,
-#             legendfontsize=6,
-#             tick_size=0.8,
-#             tick_width=0.2,
-#         ),
-#     ):
-#         """
-#         Calculates ACC, Confusion Matrix, Classification Report, and ROC-AUC score on a fold.
-#         Metrics and curves will be kept in self.folds_dict.
-#         """
-
-#         ## Preparation
-#         # for convenience
-#         true_class = scitex.gen.torch_to_arr(true_class).astype(int).reshape(-1)
-#         pred_class = (
-#             scitex.gen.torch_to_arr(pred_class).astype(np.float64).reshape(-1)
-#         )
-#         pred_proba = scitex.gen.torch_to_arr(pred_proba).astype(np.float64)
-
-#         # for curves
-#         scitex.plt.configure_mpl(
-#             plt,
-#             **auc_plt_config,
-#         )
-
-#         ## Calc metrics
-#         # Balanced ACC
-#         bacc = self.calc_bACC(true_class, pred_class, i_fold, show=show)
-#         self.folds_dict["balanced_acc"].append(bacc)
-
-#         # MCC
-#         self.folds_dict["mcc"].append(
-#             self.calc_mcc(true_class, pred_class, i_fold, show=show)
-#         )
-
-#         # Confusion Matrix
-#         self.folds_dict["conf_mat/conf_mat"].append(
-#             self.calc_conf_mat(
-#                 true_class,
-#                 pred_class,
-#                 labels,
-#                 i_fold,
-#                 show=show,
-#             )
-#         )
-
-#         # Classification Report
-#         self.folds_dict["clf_report"].append(
-#             self.calc_clf_report(
-#                 true_class, pred_class, labels, bacc, i_fold, show=show
-#             )
-#         )
-
-#         ## Curves
-#         # ROC curve
-#         self.sdir_for_roc_csv = f"{self.sdir}roc/csv/"
-#         fig_roc, metrics_roc_auc_dict = self.calc_and_plot_roc_curve(
-#             true_class,
-#             pred_proba,
-#             labels,
-#             sdir_for_csv=self.sdir_for_roc_csv + f"fold#{i_fold}/",
-#         )
-#         self.folds_dict["roc/micro"].append(
-#             metrics_roc_auc_dict["roc_auc"]["micro"]
-#         )
-#         self.folds_dict["roc/macro"].append(
-#             metrics_roc_auc_dict["roc_auc"]["macro"]
-#         )
-#         self.folds_dict["roc/figs"].append(fig_roc)
-
-#         # PRE-REC curve
-#         fig_pre_rec, metrics_pre_rec_auc_dict = (
-#             self.calc_and_plot_pre_rec_curve(true_class, pred_proba, labels)
-#         )
-#         self.folds_dict["pre_rec/micro"].append(
-#             metrics_pre_rec_auc_dict["pre_rec_auc"]["micro"]
-#         )
-#         self.folds_dict["pre_rec/macro"].append(
-#             metrics_pre_rec_auc_dict["pre_rec_auc"]["macro"]
-#         )
-#         self.folds_dict["pre_rec/figs"].append(fig_pre_rec)
-
-#     @staticmethod
-#     def _mk_cv_index(n_folds):
-#         return [
-#             f"{n_folds}-folds_CV_mean",
-#             f"{n_folds}-fold_CV_std",
-#         ] + [f"fold#{i_fold}" for i_fold in range(n_folds)]
-
-#     def summarize_roc(
-#         self,
-#     ):
-
-#         folds_dirs = _glob(self.sdir_for_roc_csv + "fold#*")
-#         n_folds = len(folds_dirs)
-
-#         # get class names
-#         _csv_files = _glob(os.path.join(folds_dirs[0], "*"))
-#         classes_str = [
-#             csv_file.split("/")[-1].split(".csv")[0] for csv_file in _csv_files
-#         ]
-
-#         # dfs_classes = []
-#         # take mean and std by each class
-#         for cls_str in classes_str:
-
-#             fpaths_cls = [
-#                 os.path.join(fold_dir, f"{cls_str}.csv")
-#                 for fold_dir in folds_dirs
-#             ]
-
-#             ys = []
-#             roc_aucs = []
-#             for fpath_cls in fpaths_cls:
-#                 loaded_df = scitex.io.load(fpath_cls)
-#                 ys.append(loaded_df["y"])
-#                 roc_aucs.append(loaded_df["roc_auc"])
-#             ys = pd.concat(ys, axis=1)
-#             roc_aucs = pd.concat(roc_aucs, axis=1)
-
-#             df_cls = loaded_df[["x"]].copy()
-#             df_cls["y_mean"] = ys.mean(axis=1)
-#             df_cls["y_std"] = ys.std(axis=1)
-#             df_cls["roc_auc_mean"] = roc_aucs.mean(axis=1)
-#             df_cls["roc_auc_std"] = roc_aucs.std(axis=1)
-
-#             spath_cls = os.path.join(
-#                 self.sdir_for_roc_csv, f"k-fold_mean_std/{cls_str}.csv"
-#             )
-#             scitex.io.save(df_cls, spath_cls)
-#             # dfs_classes.append(df_cls)
-
-#     def summarize(
-#         self,
-#         n_round=3,
-#         show=False,
-#     ):
-#         """
-#         1) Take mean and std of scalars/pd.Dataframes for folds.
-#         2) Replace self.folds_dict with the summarized DataFrames.
-#         """
-#         self.summarize_roc()
-
-#         _n_folds_all = [
-#             len(self.folds_dict[k]) for k in self.folds_dict.keys()
-#         ]  # sometimes includes 0 because AUC curves are not always defined.
-#         self.n_folds_intended = max(_n_folds_all)
-
-#         for i_k, k in enumerate(self.folds_dict.keys()):
-#             n_folds = _n_folds_all[i_k]
-
-#             if n_folds != 0:
-#                 ## listed scalars
-#                 if is_listed_X(self.folds_dict[k], [float, int]):
-#                     mm = np.mean(self.folds_dict[k])
-#                     ss = np.std(self.folds_dict[k], ddof=1)
-#                     sr = pd.DataFrame(
-#                         data=[mm, ss] + self.folds_dict[k],
-#                         index=self._mk_cv_index(n_folds),
-#                         columns=[k],
-#                     )
-#                     self.folds_dict[k] = sr.round(n_round)
-
-#                 ## listed pd.DataFrames
-#                 elif is_listed_X(self.folds_dict[k], pd.DataFrame):
-#                     zero_df_for_mm = 0 * self.folds_dict[k][0].copy()
-#                     zero_df_for_ss = 0 * self.folds_dict[k][0].copy()
-
-#                     mm = (
-#                         zero_df_for_mm
-#                         + np.stack(self.folds_dict[k]).mean(axis=0)
-#                     ).round(n_round)
-
-#                     ss = (
-#                         zero_df_for_ss
-#                         + np.stack(self.folds_dict[k]).std(axis=0, ddof=1)
-#                     ).round(n_round)
-
-#                     self.folds_dict[k] = [mm, ss] + [
-#                         df_fold.round(n_round)
-#                         for df_fold in self.folds_dict[k]
-#                     ]
-
-#                     if show:
-#                         print(
-#                             "\n----------------------------------------\n"
-#                             f"\n{k}\n"
-#                             f"\n{n_folds}-fold-CV mean:\n"
-#                         )
-#                         _pprint(self.folds_dict[k][0])
-#                         print(f"\n\n{n_folds}-fold-CV std.:\n")
-#                         _pprint(self.folds_dict[k][1])
-#                         print("\n\n----------------------------------------\n")
-
-#                 ## listed figures
-#                 elif is_listed_X(self.folds_dict[k], matplotlib.figure.Figure):
-#                     pass
-
-#                 else:
-#                     print(f"{k} was not summarized")
-#                     print(type(self.folds_dict[k][0]))
-
-#     def save(
-#         self,
-#         files_to_reproduce=None,
-#         meta_dict=None,
-#     ):
-#         """
-#         1) Saves the content of self.folds_dict.
-#         2) Plots the colormap of confusion matrices and saves them.
-#         3) Saves passed meta_dict under self.sdir
-
-#         Example:
-#             meta_df_1 = pd.DataFrame(data=np.random.rand(3,3))
-#             meta_dict_1 = {"a": 0}
-#             meta_dict_2 = {"b": 0}
-#             meta_dict = {"meta_1.csv": meta_df_1,
-#                          "meta_1.yaml": meta_dict_1,
-#                          "meta_2.yaml": meta_dict_1,
-#             }
-
-#         """
-#         if meta_dict is not None:
-#             for k, v in meta_dict.items():
-#                 scitex.io.save(v, self.sdir + k)
-
-#         for k in self.folds_dict.keys():
-
-#             ## pd.Series / pd.DataFrame
-#             if isinstance(self.folds_dict[k], pd.Series) or isinstance(
-#                 self.folds_dict[k], pd.DataFrame
-#             ):
-#                 scitex.io.save(self.folds_dict[k], self.sdir + f"{k}.csv")
-
-#             ## listed pd.DataFrame
-#             elif is_listed_X(self.folds_dict[k], pd.DataFrame):
-#                 scitex.io.save(
-#                     self.folds_dict[k],
-#                     self.sdir + f"{k}.csv",
-#                     # indi_suffix=self.cv_index,
-#                     indi_suffix=self._mk_cv_index(len(self.folds_dict[k])),
-#                 )
-
-#             ## listed figures
-#             elif is_listed_X(self.folds_dict[k], matplotlib.figure.Figure):
-#                 for i_fold, fig in enumerate(self.folds_dict[k]):
-#                     scitex.io.save(
-#                         self.folds_dict[k][i_fold],
-#                         self.sdir + f"{k}/fold#{i_fold}.png",
-#                     )
-
-#             else:
-#                 print(f"{k} was not saved")
-#                 print(type(self.folds_dict[k]))
-
-#         if files_to_reproduce is not None:
-#             if isinstance(files_to_reproduce, list):
-#                 files_to_reproduce = [files_to_reproduce]
-#             for f in files_to_reproduce:
-#                 scitex.io.save(f, self.sdir)
-
-#     def plot_and_save_conf_mats(
-#         self,
-#         plt,
-#         extend_ratio=1.0,
-#         colorbar=True,
-#         confmat_plt_config=None,
-#         sci_notation_kwargs=None,
-#     ):
-#         def _inner_plot_conf_mat(
-#             plt,
-#             cm_df,
-#             title,
-#             extend_ratio=1.0,
-#             colorbar=True,
-#             sci_notation_kwargs=None,
-#         ):
-#             labels = list(cm_df.columns)
-#             fig_conf_mat = scitex.ml.plt.confusion_matrix(
-#                 plt,
-#                 cm_df.T,
-#                 labels=labels,
-#                 title=title,
-#                 x_extend_ratio=extend_ratio,
-#                 y_extend_ratio=extend_ratio,
-#                 colorbar=colorbar,
-#             )
-
-#             if sci_notation_kwargs is not None:
-#                 fig_conf_mat.axes[-1] = scitex.plt.ax_scientific_notation(
-#                     fig_conf_mat.axes[-1], **sci_notation_kwargs
-#                 )
-#             return fig_conf_mat
-
-#         ## Configures mpl
-#         scitex.plt.configure_mpl(
-#             plt,
-#             **confmat_plt_config,
-#         )
-
-#         ########################################
-#         ## Prepares confmats dfs
-#         ########################################
-#         ## Drops mean and std for the folds
-#         try:
-#             conf_mats = self.folds_dict["conf_mat/conf_mat"][
-#                 -self.n_folds_intended :
-#             ]
-
-#         except Exception as e:
-#             print(e)
-#             conf_mats = self.folds_dict["conf_mat/conf_mat"]
-
-#         ## Prepaires conf_mat_overall_sum
-#         conf_mat_zero = 0 * conf_mats[0].copy()  # get the table format
-#         conf_mat_overall_sum = conf_mat_zero + np.stack(conf_mats).sum(axis=0)
-
-#         ########################################
-#         ## Plots & Saves
-#         ########################################
-#         # each fold's conf
-#         for i_fold, cm in enumerate(conf_mats):
-#             title = f"Test fold#{i_fold}"
-#             fig_conf_mat_fold = _inner_plot_conf_mat(
-#                 plt,
-#                 cm,
-#                 title,
-#                 extend_ratio=extend_ratio,
-#                 colorbar=colorbar,
-#                 sci_notation_kwargs=sci_notation_kwargs,
-#             )
-#             scitex.io.save(
-#                 fig_conf_mat_fold,
-#                 self.sdir + f"conf_mat/figs/fold#{i_fold}.png",
-#             )
-#             plt.close()
-
-#         ## overall_sum conf_mat
-#         title = f"{self.n_folds_intended}-CV overall sum"
-#         fig_conf_mat_overall_sum = _inner_plot_conf_mat(
-#             plt,
-#             conf_mat_overall_sum,
-#             title,
-#             extend_ratio=extend_ratio,
-#             colorbar=colorbar,
-#             sci_notation_kwargs=sci_notation_kwargs,
-#         )
-#         scitex.io.save(
-#             fig_conf_mat_overall_sum,
-#             self.sdir
-#             + f"conf_mat/figs/{self.n_folds_intended}-fold_cv_overall-sum.png",
-#         )
-#         plt.close()
-
-
-# if __name__ == "__main__":
-#     import random
-#     import sys
-
-#     import scitex
-#     import numpy as np
-#     from catboost import CatBoostClassifier, Pool
-#     from sklearn.datasets import load_digits
-#     from sklearn.model_selection import StratifiedKFold
-
-#     ################################################################################
-#     ## Sets tee
-#     ################################################################################
-#     sdir = scitex.io.mk_spath(
-#         "./tmp/sdir-ClassificationReporter/"
-#     )  # "/tmp/sdir/"
-#     sys.stdout, sys.stderr = scitex.gen.tee(sys, sdir)
-
-#     ################################################################################
-#     ## Fixes seeds
-#     ################################################################################
-#     fix_seeds(np=np)
-
-#     ## Loads
-#     mnist = load_digits()
-#     X, T = mnist.data, mnist.target
-#     labels = mnist.target_names.astype(str)
-
-#     ## Main
-#     skf = StratifiedKFold(n_splits=5, shuffle=True)
-#     # reporter = ClassificationReporter(sdir)
-#     mreporter = MultiClassificationReporter(sdir, tgts=["Test1", "Test2"])
-#     for i_fold, (indi_tra, indi_tes) in enumerate(skf.split(X, T)):
-#         X_tra, T_tra = X[indi_tra], T[indi_tra]
-#         X_tes, T_tes = X[indi_tes], T[indi_tes]
-
-#         clf = CatBoostClassifier(verbose=False)
-
-#         clf.fit(X_tra, T_tra, verbose=False)
-
-#         ## Prediction
-#         pred_proba_tes = clf.predict_proba(X_tes)
-#         pred_cls_tes = np.argmax(pred_proba_tes, axis=1)
-
-#         pred_cls_tes[pred_cls_tes == 9] = 8  # overide 9 as 8 # fixme
-
-#         ##############################
-#         ## Manually adds objects to reporter to save
-#         ##############################
-#         ## Figure
-#         fig, ax = plt.subplots()
-#         ax.plot(np.arange(10))
-#         # reporter.add("manu_figs", fig)
-#         mreporter.add("manu_figs", fig, tgt="Test1")
-#         mreporter.add("manu_figs", fig, tgt="Test2")
-
-#         ## DataFrame
-#         df = pd.DataFrame(np.random.rand(5, 3))
-#         # reporter.add("manu_dfs", df)
-#         mreporter.add("manu_dfs", df, tgt="Test1")
-#         mreporter.add("manu_dfs", df, tgt="Test2")
-
-#         ## Scalar
-#         scalar = random.random()
-#         # reporter.add(
-#         #     "manu_scalars",
-#         #     scalar,
-#         # )
-#         mreporter.add("manu_scalars", scalar, tgt="Test1")
-#         mreporter.add("manu_scalars", scalar, tgt="Test2")
-
-#         ########################################
-#         ## Metrics
-#         ########################################
-#         mreporter.calc_metrics(
-#             T_tes,
-#             pred_cls_tes,
-#             pred_proba_tes,
-#             labels=labels,
-#             i_fold=i_fold,
-#             tgt="Test1",
-#         )
-#         mreporter.calc_metrics(
-#             T_tes,
-#             pred_cls_tes,
-#             pred_proba_tes,
-#             labels=labels,
-#             i_fold=i_fold,
-#             tgt="Test2",
-#         )
-
-#     # reporter.summarize(show=True)
-#     mreporter.summarize(show=True, tgt="Test1")
-#     mreporter.summarize(show=True, tgt="Test2")
-
-#     fake_fpaths = ["fake_file_1.txt", "fake_file_2.txt"]
-#     for ff in fake_fpaths:
-#         scitex.io.touch(ff)
-
-#     files_to_reproduce = [
-#         scitex.gen.get_this_fpath(when_ipython="/dev/null"),
-#         *fake_fpaths,
-#     ]
-#     # reporter.save(files_to_reproduce=files_to_reproduce)
-#     mreporter.save(files_to_reproduce=files_to_reproduce, tgt="Test1")
-#     mreporter.save(files_to_reproduce=files_to_reproduce, tgt="Test2")
-
-#     confmat_plt_config = dict(
-#         figsize=(8, 8),
-#         # labelsize=8,
-#         # fontsize=6,
-#         # legendfontsize=6,
-#         figscale=2,
-#         tick_size=0.8,
-#         tick_width=0.2,
-#     )
-
-#     sci_notation_kwargs = dict(
-#         order=1,
-#         fformat="%1.0d",
-#         scilimits=(-3, 3),
-#         x=False,
-#         y=True,
-#     )  # "%3.1f"
-
-#     # sci_notation_kwargs = None
-#     # reporter.plot_and_save_conf_mats(
-#     #     plt,
-#     #     extend_ratio=1.0,
-#     #     confmat_plt_config=confmat_plt_config,
-#     #     sci_notation_kwargs=sci_notation_kwargs,
-#     # )
-
-#     mreporter.plot_and_save_conf_mats(
-#         plt,
-#         extend_ratio=1.0,
-#         confmat_plt_config=confmat_plt_config,
-#         sci_notation_kwargs=sci_notation_kwargs,
-#         tgt="Test1",
-#     )
-#     mreporter.plot_and_save_conf_mats(
-#         plt,
-#         extend_ratio=1.0,
-#         confmat_plt_config=confmat_plt_config,
-#         sci_notation_kwargs=sci_notation_kwargs,
-#         tgt="Test2",
-#     )
-
-# python -m scitex.ai.ClassificationReporter
-
-# EOF