scitex 2.0.0__py2.py3-none-any.whl → 2.1.0__py2.py3-none-any.whl

scitex/{ai → ml/training}/_LearningCurveLogger.py

@@ -1,23 +1,28 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
-# Time-stamp: "2024-11-20 08:49:50 (ywatanabe)"
-# File: ./scitex_repo/src/scitex/ai/_LearningCurveLogger.py
-
-THIS_FILE = "/home/ywatanabe/proj/scitex_repo/src/scitex/ai/_LearningCurveLogger.py"
+# Timestamp: "2024-11-20 08:49:50 (ywatanabe)"
+# File: /home/ywatanabe/proj/scitex_repo/src/scitex/ml/training/_LearningCurveLogger.py
+# ----------------------------------------
+from __future__ import annotations
+import os
+__FILE__ = __file__
+__DIR__ = os.path.dirname(__FILE__)
+# ----------------------------------------
 
 """
 Functionality:
     - Records and visualizes learning curves during model training
     - Supports tracking of multiple metrics across training/validation/test phases
     - Generates plots showing training progress over iterations and epochs
+    - Delegates plotting to scitex.ml.plt.plot_learning_curve for consistency
 
 Input:
     - Training metrics dictionary containing loss, accuracy, predictions etc.
     - Step information (Training/Validation/Test)
 
 Output:
-    - Learning curve plots
-    - Dataframes with recorded metrics
+    - Learning curve plots via scitex.ml.plt.plot_learning_curve
+    - DataFrames with recorded metrics
     - Training progress prints
 
 Prerequisites:
@@ -26,46 +31,19 @@ Prerequisites:
     - matplotlib
     - pandas
     - numpy
+    - scitex
 """
 
-import re as _re
-from collections import defaultdict as _defaultdict
-from pprint import pprint as _pprint
-from typing import Dict as _Dict
-from typing import List as _List
-from typing import Union as _Union
-from typing import Optional as _Optional
-from typing import Any as _Any
-
-import matplotlib as _matplotlib
-import pandas as _pd
-import numpy as _np
-import warnings as _warnings
-import torch as _torch
-
-# Import scitex modules for plotting
-try:
-    import scitex.plt.utils as _plt_utils
-    import scitex.plt.color as _plt_color
-    
-    class _plt_module:
-        configure_mpl = _plt_utils.configure_mpl
-        colors = _plt_color
-except ImportError:
-    # Mock for testing when scitex is not available
-    class _plt_module:
-        @staticmethod
-        def configure_mpl(*args, **kwargs):
-            pass
-        
-        class colors:
-            @staticmethod
-            def to_RGBA(color, alpha=1.0):
-                return color
-            
-            @staticmethod
-            def to_rgba(color, alpha=1.0):
-                return color
+import re
+import warnings
+from collections import defaultdict
+from pprint import pprint
+from typing import Any, Dict, List, Optional, Union
+
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
 
 
 class LearningCurveLogger:
@@ -84,89 +62,96 @@ class LearningCurveLogger:
     ...     "i_global": 100
     ... }
     >>> logger(metrics, "Training")
-    >>> fig = logger.plot_learning_curves(plt)
+    >>> fig = logger.plot_learning_curves()
     """
 
     def __init__(self) -> None:
-        self.logged_dict: _Dict[str, _Dict] = _defaultdict(dict)
+        self.logged_dict: Dict[str, Dict] = defaultdict(dict)
 
-        _warnings.warn(
+        warnings.warn(
             '\n"gt_label" will be removed in the future. Please use "true_class" instead.\n',
             DeprecationWarning,
         )
 
-    def __call__(self, dict_to_log: _Dict[str, _Any], step: str) -> None:
+    def __call__(self, dict_to_log: Dict[str, Any], step: str) -> None:
         """Logs metrics for a training step.
 
         Parameters
         ----------
-        dict_to_log : _Dict[str, _Any]
-            _Dictionary containing metrics to log
+        dict_to_log : Dict[str, Any]
+            Dictionary containing metrics to log
         step : str
             Phase of training ('Training', 'Validation', or 'Test')
         """
+        # Handle deprecated gt_label
         if "gt_label" in dict_to_log:
             dict_to_log["true_class"] = dict_to_log.pop("gt_label")
 
         for k_to_log in dict_to_log:
             try:
                 self.logged_dict[step][k_to_log].append(dict_to_log[k_to_log])
-            except:
+            except KeyError:
                 self.logged_dict[step][k_to_log] = [dict_to_log[k_to_log]]
 
     @property
-    def dfs(self) -> _Dict[str, _pd.DataFrame]:
+    def dfs(self) -> Dict[str, pd.DataFrame]:
         """Returns DataFrames of logged metrics.
 
         Returns
         -------
-        _Dict[str, _pd.DataFrame]
-            _Dictionary of DataFrames for each step
+        Dict[str, pd.DataFrame]
+            Dictionary of DataFrames for each step
         """
-        return self._to_dfs_pivot(
-            self.logged_dict,
-            pivot_column=None,
-        )
+        return self._to_dfs_pivot(self.logged_dict, pivot_column=None)
 
-    def get_x_of_i_epoch(self, x: str, step: str, i_epoch: int) -> _np.ndarray:
-        """Gets metric values for a specific epoch.
-
-        Parameters
-        ----------
-        x : str
-            Name of metric to retrieve
-        step : str
-            Training phase
-        i_epoch : int
-            Epoch number
+    def to_metrics_df(self) -> pd.DataFrame:
+        """Convert logged data to metrics DataFrame for plot_learning_curve.
 
         Returns
         -------
-        _np.ndarray
-            Array of metric values for specified epoch
+        pd.DataFrame
+            DataFrame with columns: step, i_global, i_epoch, i_batch, and metric columns
         """
-        indi = _np.array(self.logged_dict[step]["i_epoch"]) == i_epoch
-        x_all_arr = _np.array(self.logged_dict[step][x])
-        assert len(indi) == len(x_all_arr)
-        return x_all_arr[indi]
+        all_rows = []
+
+        for step, metrics in self.logged_dict.items():
+            n_samples = len(metrics['i_global'])
+
+            for i in range(n_samples):
+                row = {'step': step}
+                for key, values in metrics.items():
+                    # Only include scalar metrics for the DataFrame
+                    if key.endswith('_plot') or key in ['i_global', 'i_epoch', 'i_batch', 'i_fold']:
+                        row[key] = values[i]
+                all_rows.append(row)
+
+        df = pd.DataFrame(all_rows)
+
+        # Rename _plot columns
+        df.columns = [col.replace('_plot', '') if col.endswith('_plot') else col
+                      for col in df.columns]
+
+        # Ensure i_batch exists (create dummy if not logged)
+        if 'i_batch' not in df.columns:
+            df['i_batch'] = df.groupby(['step', 'i_epoch']).cumcount()
+
+        return df
 
     def plot_learning_curves(
         self,
-        plt: _Any,
-        plt_config_dict: _Optional[_Dict] = None,
-        title: _Optional[str] = None,
+        title: Optional[str] = None,
         max_n_ticks: int = 4,
         linewidth: float = 1,
-        scattersize: float = 50,
-    ) -> _matplotlib.figure.Figure:
+        scattersize: float = 3,
+        yscale: str = "linear",
+        spath: Optional[str] = None,
+    ) -> matplotlib.figure.Figure:
         """Plots learning curves from logged metrics.
 
+        Delegates to scitex.ml.plt.plot_learning_curve for consistent plotting.
+
         Parameters
         ----------
-        plt : _matplotlib.pyplot
-            _Matplotlib pyplot object
-        plt_config_dict : _Dict, optional
-            Plot configuration parameters
         title : str, optional
             Plot title
         max_n_ticks : int
@@ -175,98 +160,72 @@ class LearningCurveLogger:
             Width of plot lines
         scattersize : float
             Size of scatter points
+        yscale : str
+            Y-axis scale ('linear' or 'log')
+        spath : str, optional
+            Save path for the figure
 
         Returns
         -------
-        _matplotlib.figure.Figure
+        matplotlib.figure.Figure
             Figure containing learning curves
         """
+        from scitex.ml.plt import plot_learning_curve
 
-        if plt_config_dict is not None:
-            _plt_module.configure_mpl(plt, **plt_config_dict)
+        # Convert to metrics DataFrame
+        metrics_df = self.to_metrics_df()
 
-        self.dfs_pivot_i_global = self._to_dfs_pivot(
-            self.logged_dict, pivot_column="i_global"
-        )
-
-        COLOR_DICT = {
-            "Training": "blue",
-            "Validation": "green",
-            "Test": "red",
-        }
-
-        keys_to_plot = self._find_keys_to_plot(self.logged_dict)
+        # Find keys to plot (exclude metadata columns)
+        keys_to_plot = [
+            col for col in metrics_df.columns
+            if col not in ['step', 'i_global', 'i_epoch', 'i_batch', 'i_fold']
+        ]
 
         if len(keys_to_plot) == 0:
             # Create empty plot when no plot keys found
-            fig, axes = plt.subplots(1, 1)
-            axes.set_xlabel("Iteration#")
-            axes.set_ylabel("No metrics to plot")
-            fig.text(0.5, 0.95, title, ha="center")
+            fig, ax = plt.subplots(1, 1)
+            ax.set_xlabel("Iteration #")
+            ax.set_ylabel("No metrics to plot")
+            if title:
+                fig.text(0.5, 0.95, title, ha="center")
             return fig
 
-        fig, axes = plt.subplots(len(keys_to_plot), 1, sharex=True, sharey=False)
-        
-        # Handle both single and multiple axes cases
-        if len(keys_to_plot) == 1:
-            axes = [axes]  # Make it a list for consistent indexing
-        
-        axes[-1].set_xlabel("Iteration#")
-        fig.text(0.5, 0.95, title, ha="center")
-
-        for i_plt, plt_k in enumerate(keys_to_plot):
-            ax = axes[i_plt]
-            ax.set_ylabel(self._rename_if_key_to_plot(plt_k))
-            ax.xaxis.set_major_locator(_matplotlib.ticker.MaxNLocator(max_n_ticks))
-            ax.yaxis.set_major_locator(_matplotlib.ticker.MaxNLocator(max_n_ticks))
-
-            if _re.search("[aA][cC][cC]", plt_k):
-                ax.set_ylim(0, 1)
-                ax.set_yticks([0, 0.5, 1.0])
-
-            for step_k in self.dfs_pivot_i_global.keys():
-                if step_k == _re.search("^[Tt]rain", step_k):
-                    ax.plot(
-                        self.dfs_pivot_i_global[step_k].index,
-                        self.dfs_pivot_i_global[step_k][plt_k],
-                        label=step_k,
-                        color=_plt_module.colors.to_rgba(COLOR_DICT[step_k], alpha=0.9),
-                        linewidth=linewidth,
-                    )
-                    ax.legend()
-
-                    epoch_starts = abs(
-                        self.dfs_pivot_i_global[step_k]["i_epoch"]
-                        - self.dfs_pivot_i_global[step_k]["i_epoch"].shift(-1)
-                    )
-                    indi_global_epoch_starts = [0] + list(
-                        epoch_starts[epoch_starts == 1].index
-                    )
-
-                    for i_epoch, i_global_epoch_start in enumerate(
-                        indi_global_epoch_starts
-                    ):
-                        ax.axvline(
-                            x=i_global_epoch_start,
-                            ymin=-1e4,
-                            ymax=1e4,
-                            linestyle="--",
-                            color=_plt_module.colors.to_rgba("gray", alpha=0.5),
-                        )
-
-                if (step_k == "Validation") or (step_k == "Test"):
-                    ax.scatter(
-                        self.dfs_pivot_i_global[step_k].index,
-                        self.dfs_pivot_i_global[step_k][plt_k],
-                        label=step_k,
-                        color=_plt_module.colors.to_rgba(COLOR_DICT[step_k], alpha=0.9),
-                        s=scattersize,
-                        alpha=0.9,
-                    )
-                    ax.legend()
+        # Delegate to centralized plotting function
+        fig = plot_learning_curve(
+            metrics_df=metrics_df,
+            keys=keys_to_plot,
+            title=title or "Learning Curves",
+            max_n_ticks=max_n_ticks,
+            scattersize=scattersize,
+            linewidth=linewidth,
+            yscale=yscale,
+            spath=spath,
+        )
 
         return fig
 
+    def get_x_of_i_epoch(self, x: str, step: str, i_epoch: int) -> np.ndarray:
+        """Gets metric values for a specific epoch.
+
+        Parameters
+        ----------
+        x : str
+            Name of metric to retrieve
+        step : str
+            Training phase
+        i_epoch : int
+            Epoch number
+
+        Returns
+        -------
+        np.ndarray
+            Array of metric values for specified epoch
+        """
+        indi = np.array(self.logged_dict[step]["i_epoch"]) == i_epoch
+        x_all_arr = np.array(self.logged_dict[step][x])
+        assert len(indi) == len(x_all_arr)
+        return x_all_arr[indi]
+
     def print(self, step: str) -> None:
         """Prints metrics for given step.
 
@@ -282,48 +241,25 @@ class LearningCurveLogger:
         )
         print("\n----------------------------------------\n")
         print(f"\n{step}: (mean of batches)\n")
-        _pprint(df_pivot_i_epoch_step)
+        pprint(df_pivot_i_epoch_step)
         print("\n----------------------------------------\n")
 
     @staticmethod
-    def _find_keys_to_plot(logged_dict: _Dict) -> _List[str]:
-        """Find metrics to plot from logged dictionary.
-
-        Parameters
-        ----------
-        logged_dict : _Dict
-            _Dictionary of logged metrics
-
-        Returns
-        -------
-        _List[str]
-            _List of metric names to plot
-        """
-        for step_k in logged_dict.keys():
-            break
-
-        keys_to_plot = []
-        for k in logged_dict[step_k].keys():
-            if _re.search("_plot$", k):
-                keys_to_plot.append(k)
-        return keys_to_plot
-
-    @staticmethod
-    def _rename_if_key_to_plot(x: _Union[str, _pd.Index]) -> _Union[str, _pd.Index]:
+    def _rename_if_key_to_plot(x: Union[str, pd.Index]) -> Union[str, pd.Index]:
         """Rename metric keys for plotting.
 
         Parameters
         ----------
-        x : str or _pd.Index
+        x : str or pd.Index
             Metric name(s) to rename
 
         Returns
         -------
-        str or _pd.Index
+        str or pd.Index
             Renamed metric name(s)
         """
         if isinstance(x, str):
-            if _re.search("_plot$", x):
+            if re.search("_plot$", x):
                 return x.replace("_plot", "")
             else:
                 return x
@@ -332,31 +268,30 @@ class LearningCurveLogger:
 
     @staticmethod
     def _to_dfs_pivot(
-        logged_dict: _Dict[str, _Dict],
-        pivot_column: _Optional[str] = None,
-    ) -> _Dict[str, _pd.DataFrame]:
+        logged_dict: Dict[str, Dict],
+        pivot_column: Optional[str] = None,
+    ) -> Dict[str, pd.DataFrame]:
         """Convert logged dictionary to pivot DataFrames.
 
         Parameters
         ----------
-        logged_dict : _Dict[str, _Dict]
-            _Dictionary of logged metrics
+        logged_dict : Dict[str, Dict]
+            Dictionary of logged metrics
         pivot_column : str, optional
             Column to pivot on
 
         Returns
         -------
-        _Dict[str, _pd.DataFrame]
-            _Dictionary of pivot DataFrames
+        Dict[str, pd.DataFrame]
+            Dictionary of pivot DataFrames
         """
-
         dfs_pivot = {}
         for step_k in logged_dict.keys():
             if pivot_column is None:
-                df = _pd.DataFrame(logged_dict[step_k])
+                df = pd.DataFrame(logged_dict[step_k])
             else:
                 df = (
-                    _pd.DataFrame(logged_dict[step_k])
+                    pd.DataFrame(logged_dict[step_k])
                     .groupby(pivot_column)
                     .mean()
                     .reset_index()
@@ -366,24 +301,16 @@ class LearningCurveLogger:
         return dfs_pivot
 
 
-if __name__ == "__main__":
-    import warnings
-
-    import matplotlib.pyplot as plt
+"""Functions & Classes"""
+def main(args):
+    """Demo learning curve logger with MNIST training."""
     import torch
     import torch.nn as nn
     from sklearn.metrics import balanced_accuracy_score
     from torch.utils.data import DataLoader, TensorDataset
     from torch.utils.data.dataset import Subset
     from torchvision import datasets
-
-    import sys
-
-    ################################################################################
-    ## Sets tee
-    ################################################################################
-    sdir = scitex.io.path.mk_spath("")  # "/tmp/sdir/"
-    sys.stdout, sys.stderr = scitex.gen.tee(sys, sdir)
+    import scitex
 
     ################################################################################
     ## NN
@@ -401,33 +328,33 @@ if __name__ == "__main__":
             return x
 
     ################################################################################
-    ## Prepaires demo data
+    ## Prepare demo data
     ################################################################################
     ## Downloads
     _ds_tra_val = datasets.MNIST("/tmp/mnist", train=True, download=True)
     _ds_tes = datasets.MNIST("/tmp/mnist", train=False, download=True)
 
     ## Training-Validation splitting
-    n_samples = len(_ds_tra_val)  # n_samples is 60000
-    train_size = int(n_samples * 0.8)  # train_size is 48000
+    n_samples = len(_ds_tra_val)
+    train_size = int(n_samples * 0.8)
 
-    subset1_indices = list(range(0, train_size))  # [0,1,.....47999]
-    subset2_indices = list(range(train_size, n_samples))  # [48000,48001,.....59999]
+    subset1_indices = list(range(0, train_size))
+    subset2_indices = list(range(train_size, n_samples))
 
     _ds_tra = Subset(_ds_tra_val, subset1_indices)
     _ds_val = Subset(_ds_tra_val, subset2_indices)
 
     ## to tensors
     ds_tra = TensorDataset(
-        _ds_tra.dataset.data.to(_torch.float32),
+        _ds_tra.dataset.data.to(torch.float32),
         _ds_tra.dataset.targets,
     )
     ds_val = TensorDataset(
-        _ds_val.dataset.data.to(_torch.float32),
+        _ds_val.dataset.data.to(torch.float32),
         _ds_val.dataset.targets,
     )
     ds_tes = TensorDataset(
-        _ds_tes.data.to(_torch.float32),
+        _ds_tes.data.to(torch.float32),
         _ds_tes.targets,
     )
 
@@ -459,7 +386,7 @@ if __name__ == "__main__":
     ################################################################################
     model = Perceptron()
     loss_func = nn.CrossEntropyLoss()
-    optimizer = _torch.optim.SGD(model.parameters(), lr=1e-3)
+    optimizer = torch.optim.SGD(model.parameters(), lr=1e-3)
     softmax = nn.Softmax(dim=-1)
 
     ################################################################################
@@ -468,14 +395,12 @@ if __name__ == "__main__":
     lc_logger = LearningCurveLogger()
     i_global = 0
 
-    n_classes = len(dl_tra.dataset.tensors[1].unique())
     i_fold = 0
     max_epochs = 3
 
     for i_epoch in range(max_epochs):
         step = "Validation"
         for i_batch, batch in enumerate(dl_val):
-
             X, T = batch
             logits = model(X)
             pred_proba = softmax(logits)
@@ -490,10 +415,10 @@ if __name__ == "__main__":
                 "loss_plot": float(loss),
                 "balanced_ACC_plot": float(bACC),
                 "pred_proba": pred_proba.detach().cpu().numpy(),
-                "gt_label": T.cpu().numpy(),
-                # "true_class": T.cpu().numpy(),
+                "true_class": T.cpu().numpy(),
                 "i_fold": i_fold,
                 "i_epoch": i_epoch,
+                "i_batch": i_batch,
                 "i_global": i_global,
             }
             lc_logger(dict_to_log, step)
@@ -521,10 +446,10 @@ if __name__ == "__main__":
                 "loss_plot": float(loss),
                 "balanced_ACC_plot": float(bACC),
                 "pred_proba": pred_proba.detach().cpu().numpy(),
-                "gt_label": T.cpu().numpy(),
-                # "true_class": T.cpu().numpy(),
+                "true_class": T.cpu().numpy(),
                 "i_fold": i_fold,
                 "i_epoch": i_epoch,
+                "i_batch": i_batch,
                 "i_global": i_global,
             }
             lc_logger(dict_to_log, step)
@@ -535,7 +460,6 @@ if __name__ == "__main__":
 
     step = "Test"
     for i_batch, batch in enumerate(dl_tes):
-
         X, T = batch
         logits = model(X)
         pred_proba = softmax(logits)
@@ -550,34 +474,66 @@ if __name__ == "__main__":
             "loss_plot": float(loss),
             "balanced_ACC_plot": float(bACC),
             "pred_proba": pred_proba.detach().cpu().numpy(),
-            # "gt_label": T.cpu().numpy(),
             "true_class": T.cpu().numpy(),
             "i_fold": i_fold,
             "i_epoch": i_epoch,
+            "i_batch": i_batch,
             "i_global": i_global,
         }
         lc_logger(dict_to_log, step)
 
     lc_logger.print(step)
 
-    plt_config_dict = dict(
-        # figsize=(8.7, 10),
-        figscale=2.5,
-        labelsize=16,
-        fontsize=12,
-        legendfontsize=12,
-        tick_size=0.8,
-        tick_width=0.2,
-    )
-
+    # Plot using refactored method
     fig = lc_logger.plot_learning_curves(
-        plt,
-        plt_config_dict=plt_config_dict,
         title=f"fold#{i_fold}",
         linewidth=1,
         scattersize=50,
+        spath="learning_curve_logger_demo.jpg",
+    )
+
+    return 0
+
+
+import argparse
+def parse_args() -> argparse.Namespace:
+    """Parse command line arguments."""
+    parser = argparse.ArgumentParser(description='Demo learning curve logger')
+    return parser.parse_args()
+
+
+def run_main() -> None:
+    """Initialize scitex framework, run main function, and cleanup."""
+    global CONFIG, CC, sys, plt, rng
+
+    import sys
+    import matplotlib.pyplot as plt
+    import scitex as stx
+
+    args = parse_args()
+
+    CONFIG, sys.stdout, sys.stderr, plt, CC, rng = stx.session.start(
+        sys,
+        plt,
+        args=args,
+        file=__FILE__,
+        sdir_suffix=None,
+        verbose=False,
+        agg=True,
     )
-    fig.show()
-    # scitex.gen.save(fig, sdir + f"fold#{i_fold}.png")
+
+    exit_status = main(args)
+
+    stx.session.close(
+        CONFIG,
+        verbose=False,
+        notify=False,
+        message="",
+        exit_status=exit_status,
+    )
+
+
+if __name__ == "__main__":
+    run_main()
 
 # EOF

scitex/ml/training/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+"""Training utilities."""
+
+from ._EarlyStopping import EarlyStopping
+from ._LearningCurveLogger import LearningCurveLogger
+
+__all__ = ["EarlyStopping", "LearningCurveLogger"]