halib 0.1.66__tar.gz → 0.1.70__tar.gz

{halib-0.1.66 → halib-0.1.70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.66
+Version: 0.1.70
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -52,7 +52,12 @@ Dynamic: summary
 
 Helper package for coding and automation
 
-**Version 0.1.66**
+**Version 0.1.70**
+
++ `research/base_exp`: add base experiment class to handle common experiment tasks, including performance calculation and saving results.
+
+
+**Version 0.1.67**
 
 + now use `uv` for venv management
 + `research/perfcalc`: support both torchmetrics and custom metrics for performance calculation

{halib-0.1.66 → halib-0.1.70}/README.md

@@ -1,6 +1,11 @@
 Helper package for coding and automation
 
-**Version 0.1.66**
+**Version 0.1.70**
+
++ `research/base_exp`: add base experiment class to handle common experiment tasks, including performance calculation and saving results.
+
+
+**Version 0.1.67**
 
 + now use `uv` for venv management
 + `research/perfcalc`: support both torchmetrics and custom metrics for performance calculation

halib-0.1.70/halib/research/base_config.py

@@ -0,0 +1,100 @@
+import os
+from rich.pretty import pprint
+from abc import ABC, abstractmethod
+from dataclass_wizard import YAMLWizard
+
+
+class NamedConfig(ABC):
+    """
+    Base class for named configurations.
+    All configurations should have a name.
+    """
+
+    @abstractmethod
+    def get_name(self):
+        """
+        Get the name of the configuration.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+
+class ExpBaseConfig(ABC, YAMLWizard):
+    """
+    Base class for configuration objects.
+    What a cfg class must have:
+    1 - a dataset cfg
+    2 - a metric cfg
+    3 - a method cfg
+    """
+
+    # Save to yaml fil
+    def save_to_outdir(
+        self, filename: str = "config.yaml", outdir=None, override: bool = False
+    ) -> None:
+        """
+        Save the configuration to the output directory.
+        """
+        if outdir is not None:
+            output_dir = outdir
+        else:
+            output_dir = self.get_outdir()
+        os.makedirs(output_dir, exist_ok=True)
+        assert (output_dir is not None) and (
+            os.path.isdir(output_dir)
+        ), f"Output directory '{output_dir}' does not exist or is not a directory."
+        file_path = os.path.join(output_dir, filename)
+        if os.path.exists(file_path) and not override:
+            pprint(
+                f"File '{file_path}' already exists. Use 'override=True' to overwrite."
+            )
+        else:
+            # method of YAMLWizard to_yaml_file
+            self.to_yaml_file(file_path)
+
+    @classmethod
+    @abstractmethod
+    # load from a custom YAML file
+    def from_custom_yaml_file(cls, yaml_file: str):
+        """Load a configuration from a custom YAML file."""
+        pass
+
+    @abstractmethod
+    def get_cfg_name(self):
+        """
+        Get the name of the configuration.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def get_outdir(self):
+        """
+        Get the output directory for the configuration.
+        This method should be implemented in subclasses.
+        """
+        return None
+
+    @abstractmethod
+    def get_general_cfg(self):
+        """
+        Get the general configuration like output directory, log settings, SEED, etc.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def get_dataset_cfg(self) -> NamedConfig:
+        """
+        Get the dataset configuration.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def get_metric_cfg(self) -> NamedConfig:
+        """
+        Get the metric configuration.
+        This method should be implemented in subclasses.
+        """
+        pass

halib-0.1.70/halib/research/base_exp.py

@@ -0,0 +1,99 @@
+from abc import ABC, abstractmethod
+
+from base_config import ExpBaseConfig
+from perfcalc import PerfCalc
+from metrics import MetricsBackend
+
+# ! SEE https://github.com/hahv/base_exp for sample usage
+class BaseExperiment(PerfCalc, ABC):
+    """
+    Base class for experiments.
+    Orchestrates the experiment pipeline using a pluggable metrics backend.
+    """
+
+    def __init__(self, config: ExpBaseConfig):
+        self.config = config
+        self.metric_backend = None
+
+    # -----------------------
+    # PerfCalc Required Methods
+    # -----------------------
+    def get_dataset_name(self):
+        return self.config.get_dataset_cfg().get_name()
+
+    def get_experiment_name(self):
+        return self.config.get_cfg_name()
+
+    def get_metric_backend(self):
+        if not self.metric_backend:
+            self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+        return self.metric_backend
+
+    # -----------------------
+    # Abstract Experiment Steps
+    # -----------------------
+    @abstractmethod
+    def init_general(self, general_cfg):
+        """Setup general settings like SEED, logging, env variables."""
+        pass
+
+    @abstractmethod
+    def prepare_dataset(self, dataset_cfg):
+        """Load/prepare dataset."""
+        pass
+
+    @abstractmethod
+    def prepare_metrics(self, metric_cfg) -> MetricsBackend:
+        """
+        Prepare the metrics for the experiment.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def exec_exp(self, *args, **kwargs):
+        """Run experiment process, e.g.: training/evaluation loop.
+        Return: raw_metrics_data, and extra_data as input for calc_and_save_exp_perfs
+        """
+        pass
+
+    def eval_exp(self):
+        """Optional: re-run evaluation from saved results."""
+        pass
+
+    # -----------------------
+    # Main Experiment Runner
+    # -----------------------
+    def run_exp(self, do_calc_metrics=True, *args, **kwargs):
+        """
+        Run the whole experiment pipeline.
+        Params:
+            + 'outfile' to save csv file results,
+            + 'outdir' to set output directory for experiment results.
+            + 'return_df' to return a DataFrame of results instead of a dictionary.
+
+        Full pipeline:
+            1. Init
+            2. Dataset
+            3. Metrics Preparation
+            4. Save Config
+            5. Execute
+            6. Calculate & Save Metrics
+        """
+        self.init_general(self.config.get_general_cfg())
+        self.prepare_dataset(self.config.get_dataset_cfg())
+
+        # Save config before running
+        self.config.save_to_outdir()
+
+        # Execute experiment
+        results = self.exec_exp(*args, **kwargs)
+        if do_calc_metrics:
+            metrics_data, extra_data = results
+            # Calculate & Save metrics
+            perf_results = self.calc_and_save_exp_perfs(
+                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
+            )
+            return perf_results
+        else:
+            return results

{halib-0.1.66 → halib-0.1.70}/halib/research/metrics.py

@@ -44,6 +44,15 @@ class MetricsBackend(ABC):
     ) -> Dict[str, Any]:
         pass
 
+    def prepare_metrics_backend_data(
+        self, raw_metric_data, *args, **kwargs
+    ):
+        """
+        Prepare the data for the metrics backend.
+        This method can be overridden by subclasses to customize data preparation.
+        """
+        return raw_metric_data
+
     def calc_metrics(
         self,  metrics_data_dict: Dict[str, Any], *args, **kwargs
     ) -> Dict[str, Any]:
@@ -55,6 +64,10 @@ class MetricsBackend(ABC):
         for metric in self.metric_names:
             if metric not in metrics_data_dict:
                 raise ValueError(f"Metric '{metric}' not found in provided data.")
+        # Prepare the data for the backend
+        metrics_data_dict = self.prepare_metrics_backend_data(
+            metrics_data_dict, *args, **kwargs
+        )
         # Call the abstract method to compute metrics
         return self.compute_metrics(self.metric_info, metrics_data_dict, *args, **kwargs)
 

{halib-0.1.66 → halib-0.1.70}/halib/research/perfcalc.py

@@ -1,35 +1,27 @@
 import os
 import glob
-import inspect
+from typing import Optional, Tuple
 import pandas as pd
 
-from typing import Dict
-from functools import wraps
 from rich.pretty import pprint
 
 from abc import ABC, abstractmethod
 from collections import OrderedDict
 
 from ..filetype import csvfile
+from ..system import filesys as fs
 from ..common import now_str
 from ..research.perftb import PerfTB
 from ..research.metrics import *
 
-# # try to import torch, and torchmetrics
-# try:
-#     import torch
-#     import torchmetrics
-#     from torchmetrics import Metric
-# except ImportError:
-#     raise ImportError("Please install torch and torchmetrics to use this module.")
-
 
 REQUIRED_COLS = ["experiment", "dataset"]
 CSV_FILE_POSTFIX = "__perf"
+METRIC_PREFIX = "metric_"
 
-class PerfCalc(ABC): # Abstract base class for performance calculation
+class PerfCalc(ABC):  # Abstract base class for performance calculation
     @abstractmethod
-    def get_experiment_name(self):
+    def get_experiment_name(self) -> str:
         """
         Return the name of the experiment.
         This function should be overridden by the subclass if needed.
@@ -37,7 +29,7 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
         pass
 
     @abstractmethod
-    def get_dataset_name(self):
+    def get_dataset_name(self) -> str:
         """
         Return the name of the dataset.
         This function should be overridden by the subclass if needed.
@@ -52,96 +44,128 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
         """
         pass
 
-    # ! can be override, but ONLY if torchmetrics are used
-    # Prepare the exp data for torch metrics.
-    @abstractmethod
-    def prepare_metrics_data_dict(self, metric_names, *args, **kwargs):
-        """
-        Prepare the data for metrics.
-        This function should be overridden by the subclass if needed.
-        Must return a dictionary with keys as metric names and values as the data to be used for those metrics.
-        NOTE: that the data (for each metric) must be in the format expected by the torchmetrics instance (for that metric). E.g: {"accuracy": {"preds": [...], "target": [...]}, ...} since torchmetrics expects the data in a specific format.
-        """
-        pass
+    def valid_proc_extra_data(
+        self, proc_extra_data
+    ):
+        # make sure that all items in proc_extra_data are dictionaries, with same keys
+        if proc_extra_data is None or len(proc_extra_data) == 0:
+            return
+        if not all(isinstance(item, dict) for item in proc_extra_data):
+            raise TypeError("All items in proc_extra_data must be dictionaries")
 
-    def calc_exp_outdict_custom_fields(self, outdict, *args, **kwargs):
-        """Can be overridden by the subclass to add custom fields to the output dictionary.
-        ! must return the modified outdict, and a ordered list of custom fields to be added to the output dictionary.
-        """
-        return outdict, []
-
-    def __valid_calc_custom_fields(self, fun_results):
-        if not isinstance(fun_results, tuple) or len(fun_results) != 2:
-            raise ValueError("Function must return a tuple (outdict, custom_fields)")
-        outdict, custom_fields = fun_results
-        if not isinstance(outdict, dict):
-            raise TypeError("Output must be a dictionary")
-        if not isinstance(custom_fields, list):
-            raise TypeError("Custom fields must be a list")
-        for field in custom_fields:
-            if not isinstance(field, str):
-                raise TypeError(f"Custom field '{field}' is not a string")
-        return outdict, custom_fields
+        if not all(item.keys() == proc_extra_data[0].keys() for item in proc_extra_data):
+            raise ValueError("All dictionaries in proc_extra_data must have the same keys")
+
+    def valid_proc_metric_raw_data(
+        self, metric_names, proc_metric_raw_data
+    ):
+        # make sure that all items in proc_metric_raw_data are dictionaries, with same keys as metric_names
+        assert isinstance(proc_metric_raw_data, list) and len(proc_metric_raw_data) > 0, \
+            "raw_data_for_metrics must be a non-empty list of dictionaries"
+
+        # make sure that all items in proc_metric_raw_data are dictionaries with keys as metric_names
+        if not all(isinstance(item, dict) for item in proc_metric_raw_data):
+            raise TypeError("All items in raw_data_for_metrics must be dictionaries")
+        if not all( set(item.keys()) == set(metric_names) for item in proc_metric_raw_data):
+            raise ValueError(
+                "All dictionaries in raw_data_for_metrics must have the same keys as metric_names"
+            )
 
     # ! only need to override this method if torchmetrics are not used
-    def calc_exp_perf_metrics(self, metric_names, *args, **kwargs):
+    def calc_exp_perf_metrics(
+        self, metric_names, raw_metrics_data, extra_data=None, *args, **kwargs
+    ):
+        assert isinstance(raw_metrics_data, dict) or isinstance(raw_metrics_data, list), \
+            "raw_data_for_metrics must be a dictionary or a list"
+
+        if extra_data is not None:
+            assert isinstance(extra_data, type(raw_metrics_data)), \
+                "extra_data must be of the same type as raw_data_for_metrics (dict or list)"
+        # prepare raw_metric data for processing
+        proc_metric_raw_data_ls = raw_metrics_data if isinstance(raw_metrics_data, list) else [raw_metrics_data.copy()]
+        self.valid_proc_metric_raw_data(metric_names, proc_metric_raw_data_ls)
+        # prepare extra data for processing
+        proc_extra_data_ls = []
+        if extra_data is not None:
+            proc_extra_data_ls = extra_data if isinstance(extra_data, list) else [extra_data.copy()]
+            assert len(proc_extra_data_ls) == len(proc_metric_raw_data_ls), \
+                "extra_data must have the same length as raw_data_for_metrics if it is a list"
+        # validate the extra_data
+        self.valid_proc_extra_data(proc_extra_data_ls)
+
+        # calculate the metrics output results
         metrics_backend = self.get_metric_backend()
-        out_dict = {"dataset": self.get_dataset_name(), "experiment": self.get_experiment_name()}
-        out_dict, custom_fields = self.__valid_calc_custom_fields(self.calc_exp_outdict_custom_fields(
-            outdict=out_dict, *args, **kwargs
-        ))
-        metrics_data_dict = self.prepare_metrics_data_dict(
-            metric_names, *args, **kwargs
-        )
-        metric_results = metrics_backend.calc_metrics(
-            metrics_data_dict=metrics_data_dict, *args, **kwargs
-        )
-        metric_results_prefix = {
-            f"metric_{k}": v for k, v in metric_results.items()
-        }
-        out_dict.update(metric_results_prefix)
-        ordered_cols = REQUIRED_COLS + custom_fields + list(metric_results_prefix.keys())
-        out_dict = OrderedDict(
-            (col, out_dict[col]) for col in ordered_cols if col in out_dict
-        )
-        return out_dict
+        proc_outdict_list = []
+        for idx, raw_metrics_data in enumerate(proc_metric_raw_data_ls):
+            out_dict = {
+                "dataset": self.get_dataset_name(),
+                "experiment": self.get_experiment_name(),
+            }
+            custom_fields = []
+            if len(proc_extra_data_ls)> 0:
+                # add extra data to the output dictionary
+                extra_data_item = proc_extra_data_ls[idx]
+                out_dict.update(extra_data_item)
+                custom_fields = list(extra_data_item.keys())
+            metric_results = metrics_backend.calc_metrics(
+                metrics_data_dict=raw_metrics_data, *args, **kwargs
+            )
+            metric_results_prefix = {f"metric_{k}": v for k, v in metric_results.items()}
+            out_dict.update(metric_results_prefix)
+            ordered_cols = (
+                REQUIRED_COLS + custom_fields + list(metric_results_prefix.keys())
+            )
+            out_dict = OrderedDict(
+                (col, out_dict[col]) for col in ordered_cols if col in out_dict
+            )
+            proc_outdict_list.append(out_dict)
+
+        return proc_outdict_list
 
     #! custom kwargs:
     #! outfile - if provided, will save the output to a CSV file with the given path
     #! outdir - if provided, will save the output to a CSV file in the given directory with a generated filename
     #! return_df - if True, will return a DataFrame instead of a dictionary
-
-    def calc_save_exp_perfs(self, *args, **kwargs):
+    def calc_and_save_exp_perfs(
+        self,
+        raw_metrics_data: Union[List[dict], dict],
+        extra_data: Optional[Union[List[dict], dict]] = None,
+        *args,
+        **kwargs,
+    ) -> Tuple[Union[List[OrderedDict], pd.DataFrame], Optional[str]]:
         """
         Calculate the metrics.
         This function should be overridden by the subclass if needed.
         Must return a dictionary with keys as metric names and values as the calculated metrics.
         """
         metric_names = self.get_metric_backend().metric_names
-        out_dict = self.calc_exp_perf_metrics(metric_names=metric_names, *args, **kwargs)
+        out_dict_list = self.calc_exp_perf_metrics(
+            metric_names=metric_names, raw_metrics_data=raw_metrics_data,
+            extra_data=extra_data,
+            *args, **kwargs
+        )
         csv_outfile = kwargs.get("outfile", None)
         if csv_outfile is not None:
             filePathNoExt, _ = os.path.splitext(csv_outfile)
             # pprint(f"CSV Outfile Path (No Ext): {filePathNoExt}")
-            csv_outfile = f'{filePathNoExt}{CSV_FILE_POSTFIX}.csv'
+            csv_outfile = f"{filePathNoExt}{CSV_FILE_POSTFIX}.csv"
         elif "outdir" in kwargs:
             csvoutdir = kwargs["outdir"]
             csvfilename = f"{now_str()}_{self.get_dataset_name()}_{self.get_experiment_name()}_{CSV_FILE_POSTFIX}.csv"
             csv_outfile = os.path.join(csvoutdir, csvfilename)
 
         # convert out_dict to a DataFrame
-        df = pd.DataFrame([out_dict])
+        df = pd.DataFrame(out_dict_list)
         # get the orders of the columns as the orders or the keys in out_dict
-        ordered_cols = list(out_dict.keys())
+        ordered_cols = list(out_dict_list[0].keys())
         df = df[ordered_cols]  # reorder columns
-
         if csv_outfile:
             df.to_csv(csv_outfile, index=False, sep=";", encoding="utf-8")
         return_df = kwargs.get("return_df", False)
-        if return_df: # return DataFrame instead of dict if requested
+        if return_df:  # return DataFrame instead of dict if requested
             return df, csv_outfile
         else:
-            return out_dict, csv_outfile
+            return out_dict_list, csv_outfile
 
     @staticmethod
     def default_exp_csv_filter_fn(exp_file_name: str) -> bool:
@@ -153,29 +177,37 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
 
     @classmethod
     def gen_perf_report_for_multip_exps(
-        cls, indir: str, exp_csv_filter_fn=default_exp_csv_filter_fn, csv_sep=";"
+        cls, indir: str, exp_csv_filter_fn=default_exp_csv_filter_fn, include_file_name=False, csv_sep=";"
     ) -> PerfTB:
         """
         Generate a performance report by scanning experiment subdirectories.
         Must return a dictionary with keys as metric names and values as performance tables.
         """
-        def get_df_for_all_exp_perf(csv_perf_files, csv_sep=';'):
+        def get_df_for_all_exp_perf(csv_perf_files, csv_sep=";"):
             """
             Create a single DataFrame from all CSV files.
             Assumes all CSV files MAY have different metrics
             """
             cols = []
+            FILE_NAME_COL = "file_name" if include_file_name else None
+
             for csv_file in csv_perf_files:
                 temp_df = pd.read_csv(csv_file, sep=csv_sep)
+                if FILE_NAME_COL:
+                    temp_df[FILE_NAME_COL] = fs.get_file_name(csv_file, split_file_ext=False)
+                    # csvfile.fn_display_df(temp_df)
                 temp_df_cols = temp_df.columns.tolist()
                 for col in temp_df_cols:
                     if col not in cols:
                         cols.append(col)
+
             df = pd.DataFrame(columns=cols)
             for csv_file in csv_perf_files:
                 temp_df = pd.read_csv(csv_file, sep=csv_sep)
+                if FILE_NAME_COL:
+                    temp_df[FILE_NAME_COL] = fs.get_file_name(csv_file, split_file_ext=False)
                 # Drop all-NA columns to avoid dtype inconsistency
-                temp_df = temp_df.dropna(axis=1, how='all')
+                temp_df = temp_df.dropna(axis=1, how="all")
                 # ensure all columns are present in the final DataFrame
                 for col in cols:
                     if col not in temp_df.columns:
@@ -183,27 +215,36 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
                 df = pd.concat([df, temp_df], ignore_index=True)
             # assert that REQUIRED_COLS are present in the DataFrame
             # pprint(df.columns.tolist())
-            for col in REQUIRED_COLS:
+            sticky_cols = REQUIRED_COLS + ([FILE_NAME_COL] if include_file_name else []) # columns that must always be present
+            for col in sticky_cols:
                 if col not in df.columns:
-                    raise ValueError(f"Required column '{col}' is missing from the DataFrame. REQUIRED_COLS = {REQUIRED_COLS}")
-            metric_cols = [col for col in df.columns if col.startswith('metric_')]
-            assert len(metric_cols) > 0, "No metric columns found in the DataFrame. Ensure that the CSV files contain metric columns starting with 'metric_'."
-            final_cols = REQUIRED_COLS + metric_cols
+                    raise ValueError(
+                        f"Required column '{col}' is missing from the DataFrame. REQUIRED_COLS = {sticky_cols}"
+                    )
+            metric_cols = [col for col in df.columns if col.startswith(METRIC_PREFIX)]
+            assert (
+                len(metric_cols) > 0
+            ), "No metric columns found in the DataFrame. Ensure that the CSV files contain metric columns starting with 'metric_'."
+            final_cols = sticky_cols + metric_cols
             df = df[final_cols]
             # !hahv debug
-            pprint('------ Final DataFrame Columns ------')
+            pprint("------ Final DataFrame Columns ------")
             csvfile.fn_display_df(df)
             # ! validate all rows in df before returning
             # make sure all rows will have at least values for REQUIRED_COLS and at least one metric column
             for index, row in df.iterrows():
-                if not all(col in row and pd.notna(row[col]) for col in REQUIRED_COLS):
-                    raise ValueError(f"Row {index} is missing required columns or has NaN values in required columns: {row}")
+                if not all(col in row and pd.notna(row[col]) for col in sticky_cols):
+                    raise ValueError(
+                        f"Row {index} is missing required columns or has NaN values in required columns: {row}"
+                    )
                 if not any(pd.notna(row[col]) for col in metric_cols):
                     raise ValueError(f"Row {index} has no metric values: {row}")
             # make sure these is no (experiment, dataset) pair that is duplicated
-            duplicates = df.duplicated(subset=['experiment', 'dataset'], keep=False)
+            duplicates = df.duplicated(subset=sticky_cols, keep=False)
             if duplicates.any():
-                raise ValueError("Duplicate (experiment, dataset) pairs found in the DataFrame. Please ensure that each experiment-dataset combination is unique.")
+                raise ValueError(
+                    "Duplicate (experiment, dataset) pairs found in the DataFrame. Please ensure that each experiment-dataset combination is unique."
+                )
             return df
 
         def mk_perftb_report(df):
@@ -213,9 +254,9 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
             """
             perftb = PerfTB()
             # find all "dataset" values (unique)
-            dataset_names = list(df['dataset'].unique())
-            # find all columns that start with "metric_"
-            metric_cols = [col for col in df.columns if col.startswith('metric_')]
+            dataset_names = list(df["dataset"].unique())
+            # find all columns that start with METRIC_PREFIX
+            metric_cols = [col for col in df.columns if col.startswith(METRIC_PREFIX)]
 
             # Determine which metrics are associated with each dataset.
             # Since a dataset may appear in multiple rows and may not include all metrics in each, identify the row with the same dataset that contains the most non-NaN metric values. The set of metrics for that dataset is defined by the non-NaN metrics in that row.
@@ -225,7 +266,11 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
                 dataset_rows = df[df["dataset"] == dataset_name]
                 # Find the row with the most non-NaN metric values
                 max_non_nan_row = dataset_rows[metric_cols].count(axis=1).idxmax()
-                metrics_for_dataset = dataset_rows.loc[max_non_nan_row, metric_cols].dropna().index.tolist()
+                metrics_for_dataset = (
+                    dataset_rows.loc[max_non_nan_row, metric_cols]
+                    .dropna()
+                    .index.tolist()
+                )
                 dataset_metrics[dataset_name] = metrics_for_dataset
 
             for dataset_name, metrics in dataset_metrics.items():
@@ -233,11 +278,11 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
                 perftb.add_dataset(dataset_name, metrics)
 
             for _, row in df.iterrows():
-                dataset_name = row['dataset']
+                dataset_name = row["dataset"]
                 ds_metrics = dataset_metrics.get(dataset_name)
                 if dataset_name in dataset_metrics:
                     # Add the metrics for this row to the performance table
-                    exp_name = row.get('experiment')
+                    exp_name = row.get("experiment")
                     exp_metric_values = {}
                     for metric in ds_metrics:
                         if metric in row and pd.notna(row[metric]):
@@ -245,7 +290,7 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
                     perftb.add_experiment(
                         experiment_name=exp_name,
                         dataset_name=dataset_name,
-                        metrics=exp_metric_values
+                        metrics=exp_metric_values,
                     )
 
             return perftb
@@ -260,9 +305,7 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
             if os.path.isdir(os.path.join(indir, d))
         ]
         if len(exp_dirs) == 0:
-            csv_perf_files = glob.glob(
-                os.path.join(indir, f"*.csv")
-            )
+            csv_perf_files = glob.glob(os.path.join(indir, f"*.csv"))
             csv_perf_files = [
                 file_item
                 for file_item in csv_perf_files
@@ -273,13 +316,9 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
             # Collect all matching CSV files in those subdirs
             for exp_dir in exp_dirs:
                 # pprint(f"Searching in experiment directory: {exp_dir}")
-                matched = glob.glob(
-                    os.path.join(exp_dir, f"*.csv")
-                )
+                matched = glob.glob(os.path.join(exp_dir, f"*.csv"))
                 matched = [
-                    file_item
-                    for file_item in matched
-                    if exp_csv_filter_fn(file_item)
+                    file_item for file_item in matched if exp_csv_filter_fn(file_item)
                 ]
                 csv_perf_files.extend(matched)
 
@@ -287,9 +326,11 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
             len(csv_perf_files) > 0
         ), f"No CSV files matching pattern '{exp_csv_filter_fn}' found in the experiment directories."
 
-        assert len(csv_perf_files) > 0, f"No CSV files matching pattern '{exp_csv_filter_fn}' found in the experiment directories."
+        assert (
+            len(csv_perf_files) > 0
+        ), f"No CSV files matching pattern '{exp_csv_filter_fn}' found in the experiment directories."
 
         all_exp_perf_df = get_df_for_all_exp_perf(csv_perf_files, csv_sep=csv_sep)
-        csvfile.fn_display_df(all_exp_perf_df)
+        # csvfile.fn_display_df(all_exp_perf_df)
         perf_tb = mk_perftb_report(all_exp_perf_df)
         return perf_tb

{halib-0.1.66 → halib-0.1.70}/halib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.66
+Version: 0.1.70
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -52,7 +52,12 @@ Dynamic: summary
 
 Helper package for coding and automation
 
-**Version 0.1.66**
+**Version 0.1.70**
+
++ `research/base_exp`: add base experiment class to handle common experiment tasks, including performance calculation and saving results.
+
+
+**Version 0.1.67**
 
 + now use `uv` for venv management
 + `research/perfcalc`: support both torchmetrics and custom metrics for performance calculation

{halib-0.1.66 → halib-0.1.70}/halib.egg-info/SOURCES.txt

@@ -26,6 +26,8 @@ halib/online/gdrive_mkdir.py
 halib/online/gdrive_test.py
 halib/online/projectmake.py
 halib/research/__init__.py
+halib/research/base_config.py
+halib/research/base_exp.py
 halib/research/dataset.py
 halib/research/metrics.py
 halib/research/perfcalc.py

{halib-0.1.66 → halib-0.1.70}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt") as f:
 
 setuptools.setup(
     name="halib",
-    version="0.1.66",
+    version="0.1.70",
     author="Hoang Van Ha",
     author_email="hoangvanhauit@gmail.com",
     description="Small library for common tasks",