halib 0.1.99__tar.gz → 0.2.1__tar.gz

{halib-0.1.99 → halib-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.99
+Version: 0.2.1
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -53,6 +53,9 @@ Dynamic: summary
 
 # Helper package for coding and automation
 
+**Version 0.2.01**
++ `research/base_exp`: add `eval_exp` method to evaluate experiment (e.g., model evaluation on test set) after experiment running is done.
+
 **Version 0.1.99**
 + `filetype/ipynb`: add `gen_ipynb_name` generator to create file name based on current notebook name as prefix (with optional timestamp)
 

{halib-0.1.99 → halib-0.2.1}/README.md

@@ -1,5 +1,8 @@
 # Helper package for coding and automation
 
+**Version 0.2.01**
++ `research/base_exp`: add `eval_exp` method to evaluate experiment (e.g., model evaluation on test set) after experiment running is done.
+
 **Version 0.1.99**
 + `filetype/ipynb`: add `gen_ipynb_name` generator to create file name based on current notebook name as prefix (with optional timestamp)
 

halib-0.2.1/halib/research/base_exp.py

@@ -0,0 +1,157 @@
+from abc import ABC, abstractmethod
+from typing import Tuple, Any, Optional
+from ..research.base_config import ExpBaseConfig
+from ..research.perfcalc import PerfCalc
+from ..research.metrics import MetricsBackend
+
+# ! SEE https://github.com/hahv/base_exp for sample usage
+class BaseExperiment(PerfCalc, ABC):
+    """
+    Base class for experiments.
+    Orchestrates the experiment pipeline using a pluggable metrics backend.
+    """
+
+    def __init__(self, config: ExpBaseConfig):
+        self.config = config
+        self.metric_backend = None
+        # Flag to track if init_general/prepare_dataset has run
+        self._is_env_ready = False
+
+    # -----------------------
+    # PerfCalc Required Methods
+    # -----------------------
+    def get_dataset_name(self):
+        return self.config.get_dataset_cfg().get_name()
+
+    def get_experiment_name(self):
+        return self.config.get_cfg_name()
+
+    def get_metric_backend(self):
+        if not self.metric_backend:
+            self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+        return self.metric_backend
+
+    # -----------------------
+    # Abstract Experiment Steps
+    # -----------------------
+    @abstractmethod
+    def init_general(self, general_cfg):
+        """Setup general settings like SEED, logging, env variables."""
+        pass
+
+    @abstractmethod
+    def prepare_dataset(self, dataset_cfg):
+        """Load/prepare dataset."""
+        pass
+
+    @abstractmethod
+    def prepare_metrics(self, metric_cfg) -> MetricsBackend:
+        """
+        Prepare the metrics for the experiment.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def before_exec_exp_once(self, *args, **kwargs):
+        """Optional: any setup before exec_exp. Note this is called once per run_exp."""
+        pass
+
+    @abstractmethod
+    def exec_exp(self, *args, **kwargs) -> Optional[Tuple[Any, Any]]:
+        """Run experiment process, e.g.: training/evaluation loop.
+        Return: either `None` or a tuple of (raw_metrics_data, extra_data) for calc_and_save_exp_perfs
+        """
+        pass
+
+    @abstractmethod
+    def exec_eval(self, *args, **kwargs) -> Optional[Tuple[Any, Any]]:
+        """Run evaluation process.
+        Return: either `None` or a tuple of (raw_metrics_data, extra_data) for calc_and_save_exp_perfs
+        """
+        pass
+
+    # -----------------------
+    # Internal Helpers
+    # -----------------------
+    def _validate_and_unpack(self, results):
+        if results is None:
+            return None
+        if not isinstance(results, (tuple, list)) or len(results) != 2:
+            raise ValueError("exec must return (metrics_data, extra_data)")
+        return results[0], results[1]
+
+    def _prepare_environment(self, force_reload: bool = False):
+        """
+        Common setup. Skips if already initialized, unless force_reload is True.
+        """
+        if self._is_env_ready and not force_reload:
+            # Environment is already prepared, skipping setup.
+            return
+
+        # 1. Run Setup
+        self.init_general(self.config.get_general_cfg())
+        self.prepare_dataset(self.config.get_dataset_cfg())
+
+        # 2. Update metric backend (refresh if needed)
+        self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+
+        # 3. Mark as ready
+        self._is_env_ready = True
+
+    # -----------------------
+    # Main Experiment Runner
+    # -----------------------
+    def run_exp(self, should_calc_metrics=True, reload_env=False, *args, **kwargs):
+        """
+        Run the whole experiment pipeline.
+        :param reload_env: If True, forces dataset/general init to run again.
+        :param should_calc_metrics: Whether to calculate and save metrics after execution.
+        :kwargs Params:
+            + 'outfile' to save csv file results,
+            + 'outdir' to set output directory for experiment results.
+            + 'return_df' to return a DataFrame of results instead of a dictionary.
+
+        Full pipeline:
+            1. Init
+            2. Prepare Environment (General + Dataset + Metrics)
+            3. Save Config
+            4. Execute
+            5. Calculate & Save Metrics
+        """
+        self._prepare_environment(force_reload=reload_env)
+
+        # Any pre-exec setup (loading models, etc)
+        self.before_exec_exp_once(*args, **kwargs)
+        # Save config before running
+        self.config.save_to_outdir()
+
+        # Execute experiment
+        results = self.exec_exp(*args, **kwargs)
+
+        if should_calc_metrics and results is not None:
+            metrics_data, extra_data = self._validate_and_unpack(results)
+            # Calculate & Save metrics
+            perf_results = self.calc_perfs(
+                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
+            )
+            return perf_results
+        else:
+            return results
+
+    # -----------------------
+    # Main Experiment Evaluator
+    # -----------------------
+    def eval_exp(self, reload_env=False, *args, **kwargs):
+        """
+        Run evaluation only.
+        :param reload_env: If True, forces dataset/general init to run again.
+        """
+        self._prepare_environment(force_reload=reload_env)
+        results = self.exec_eval(*args, **kwargs)
+        if results is not None:
+            metrics_data, extra_data = self._validate_and_unpack(results)
+            return self.calc_perfs(
+                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
+            )
+        return None

{halib-0.1.99 → halib-0.2.1}/halib/research/dataset.py

@@ -13,8 +13,8 @@ from rich.pretty import pprint
 from torchvision.datasets import ImageFolder
 from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
 
-from ..system import filesys as fs
 from ..common import console, seed_everything, ConsoleLog
+from ..system import filesys as fs
 
 def parse_args():
     parser = ArgumentParser(description="desc text")

{halib-0.1.99 → halib-0.2.1}/halib/research/metrics.py

@@ -11,6 +11,10 @@ class MetricsBackend(ABC):
     def __init__(self, metrics_info: Union[List[str], Dict[str, Any]]):
         """
         Initialize the backend with optional metrics_info.
+        `metrics_info` can be either:
+        - A list of metric names (strings). e.g., ["accuracy", "precision"]
+        - A dict mapping metric names with object that defines how to compute them. e.g: {"accuracy": torchmetrics.Accuracy(), "precision": torchmetrics.Precision()}
+
         """
         self.metric_info = metrics_info
         self.validate_metrics_info(self.metric_info)

{halib-0.1.99 → halib-0.2.1}/halib/research/mics.py

@@ -9,7 +9,7 @@ PC_NAME_TO_ABBR = {
     "DESKTOP-5IRHU87": "MSI_Laptop",
     "DESKTOP-96HQCNO": "4090_SV",
     "DESKTOP-Q2IKLC0": "4GPU_SV",
-    "DESKTOP-QNS3DNF": "1GPU_SV"
+    "DESKTOP-QNS3DNF": "1GPU_SV",
 }
 
 DEFAULT_ABBR_WORKING_DISK = {
@@ -19,19 +19,25 @@ DEFAULT_ABBR_WORKING_DISK = {
     "4GPU_SV": "D:",
 }
 
+
 def list_PCs(show=True):
-    df = pd.DataFrame(list(PC_NAME_TO_ABBR.items()), columns=["PC Name", "Abbreviation"])
+    df = pd.DataFrame(
+        list(PC_NAME_TO_ABBR.items()), columns=["PC Name", "Abbreviation"]
+    )
     if show:
         csvfile.fn_display_df(df)
     return df
 
+
 def get_PC_name():
     return platform.node()
 
+
 def get_PC_abbr_name():
     pc_name = get_PC_name()
     return PC_NAME_TO_ABBR.get(pc_name, "Unknown")
 
+
 # ! This funcction search for full paths in the obj and normalize them according to the current platform and working disk
 # ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "windows" => "D:/zdataset/DFire"
 # ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "linux" => "/mnt/d/zdataset/DFire"

{halib-0.1.99 → halib-0.2.1}/halib/research/perfcalc.py

@@ -3,12 +3,9 @@ import glob
 from typing import Optional, Tuple
 import pandas as pd
 
-from rich.pretty import pprint
-
 from abc import ABC, abstractmethod
 from collections import OrderedDict
 
-from ..filetype import csvfile
 from ..system import filesys as fs
 from ..common import now_str
 from ..research.perftb import PerfTB
@@ -19,6 +16,7 @@ REQUIRED_COLS = ["experiment", "dataset"]
 CSV_FILE_POSTFIX = "__perf"
 METRIC_PREFIX = "metric_"
 
+
 class PerfCalc(ABC):  # Abstract base class for performance calculation
     @abstractmethod
     def get_experiment_name(self) -> str:
@@ -44,29 +42,32 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
         """
         pass
 
-    def valid_proc_extra_data(
-        self, proc_extra_data
-    ):
+    def valid_proc_extra_data(self, proc_extra_data):
         # make sure that all items in proc_extra_data are dictionaries, with same keys
         if proc_extra_data is None or len(proc_extra_data) == 0:
             return
         if not all(isinstance(item, dict) for item in proc_extra_data):
             raise TypeError("All items in proc_extra_data must be dictionaries")
 
-        if not all(item.keys() == proc_extra_data[0].keys() for item in proc_extra_data):
-            raise ValueError("All dictionaries in proc_extra_data must have the same keys")
+        if not all(
+            item.keys() == proc_extra_data[0].keys() for item in proc_extra_data
+        ):
+            raise ValueError(
+                "All dictionaries in proc_extra_data must have the same keys"
+            )
 
-    def valid_proc_metric_raw_data(
-        self, metric_names, proc_metric_raw_data
-    ):
+    def valid_proc_metric_raw_data(self, metric_names, proc_metric_raw_data):
         # make sure that all items in proc_metric_raw_data are dictionaries, with same keys as metric_names
-        assert isinstance(proc_metric_raw_data, list) and len(proc_metric_raw_data) > 0, \
-            "raw_data_for_metrics must be a non-empty list of dictionaries"
+        assert (
+            isinstance(proc_metric_raw_data, list) and len(proc_metric_raw_data) > 0
+        ), "raw_data_for_metrics must be a non-empty list of dictionaries"
 
         # make sure that all items in proc_metric_raw_data are dictionaries with keys as metric_names
         if not all(isinstance(item, dict) for item in proc_metric_raw_data):
             raise TypeError("All items in raw_data_for_metrics must be dictionaries")
-        if not all( set(item.keys()) == set(metric_names) for item in proc_metric_raw_data):
+        if not all(
+            set(item.keys()) == set(metric_names) for item in proc_metric_raw_data
+        ):
             raise ValueError(
                 "All dictionaries in raw_data_for_metrics must have the same keys as metric_names"
             )
@@ -75,21 +76,30 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
     def calc_exp_perf_metrics(
         self, metric_names, raw_metrics_data, extra_data=None, *args, **kwargs
     ):
-        assert isinstance(raw_metrics_data, dict) or isinstance(raw_metrics_data, list), \
-            "raw_data_for_metrics must be a dictionary or a list"
+        assert isinstance(raw_metrics_data, dict) or isinstance(
+            raw_metrics_data, list
+        ), "raw_data_for_metrics must be a dictionary or a list"
 
         if extra_data is not None:
-            assert isinstance(extra_data, type(raw_metrics_data)), \
-                "extra_data must be of the same type as raw_data_for_metrics (dict or list)"
+            assert isinstance(
+                extra_data, type(raw_metrics_data)
+            ), "extra_data must be of the same type as raw_data_for_metrics (dict or list)"
         # prepare raw_metric data for processing
-        proc_metric_raw_data_ls = raw_metrics_data if isinstance(raw_metrics_data, list) else [raw_metrics_data.copy()]
+        proc_metric_raw_data_ls = (
+            raw_metrics_data
+            if isinstance(raw_metrics_data, list)
+            else [raw_metrics_data.copy()]
+        )
         self.valid_proc_metric_raw_data(metric_names, proc_metric_raw_data_ls)
         # prepare extra data for processing
         proc_extra_data_ls = []
         if extra_data is not None:
-            proc_extra_data_ls = extra_data if isinstance(extra_data, list) else [extra_data.copy()]
-            assert len(proc_extra_data_ls) == len(proc_metric_raw_data_ls), \
-                "extra_data must have the same length as raw_data_for_metrics if it is a list"
+            proc_extra_data_ls = (
+                extra_data if isinstance(extra_data, list) else [extra_data.copy()]
+            )
+            assert len(proc_extra_data_ls) == len(
+                proc_metric_raw_data_ls
+            ), "extra_data must have the same length as raw_data_for_metrics if it is a list"
         # validate the extra_data
         self.valid_proc_extra_data(proc_extra_data_ls)
 
@@ -102,7 +112,7 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
                 "experiment": self.get_experiment_name(),
             }
             custom_fields = []
-            if len(proc_extra_data_ls)> 0:
+            if len(proc_extra_data_ls) > 0:
                 # add extra data to the output dictionary
                 extra_data_item = proc_extra_data_ls[idx]
                 out_dict.update(extra_data_item)
@@ -110,7 +120,9 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
             metric_results = metrics_backend.calc_metrics(
                 metrics_data_dict=raw_metrics_data, *args, **kwargs
             )
-            metric_results_prefix = {f"metric_{k}": v for k, v in metric_results.items()}
+            metric_results_prefix = {
+                f"metric_{k}": v for k, v in metric_results.items()
+            }
             out_dict.update(metric_results_prefix)
             ordered_cols = (
                 REQUIRED_COLS + custom_fields + list(metric_results_prefix.keys())
@@ -126,7 +138,7 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
     #! outfile - if provided, will save the output to a CSV file with the given path
     #! outdir - if provided, will save the output to a CSV file in the given directory with a generated filename
     #! return_df - if True, will return a DataFrame instead of a dictionary
-    def calc_and_save_exp_perfs(
+    def calc_perfs(
         self,
         raw_metrics_data: Union[List[dict], dict],
         extra_data: Optional[Union[List[dict], dict]] = None,
@@ -140,9 +152,11 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
         """
         metric_names = self.get_metric_backend().metric_names
         out_dict_list = self.calc_exp_perf_metrics(
-            metric_names=metric_names, raw_metrics_data=raw_metrics_data,
+            metric_names=metric_names,
+            raw_metrics_data=raw_metrics_data,
             extra_data=extra_data,
-            *args, **kwargs
+            *args,
+            **kwargs,
         )
         csv_outfile = kwargs.get("outfile", None)
         if csv_outfile is not None:
@@ -176,13 +190,18 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
         return "__perf.csv" in exp_file_name
 
     @classmethod
-    def gen_perf_report_for_multip_exps(
-        cls, indir: str, exp_csv_filter_fn=default_exp_csv_filter_fn, include_file_name=False, csv_sep=";"
+    def get_perftb_for_multi_exps(
+        cls,
+        indir: str,
+        exp_csv_filter_fn=default_exp_csv_filter_fn,
+        include_file_name=False,
+        csv_sep=";",
     ) -> PerfTB:
         """
         Generate a performance report by scanning experiment subdirectories.
         Must return a dictionary with keys as metric names and values as performance tables.
         """
+
         def get_df_for_all_exp_perf(csv_perf_files, csv_sep=";"):
             """
             Create a single DataFrame from all CSV files.
@@ -194,7 +213,9 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
             for csv_file in csv_perf_files:
                 temp_df = pd.read_csv(csv_file, sep=csv_sep)
                 if FILE_NAME_COL:
-                    temp_df[FILE_NAME_COL] = fs.get_file_name(csv_file, split_file_ext=False)
+                    temp_df[FILE_NAME_COL] = fs.get_file_name(
+                        csv_file, split_file_ext=False
+                    )
                     # csvfile.fn_display_df(temp_df)
                 temp_df_cols = temp_df.columns.tolist()
                 for col in temp_df_cols:
@@ -205,7 +226,9 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
             for csv_file in csv_perf_files:
                 temp_df = pd.read_csv(csv_file, sep=csv_sep)
                 if FILE_NAME_COL:
-                    temp_df[FILE_NAME_COL] = fs.get_file_name(csv_file, split_file_ext=False)
+                    temp_df[FILE_NAME_COL] = fs.get_file_name(
+                        csv_file, split_file_ext=False
+                    )
                 # Drop all-NA columns to avoid dtype inconsistency
                 temp_df = temp_df.dropna(axis=1, how="all")
                 # ensure all columns are present in the final DataFrame
@@ -215,7 +238,9 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
                 df = pd.concat([df, temp_df], ignore_index=True)
             # assert that REQUIRED_COLS are present in the DataFrame
             # pprint(df.columns.tolist())
-            sticky_cols = REQUIRED_COLS + ([FILE_NAME_COL] if include_file_name else []) # columns that must always be present
+            sticky_cols = REQUIRED_COLS + (
+                [FILE_NAME_COL] if include_file_name else []
+            )  # columns that must always be present
             for col in sticky_cols:
                 if col not in df.columns:
                     raise ValueError(

{halib-0.1.99 → halib-0.2.1}/halib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.99
+Version: 0.2.1
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -53,6 +53,9 @@ Dynamic: summary
 
 # Helper package for coding and automation
 
+**Version 0.2.01**
++ `research/base_exp`: add `eval_exp` method to evaluate experiment (e.g., model evaluation on test set) after experiment running is done.
+
 **Version 0.1.99**
 + `filetype/ipynb`: add `gen_ipynb_name` generator to create file name based on current notebook name as prefix (with optional timestamp)
 

{halib-0.1.99 → halib-0.2.1}/halib.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@ GDriveFolder.txt
 LICENSE.txt
 MANIFEST.in
 README.md
-guide_publish_pip.pdf
 setup.py
 halib/__init__.py
 halib/common.py

{halib-0.1.99 → halib-0.2.1}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt") as f:
 
 setuptools.setup(
     name="halib",
-    version="0.1.99",
+    version="0.2.01",
     author="Hoang Van Ha",
     author_email="hoangvanhauit@gmail.com",
     description="Small library for common tasks",

halib-0.1.99/halib/research/base_exp.py

@@ -1,100 +0,0 @@
-from abc import ABC, abstractmethod
-
-from ..research.base_config import ExpBaseConfig
-from ..research.perfcalc import PerfCalc
-from ..research.metrics import MetricsBackend
-
-# ! SEE https://github.com/hahv/base_exp for sample usage
-class BaseExperiment(PerfCalc, ABC):
-    """
-    Base class for experiments.
-    Orchestrates the experiment pipeline using a pluggable metrics backend.
-    """
-
-    def __init__(self, config: ExpBaseConfig):
-        self.config = config
-        self.metric_backend = None
-
-    # -----------------------
-    # PerfCalc Required Methods
-    # -----------------------
-    def get_dataset_name(self):
-        return self.config.get_dataset_cfg().get_name()
-
-    def get_experiment_name(self):
-        return self.config.get_cfg_name()
-
-    def get_metric_backend(self):
-        if not self.metric_backend:
-            self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
-        return self.metric_backend
-
-    # -----------------------
-    # Abstract Experiment Steps
-    # -----------------------
-    @abstractmethod
-    def init_general(self, general_cfg):
-        """Setup general settings like SEED, logging, env variables."""
-        pass
-
-    @abstractmethod
-    def prepare_dataset(self, dataset_cfg):
-        """Load/prepare dataset."""
-        pass
-
-    @abstractmethod
-    def prepare_metrics(self, metric_cfg) -> MetricsBackend:
-        """
-        Prepare the metrics for the experiment.
-        This method should be implemented in subclasses.
-        """
-        pass
-
-    @abstractmethod
-    def exec_exp(self, *args, **kwargs):
-        """Run experiment process, e.g.: training/evaluation loop.
-        Return: raw_metrics_data, and extra_data as input for calc_and_save_exp_perfs
-        """
-        pass
-
-    def eval_exp(self):
-        """Optional: re-run evaluation from saved results."""
-        pass
-
-    # -----------------------
-    # Main Experiment Runner
-    # -----------------------
-    def run_exp(self, do_calc_metrics=True, *args, **kwargs):
-        """
-        Run the whole experiment pipeline.
-        Params:
-            + 'outfile' to save csv file results,
-            + 'outdir' to set output directory for experiment results.
-            + 'return_df' to return a DataFrame of results instead of a dictionary.
-
-        Full pipeline:
-            1. Init
-            2. Dataset
-            3. Metrics Preparation
-            4. Save Config
-            5. Execute
-            6. Calculate & Save Metrics
-        """
-        self.init_general(self.config.get_general_cfg())
-        self.prepare_dataset(self.config.get_dataset_cfg())
-        self.prepare_metrics(self.config.get_metric_cfg())
-
-        # Save config before running
-        self.config.save_to_outdir()
-
-        # Execute experiment
-        results = self.exec_exp(*args, **kwargs)
-        if do_calc_metrics:
-            metrics_data, extra_data = results
-            # Calculate & Save metrics
-            perf_results = self.calc_and_save_exp_perfs(
-                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
-            )
-            return perf_results
-        else:
-            return results