halib 0.1.99__py3-none-any.whl → 0.2.1__py3-none-any.whl

halib/research/base_exp.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-
+from typing import Tuple, Any, Optional
 from ..research.base_config import ExpBaseConfig
 from ..research.perfcalc import PerfCalc
 from ..research.metrics import MetricsBackend
@@ -14,6 +14,8 @@ class BaseExperiment(PerfCalc, ABC):
     def __init__(self, config: ExpBaseConfig):
         self.config = config
         self.metric_backend = None
+        # Flag to track if init_general/prepare_dataset has run
+        self._is_env_ready = False
 
     # -----------------------
     # PerfCalc Required Methods
@@ -51,50 +53,105 @@ class BaseExperiment(PerfCalc, ABC):
         pass
 
     @abstractmethod
-    def exec_exp(self, *args, **kwargs):
+    def before_exec_exp_once(self, *args, **kwargs):
+        """Optional: any setup before exec_exp. Note this is called once per run_exp."""
+        pass
+
+    @abstractmethod
+    def exec_exp(self, *args, **kwargs) -> Optional[Tuple[Any, Any]]:
         """Run experiment process, e.g.: training/evaluation loop.
-        Return: raw_metrics_data, and extra_data as input for calc_and_save_exp_perfs
+        Return: either `None` or a tuple of (raw_metrics_data, extra_data) for calc_and_save_exp_perfs
         """
         pass
 
-    def eval_exp(self):
-        """Optional: re-run evaluation from saved results."""
+    @abstractmethod
+    def exec_eval(self, *args, **kwargs) -> Optional[Tuple[Any, Any]]:
+        """Run evaluation process.
+        Return: either `None` or a tuple of (raw_metrics_data, extra_data) for calc_and_save_exp_perfs
+        """
         pass
 
+    # -----------------------
+    # Internal Helpers
+    # -----------------------
+    def _validate_and_unpack(self, results):
+        if results is None:
+            return None
+        if not isinstance(results, (tuple, list)) or len(results) != 2:
+            raise ValueError("exec must return (metrics_data, extra_data)")
+        return results[0], results[1]
+
+    def _prepare_environment(self, force_reload: bool = False):
+        """
+        Common setup. Skips if already initialized, unless force_reload is True.
+        """
+        if self._is_env_ready and not force_reload:
+            # Environment is already prepared, skipping setup.
+            return
+
+        # 1. Run Setup
+        self.init_general(self.config.get_general_cfg())
+        self.prepare_dataset(self.config.get_dataset_cfg())
+
+        # 2. Update metric backend (refresh if needed)
+        self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+
+        # 3. Mark as ready
+        self._is_env_ready = True
+
     # -----------------------
     # Main Experiment Runner
     # -----------------------
-    def run_exp(self, do_calc_metrics=True, *args, **kwargs):
+    def run_exp(self, should_calc_metrics=True, reload_env=False, *args, **kwargs):
         """
         Run the whole experiment pipeline.
-        Params:
+        :param reload_env: If True, forces dataset/general init to run again.
+        :param should_calc_metrics: Whether to calculate and save metrics after execution.
+        :kwargs Params:
             + 'outfile' to save csv file results,
             + 'outdir' to set output directory for experiment results.
             + 'return_df' to return a DataFrame of results instead of a dictionary.
 
         Full pipeline:
             1. Init
-            2. Dataset
-            3. Metrics Preparation
-            4. Save Config
-            5. Execute
-            6. Calculate & Save Metrics
+            2. Prepare Environment (General + Dataset + Metrics)
+            3. Save Config
+            4. Execute
+            5. Calculate & Save Metrics
         """
-        self.init_general(self.config.get_general_cfg())
-        self.prepare_dataset(self.config.get_dataset_cfg())
-        self.prepare_metrics(self.config.get_metric_cfg())
+        self._prepare_environment(force_reload=reload_env)
 
+        # Any pre-exec setup (loading models, etc)
+        self.before_exec_exp_once(*args, **kwargs)
         # Save config before running
         self.config.save_to_outdir()
 
         # Execute experiment
         results = self.exec_exp(*args, **kwargs)
-        if do_calc_metrics:
-            metrics_data, extra_data = results
+
+        if should_calc_metrics and results is not None:
+            metrics_data, extra_data = self._validate_and_unpack(results)
             # Calculate & Save metrics
-            perf_results = self.calc_and_save_exp_perfs(
+            perf_results = self.calc_perfs(
                 raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
             )
             return perf_results
         else:
             return results
+
+    # -----------------------
+    # Main Experiment Evaluator
+    # -----------------------
+    def eval_exp(self, reload_env=False, *args, **kwargs):
+        """
+        Run evaluation only.
+        :param reload_env: If True, forces dataset/general init to run again.
+        """
+        self._prepare_environment(force_reload=reload_env)
+        results = self.exec_eval(*args, **kwargs)
+        if results is not None:
+            metrics_data, extra_data = self._validate_and_unpack(results)
+            return self.calc_perfs(
+                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
+            )
+        return None

halib/research/dataset.py

@@ -13,8 +13,8 @@ from rich.pretty import pprint
 from torchvision.datasets import ImageFolder
 from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
 
-from ..system import filesys as fs
 from ..common import console, seed_everything, ConsoleLog
+from ..system import filesys as fs
 
 def parse_args():
     parser = ArgumentParser(description="desc text")

halib/research/metrics.py

@@ -11,6 +11,10 @@ class MetricsBackend(ABC):
     def __init__(self, metrics_info: Union[List[str], Dict[str, Any]]):
         """
         Initialize the backend with optional metrics_info.
+        `metrics_info` can be either:
+        - A list of metric names (strings). e.g., ["accuracy", "precision"]
+        - A dict mapping metric names with object that defines how to compute them. e.g: {"accuracy": torchmetrics.Accuracy(), "precision": torchmetrics.Precision()}
+
         """
         self.metric_info = metrics_info
         self.validate_metrics_info(self.metric_info)

halib/research/mics.py

@@ -9,7 +9,7 @@ PC_NAME_TO_ABBR = {
     "DESKTOP-5IRHU87": "MSI_Laptop",
     "DESKTOP-96HQCNO": "4090_SV",
     "DESKTOP-Q2IKLC0": "4GPU_SV",
-    "DESKTOP-QNS3DNF": "1GPU_SV"
+    "DESKTOP-QNS3DNF": "1GPU_SV",
 }
 
 DEFAULT_ABBR_WORKING_DISK = {
@@ -19,19 +19,25 @@ DEFAULT_ABBR_WORKING_DISK = {
     "4GPU_SV": "D:",
 }
 
+
 def list_PCs(show=True):
-    df = pd.DataFrame(list(PC_NAME_TO_ABBR.items()), columns=["PC Name", "Abbreviation"])
+    df = pd.DataFrame(
+        list(PC_NAME_TO_ABBR.items()), columns=["PC Name", "Abbreviation"]
+    )
     if show:
         csvfile.fn_display_df(df)
     return df
 
+
 def get_PC_name():
     return platform.node()
 
+
 def get_PC_abbr_name():
     pc_name = get_PC_name()
     return PC_NAME_TO_ABBR.get(pc_name, "Unknown")
 
+
 # ! This funcction search for full paths in the obj and normalize them according to the current platform and working disk
 # ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "windows" => "D:/zdataset/DFire"
 # ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "linux" => "/mnt/d/zdataset/DFire"

halib/research/perfcalc.py

@@ -3,12 +3,9 @@ import glob
 from typing import Optional, Tuple
 import pandas as pd
 
-from rich.pretty import pprint
-
 from abc import ABC, abstractmethod
 from collections import OrderedDict
 
-from ..filetype import csvfile
 from ..system import filesys as fs
 from ..common import now_str
 from ..research.perftb import PerfTB
@@ -19,6 +16,7 @@ REQUIRED_COLS = ["experiment", "dataset"]
 CSV_FILE_POSTFIX = "__perf"
 METRIC_PREFIX = "metric_"
 
+
 class PerfCalc(ABC):  # Abstract base class for performance calculation
     @abstractmethod
     def get_experiment_name(self) -> str:
@@ -44,29 +42,32 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
         """
         pass
 
-    def valid_proc_extra_data(
-        self, proc_extra_data
-    ):
+    def valid_proc_extra_data(self, proc_extra_data):
         # make sure that all items in proc_extra_data are dictionaries, with same keys
         if proc_extra_data is None or len(proc_extra_data) == 0:
             return
         if not all(isinstance(item, dict) for item in proc_extra_data):
             raise TypeError("All items in proc_extra_data must be dictionaries")
 
-        if not all(item.keys() == proc_extra_data[0].keys() for item in proc_extra_data):
-            raise ValueError("All dictionaries in proc_extra_data must have the same keys")
+        if not all(
+            item.keys() == proc_extra_data[0].keys() for item in proc_extra_data
+        ):
+            raise ValueError(
+                "All dictionaries in proc_extra_data must have the same keys"
+            )
 
-    def valid_proc_metric_raw_data(
-        self, metric_names, proc_metric_raw_data
-    ):
+    def valid_proc_metric_raw_data(self, metric_names, proc_metric_raw_data):
         # make sure that all items in proc_metric_raw_data are dictionaries, with same keys as metric_names
-        assert isinstance(proc_metric_raw_data, list) and len(proc_metric_raw_data) > 0, \
-            "raw_data_for_metrics must be a non-empty list of dictionaries"
+        assert (
+            isinstance(proc_metric_raw_data, list) and len(proc_metric_raw_data) > 0
+        ), "raw_data_for_metrics must be a non-empty list of dictionaries"
 
         # make sure that all items in proc_metric_raw_data are dictionaries with keys as metric_names
         if not all(isinstance(item, dict) for item in proc_metric_raw_data):
             raise TypeError("All items in raw_data_for_metrics must be dictionaries")
-        if not all( set(item.keys()) == set(metric_names) for item in proc_metric_raw_data):
+        if not all(
+            set(item.keys()) == set(metric_names) for item in proc_metric_raw_data
+        ):
             raise ValueError(
                 "All dictionaries in raw_data_for_metrics must have the same keys as metric_names"
             )
@@ -75,21 +76,30 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
     def calc_exp_perf_metrics(
         self, metric_names, raw_metrics_data, extra_data=None, *args, **kwargs
     ):
-        assert isinstance(raw_metrics_data, dict) or isinstance(raw_metrics_data, list), \
-            "raw_data_for_metrics must be a dictionary or a list"
+        assert isinstance(raw_metrics_data, dict) or isinstance(
+            raw_metrics_data, list
+        ), "raw_data_for_metrics must be a dictionary or a list"
 
         if extra_data is not None:
-            assert isinstance(extra_data, type(raw_metrics_data)), \
-                "extra_data must be of the same type as raw_data_for_metrics (dict or list)"
+            assert isinstance(
+                extra_data, type(raw_metrics_data)
+            ), "extra_data must be of the same type as raw_data_for_metrics (dict or list)"
         # prepare raw_metric data for processing
-        proc_metric_raw_data_ls = raw_metrics_data if isinstance(raw_metrics_data, list) else [raw_metrics_data.copy()]
+        proc_metric_raw_data_ls = (
+            raw_metrics_data
+            if isinstance(raw_metrics_data, list)
+            else [raw_metrics_data.copy()]
+        )
         self.valid_proc_metric_raw_data(metric_names, proc_metric_raw_data_ls)
         # prepare extra data for processing
         proc_extra_data_ls = []
         if extra_data is not None:
-            proc_extra_data_ls = extra_data if isinstance(extra_data, list) else [extra_data.copy()]
-            assert len(proc_extra_data_ls) == len(proc_metric_raw_data_ls), \
-                "extra_data must have the same length as raw_data_for_metrics if it is a list"
+            proc_extra_data_ls = (
+                extra_data if isinstance(extra_data, list) else [extra_data.copy()]
+            )
+            assert len(proc_extra_data_ls) == len(
+                proc_metric_raw_data_ls
+            ), "extra_data must have the same length as raw_data_for_metrics if it is a list"
         # validate the extra_data
         self.valid_proc_extra_data(proc_extra_data_ls)
 
@@ -102,7 +112,7 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
                 "experiment": self.get_experiment_name(),
             }
             custom_fields = []
-            if len(proc_extra_data_ls)> 0:
+            if len(proc_extra_data_ls) > 0:
                 # add extra data to the output dictionary
                 extra_data_item = proc_extra_data_ls[idx]
                 out_dict.update(extra_data_item)
@@ -110,7 +120,9 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
             metric_results = metrics_backend.calc_metrics(
                 metrics_data_dict=raw_metrics_data, *args, **kwargs
             )
-            metric_results_prefix = {f"metric_{k}": v for k, v in metric_results.items()}
+            metric_results_prefix = {
+                f"metric_{k}": v for k, v in metric_results.items()
+            }
             out_dict.update(metric_results_prefix)
             ordered_cols = (
                 REQUIRED_COLS + custom_fields + list(metric_results_prefix.keys())
@@ -126,7 +138,7 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
     #! outfile - if provided, will save the output to a CSV file with the given path
     #! outdir - if provided, will save the output to a CSV file in the given directory with a generated filename
     #! return_df - if True, will return a DataFrame instead of a dictionary
-    def calc_and_save_exp_perfs(
+    def calc_perfs(
         self,
         raw_metrics_data: Union[List[dict], dict],
         extra_data: Optional[Union[List[dict], dict]] = None,
@@ -140,9 +152,11 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
         """
         metric_names = self.get_metric_backend().metric_names
         out_dict_list = self.calc_exp_perf_metrics(
-            metric_names=metric_names, raw_metrics_data=raw_metrics_data,
+            metric_names=metric_names,
+            raw_metrics_data=raw_metrics_data,
             extra_data=extra_data,
-            *args, **kwargs
+            *args,
+            **kwargs,
         )
         csv_outfile = kwargs.get("outfile", None)
         if csv_outfile is not None:
@@ -176,13 +190,18 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
         return "__perf.csv" in exp_file_name
 
     @classmethod
-    def gen_perf_report_for_multip_exps(
-        cls, indir: str, exp_csv_filter_fn=default_exp_csv_filter_fn, include_file_name=False, csv_sep=";"
+    def get_perftb_for_multi_exps(
+        cls,
+        indir: str,
+        exp_csv_filter_fn=default_exp_csv_filter_fn,
+        include_file_name=False,
+        csv_sep=";",
     ) -> PerfTB:
         """
         Generate a performance report by scanning experiment subdirectories.
         Must return a dictionary with keys as metric names and values as performance tables.
         """
+
         def get_df_for_all_exp_perf(csv_perf_files, csv_sep=";"):
             """
             Create a single DataFrame from all CSV files.
@@ -194,7 +213,9 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
             for csv_file in csv_perf_files:
                 temp_df = pd.read_csv(csv_file, sep=csv_sep)
                 if FILE_NAME_COL:
-                    temp_df[FILE_NAME_COL] = fs.get_file_name(csv_file, split_file_ext=False)
+                    temp_df[FILE_NAME_COL] = fs.get_file_name(
+                        csv_file, split_file_ext=False
+                    )
                     # csvfile.fn_display_df(temp_df)
                 temp_df_cols = temp_df.columns.tolist()
                 for col in temp_df_cols:
@@ -205,7 +226,9 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
             for csv_file in csv_perf_files:
                 temp_df = pd.read_csv(csv_file, sep=csv_sep)
                 if FILE_NAME_COL:
-                    temp_df[FILE_NAME_COL] = fs.get_file_name(csv_file, split_file_ext=False)
+                    temp_df[FILE_NAME_COL] = fs.get_file_name(
+                        csv_file, split_file_ext=False
+                    )
                 # Drop all-NA columns to avoid dtype inconsistency
                 temp_df = temp_df.dropna(axis=1, how="all")
                 # ensure all columns are present in the final DataFrame
@@ -215,7 +238,9 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
                 df = pd.concat([df, temp_df], ignore_index=True)
             # assert that REQUIRED_COLS are present in the DataFrame
             # pprint(df.columns.tolist())
-            sticky_cols = REQUIRED_COLS + ([FILE_NAME_COL] if include_file_name else []) # columns that must always be present
+            sticky_cols = REQUIRED_COLS + (
+                [FILE_NAME_COL] if include_file_name else []
+            )  # columns that must always be present
             for col in sticky_cols:
                 if col not in df.columns:
                     raise ValueError(

{halib-0.1.99.dist-info → halib-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.99
+Version: 0.2.1
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -53,6 +53,9 @@ Dynamic: summary
 
 # Helper package for coding and automation
 
+**Version 0.2.01**
++ `research/base_exp`: add `eval_exp` method to evaluate experiment (e.g., model evaluation on test set) after experiment running is done.
+
 **Version 0.1.99**
 + `filetype/ipynb`: add `gen_ipynb_name` generator to create file name based on current notebook name as prefix (with optional timestamp)
 

{halib-0.1.99.dist-info → halib-0.2.1.dist-info}/RECORD

@@ -30,15 +30,15 @@ halib/online/gdrive_test.py,sha256=hMWzz4RqZwETHp4GG4WwVNFfYvFQhp2Boz5t-DqwMo0,1
 halib/online/projectmake.py,sha256=Zrs96WgXvO4nIrwxnCOletL4aTBge-EoF0r7hpKO1w8,4034
 halib/research/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/research/base_config.py,sha256=AqZHZ0NNQ3WmUOfRzs36lf3o0FrehSdVLbdmgNpbV7A,2833
-halib/research/base_exp.py,sha256=hiO2flt_I0iJJ4bWcQwyh2ISezoC8t2k3PtxHeVr0eI,3278
+halib/research/base_exp.py,sha256=mC2bcG3M-dIvAkLw3d4O4Nu0HyvfrmvHvHX4iW8qurY,5749
 halib/research/benchquery.py,sha256=FuKnbWQtCEoRRtJAfN-zaN-jPiO_EzsakmTOMiqi7GQ,4626
-halib/research/dataset.py,sha256=avMRM9Jk3tq0P7iX1Jq-kq5d_3en3lcUlAy1dAtGSEU,6734
+halib/research/dataset.py,sha256=X6lob-pcndC2aOP0fQ3z8aUnNVVR95ia4tiuxIAXppk,6734
 halib/research/flop_csv.py,sha256=JeIUWgPFmhkPqvmhe-MLwwvAu9yR5F2k3qaViJCJJD4,1148
 halib/research/flops.py,sha256=Us0VudX8QMOm7YenICGf-Tq57C_l9x9hj-MUGA8_hCg,5773
-halib/research/metrics.py,sha256=PXPCy8r1_0lpMKfjc5SjIpRHnX80gHmeZ1C4eVj9U_s,5200
-halib/research/mics.py,sha256=nZyric8d0yKP5HrwwLsN4AjszrdxAhpJCRo1oy-EKJI,2612
+halib/research/metrics.py,sha256=qRiNiCKGUSTLY7gPMVMuVHGAAyeosfGWup2eM4490aw,5485
+halib/research/mics.py,sha256=kFl7IsZP1N-qkogvuFojkDNr1Qf0kce9h--qOnJ_3gk,2637
 halib/research/params_gen.py,sha256=GcTMlniL0iE3HalJY-gVRiYa8Qy8u6nX4LkKZeMkct8,4262
-halib/research/perfcalc.py,sha256=G8WpGB95AY5KQCt0__bPK1yUa2M1onNhXLM7twkElxg,15904
+halib/research/perfcalc.py,sha256=gjQh1D7CyEjocX5rusAzygyGFrb_s9Ob3Lqs4lIDGxg,16228
 halib/research/perftb.py,sha256=YlBXMeWn8S0LhsgxONEQZrKomRTju2T8QGGspUOy_6Y,31100
 halib/research/plot.py,sha256=GBCXP1QnzRlNqjAl9UvGvW3I9II61DBStJNQThrLy38,28578
 halib/research/profiler.py,sha256=GRAewTo0jGkOputjmRwtYVfJYBze_ivsOnrW9exWkPQ,11772
@@ -57,8 +57,8 @@ halib/utils/gpu_mon.py,sha256=vD41_ZnmPLKguuq9X44SB_vwd9JrblO4BDzHLXZhhFY,2233
 halib/utils/listop.py,sha256=Vpa8_2fI0wySpB2-8sfTBkyi_A4FhoFVVvFiuvW8N64,339
 halib/utils/tele_noti.py,sha256=-4WXZelCA4W9BroapkRyIdUu9cUVrcJJhegnMs_WpGU,5928
 halib/utils/video.py,sha256=zLoj5EHk4SmP9OnoHjO8mLbzPdtq6gQPzTQisOEDdO8,3261
-halib-0.1.99.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.1.99.dist-info/METADATA,sha256=1v3WnxBK6Aj36jA7tIycuFGQwJs19uLMi_EhSMpvmaU,6387
-halib-0.1.99.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-halib-0.1.99.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.1.99.dist-info/RECORD,,
+halib-0.2.1.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.2.1.dist-info/METADATA,sha256=VZ-msDI3P56gyQR12cQm1Ea6HZ7A6c1tXAHtuSzadJU,6548
+halib-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+halib-0.2.1.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.2.1.dist-info/RECORD,,