halib 0.1.67__py3-none-any.whl → 0.1.70__py3-none-any.whl

halib/research/base_config.py

@@ -0,0 +1,100 @@
+import os
+from rich.pretty import pprint
+from abc import ABC, abstractmethod
+from dataclass_wizard import YAMLWizard
+
+
+class NamedConfig(ABC):
+    """
+    Base class for named configurations.
+    All configurations should have a name.
+    """
+
+    @abstractmethod
+    def get_name(self):
+        """
+        Get the name of the configuration.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+
+class ExpBaseConfig(ABC, YAMLWizard):
+    """
+    Base class for configuration objects.
+    What a cfg class must have:
+    1 - a dataset cfg
+    2 - a metric cfg
+    3 - a method cfg
+    """
+
+    # Save to yaml fil
+    def save_to_outdir(
+        self, filename: str = "config.yaml", outdir=None, override: bool = False
+    ) -> None:
+        """
+        Save the configuration to the output directory.
+        """
+        if outdir is not None:
+            output_dir = outdir
+        else:
+            output_dir = self.get_outdir()
+        os.makedirs(output_dir, exist_ok=True)
+        assert (output_dir is not None) and (
+            os.path.isdir(output_dir)
+        ), f"Output directory '{output_dir}' does not exist or is not a directory."
+        file_path = os.path.join(output_dir, filename)
+        if os.path.exists(file_path) and not override:
+            pprint(
+                f"File '{file_path}' already exists. Use 'override=True' to overwrite."
+            )
+        else:
+            # method of YAMLWizard to_yaml_file
+            self.to_yaml_file(file_path)
+
+    @classmethod
+    @abstractmethod
+    # load from a custom YAML file
+    def from_custom_yaml_file(cls, yaml_file: str):
+        """Load a configuration from a custom YAML file."""
+        pass
+
+    @abstractmethod
+    def get_cfg_name(self):
+        """
+        Get the name of the configuration.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def get_outdir(self):
+        """
+        Get the output directory for the configuration.
+        This method should be implemented in subclasses.
+        """
+        return None
+
+    @abstractmethod
+    def get_general_cfg(self):
+        """
+        Get the general configuration like output directory, log settings, SEED, etc.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def get_dataset_cfg(self) -> NamedConfig:
+        """
+        Get the dataset configuration.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def get_metric_cfg(self) -> NamedConfig:
+        """
+        Get the metric configuration.
+        This method should be implemented in subclasses.
+        """
+        pass

halib/research/base_exp.py

@@ -0,0 +1,99 @@
+from abc import ABC, abstractmethod
+
+from base_config import ExpBaseConfig
+from perfcalc import PerfCalc
+from metrics import MetricsBackend
+
+# ! SEE https://github.com/hahv/base_exp for sample usage
+class BaseExperiment(PerfCalc, ABC):
+    """
+    Base class for experiments.
+    Orchestrates the experiment pipeline using a pluggable metrics backend.
+    """
+
+    def __init__(self, config: ExpBaseConfig):
+        self.config = config
+        self.metric_backend = None
+
+    # -----------------------
+    # PerfCalc Required Methods
+    # -----------------------
+    def get_dataset_name(self):
+        return self.config.get_dataset_cfg().get_name()
+
+    def get_experiment_name(self):
+        return self.config.get_cfg_name()
+
+    def get_metric_backend(self):
+        if not self.metric_backend:
+            self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+        return self.metric_backend
+
+    # -----------------------
+    # Abstract Experiment Steps
+    # -----------------------
+    @abstractmethod
+    def init_general(self, general_cfg):
+        """Setup general settings like SEED, logging, env variables."""
+        pass
+
+    @abstractmethod
+    def prepare_dataset(self, dataset_cfg):
+        """Load/prepare dataset."""
+        pass
+
+    @abstractmethod
+    def prepare_metrics(self, metric_cfg) -> MetricsBackend:
+        """
+        Prepare the metrics for the experiment.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def exec_exp(self, *args, **kwargs):
+        """Run experiment process, e.g.: training/evaluation loop.
+        Return: raw_metrics_data, and extra_data as input for calc_and_save_exp_perfs
+        """
+        pass
+
+    def eval_exp(self):
+        """Optional: re-run evaluation from saved results."""
+        pass
+
+    # -----------------------
+    # Main Experiment Runner
+    # -----------------------
+    def run_exp(self, do_calc_metrics=True, *args, **kwargs):
+        """
+        Run the whole experiment pipeline.
+        Params:
+            + 'outfile' to save csv file results,
+            + 'outdir' to set output directory for experiment results.
+            + 'return_df' to return a DataFrame of results instead of a dictionary.
+
+        Full pipeline:
+            1. Init
+            2. Dataset
+            3. Metrics Preparation
+            4. Save Config
+            5. Execute
+            6. Calculate & Save Metrics
+        """
+        self.init_general(self.config.get_general_cfg())
+        self.prepare_dataset(self.config.get_dataset_cfg())
+
+        # Save config before running
+        self.config.save_to_outdir()
+
+        # Execute experiment
+        results = self.exec_exp(*args, **kwargs)
+        if do_calc_metrics:
+            metrics_data, extra_data = results
+            # Calculate & Save metrics
+            perf_results = self.calc_and_save_exp_perfs(
+                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
+            )
+            return perf_results
+        else:
+            return results

{halib-0.1.67.dist-info → halib-0.1.70.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.67
+Version: 0.1.70
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -52,6 +52,11 @@ Dynamic: summary
 
 Helper package for coding and automation
 
+**Version 0.1.70**
+
++ `research/base_exp`: add base experiment class to handle common experiment tasks, including performance calculation and saving results.
+
+
 **Version 0.1.67**
 
 + now use `uv` for venv management

{halib-0.1.67.dist-info → halib-0.1.70.dist-info}/RECORD

@@ -28,6 +28,8 @@ halib/online/gdrive_mkdir.py,sha256=wSJkQMJCDuS1gxQ2lHQHq_IrJ4xR_SEoPSo9n_2WNFU,
 halib/online/gdrive_test.py,sha256=hMWzz4RqZwETHp4GG4WwVNFfYvFQhp2Boz5t-DqwMo0,1342
 halib/online/projectmake.py,sha256=Zrs96WgXvO4nIrwxnCOletL4aTBge-EoF0r7hpKO1w8,4034
 halib/research/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+halib/research/base_config.py,sha256=AIjVzl2ZJ9b8yIGb2X5EZwLmyGJ_9wNWqrib1nU3Wj0,2831
+halib/research/base_exp.py,sha256=mIyxVDKb40lngrEgW3_hUV5KPon25nxQeaWCUU_AWjQ,3185
 halib/research/benchquery.py,sha256=FuKnbWQtCEoRRtJAfN-zaN-jPiO_EzsakmTOMiqi7GQ,4626
 halib/research/dataset.py,sha256=QU0Hr5QFb8_XlvnOMgC9QJGIpwXAZ9lDd0RdQi_QRec,6743
 halib/research/metrics.py,sha256=Xgv0GUGo-o-RJaBOmkRCRpQJaYijF_1xeKkyYU_Bv4U,5249
@@ -49,8 +51,8 @@ halib/utils/gpu_mon.py,sha256=vD41_ZnmPLKguuq9X44SB_vwd9JrblO4BDzHLXZhhFY,2233
 halib/utils/listop.py,sha256=Vpa8_2fI0wySpB2-8sfTBkyi_A4FhoFVVvFiuvW8N64,339
 halib/utils/tele_noti.py,sha256=-4WXZelCA4W9BroapkRyIdUu9cUVrcJJhegnMs_WpGU,5928
 halib/utils/video.py,sha256=ZqzNVPgc1RZr_T0OlHvZ6SzyBpL7O27LtB86JMbBuR0,3059
-halib-0.1.67.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.1.67.dist-info/METADATA,sha256=Zk22ct5W95qBzGkz0tNepuAdfUwPJTbVO7Nb4L_hFTQ,5541
-halib-0.1.67.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-halib-0.1.67.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.1.67.dist-info/RECORD,,
+halib-0.1.70.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.1.70.dist-info/METADATA,sha256=C5ei-WhAmt6SuG-vR8pIQd2Uat5HrlT20RX9JP3D0Q4,5706
+halib-0.1.70.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+halib-0.1.70.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.1.70.dist-info/RECORD,,