halib 0.2.8__py3-none-any.whl → 0.2.9__py3-none-any.whl

halib/exp/core/base_exp.py

@@ -1,168 +1,147 @@
-import os
-from rich.pretty import pprint
 from abc import ABC, abstractmethod
-from typing import List, Optional, TypeVar, Generic
-
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from dataclass_wizard import YAMLWizard
-
-
-class NamedCfg(ABC):
-    """
-    Base class for named configurations.
-    All configurations should have a name.
-    """
-
-    @abstractmethod
-    def get_name(self):
-        """
-        Get the name of the configuration.
-        This method should be implemented in subclasses.
-        """
-        pass
-
-
-@dataclass
-class AutoNamedCfg(YAMLWizard, NamedCfg):
-    """
-    Mixin that automatically implements get_name() by returning self.name.
-    Classes using this MUST have a 'name' field.
-    """
-
-    name: Optional[str] = None
-
-    def get_name(self):
-        return self.name
-
-    def __post_init__(self):
-        # Enforce the "MUST" rule here
-        if self.name is None:
-            # We allow None during initial load, but it must be set before usage
-            # or handled by the loader.
-            pass
-
-
-T = TypeVar("T", bound=AutoNamedCfg)
+from typing import Tuple, Any, Optional
+from .base_config import ExpBaseCfg
+from ..perf.perfcalc import PerfCalc
+from ..perf.perfmetrics import MetricsBackend
 
 
-class BaseSelectorCfg(Generic[T]):
-    """
-    Base class to handle the logic of selecting an item from a list by name.
-    """
-
-    def _resolve_selection(self, items: List[T], selected_name: str, context: str) -> T:
-        if selected_name is None:
-            raise ValueError(f"No {context} selected in the configuration.")
-
-        # Create a lookup dict for O(1) access, or just iterate if list is short
-        for item in items:
-            if item.name == selected_name:
-                return item
+class ExpHook:
+    """Base interface for all experiment hooks."""
+    def on_before_run(self, exp): pass
+    def on_after_run(self, exp, results): pass
 
-        raise ValueError(
-            f"{context.capitalize()} '{selected_name}' not found in the configuration list."
-        )
 
-
-class ExpBaseCfg(ABC, YAMLWizard):
+# ! SEE https://github.com/hahv/base_exp for sample usage
+class BaseExp(PerfCalc, ABC):
     """
-    Base class for configuration objects.
-    What a cfg class must have:
-    1 - a dataset cfg
-    2 - a metric cfg
-    3 - a method cfg
+    Base class for experiments.
+    Orchestrates the experiment pipeline using a pluggable metrics backend.
     """
 
-    cfg_name: Optional[str] = None
-
-    # Save to yaml fil
-    def save_to_outdir(
-        self, filename: str = "__config.yaml", outdir=None, override: bool = False
-    ) -> None:
-        """
-        Save the configuration to the output directory.
-        """
-        if outdir is not None:
-            output_dir = outdir
-        else:
-            output_dir = self.get_outdir()
-        os.makedirs(output_dir, exist_ok=True)
-        assert (output_dir is not None) and (
-            os.path.isdir(output_dir)
-        ), f"Output directory '{output_dir}' does not exist or is not a directory."
-        file_path = os.path.join(output_dir, filename)
-        if os.path.exists(file_path) and not override:
-            pprint(
-                f"File '{file_path}' already exists. Use 'override=True' to overwrite."
-            )
-        else:
-            # method of YAMLWizard to_yaml_file
-            self.to_yaml_file(file_path)
-
-    @classmethod
+    def __init__(self, config: ExpBaseCfg):
+        self.config = config
+        self.metric_backend = None
+        # Flag to track if init_general/prepare_dataset has run
+        self._is_env_ready = False
+        self.hooks = []
+
+    def register_hook(self, hook: ExpHook):
+        self.hooks.append(hook)
+
+    def _trigger_hooks(self, method_name: str, *args, **kwargs):
+        for hook in self.hooks:
+            method = getattr(hook, method_name, None)
+            if callable(method):
+                method(*args, **kwargs)
+
+    # -----------------------
+    # PerfCalc Required Methods
+    # -----------------------
+    def get_dataset_name(self):
+        return self.config.get_dataset_cfg().get_name()
+
+    def get_experiment_name(self):
+        return self.config.get_cfg_name()
+
+    def get_metric_backend(self):
+        if not self.metric_backend:
+            self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+        return self.metric_backend
+
+    # -----------------------
+    # Abstract Experiment Steps
+    # -----------------------
     @abstractmethod
-    # load from a custom YAML file
-    def from_custom_yaml_file(cls, yaml_file: str):
-        """Load a configuration from a custom YAML file."""
+    def init_general(self, general_cfg):
+        """Setup general settings like SEED, logging, env variables."""
         pass
 
-    def get_cfg_name(self, sep: str = "__", *args, **kwargs) -> str:
-        if self.cfg_name is None:
-            # auto get the config name from dataset, method, metric
-            # 2. Generate the canonical Config Name
-            name_parts = []
-            general_info = self.get_general_cfg().get_name()
-            dataset_info = self.get_dataset_cfg().get_name()
-            method_info = self.get_method_cfg().get_name()
-            name_parts = [
-                general_info,
-                f"ds_{dataset_info}",
-                f"mt_{method_info}",
-            ]
-            if "extra" in kwargs:
-                extra_info = kwargs["extra"]
-                assert isinstance(extra_info, str), "'extra' kwarg must be a string."
-                name_parts.append(extra_info)
-            self.cfg_name = sep.join(name_parts)
-        return self.cfg_name
-
     @abstractmethod
-    def get_outdir(self):
-        """
-        Get the output directory for the configuration.
-        This method should be implemented in subclasses.
-        """
-        return None
+    def prepare_dataset(self, dataset_cfg):
+        """Load/prepare dataset."""
+        pass
 
     @abstractmethod
-    def get_general_cfg(self) -> NamedCfg:
+    def prepare_metrics(self, metric_cfg) -> MetricsBackend:
         """
-        Get the general configuration like output directory, log settings, SEED, etc.
+        Prepare the metrics for the experiment.
         This method should be implemented in subclasses.
         """
         pass
 
     @abstractmethod
-    def get_dataset_cfg(self) -> NamedCfg:
-        """
-        Get the dataset configuration.
-        This method should be implemented in subclasses.
+    def exec_exp(self, *args, **kwargs) -> Optional[Tuple[Any, Any]]:
+        """Run experiment process, e.g.: training/evaluation loop.
+        Return: either `None` or a tuple of (raw_metrics_data, extra_data) for calc_and_save_exp_perfs
         """
         pass
 
-    @abstractmethod
-    def get_method_cfg(self) -> NamedCfg:
+    # -----------------------
+    # Internal Helpers
+    # -----------------------
+    def _validate_and_unpack(self, results):
+        if results is None:
+            return None
+        if not isinstance(results, (tuple, list)) or len(results) != 2:
+            raise ValueError("exec must return (metrics_data, extra_data)")
+        return results[0], results[1]
+
+    def _prepare_environment(self, force_reload: bool = False):
         """
-        Get the method configuration.
-        This method should be implemented in subclasses.
+        Common setup. Skips if already initialized, unless force_reload is True.
         """
-        pass
+        if self._is_env_ready and not force_reload:
+            # Environment is already prepared, skipping setup.
+            return
 
-    @abstractmethod
-    def get_metric_cfg(self) -> NamedCfg:
+        # 1. Run Setup
+        self.init_general(self.config.get_general_cfg())
+        self.prepare_dataset(self.config.get_dataset_cfg())
+
+        # 2. Update metric backend (refresh if needed)
+        self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+
+        # 3. Mark as ready
+        self._is_env_ready = True
+
+    # -----------------------
+    # Main Experiment Runner
+    # -----------------------
+    def run_exp(self, should_calc_metrics=True, reload_env=False, *args, **kwargs):
         """
-        Get the metric configuration.
-        This method should be implemented in subclasses.
+        Run the whole experiment pipeline.
+        :param reload_env: If True, forces dataset/general init to run again.
+        :param should_calc_metrics: Whether to calculate and save metrics after execution.
+        :kwargs Params:
+            + 'outfile' to save csv file results,
+            + 'outdir' to set output directory for experiment results.
+            + 'return_df' to return a DataFrame of results instead of a dictionary.
+
+        Full pipeline:
+            1. Init
+            2. Prepare Environment (General + Dataset + Metrics)
+            3. Save Config
+            4. Execute
+            5. Calculate & Save Metrics
         """
-        pass
+        self._prepare_environment(force_reload=reload_env)
+
+        self._trigger_hooks("before_run", self)
+
+        # Save config before running
+        self.config.save_to_outdir()
+
+        # Execute experiment
+        results = self.exec_exp(*args, **kwargs)
+
+        if should_calc_metrics and results is not None:
+            metrics_data, extra_data = self._validate_and_unpack(results)
+            # Calculate & Save metrics
+            perf_results = self.calc_perfs(
+                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
+            )
+            self._trigger_hooks("after_run", self, perf_results)
+            return perf_results
+        else:
+            self._trigger_hooks("after_run", self, results)
+            return results

{halib-0.2.8.dist-info → halib-0.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.2.8
+Version: 0.2.9
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -53,7 +53,7 @@ Dynamic: summary
 
 # Helper package for coding and automation
 
-**Version 0.2.8**
+**Version 0.2.9**
 + reorganize packages with most changes in `research` package; also rename `research` to `exp` (package for experiment management and utilities)
 + update `exp/perfcalc.py` to allow save computed performance to csv file (without explicit calling method `calc_perfs`)
 

{halib-0.2.8.dist-info → halib-0.2.9.dist-info}/RECORD

@@ -22,7 +22,7 @@ halib/common/rich_color.py,sha256=tyK5fl3Dtv1tKsfFzt_5Rco4Fj72QliA-w5aGXaVuqQ,63
 halib/exp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/exp/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/exp/core/base_config.py,sha256=Js2oVDt7qwT7eV_sOUWw6XXl569G1bX6ls-VYAx2gWY,5032
-halib/exp/core/base_exp.py,sha256=XjRHXbUHE-DCZLRDTteDF5gsxKN3mhGEe2zWL24JP80,5131
+halib/exp/core/base_exp.py,sha256=fknJVmW6ubbapOggbkrbNWgc1ZXcUz_FE3wMyuIGX7M,5180
 halib/exp/core/param_gen.py,sha256=I9JHrDCaep4CjvApDoX0QzFuw38zMC2PsDFueuA7pjM,4271
 halib/exp/core/wandb_op.py,sha256=powL2QyLBqF-6PUGAOqd60s1npHLLKJxPns3S4hKeNo,4160
 halib/exp/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -101,8 +101,8 @@ halib/utils/list.py,sha256=BM-8sRhYyqF7bh4p7TQtV7P_gnFruUCA6DTUOombaZg,337
 halib/utils/listop.py,sha256=Vpa8_2fI0wySpB2-8sfTBkyi_A4FhoFVVvFiuvW8N64,339
 halib/utils/tele_noti.py,sha256=-4WXZelCA4W9BroapkRyIdUu9cUVrcJJhegnMs_WpGU,5928
 halib/utils/video.py,sha256=zLoj5EHk4SmP9OnoHjO8mLbzPdtq6gQPzTQisOEDdO8,3261
-halib-0.2.8.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.2.8.dist-info/METADATA,sha256=QU-YIW5BNbgDy9j9ttQsleQ2zkBmWjpVazjjb23vIo8,6836
-halib-0.2.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-halib-0.2.8.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.2.8.dist-info/RECORD,,
+halib-0.2.9.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.2.9.dist-info/METADATA,sha256=qO311QkpE58ZSiy6NOMsRowRhunVHWa0gr__QW_6iQY,6836
+halib-0.2.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+halib-0.2.9.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.2.9.dist-info/RECORD,,