halib 0.1.65__tar.gz → 0.1.66__tar.gz

{halib-0.1.65 → halib-0.1.66}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.65
+Version: 0.1.66
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -52,7 +52,7 @@ Dynamic: summary
 
 Helper package for coding and automation
 
-**Version 0.1.65**
+**Version 0.1.66**
 
 + now use `uv` for venv management
 + `research/perfcalc`: support both torchmetrics and custom metrics for performance calculation

{halib-0.1.65 → halib-0.1.66}/README.md

@@ -1,6 +1,6 @@
 Helper package for coding and automation
 
-**Version 0.1.65**
+**Version 0.1.66**
 
 + now use `uv` for venv management
 + `research/perfcalc`: support both torchmetrics and custom metrics for performance calculation

halib-0.1.66/halib/research/metrics.py

@@ -0,0 +1,121 @@
+# -------------------------------
+# Metrics Backend Interface
+# -------------------------------
+import inspect
+from typing import Dict, Union, List, Any
+from abc import ABC, abstractmethod
+
+class MetricsBackend(ABC):
+    """Interface for pluggable metrics computation backends."""
+
+    def __init__(self, metrics_info: Union[List[str], Dict[str, Any]]):
+        """
+        Initialize the backend with optional metrics_info.
+        """
+        self.metric_info = metrics_info
+        self.validate_metrics_info(self.metric_info)
+
+    @property
+    def metric_names(self) -> List[str]:
+        """
+        Return a list of metric names.
+        If metric_info is a dict, return its keys; if it's a list, return it directly.
+        """
+        if isinstance(self.metric_info, dict):
+            return list(self.metric_info.keys())
+        elif isinstance(self.metric_info, list):
+            return self.metric_info
+        else:
+            raise TypeError("metric_info must be a list or a dict")
+
+    def validate_metrics_info(self, metrics_info):
+        if isinstance(metrics_info, list):
+            return metrics_info
+        elif isinstance(metrics_info, dict):
+            return {k: v for k, v in metrics_info.items() if isinstance(k, str)}
+        else:
+            raise TypeError(
+                "metrics_info must be a list of strings or a dict with string keys"
+            )
+
+    @abstractmethod
+    def compute_metrics(
+        self, metrics_info: Union[List[str], Dict[str, Any]],  metrics_data_dict: Dict[str, Any], *args, **kwargs
+    ) -> Dict[str, Any]:
+        pass
+
+    def calc_metrics(
+        self,  metrics_data_dict: Dict[str, Any], *args, **kwargs
+    ) -> Dict[str, Any]:
+        """
+        Calculate metrics based on the provided metrics_info and data.
+        This method should be overridden by subclasses to implement specific metric calculations.
+        """
+        # prevalidate the metrics_data_dict
+        for metric in self.metric_names:
+            if metric not in metrics_data_dict:
+                raise ValueError(f"Metric '{metric}' not found in provided data.")
+        # Call the abstract method to compute metrics
+        return self.compute_metrics(self.metric_info, metrics_data_dict, *args, **kwargs)
+
+class TorchMetricsBackend(MetricsBackend):
+    """TorchMetrics-based backend implementation."""
+
+    def __init__(self, metrics_info: Union[List[str], Dict[str, Any]]):
+        try:
+            import torch
+            from torchmetrics import Metric
+        except ImportError:
+            raise ImportError(
+                "TorchMetricsBackend requires torch and torchmetrics to be installed."
+            )
+        self.metric_info = metrics_info
+        self.torch = torch
+        self.Metric = Metric
+        self.validate_metrics_info(metrics_info)
+
+    def validate_metrics_info(self, metrics_info):
+        if not isinstance(metrics_info, dict):
+            raise TypeError(
+                "TorchMetricsBackend requires metrics_info as a dict {name: MetricInstance}"
+            )
+        for k, v in metrics_info.items():
+            if not isinstance(k, str):
+                raise TypeError(f"Key '{k}' is not a string")
+            if not isinstance(v, self.Metric):
+                raise TypeError(f"Value for key '{k}' must be a torchmetrics.Metric")
+        return metrics_info
+
+    def compute_metrics(self, metrics_info, metrics_data_dict, *args, **kwargs):
+        out_dict = {}
+        for metric, metric_instance in metrics_info.items():
+            if metric not in metrics_data_dict:
+                raise ValueError(f"Metric '{metric}' not found in provided data.")
+
+            metric_data = metrics_data_dict[metric]
+            sig = inspect.signature(metric_instance.update)
+            expected_args = list(sig.parameters.values())
+
+            if isinstance(metric_data, dict):
+                args = [metric_data[param.name] for param in expected_args]
+            elif isinstance(metric_data, (list, tuple)):
+                args = metric_data
+            else:
+                raise TypeError(f"Unsupported data format for metric '{metric}'")
+
+            if len(expected_args) == 1:
+                metric_instance.update(args)
+            else:
+                metric_instance.update(*args)
+
+            computed_value = metric_instance.compute()
+            if isinstance(computed_value, self.torch.Tensor):
+                computed_value = (
+                    computed_value.item()
+                    if computed_value.numel() == 1
+                    else computed_value.tolist()
+                )
+
+
+            out_dict[metric] = computed_value
+        return out_dict

{halib-0.1.65 → halib-0.1.66}/halib/research/perfcalc.py

@@ -8,62 +8,26 @@ from functools import wraps
 from rich.pretty import pprint
 
 from abc import ABC, abstractmethod
+from collections import OrderedDict
 
 from ..filetype import csvfile
 from ..common import now_str
 from ..research.perftb import PerfTB
-from collections import OrderedDict
-
-# try to import torch, and torchmetrics
-try:
-    import torch
-    import torchmetrics
-    from torchmetrics import Metric
-except ImportError:
-    raise ImportError("Please install torch and torchmetrics to use this module.")
-
-def validate_torch_metrics(fn):
-    @wraps(fn)
-    def wrapper(self, *args, **kwargs):
-        result = fn(self, *args, **kwargs)
+from ..research.metrics import *
 
-        if not isinstance(result, dict):
-            raise TypeError("torch_metrics() must return a dictionary")
+# # try to import torch, and torchmetrics
+# try:
+#     import torch
+#     import torchmetrics
+#     from torchmetrics import Metric
+# except ImportError:
+#     raise ImportError("Please install torch and torchmetrics to use this module.")
 
-        for k, v in result.items():
-            if not isinstance(k, str):
-                raise TypeError(f"Key '{k}' is not a string")
-            if not isinstance(v, Metric):
-                raise TypeError(
-                    f"Value for key '{k}' is not a torchmetrics.Metric (got {type(v).__name__})"
-                )
-
-        return result
-
-    return wrapper
-def valid_custom_fields(fn):
-    @wraps(fn)
-    def wrapper(self, *args, **kwargs):
-        rs = fn(self, *args, **kwargs)
-        if not isinstance(rs, tuple) or len(rs) != 2:
-            raise ValueError("Function must return a tuple (outdict, custom_fields)")
-        outdict, custom_fields = rs
-        if not isinstance(outdict, dict):
-            raise TypeError("Output must be a dictionary")
-        if not isinstance(custom_fields, list):
-            raise TypeError("Custom fields must be a list")
-        for field in custom_fields:
-            if not isinstance(field, str):
-                raise TypeError(f"Custom field '{field}' is not a string")
-        return outdict, custom_fields
-
-    return wrapper
 
 REQUIRED_COLS = ["experiment", "dataset"]
 CSV_FILE_POSTFIX = "__perf"
 
 class PerfCalc(ABC): # Abstract base class for performance calculation
-
     @abstractmethod
     def get_experiment_name(self):
         """
@@ -81,22 +45,17 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
         pass
 
     @abstractmethod
-    def get_metrics_info(self):
+    def get_metric_backend(self) -> MetricsBackend:
         """
         Return a list of metric names to be used for performance calculation OR a dictionaray with keys as metric names and values as metric instances of torchmetrics.Metric. For example: {"accuracy": Accuracy(), "precision": Precision()}
 
         """
         pass
 
-    def calc_exp_outdict_custom_fields(self, outdict, *args, **kwargs):
-        """Can be overridden by the subclass to add custom fields to the output dictionary.
-        ! must return the modified outdict, and a ordered list of custom fields to be added to the output dictionary.
-        """
-        return outdict, []
-
     # ! can be override, but ONLY if torchmetrics are used
     # Prepare the exp data for torch metrics.
-    def prepare_torch_metrics_exp_data(self, metric_names, *args, **kwargs):
+    @abstractmethod
+    def prepare_metrics_data_dict(self, metric_names, *args, **kwargs):
         """
         Prepare the data for metrics.
         This function should be overridden by the subclass if needed.
@@ -105,99 +64,47 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
         """
         pass
 
-    def __validate_metrics_info(self, metrics_info):
-        """
-        Validate the metrics_info to ensure it is a list or a dictionary with valid metric names and instances.
-        """
-        if not isinstance(metrics_info, (list, dict)):
-            raise TypeError(f"Metrics info must be a list or a dictionary, got {type(metrics_info).__name__}")
-
-        if isinstance(metrics_info, dict):
-            for k, v in metrics_info.items():
-                if not isinstance(k, str):
-                    raise TypeError(f"Key '{k}' is not a string")
-                if not isinstance(v, Metric):
-                    raise TypeError(f"Value for key '{k}' is not a torchmetrics.Metric (got {type(v).__name__})")
-        elif isinstance(metrics_info, list):
-            for metric in metrics_info:
-                if not isinstance(metric, str):
-                    raise TypeError(f"Metric '{metric}' is not a string")
-        return metrics_info
-    def __calc_exp_perf_metrics(self, *args, **kwargs):
-        """
-        Calculate the performance metrics for the experiment.
+    def calc_exp_outdict_custom_fields(self, outdict, *args, **kwargs):
+        """Can be overridden by the subclass to add custom fields to the output dictionary.
+        ! must return the modified outdict, and a ordered list of custom fields to be added to the output dictionary.
         """
-        metrics_info = self.__validate_metrics_info(self.get_metrics_info())
-        USED_TORCHMETRICS = isinstance(metrics_info, dict)
-        metric_names = metrics_info if isinstance(metrics_info, list) else list(metrics_info.keys())
-        out_dict = {metric: None for metric in metric_names}
-        out_dict["dataset"] = self.get_dataset_name()
-        out_dict["experiment"] = self.get_experiment_name()
-        out_dict, custom_fields = self.calc_exp_outdict_custom_fields(
-            outdict=out_dict, *args, **kwargs
-        )
-        if USED_TORCHMETRICS:
-            torch_metrics_dict = self.get_metrics_info()
-            all_metric_data = self.prepare_torch_metrics_exp_data(
-                metric_names, *args, **kwargs
-            )
-            metric_col_names = []
-            for metric in metric_names:
-                if metric not in all_metric_data:
-                    raise ValueError(f"Metric '{metric}' not found in provided data.")
-                tmetric = torch_metrics_dict[metric]  # torchmetrics instance
-                metric_data = all_metric_data[metric]  # should be a dict of args/kwargs
-                # Inspect expected parameters for the metric's update() method
-                sig = inspect.signature(tmetric.update)
-                expected_args = list(sig.parameters.values())
-                # Prepare args in correct order
-                if isinstance(metric_data, dict):
-                    # Match dict keys to parameter names
-                    args = [metric_data[param.name] for param in expected_args]
-                elif isinstance(metric_data, (list, tuple)):
-                    args = metric_data
-                else:
-                    raise TypeError(f"Unsupported data format for metric '{metric}'")
+        return outdict, []
 
-                # Call update and compute
-                if len(expected_args) == 1:
-                    tmetric.update(args)  # pass as single argument
-                else:
-                    tmetric.update(*args)  # unpack multiple arguments
-                computed_value = tmetric.compute()
-                # ensure the computed value converted to a scala value or list array
-                if isinstance(computed_value, torch.Tensor):
-                    if computed_value.numel() == 1:
-                        computed_value = computed_value.item()
-                    else:
-                        computed_value = computed_value.tolist()
-                col_name = f"metric_{metric}" if "metric_" not in metric else metric
-                metric_col_names.append(col_name)
-                out_dict[col_name] = computed_value
-        else:
-            # If torchmetrics are not used, calculate metrics using the custom method
-            metric_rs_dict = self.calc_exp_perf_metrics(
-                metric_names, *args, **kwargs)
-            for metric in metric_names:
-                if metric not in metric_rs_dict:
-                    raise ValueError(f"Metric '{metric}' not found in provided data.")
-                col_name = f"metric_{metric}" if "metric_" not in metric else metric
-                out_dict[col_name] = metric_rs_dict[metric]
-            metric_col_names = [f"metric_{metric}" for metric in metric_names]
-        ordered_cols = REQUIRED_COLS + custom_fields + metric_col_names
-        # create a new ordered dictionary with the correct order
-        out_dict = OrderedDict((col, out_dict[col]) for col in ordered_cols if col in out_dict)
-        return out_dict
+    def __valid_calc_custom_fields(self, fun_results):
+        if not isinstance(fun_results, tuple) or len(fun_results) != 2:
+            raise ValueError("Function must return a tuple (outdict, custom_fields)")
+        outdict, custom_fields = fun_results
+        if not isinstance(outdict, dict):
+            raise TypeError("Output must be a dictionary")
+        if not isinstance(custom_fields, list):
+            raise TypeError("Custom fields must be a list")
+        for field in custom_fields:
+            if not isinstance(field, str):
+                raise TypeError(f"Custom field '{field}' is not a string")
+        return outdict, custom_fields
 
     # ! only need to override this method if torchmetrics are not used
     def calc_exp_perf_metrics(self, metric_names, *args, **kwargs):
-        """
-        Calculate the performance metrics for the experiment, but not using torchmetrics.
-        This function should be overridden by the subclass if needed.
-        Must return a dictionary with keys as metric names and values as the calculated metrics.
-        """
-        raise NotImplementedError("calc_exp_perf_metrics() must be overridden by the subclass if torchmetrics are not used.")
-
+        metrics_backend = self.get_metric_backend()
+        out_dict = {"dataset": self.get_dataset_name(), "experiment": self.get_experiment_name()}
+        out_dict, custom_fields = self.__valid_calc_custom_fields(self.calc_exp_outdict_custom_fields(
+            outdict=out_dict, *args, **kwargs
+        ))
+        metrics_data_dict = self.prepare_metrics_data_dict(
+            metric_names, *args, **kwargs
+        )
+        metric_results = metrics_backend.calc_metrics(
+            metrics_data_dict=metrics_data_dict, *args, **kwargs
+        )
+        metric_results_prefix = {
+            f"metric_{k}": v for k, v in metric_results.items()
+        }
+        out_dict.update(metric_results_prefix)
+        ordered_cols = REQUIRED_COLS + custom_fields + list(metric_results_prefix.keys())
+        out_dict = OrderedDict(
+            (col, out_dict[col]) for col in ordered_cols if col in out_dict
+        )
+        return out_dict
 
     #! custom kwargs:
     #! outfile - if provided, will save the output to a CSV file with the given path
@@ -210,9 +117,8 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
         This function should be overridden by the subclass if needed.
         Must return a dictionary with keys as metric names and values as the calculated metrics.
         """
-        out_dict = self.__calc_exp_perf_metrics(*args, **kwargs)
-        # pprint(f"Output Dictionary: {out_dict}")
-        # check if any kwargs named "outfile"
+        metric_names = self.get_metric_backend().metric_names
+        out_dict = self.calc_exp_perf_metrics(metric_names=metric_names, *args, **kwargs)
         csv_outfile = kwargs.get("outfile", None)
         if csv_outfile is not None:
             filePathNoExt, _ = os.path.splitext(csv_outfile)
@@ -284,6 +190,9 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
             assert len(metric_cols) > 0, "No metric columns found in the DataFrame. Ensure that the CSV files contain metric columns starting with 'metric_'."
             final_cols = REQUIRED_COLS + metric_cols
             df = df[final_cols]
+            # !hahv debug
+            pprint('------ Final DataFrame Columns ------')
+            csvfile.fn_display_df(df)
             # ! validate all rows in df before returning
             # make sure all rows will have at least values for REQUIRED_COLS and at least one metric column
             for index, row in df.iterrows():
@@ -383,4 +292,4 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
         all_exp_perf_df = get_df_for_all_exp_perf(csv_perf_files, csv_sep=csv_sep)
         csvfile.fn_display_df(all_exp_perf_df)
         perf_tb = mk_perftb_report(all_exp_perf_df)
-        return perf_tb
+        return perf_tb

{halib-0.1.65 → halib-0.1.66}/halib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.65
+Version: 0.1.66
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -52,7 +52,7 @@ Dynamic: summary
 
 Helper package for coding and automation
 
-**Version 0.1.65**
+**Version 0.1.66**
 
 + now use `uv` for venv management
 + `research/perfcalc`: support both torchmetrics and custom metrics for performance calculation

{halib-0.1.65 → halib-0.1.66}/halib.egg-info/SOURCES.txt

@@ -27,6 +27,7 @@ halib/online/gdrive_test.py
 halib/online/projectmake.py
 halib/research/__init__.py
 halib/research/dataset.py
+halib/research/metrics.py
 halib/research/perfcalc.py
 halib/research/perftb.py
 halib/research/plot.py

{halib-0.1.65 → halib-0.1.66}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt") as f:
 
 setuptools.setup(
     name="halib",
-    version="0.1.65",
+    version="0.1.66",
     author="Hoang Van Ha",
     author_email="hoangvanhauit@gmail.com",
     description="Small library for common tasks",