halib 0.2.21__py3-none-any.whl → 0.2.27__py3-none-any.whl

halib/exp/core/param_gen.py

@@ -4,6 +4,11 @@ import numpy as np
 from itertools import product
 from typing import Dict, Any, List, Iterator, Optional
 from ...filetype import yamlfile
+from ...utils.dict import DictUtils
+
+# Assuming DictUtils is available in the scope or imported
+# from .dict_utils import DictUtils
+
 
 class ParamGen:
     """
@@ -38,6 +43,7 @@ class ParamGen:
         keys (List[str]): List of flattened dot-notation keys being swept.
         values (List[List[Any]]): List of value options for each key.
     """
+
     def __init__(
         self, sweep_cfg: Dict[str, Any], base_cfg: Optional[Dict[str, Any]] = None
     ):
@@ -50,7 +56,12 @@ class ParamGen:
         self.base_cfg = base_cfg if base_cfg is not None else {}
 
         # Recursively flatten the nested sweep config into dot-notation keys
-        self.param_space = self._flatten_params(sweep_cfg)
+        # Refactored to use DictUtils, passing our custom leaf logic
+        flat_sweep = DictUtils.flatten(sweep_cfg, is_leaf_predicate=self._is_sweep_leaf)
+
+        # Expand values (ranges, strings) which DictUtils leaves as-is
+        self.param_space = {k: self._expand_val(v) for k, v in flat_sweep.items()}
+
         self.keys = list(self.param_space.keys())
         self.values = list(self.param_space.values())
 
@@ -66,13 +77,16 @@ class ParamGen:
 
             # 2. Deep copy base and update with current params
             new_cfg = copy.deepcopy(self.base_cfg)
-            new_cfg = self._apply_updates(new_cfg, flat_params)
+
+            # Refactored: Unflatten the specific params, then deep merge
+            update_structure = DictUtils.unflatten(flat_params)
+            DictUtils.deep_update(new_cfg, update_structure)
 
             # 3. Store metadata (Optional)
             # if "_meta" not in new_cfg:
             #     new_cfg["_meta"] = {}
             # We unflatten the sweep params here so the log is readable
-            # new_cfg["_meta"]["sweep_params"] = self._unflatten(flat_params)
+            # new_cfg["_meta"]["sweep_params"] = DictUtils.unflatten(flat_params)
 
             yield new_cfg
 
@@ -124,27 +138,8 @@ class ParamGen:
             combinations.append(flat_dict)
         return combinations
 
-    def _unflatten(self, flat_dict: Dict[str, Any]) -> Dict[str, Any]:
-        """Converts {'a.b': 1} back to {'a': {'b': 1}}."""
-        nested = {}
-        self._apply_updates(nested, flat_dict)
-        return nested
-
-    def _flatten_params(
-        self, cfg: Dict[str, Any], parent_key: str = ""
-    ) -> Dict[str, List[Any]]:
-        """Recursively converts nested dicts into flat dot-notation keys."""
-        flat = {}
-        for key, val in cfg.items():
-            current_key = f"{parent_key}.{key}" if parent_key else key
-
-            if self._is_sweep_leaf(val):
-                flat[current_key] = self._expand_val(val)
-            elif isinstance(val, dict):
-                flat.update(self._flatten_params(val, current_key))
-            else:
-                flat[current_key] = [val]
-        return flat
+    # Note: _unflatten, _flatten_params, and _apply_updates have been removed
+    # as they are replaced by DictUtils methods.
 
     def _is_sweep_leaf(self, val: Any) -> bool:
         if isinstance(val, list):
@@ -173,17 +168,3 @@ class ParamGen:
             return np.arange(val["start"], val["stop"], step).tolist()
 
         return [val]
-
-    def _apply_updates(
-        self, cfg: Dict[str, Any], updates: Dict[str, Any]
-    ) -> Dict[str, Any]:
-        """Deep merges dot-notation updates into cfg."""
-        for key, val in updates.items():
-            parts = key.split(".")
-            target = cfg
-            for part in parts[:-1]:
-                if part not in target:
-                    target[part] = {}
-                target = target[part]
-            target[parts[-1]] = val
-        return cfg

halib/exp/perf/perfcalc.py

@@ -210,7 +210,37 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
         **kwargs,
     ) -> Tuple[Union[List[OrderedDict], pd.DataFrame], Optional[str]]:
         """
-        Standard use case: Calculate metrics AND save to CSV.
+        Orchestrates the calculation of performance metrics and saves the results to a CSV.
+
+        This method acts as a pipeline:
+        1. Calculates metrics (Accuracy, F1, etc.) using `raw_metrics_data`.
+        2. Merges in `extra_data` (metadata) to the results.
+        3. Saves the combined data to a CSV file.
+
+        Args:
+            raw_metrics_data (Union[List[dict], dict]):
+                The input data required for metric calculations.
+                Structure: A dictionary where keys are metric names (e.g., 'accuracy')
+                and values are inputs (e.g., {'preds': ..., 'target': ...}).
+
+            extra_data (Optional[Union[List[dict], dict]]):
+                Static metadata or context to attach to the result rows.
+                These fields are NOT used for calculation but are passed through
+                to the final output (DataFrame/CSV).
+
+                Example:
+                    If extra_data={'model_version': 'v1.0', 'epoch': 5},
+                    the output CSV will include columns 'model_version' and 'epoch'.
+
+            *args, **kwargs:
+                Additional arguments passed to `calc_exp_perf_metrics` or `save_results_to_csv`
+                (e.g., `outfile`, `return_df`).
+
+        Returns:
+            Tuple[Union[List[OrderedDict], pd.DataFrame], Optional[str]]:
+                A tuple containing:
+                1. The results as a DataFrame (if return_df=True) or a list of dicts.
+                2. The path to the saved output file (or None if save failed).
         """
         metric_names = self.get_metric_backend().metric_names
 

halib/utils/dict.py

@@ -1,9 +1,203 @@
-def flatten_dict(d, parent_key="", sep="."):
-    items = {}
-    for k, v in d.items():
-        key = f"{parent_key}{sep}{k}" if parent_key else k
-        if isinstance(v, dict):
-            items.update(flatten_dict(v, key, sep=sep))
-        else:
-            items[key] = v
-    return items
+from pygments.token import Other
+from typing import Dict, Any, Callable, Optional
+from rich.pretty import pprint
+import json
+import hashlib
+
+
+class DictUtils:
+    """
+    General-purpose dictionary manipulation utilities.
+    """
+
+    @staticmethod
+    def flatten(
+        d: Dict[str, Any],
+        parent_key: str = "",
+        sep: str = ".",
+        is_leaf_predicate: Optional[Callable[[Any], bool]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Recursively flattens a nested dictionary.
+
+        Args:
+            d: The dictionary to flatten.
+            parent_key: Prefix for keys (used during recursion).
+            sep: Separator for dot-notation keys.
+            is_leaf_predicate: Optional function that returns True if a value should
+                               be treated as a leaf (value) rather than a branch to recurse.
+                               Useful if you have dicts you don't want flattened.
+        """
+        items = []
+        for k, v in d.items():
+            new_key = f"{parent_key}{sep}{k}" if parent_key else k
+
+            # Check if we should treat this as a leaf (custom logic)
+            if is_leaf_predicate and is_leaf_predicate(v):
+                items.append((new_key, v))
+            # Standard recursion
+            elif isinstance(v, dict):
+                items.extend(
+                    DictUtils.flatten(
+                        v, new_key, sep=sep, is_leaf_predicate=is_leaf_predicate
+                    ).items()
+                )
+            else:
+                items.append((new_key, v))
+        return dict(items)
+
+    @staticmethod
+    def unflatten(flat_dict: Dict[str, Any], sep: str = ".") -> Dict[str, Any]:
+        """
+        Converts flat dot-notation keys back to nested dictionaries.
+        e.g., {'a.b': 1} -> {'a': {'b': 1}}
+        """
+        nested = {}
+        for key, value in flat_dict.items():
+            DictUtils.deep_set(nested, key, value, sep=sep)
+        return nested
+
+    @staticmethod
+    def deep_update(base: Dict[str, Any], update: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Recursively merges 'update' dict into 'base' dict.
+
+        Unlike the standard `dict.update()`, which replaces nested dictionaries entirely,
+        this method enters nested dictionaries and updates them key-by-key. This preserves
+        existing keys in 'base' that are not present in 'update'.
+
+        Args:
+            base: The original dictionary to modify.
+            update: The dictionary containing new values.
+
+        Returns:
+            The modified 'base' dictionary.
+
+        Example:
+            >>> base = {'model': {'name': 'v1', 'dropout': 0.5}}
+            >>> new_vals = {'model': {'name': 'v2'}}
+            >>> # Standard update would delete 'dropout'. deep_update keeps it:
+            >>> DictUtils.deep_update(base, new_vals)
+            {'model': {'name': 'v2', 'dropout': 0.5}}
+        """
+        for k, v in update.items():
+            if isinstance(v, dict) and k in base and isinstance(base[k], dict):
+                DictUtils.deep_update(base[k], v)
+            else:
+                base[k] = v
+        return base
+
+    @staticmethod
+    def deep_set(d: Dict[str, Any], dot_key: str, value: Any, sep: str = ".") -> None:
+        """
+        Sets a value in a nested dictionary using a dot-notation key path.
+        Automatically creates any missing intermediate dictionaries.
+
+        Args:
+            d: The dictionary to modify.
+            dot_key: The path to the value (e.g., "model.backbone.layers").
+            value: The value to set.
+            sep: The separator used in the key (default is ".").
+
+        Example:
+            >>> cfg = {}
+            >>> DictUtils.deep_set(cfg, "a.b.c", 10)
+            >>> print(cfg)
+            {'a': {'b': {'c': 10}}}
+        """
+        parts = dot_key.split(sep)
+        target = d
+        for part in parts[:-1]:
+            if part not in target:
+                target[part] = {}
+            target = target[part]
+            if not isinstance(target, dict):
+                # Handle conflict if a path was previously a value (e.g. overwriting a leaf)
+                target = {}
+        target[parts[-1]] = value
+
+    @staticmethod
+    def get_unique_hash(input_dict, length=12):
+        """
+        Returns a unique hash string for a dictionary.
+
+        :param input_dict: The dictionary params
+        :param length: The desired length of the hash string (default 12)
+        """
+        assert length >= 12, "Hash length must be at least 12 to ensure uniqueness."
+        # 1. Sort keys to ensure {a:1, b:2} == {b:2, a:1}
+        config_str = json.dumps(input_dict, sort_keys=True)
+
+        # 2. Generate full SHA-256 hash (64 chars long)
+        full_hash = hashlib.sha256(config_str.encode("utf-8")).hexdigest()
+
+        # 3. Truncate to desired length
+        return full_hash[:length]
+
+
+def test_update():
+    # --- Setup ---
+    base_config = {
+        "model": {
+            "name": "ResNet50",
+            "layers": 50,
+            "details": {
+                "activation": "relu",
+                "dropout": 0.5,  # <--- We want to keep this
+            },
+        },
+        "epochs": 10,
+    }
+
+    new_settings = {
+        "model": {"details": {"activation": "gelu"}}  # <--- We only want to change this
+    }
+
+    b1 = base_config.copy()
+    b2 = base_config.copy()
+    n1 = new_settings.copy()
+    n2 = new_settings.copy()
+
+    pprint("Base Config:")
+    pprint(base_config)
+    pprint("New Settings:")
+    pprint(new_settings)
+    print("*" * 40)
+    pprint(
+        "Task: Update base_config with new_settings, preserving unspecified nested keys."
+    )
+    print("*" * 40)
+
+    # --- Standard Update (The Problem) ---
+    pprint("Normal Update Result:")
+    b1.update(n1)
+    pprint(b1)  # type: ignore[return-value]
+
+    # --- Deep Update (The Solution) ---
+    pprint("Deep Update Result:")
+    pprint(DictUtils.deep_update(b2, n2))
+
+
+def test_hash():
+    # --- Usage ---
+    cfg1 = {"learning_rate": 0.01, "batch_size": 32, "optimizer": "adam"}
+    cfg1_shuffle = {
+        "batch_size": 32,
+        "optimizer": "adam",
+        "learning_rate": 0.01,
+    }
+    cfg2 = {"learning_rate": 0.02, "batch_size": 32, "optimizer": "adam"}
+    hash1 = DictUtils.get_unique_hash(cfg1)
+    hash2 = DictUtils.get_unique_hash(cfg1_shuffle)
+    hash3 = DictUtils.get_unique_hash(cfg2)
+    pprint(f"Config 1 Hash: {hash1}")
+    pprint(f"Config 1_shuffle Hash: {hash2}")
+    pprint(f"Config 2 Hash: {hash3}")
+
+    assert hash1 == hash2, "Hashes should match for identical dicts."
+    assert hash1 != hash3, "Hashes should differ for different dicts."
+
+
+if __name__ == "__main__":
+    test_update()
+    test_hash()

{halib-0.2.21.dist-info → halib-0.2.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.2.21
+Version: 0.2.27
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -56,7 +56,9 @@ Dynamic: summary
 
 ## v0.2.x (Experiment & Core Updates)
 
-### **v0.2.21**
+### **v0.2.27**
+- ✨ **New Feature:** Added `utils.dict.DictUtils` for advanced dictionary manipulations (merging, filtering, transforming).
+
 - ✨ **New Feature:** Added `common.common.pprint_stack_trace` to print stack traces with optional custom messages and force stop capability.
 
 - 🚀 **Improvement:**  `exp.perf.profiler` - allow to export *report dict* as csv files for further analysis

{halib-0.2.21.dist-info → halib-0.2.27.dist-info}/RECORD

@@ -23,7 +23,7 @@ halib/exp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/exp/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/exp/core/base_config.py,sha256=Js2oVDt7qwT7eV_sOUWw6XXl569G1bX6ls-VYAx2gWY,5032
 halib/exp/core/base_exp.py,sha256=fknJVmW6ubbapOggbkrbNWgc1ZXcUz_FE3wMyuIGX7M,5180
-halib/exp/core/param_gen.py,sha256=_JjakBOr0UuJOxR11ZC-mrX7ye5kdc1SfGLZuYFmG1o,7385
+halib/exp/core/param_gen.py,sha256=y4elw6bmoBMKRzS7KDLXp0xqC3x27gg0r6wrcTBfidc,6725
 halib/exp/core/wandb_op.py,sha256=powL2QyLBqF-6PUGAOqd60s1npHLLKJxPns3S4hKeNo,4160
 halib/exp/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/exp/data/dataclass_util.py,sha256=OPZzqmug0be4JEq0hJ68pKjnyl0PRYQMVJGhKw1kvyk,1382
@@ -32,7 +32,7 @@ halib/exp/data/torchloader.py,sha256=oWUplXlGd1IB6CqdRd-mGe-DfMjjZxz9hQ7SWONb-0s
 halib/exp/perf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/exp/perf/flop_calc.py,sha256=Kb3Gwqc7QtGALZzfyYXBA_9SioReJpTJdUX84kqj-Aw,6907
 halib/exp/perf/gpu_mon.py,sha256=vD41_ZnmPLKguuq9X44SB_vwd9JrblO4BDzHLXZhhFY,2233
-halib/exp/perf/perfcalc.py,sha256=p7rhVShiie7DT_s50lbvbGftVCkrWE0tQGFLUEmTXi0,18326
+halib/exp/perf/perfcalc.py,sha256=zb0eGt24kPVC2HTq9M095wP6y8TqOicWy52BxAigap0,19834
 halib/exp/perf/perfmetrics.py,sha256=qRiNiCKGUSTLY7gPMVMuVHGAAyeosfGWup2eM4490aw,5485
 halib/exp/perf/perftb.py,sha256=IWElg3OB5dmhfxnY8pMZvkL2y_EnvLmEx3gJlpUR1Fs,31066
 halib/exp/perf/profiler.py,sha256=Nx-Y1V3pCbaEOisY1sDT8s0yGFs3J6TUmutZslseoNI,19201
@@ -95,15 +95,15 @@ halib/system/filesys.py,sha256=102J2fkQhmH1_-HQVy2FQ4NOU8LTjMWV3hToT_APtq8,4401
 halib/system/path.py,sha256=k_pveq41uXEzKPU2KTIdqjUSb4MVM-hCFXHGeO-6x6Q,3694
 halib/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/utils/dataclass_util.py,sha256=rj2IMLlUzbm2OlF5_B2dRTk9njZOaF7tTjYkOsq8uLY,1416
-halib/utils/dict.py,sha256=wYE6Iw-_CnCWdMg9tpJ2Y2-e2ESkW9FxmdBkZkbUh80,299
+halib/utils/dict.py,sha256=tUmwKNbTLOyGLl5nKYdT8QIoS4pBFfdsZHCJ6NQCgp4,6922
 halib/utils/dict_op.py,sha256=wYE6Iw-_CnCWdMg9tpJ2Y2-e2ESkW9FxmdBkZkbUh80,299
 halib/utils/gpu_mon.py,sha256=vD41_ZnmPLKguuq9X44SB_vwd9JrblO4BDzHLXZhhFY,2233
 halib/utils/list.py,sha256=BM-8sRhYyqF7bh4p7TQtV7P_gnFruUCA6DTUOombaZg,337
 halib/utils/listop.py,sha256=Vpa8_2fI0wySpB2-8sfTBkyi_A4FhoFVVvFiuvW8N64,339
 halib/utils/tele_noti.py,sha256=-4WXZelCA4W9BroapkRyIdUu9cUVrcJJhegnMs_WpGU,5928
 halib/utils/video.py,sha256=zLoj5EHk4SmP9OnoHjO8mLbzPdtq6gQPzTQisOEDdO8,3261
-halib-0.2.21.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.2.21.dist-info/METADATA,sha256=cADukwK5Nxve5oqMTadeSYQUBOuEHUhkePS3i8dtqrs,7590
-halib-0.2.21.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-halib-0.2.21.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.2.21.dist-info/RECORD,,
+halib-0.2.27.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.2.27.dist-info/METADATA,sha256=Or9W2uqrbMh_PQd3KCda2KZRuuH4Wi7I4aicUPq2Ks0,7719
+halib-0.2.27.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+halib-0.2.27.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.2.27.dist-info/RECORD,,