halib 0.2.30__py3-none-any.whl

halib/exp/core/base_exp.py

@@ -0,0 +1,147 @@
+from abc import ABC, abstractmethod
+from typing import Tuple, Any, Optional
+from .base_config import ExpBaseCfg
+from ..perf.perfcalc import PerfCalc
+from ..perf.perfmetrics import MetricsBackend
+
+
+class ExpHook:
+    """Base interface for all experiment hooks."""
+    def on_before_run(self, exp): pass
+    def on_after_run(self, exp, results): pass
+
+
+# ! SEE https://github.com/hahv/base_exp for sample usage
+class BaseExp(PerfCalc, ABC):
+    """
+    Base class for experiments.
+    Orchestrates the experiment pipeline using a pluggable metrics backend.
+    """
+
+    def __init__(self, config: ExpBaseCfg):
+        self.config = config
+        self.metric_backend = None
+        # Flag to track if init_general/prepare_dataset has run
+        self._is_env_ready = False
+        self.hooks = []
+
+    def register_hook(self, hook: ExpHook):
+        self.hooks.append(hook)
+
+    def _trigger_hooks(self, method_name: str, *args, **kwargs):
+        for hook in self.hooks:
+            method = getattr(hook, method_name, None)
+            if callable(method):
+                method(*args, **kwargs)
+
+    # -----------------------
+    # PerfCalc Required Methods
+    # -----------------------
+    def get_dataset_name(self):
+        return self.config.get_dataset_cfg().get_name()
+
+    def get_experiment_name(self):
+        return self.config.get_cfg_name()
+
+    def get_metric_backend(self):
+        if not self.metric_backend:
+            self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+        return self.metric_backend
+
+    # -----------------------
+    # Abstract Experiment Steps
+    # -----------------------
+    @abstractmethod
+    def init_general(self, general_cfg):
+        """Setup general settings like SEED, logging, env variables."""
+        pass
+
+    @abstractmethod
+    def prepare_dataset(self, dataset_cfg):
+        """Load/prepare dataset."""
+        pass
+
+    @abstractmethod
+    def prepare_metrics(self, metric_cfg) -> MetricsBackend:
+        """
+        Prepare the metrics for the experiment.
+        This method should be implemented in subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def exec_exp(self, *args, **kwargs) -> Optional[Tuple[Any, Any]]:
+        """Run experiment process, e.g.: training/evaluation loop.
+        Return: either `None` or a tuple of (raw_metrics_data, extra_data) for calc_and_save_exp_perfs
+        """
+        pass
+
+    # -----------------------
+    # Internal Helpers
+    # -----------------------
+    def _validate_and_unpack(self, results):
+        if results is None:
+            return None
+        if not isinstance(results, (tuple, list)) or len(results) != 2:
+            raise ValueError("exec must return (metrics_data, extra_data)")
+        return results[0], results[1]
+
+    def _prepare_environment(self, force_reload: bool = False):
+        """
+        Common setup. Skips if already initialized, unless force_reload is True.
+        """
+        if self._is_env_ready and not force_reload:
+            # Environment is already prepared, skipping setup.
+            return
+
+        # 1. Run Setup
+        self.init_general(self.config.get_general_cfg())
+        self.prepare_dataset(self.config.get_dataset_cfg())
+
+        # 2. Update metric backend (refresh if needed)
+        self.metric_backend = self.prepare_metrics(self.config.get_metric_cfg())
+
+        # 3. Mark as ready
+        self._is_env_ready = True
+
+    # -----------------------
+    # Main Experiment Runner
+    # -----------------------
+    def run_exp(self, should_calc_metrics=True, reload_env=False, *args, **kwargs):
+        """
+        Run the whole experiment pipeline.
+        :param reload_env: If True, forces dataset/general init to run again.
+        :param should_calc_metrics: Whether to calculate and save metrics after execution.
+        :kwargs Params:
+            + 'outfile' to save csv file results,
+            + 'outdir' to set output directory for experiment results.
+            + 'return_df' to return a DataFrame of results instead of a dictionary.
+
+        Full pipeline:
+            1. Init
+            2. Prepare Environment (General + Dataset + Metrics)
+            3. Save Config
+            4. Execute
+            5. Calculate & Save Metrics
+        """
+        self._prepare_environment(force_reload=reload_env)
+
+        self._trigger_hooks("before_run", self)
+
+        # Save config before running
+        self.config.save_to_outdir()
+
+        # Execute experiment
+        results = self.exec_exp(*args, **kwargs)
+
+        if should_calc_metrics and results is not None:
+            metrics_data, extra_data = self._validate_and_unpack(results)
+            # Calculate & Save metrics
+            perf_results = self.calc_perfs(
+                raw_metrics_data=metrics_data, extra_data=extra_data, *args, **kwargs
+            )
+            self._trigger_hooks("after_run", self, perf_results)
+            return perf_results
+        else:
+            self._trigger_hooks("after_run", self, results)
+            return results

halib/exp/core/param_gen.py

@@ -0,0 +1,170 @@
+import os
+import copy
+import numpy as np
+from itertools import product
+from typing import Dict, Any, List, Iterator, Optional
+from ...filetype import yamlfile
+from ...utils.dict import DictUtils
+
+# Assuming DictUtils is available in the scope or imported
+# from .dict_utils import DictUtils
+
+
+class ParamGen:
+    """
+    A flexible parameter grid generator for hyperparameter tuning and experiment management.
+
+    This class generates a Cartesian product of parameters from a "sweep configuration"
+    and optionally merges them into a "base configuration". It abstracts away the complexity
+    of handling nested dictionaries and range generation.
+
+    Key Features:
+    -----------
+    1. **Flexible Syntax**: Define parameters using standard nested dictionaries or
+       dot-notation keys (e.g., `'model.backbone.layers'`).
+    2. **Range Shortcuts**:
+       - **Choices**: Standard lists `[1, 2, 3]`.
+       - **String Ranges**: `"start:stop:step"` (e.g., `"0:10:2"` -> `[0, 2, 4, 6, 8]`).
+       - **Dict Ranges**: `{'start': 0, 'stop': 1, 'step': 0.1}`.
+    3. **Deep Merging**: Automatically updates deep keys in `base_cfg` without overwriting siblings.
+
+    Example:
+    --------
+    >>> base = {'model': {'name': 'resnet', 'dropout': 0.1}, 'seed': 42}
+    >>> sweep = {
+    ...     'model.name': ['resnet', 'vit'],      # Dot notation
+    ...     'model.dropout': "0.1:0.3:0.1",       # Range string
+    ...     'seed': [42, 100]                     # Simple choice
+    ... }
+    >>> grid = ParamGen(sweep, base)
+    >>> configs = grid.expand()
+    >>> print(len(configs))  # Outputs: 8 (2 models * 2 dropouts * 2 seeds)
+    Attributes:
+        keys (List[str]): List of flattened dot-notation keys being swept.
+        values (List[List[Any]]): List of value options for each key.
+    """
+
+    def __init__(
+        self, sweep_cfg: Dict[str, Any], base_cfg: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Args:
+            sweep_cfg: The dictionary defining parameters to sweep.
+            base_cfg: (Optional) The base config to merge sweep parameters into.
+                      If None, expand() behaves like expand_sweep().
+        """
+        self.base_cfg = base_cfg if base_cfg is not None else {}
+
+        # Recursively flatten the nested sweep config into dot-notation keys
+        # Refactored to use DictUtils, passing our custom leaf logic
+        flat_sweep = DictUtils.flatten(sweep_cfg, is_leaf_predicate=self._is_sweep_leaf)
+
+        # Expand values (ranges, strings) which DictUtils leaves as-is
+        self.param_space = {k: self._expand_val(v) for k, v in flat_sweep.items()}
+
+        self.keys = list(self.param_space.keys())
+        self.values = list(self.param_space.values())
+
+    def get_param_space(self) -> Dict[str, List[Any]]:
+        """Returns the parameter space as a dictionary of dot-notation keys to value lists."""
+        return self.param_space
+
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
+        """Yields fully merged configurations one by one."""
+        for combination in product(*self.values):
+            # 1. Create the flat sweep dict (dot notation)
+            flat_params = dict(zip(self.keys, combination))
+
+            # 2. Deep copy base and update with current params
+            new_cfg = copy.deepcopy(self.base_cfg)
+
+            # Refactored: Unflatten the specific params, then deep merge
+            update_structure = DictUtils.unflatten(flat_params)
+            DictUtils.deep_update(new_cfg, update_structure)
+
+            # 3. Store metadata (Optional)
+            # if "_meta" not in new_cfg:
+            #     new_cfg["_meta"] = {}
+            # We unflatten the sweep params here so the log is readable
+            # new_cfg["_meta"]["sweep_params"] = DictUtils.unflatten(flat_params)
+
+            yield new_cfg
+
+    # ! --- Factory Methods ---
+    @classmethod
+    def from_dicts(
+        cls, sweep_cfg: Dict[str, Any], base_cfg: Optional[Dict[str, Any]] = None
+    ):
+        """
+        Load from dictionaries.
+        Args:
+            sweep_cfg: The dictionary defining parameters to sweep.
+            base_cfg: (Optional) The base config to merge sweep parameters into.
+        """
+        return cls(sweep_cfg, base_cfg)
+
+    @classmethod
+    def from_files(cls, sweep_yaml: str, base_yaml: Optional[str] = None):
+        """
+        Load from files.
+        Args:
+            sweep_yaml: Path to sweep config.
+            base_yaml: (Optional) Path to base config.
+        """
+        assert os.path.isfile(sweep_yaml), f"Sweep file not found: {sweep_yaml}"
+        sweep_dict = yamlfile.load_yaml(sweep_yaml, to_dict=True)
+        base_dict = None
+        if base_yaml:
+            base_dict = yamlfile.load_yaml(base_yaml, to_dict=True)
+            if "__base__" in base_dict:
+                del base_dict["__base__"]
+
+        return cls(sweep_dict, base_dict)
+
+    def expand(self) -> List[Dict[str, Any]]:
+        """Generates and returns the full list of MERGED configurations."""
+        return list(self)
+
+    def expand_sweep_flat(self) -> List[Dict[str, Any]]:
+        """
+        Returns a list of ONLY the sweep parameters, formatted as FLAT dot-notation dictionaries.
+
+        Returns:
+            [{'exp_params.model': 'resnet', 'exp_params.lr': 0.01}, ...]
+        """
+        combinations = []
+        for combination in product(*self.values):
+            flat_dict = dict(zip(self.keys, combination))
+            combinations.append(flat_dict)
+        return combinations
+
+    # Note: _unflatten, _flatten_params, and _apply_updates have been removed
+    # as they are replaced by DictUtils methods.
+
+    def _is_sweep_leaf(self, val: Any) -> bool:
+        if isinstance(val, list):
+            return True
+        if isinstance(val, str) and ":" in val:
+            return True
+        if isinstance(val, dict) and "start" in val and "stop" in val:
+            return True
+        return False
+
+    def _expand_val(self, val: Any) -> List[Any]:
+        if isinstance(val, list):
+            return val
+
+        if isinstance(val, str) and ":" in val:
+            try:
+                parts = [float(x) for x in val.split(":")]
+                if len(parts) == 3:
+                    arr = np.arange(parts[0], parts[1], parts[2])
+                    return [float(f"{x:.6g}") for x in arr]
+            except ValueError:
+                pass
+
+        if isinstance(val, dict) and "start" in val:
+            step = val.get("step", 1)
+            return np.arange(val["start"], val["stop"], step).tolist()
+
+        return [val]

halib/exp/core/wandb_op.py

@@ -0,0 +1,117 @@
+import os
+import glob
+import wandb
+import argparse
+import subprocess
+
+from tqdm import tqdm
+from rich.console import Console
+
+console = Console()
+
+def sync_runs(outdir):
+    outdir = os.path.abspath(outdir)
+    assert os.path.exists(outdir), f"Output directory {outdir} does not exist."
+    sub_dirs = [name for name in os.listdir(outdir) if os.path.isdir(os.path.join(outdir, name))]
+    assert len(sub_dirs) > 0, f"No subdirectories found in {outdir}."
+    console.rule("Parent Directory")
+    console.print(f"[yellow]{outdir}[/yellow]")
+
+    exp_dirs = [os.path.join(outdir, sub_dir) for sub_dir in sub_dirs]
+    wandb_dirs = []
+    for exp_dir in exp_dirs:
+        wandb_dirs.extend(glob.glob(f"{exp_dir}/wandb/*run-*"))
+    if len(wandb_dirs) == 0:
+        console.print(f"No wandb runs found in {outdir}.")
+        return
+    else:
+        console.print(f"Found [bold]{len(wandb_dirs)}[/bold] wandb runs in {outdir}.")
+        for i, wandb_dir in enumerate(wandb_dirs):
+            console.rule(f"Syncing wandb run {i + 1}/{len(wandb_dirs)}")
+            console.print(f"Syncing: {wandb_dir}")
+            process = subprocess.Popen(
+                ["wandb", "sync", wandb_dir],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+            )
+
+            for line in process.stdout:
+                console.print(line.strip())
+                if " ERROR Error while calling W&B API" in line:
+                    break
+            process.stdout.close()
+            process.wait()
+            if process.returncode != 0:
+                console.print(f"[red]Error syncing {wandb_dir}. Return code: {process.returncode}[/red]")
+            else:
+                console.print(f"Successfully synced {wandb_dir}.")
+
+def delete_runs(project, pattern=None):
+    console.rule("Delete W&B Runs")
+    confirm_msg = f"Are you sure you want to delete all runs in"
+    confirm_msg += f" \n\tproject: [red]{project}[/red]"
+    if pattern:
+        confirm_msg += f"\n\tpattern: [blue]{pattern}[/blue]"
+
+    console.print(confirm_msg)
+    confirmation = input(f"This action cannot be undone. [y/N]: ").strip().lower()
+    if confirmation != "y":
+        print("Cancelled.")
+        return
+
+    print("Confirmed. Proceeding...")
+    api = wandb.Api()
+    runs = api.runs(project)
+
+    deleted = 0
+    console.rule("Deleting W&B Runs")
+    if len(runs) == 0:
+        print("No runs found in the project.")
+        return
+    for run in tqdm(runs):
+        if pattern is None or pattern in run.name:
+            run.delete()
+            console.print(f"Deleted run: [red]{run.name}[/red]")
+            deleted += 1
+
+    console.print(f"Total runs deleted: {deleted}")
+
+
+def valid_argument(args):
+    if args.op == "sync":
+        assert os.path.exists(args.outdir), f"Output directory {args.outdir} does not exist."
+    elif args.op == "delete":
+        assert isinstance(args.project, str) and len(args.project.strip()) > 0, "Project name must be a non-empty string."
+    else:
+        raise ValueError(f"Unknown operation: {args.op}")
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Operations on W&B runs")
+    parser.add_argument("-op", "--op", type=str, help="Operation to perform", default="sync", choices=["delete", "sync"])
+    parser.add_argument("-prj", "--project", type=str, default="fire-paper2-2025", help="W&B project name")
+    parser.add_argument("-outdir", "--outdir", type=str, help="arg1 description", default="./zout/train")
+    parser.add_argument("-pt", "--pattern",
+        type=str,
+        default=None,
+        help="Run name pattern to match for deletion",
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    # Validate arguments, stop if invalid
+    valid_argument(args)
+
+    op = args.op
+    if op == "sync":
+        sync_runs(args.outdir)
+    elif op == "delete":
+        delete_runs(args.project, args.pattern)
+    else:
+        raise ValueError(f"Unknown operation: {op}")
+
+if __name__ == "__main__":
+    main()

halib/exp/data/dataclass_util.py

@@ -0,0 +1,41 @@
+import yaml
+from typing import Any
+
+from rich.pretty import pprint
+from dataclasses import make_dataclass
+
+from ...filetype import yamlfile
+
+def dict_to_dataclass(name: str, data: dict):
+    fields = []
+    values = {}
+
+    for key, value in data.items():
+        if isinstance(value, dict):
+            sub_dc = dict_to_dataclass(key.capitalize(), value)
+            fields.append((key, type(sub_dc)))
+            values[key] = sub_dc
+        else:
+            field_type = type(value) if value is not None else Any
+            fields.append((key, field_type))
+            values[key] = value
+
+    DC = make_dataclass(name.capitalize(), fields)
+    return DC(**values)
+
+def yaml_to_dataclass(name: str, yaml_str: str):
+    data = yaml.safe_load(yaml_str)
+    return dict_to_dataclass(name, data)
+
+
+def yamlfile_to_dataclass(name: str, file_path: str):
+    data_dict = yamlfile.load_yaml(file_path, to_dict=True)
+    if "__base__" in data_dict:
+        del data_dict["__base__"]
+    return dict_to_dataclass(name, data_dict)
+
+if __name__ == "__main__":
+    cfg = yamlfile_to_dataclass("Config", "test/dataclass_util_test_cfg.yaml")
+
+    # ! NOTICE: after print out this dataclass, we can copy the output and paste it into CHATGPT to generate a list of needed dataclass classes using `from dataclass_wizard import YAMLWizard`
+    pprint(cfg)

halib/exp/data/dataset.py

@@ -0,0 +1,208 @@
+# This script create a test version
+# of the watcam (wc) dataset
+# for testing the tflite model
+
+from argparse import ArgumentParser
+
+import os
+import click
+import shutil
+from tqdm import tqdm
+from rich import inspect
+from rich.pretty import pprint
+from torchvision.datasets import ImageFolder
+from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
+
+from ...common.common import console, seed_everything, ConsoleLog
+from ...system import filesys as fs
+
+def parse_args():
+    parser = ArgumentParser(description="desc text")
+    parser.add_argument(
+        "-indir",
+        "--indir",
+        type=str,
+        help="orignal dataset path",
+    )
+    parser.add_argument(
+        "-outdir",
+        "--outdir",
+        type=str,
+        help="dataset out path",
+        default=".",  # default to current dir
+    )
+    parser.add_argument(
+        "-val_size",
+        "--val_size",
+        type=float,
+        help="validation size",  # no default value to force user to input
+        default=0.2,
+    )
+    # add using StratifiedShuffleSplit or ShuffleSplit
+    parser.add_argument(
+        "-seed",
+        "--seed",
+        type=int,
+        help="random seed",
+        default=42,
+    )
+    parser.add_argument(
+        "-inplace",
+        "--inplace",
+        action="store_true",
+        help="inplace operation, will overwrite the outdir if exists",
+    )
+
+    parser.add_argument(
+        "-stratified",
+        "--stratified",
+        action="store_true",
+        help="use StratifiedShuffleSplit instead of ShuffleSplit",
+    )
+    parser.add_argument(
+        "-no_train",
+        "--no_train",
+        action="store_true",
+        help="only create test set, no train set",
+    )
+    parser.add_argument(
+        "-reverse",
+        "--reverse",
+        action="store_true",
+        help="combine train and val set back to original dataset",
+    )
+    return parser.parse_args()
+
+
+def move_images(image_paths, target_set_dir):
+    for img_path in tqdm(image_paths):
+        # get folder name of the image
+        img_dir = os.path.dirname(img_path)
+        out_cls_dir = os.path.join(target_set_dir, os.path.basename(img_dir))
+        if not os.path.exists(out_cls_dir):
+            os.makedirs(out_cls_dir)
+        # move the image to the class folder
+        shutil.move(img_path, out_cls_dir)
+
+
+def split_dataset_cls(
+    indir, outdir, val_size, seed, inplace, stratified_split, no_train
+):
+    seed_everything(seed)
+    console.rule("Config confirm?")
+    pprint(locals())
+    click.confirm("Continue?", abort=True)
+    assert os.path.exists(indir), f"{indir} does not exist"
+
+    if not inplace:
+        assert (not inplace) and (
+            not os.path.exists(outdir)
+        ), f"{outdir} already exists; SKIP ...."
+
+    if inplace:
+        outdir = indir
+    if not os.path.exists(outdir):
+        os.makedirs(outdir)
+
+    console.rule(f"Creating train/val dataset")
+
+    sss = (
+        ShuffleSplit(n_splits=1, test_size=val_size)
+        if not stratified_split
+        else StratifiedShuffleSplit(n_splits=1, test_size=val_size)
+    )
+
+    pprint({"split strategy": sss, "indir": indir, "outdir": outdir})
+    dataset = ImageFolder(
+        root=indir,
+        transform=None,
+    )
+    train_dataset_indices = None
+    val_dataset_indices = None  # val here means test
+    for train_indices, val_indices in sss.split(dataset.samples, dataset.targets):
+        train_dataset_indices = train_indices
+        val_dataset_indices = val_indices
+
+    # get image paths for train/val split dataset
+    train_image_paths = [dataset.imgs[i][0] for i in train_dataset_indices]
+    val_image_paths = [dataset.imgs[i][0] for i in val_dataset_indices]
+
+    # start creating train/val folders then move images
+    out_train_dir = os.path.join(outdir, "train")
+    out_val_dir = os.path.join(outdir, "val")
+    if inplace:
+        assert os.path.exists(out_train_dir) == False, f"{out_train_dir} already exists"
+        assert os.path.exists(out_val_dir) == False, f"{out_val_dir} already exists"
+
+    os.makedirs(out_train_dir)
+    os.makedirs(out_val_dir)
+
+    if not no_train:
+        with ConsoleLog(f"Moving train images to {out_train_dir} "):
+            move_images(train_image_paths, out_train_dir)
+    else:
+        pprint("test only, skip moving train images")
+        # remove out_train_dir
+        shutil.rmtree(out_train_dir)
+
+    with ConsoleLog(f"Moving val images to {out_val_dir} "):
+        move_images(val_image_paths, out_val_dir)
+
+    if inplace:
+        pprint(f"remove all folders, except train and val")
+        for cls_dir in os.listdir(outdir):
+            if cls_dir not in ["train", "val"]:
+                shutil.rmtree(os.path.join(indir, cls_dir))
+
+
+def reverse_split_ds(indir):
+    console.rule(f"Reversing split dataset <{indir}>...")
+    ls_dirs = os.listdir(indir)
+    # make sure there are only two dirs 'train' and 'val'
+    assert len(ls_dirs) == 2, f"Found more than 2 dirs: {len(ls_dirs) } dirs"
+    assert "train" in ls_dirs, f"train dir not found in {indir}"
+    assert "val" in ls_dirs, f"val dir not found in {indir}"
+    train_dir = os.path.join(indir, "train")
+    val_dir = os.path.join(indir, "val")
+    all_train_files = fs.filter_files_by_extension(
+        train_dir, ["jpg", "jpeg", "png", "bmp", "gif", "tiff"]
+    )
+    all_val_files = fs.filter_files_by_extension(
+        val_dir, ["jpg", "jpeg", "png", "bmp", "gif", "tiff"]
+    )
+    # move all files from train to indir
+    with ConsoleLog(f"Moving train images to {indir} "):
+        move_images(all_train_files, indir)
+    with ConsoleLog(f"Moving val images to {indir} "):
+        move_images(all_val_files, indir)
+    with ConsoleLog(f"Removing train and val dirs"):
+        # remove train and val dirs
+        shutil.rmtree(train_dir)
+        shutil.rmtree(val_dir)
+
+
+def main():
+    args = parse_args()
+    indir = args.indir
+    outdir = args.outdir
+    if outdir == ".":
+        # get current folder of the indir
+        indir_parent_dir = os.path.dirname(os.path.normpath(indir))
+        indir_name = os.path.basename(indir)
+        outdir = os.path.join(indir_parent_dir, f"{indir_name}_split")
+    val_size = args.val_size
+    seed = args.seed
+    inplace = args.inplace
+    stratified_split = args.stratified
+    no_train = args.no_train
+    reverse = args.reverse
+    if not reverse:
+        split_dataset_cls(
+            indir, outdir, val_size, seed, inplace, stratified_split, no_train
+        )
+    else:
+        reverse_split_ds(indir)
+
+
+if __name__ == "__main__":
+    main()