halib 0.1.91__py3-none-any.whl → 0.1.99__py3-none-any.whl

halib/common.py

@@ -1,3 +1,4 @@
+import os
 import re
 import rich
 import arrow
@@ -9,6 +10,8 @@ from rich import print
 from rich.panel import Panel
 from rich.console import Console
 from rich.pretty import pprint, Pretty
+from pathlib import PureWindowsPath
+
 
 console = Console()
 
@@ -91,18 +94,58 @@ class ConsoleLog:
             print(f"Exception message: {exc_value}")
 
 
-def pprint_local_path(local_path, tag=""):
-    # Create a file URI
-    file_path = Path(local_path).resolve()
-    is_file = file_path.is_file()
-    is_dir = file_path.is_dir()
-    type_str = "📄" if is_file else "📁" if is_dir else "❓"
-    file_uri = file_path.as_uri()
+def linux_to_wins_path(path: str) -> str:
+    """
+    Convert a Linux-style WSL path (/mnt/c/... or /mnt/d/...) to a Windows-style path (C:\...).
+    """
+    # Handle only /mnt/<drive>/... style
+    if (
+        path.startswith("/mnt/")
+        and len(path) > 6
+        and path[5].isalpha()
+        and path[6] == "/"
+    ):
+        drive = path[5].upper()  # Extract drive letter
+        win_path = f"{drive}:{path[6:]}"  # Replace "/mnt/c/" with "C:/"
+    else:
+        win_path = path  # Return unchanged if not a WSL-style path
+    # Normalize to Windows-style backslashes
+    return str(PureWindowsPath(win_path))
+
+
+def pprint_local_path(
+    local_path: str, get_wins_path: bool = False, tag: str = ""
+) -> str:
+    """
+    Pretty-print a local path with emoji and clickable file:// URI.
+
+    Args:
+        local_path: Path to file or directory (Linux or Windows style).
+        get_wins_path: If True on Linux, convert WSL-style path to Windows style before printing.
+        tag: Optional console log tag.
+
+    Returns:
+        The file URI string.
+    """
+    p = Path(local_path).resolve()
+    type_str = "📄" if p.is_file() else "📁" if p.is_dir() else "❓"
+
+    if get_wins_path and os.name == "posix":
+        # Try WSL → Windows conversion
+        converted = linux_to_wins_path(str(p))
+        if converted != str(p):  # Conversion happened
+            file_uri = str(PureWindowsPath(converted).as_uri())
+        else:
+            file_uri = p.as_uri()
+    else:
+        file_uri = p.as_uri()
+
     content_str = f"{type_str} [link={file_uri}]{file_uri}[/link]"
-    if isinstance(tag, str) and len(tag) > 0:
+
+    if tag:
         with ConsoleLog(tag):
             console.print(content_str)
     else:
-        # If tag is not provided, just print the link
-        console.print(f"{content_str}")
+        console.print(content_str)
+
     return file_uri

halib/filetype/ipynb.py

@@ -0,0 +1,63 @@
+from contextlib import contextmanager
+from pathlib import Path
+
+import ipynbname
+
+from ..common import console, now_str
+
+
+@contextmanager
+def gen_ipynb_name(
+    filename,
+    add_time_stamp=False,
+    nb_prefix="nb__",
+    separator="__",
+):
+    """
+    Context manager that prefixes the filename with the notebook name.
+    Output: NotebookName_OriginalName.ext
+    """
+    try:
+        nb_name = ipynbname.name()
+    except FileNotFoundError:
+        nb_name = "script"  # Fallback
+
+    p = Path(filename)
+
+    # --- FIX START ---
+
+    # 1. Get the parts separately
+    original_stem = p.stem  # "test" (no extension)
+    extension = p.suffix  # ".csv"
+
+    now_string = now_str() if add_time_stamp else ""
+
+    # 2. Construct the base name (Notebook + Separator + OriginalName)
+    base_name = f"{nb_prefix}{nb_name}{separator}{original_stem}"
+
+    # 3. Append timestamp if needed
+    if now_string:
+        base_name = f"{base_name}{separator}{now_string}"
+
+    # 4. Add the extension at the VERY END
+    new_filename = f"{base_name}{extension}"
+
+    # --- FIX END ---
+
+    final_path = p.parent / new_filename
+
+    # Assuming you use 'rich' console based on your snippet
+    # console.rule()
+    # print(f"📝 Saving as: {final_path}")
+
+    yield str(final_path)
+
+
+if __name__ == "__main__":
+    # --- Usage Example ---
+    # Assume Notebook Name is: "MyThesisWork"
+    filename = "results.csv"
+    with gen_ipynb_name(filename) as filename_ipynb:
+        # filename_ipynb is now: "MyThesisWork_results.csv"
+        print(f"File to save: {filename_ipynb}")
+        # df.to_csv(filename_ipynb)

halib/filetype/yamlfile.py

@@ -6,6 +6,8 @@ from omegaconf import OmegaConf
 from rich.console import Console
 from argparse import ArgumentParser
 
+from ..research.mics import *
+
 console = Console()
 
 
@@ -51,6 +53,27 @@ def load_yaml(yaml_file, to_dict=False, log_info=False):
     else:
         return omgconf
 
+def load_yaml_with_PC_abbr(
+    yaml_file, pc_abbr_to_working_disk=DEFAULT_ABBR_WORKING_DISK
+):
+    # current PC abbreviation
+    pc_abbr = get_PC_abbr_name()
+
+    # current plaftform: windows or linux
+    current_platform = platform.system().lower()
+
+    assert pc_abbr in pc_abbr_to_working_disk, f"The is no mapping for {pc_abbr} to <working_disk>"
+
+    # working disk
+    working_disk = pc_abbr_to_working_disk.get(pc_abbr)
+
+    # load yaml file
+    data_dict = load_yaml(yaml_file=yaml_file, to_dict=True)
+
+    # Normalize paths in the loaded data
+    data_dict = normalize_paths(data_dict, working_disk, current_platform)
+    return data_dict
+
 
 def parse_args():
     parser = ArgumentParser(description="desc text")

halib/research/dataset.py

@@ -4,18 +4,17 @@
 
 from argparse import ArgumentParser
 
-from rich import inspect
-from common import console, seed_everything, ConsoleLog
-from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
-from tqdm import tqdm
 import os
 import click
-from torchvision.datasets import ImageFolder
 import shutil
+from tqdm import tqdm
+from rich import inspect
 from rich.pretty import pprint
-from system import filesys as fs
-import glob
+from torchvision.datasets import ImageFolder
+from sklearn.model_selection import StratifiedShuffleSplit, ShuffleSplit
 
+from ..system import filesys as fs
+from ..common import console, seed_everything, ConsoleLog
 
 def parse_args():
     parser = ArgumentParser(description="desc text")

halib/research/flop_csv.py

@@ -0,0 +1,34 @@
+from halib import *
+from flops import _calculate_flops_for_model
+
+from halib import *
+from argparse import ArgumentParser
+
+
+def main():
+    csv_file = "./results-imagenet.csv"
+    df = pd.read_csv(csv_file)
+    # make param_count column as float
+    # df['param_count'] = df['param_count'].astype(float)
+    df['param_count'] = pd.to_numeric(df['param_count'], errors='coerce').fillna(99999).astype(float)
+    df = df[df['param_count'] < 5.0]  # filter models with param_count < 20M
+
+    dict_ls = []
+
+    for index, row in tqdm(df.iterrows()):
+        console.rule(f"Row {index+1}/{len(df)}")
+        model = row['model']
+        num_class = 2
+        _, _, mflops = _calculate_flops_for_model(model, num_class)
+        dict_ls.append({'model': model, 'param_count': row['param_count'], 'mflops': mflops})
+
+    # Create a DataFrame from the list of dictionaries
+    result_df = pd.DataFrame(dict_ls)
+
+    final_df = pd.merge(df, result_df, on=['model', 'param_count'])
+    final_df.sort_values(by='mflops', inplace=True, ascending=True)
+    csvfile.fn_display_df(final_df)
+
+
+if __name__ == "__main__":
+    main()

halib/research/flops.py

@@ -0,0 +1,156 @@
+import os
+import sys
+import torch
+import timm
+from argparse import ArgumentParser
+from fvcore.nn import FlopCountAnalysis
+from halib import *
+from halib.filetype import csvfile
+from curriculum.utils.config import *
+from curriculum.utils.model_helper import *
+
+
+# ---------------------------------------------------------------------
+# Argument Parser
+# ---------------------------------------------------------------------
+def parse_args():
+    parser = ArgumentParser(description="Calculate FLOPs for TIMM or trained models")
+
+    # Option 1: Direct TIMM model
+    parser.add_argument(
+        "--model_name", type=str, help="TIMM model name (e.g., efficientnet_b0)"
+    )
+    parser.add_argument(
+        "--num_classes", type=int, default=1000, help="Number of output classes"
+    )
+
+    # Option 2: Experiment directory
+    parser.add_argument(
+        "--indir",
+        type=str,
+        default=None,
+        help="Directory containing trained experiment (with .yaml and .pth)",
+    )
+    parser.add_argument(
+        "-o", "--o", action="store_true", help="Open output CSV after saving"
+    )
+    return parser.parse_args()
+
+
+# ---------------------------------------------------------------------
+# Helper Functions
+# ---------------------------------------------------------------------
+def _get_list_of_proc_dirs(indir):
+    assert os.path.exists(indir), f"Input directory {indir} does not exist."
+    pth_files = [f for f in os.listdir(indir) if f.endswith(".pth")]
+    if len(pth_files) > 0:
+        return [indir]
+    return [
+        os.path.join(indir, f)
+        for f in os.listdir(indir)
+        if os.path.isdir(os.path.join(indir, f))
+    ]
+
+
+def _calculate_flops_for_model(model_name, num_classes):
+    """Calculate FLOPs for a plain TIMM model."""
+    try:
+        model = timm.create_model(model_name, pretrained=False, num_classes=num_classes)
+        input_size = timm.data.resolve_data_config(model.default_cfg)["input_size"]
+        dummy_input = torch.randn(1, *input_size)
+        model.eval() # ! set to eval mode to avoid some warnings or errors
+        flops = FlopCountAnalysis(model, dummy_input)
+        gflops = flops.total() / 1e9
+        mflops = flops.total() / 1e6
+        print(f"\nModel: **{model_name}**, Classes: {num_classes}")
+        print(f"Input size: {input_size}, FLOPs: **{gflops:.3f} GFLOPs**, **{mflops:.3f} MFLOPs**\n")
+        return model_name, gflops, mflops
+    except Exception as e:
+        print(f"[Error] Could not calculate FLOPs for {model_name}: {e}")
+        return model_name, -1, -1
+
+
+def _calculate_flops_for_experiment(exp_dir):
+    """Calculate FLOPs for a trained experiment directory."""
+    yaml_files = [f for f in os.listdir(exp_dir) if f.endswith(".yaml")]
+    pth_files = [f for f in os.listdir(exp_dir) if f.endswith(".pth")]
+
+    assert (
+        len(yaml_files) == 1
+    ), f"Expected 1 YAML file in {exp_dir}, found {len(yaml_files)}"
+    assert (
+        len(pth_files) == 1
+    ), f"Expected 1 PTH file in {exp_dir}, found {len(pth_files)}"
+
+    exp_cfg_yaml = os.path.join(exp_dir, yaml_files[0])
+    cfg = ExpConfig.from_yaml(exp_cfg_yaml)
+    ds_label_list = cfg.dataset.get_label_list()
+
+    try:
+        model = build_model(
+            cfg.model.name, num_classes=len(ds_label_list), pretrained=True
+        )
+        model_weights_path = os.path.join(exp_dir, pth_files[0])
+        model.load_state_dict(torch.load(model_weights_path, map_location="cpu"))
+        model.eval()
+
+        input_size = timm.data.resolve_data_config(model.default_cfg)["input_size"]
+        dummy_input = torch.randn(1, *input_size)
+        flops = FlopCountAnalysis(model, dummy_input)
+        gflops = flops.total() / 1e9
+        mflops = flops.total() / 1e6
+
+        return str(cfg), cfg.model.name, gflops, mflops
+    except Exception as e:
+        console.print(f"[red] Error processing {exp_dir}: {e}[/red]")
+        return str(cfg), cfg.model.name, -1, -1
+
+
+# ---------------------------------------------------------------------
+# Main Entry
+# ---------------------------------------------------------------------
+def main():
+    args = parse_args()
+
+    # Case 1: Direct TIMM model input
+    if args.model_name:
+        _calculate_flops_for_model(args.model_name, args.num_classes)
+        return
+
+    # Case 2: Experiment directory input
+    if args.indir is None:
+        print("[Error] Either --model_name or --indir must be specified.")
+        return
+
+    proc_dirs = _get_list_of_proc_dirs(args.indir)
+    pprint(proc_dirs)
+
+    dfmk = csvfile.DFCreator()
+    TABLE_NAME = "model_flops_results"
+    dfmk.create_table(TABLE_NAME, ["exp_name", "model_name", "gflops", "mflops"])
+
+    console.rule(f"Calculating FLOPs for models in {len(proc_dirs)} dir(s)...")
+    rows = []
+    for exp_dir in tqdm(proc_dirs):
+        dir_name = os.path.basename(exp_dir)
+        console.rule(f"{dir_name}")
+        exp_name, model_name, gflops, mflops = _calculate_flops_for_experiment(exp_dir)
+        rows.append([exp_name, model_name, gflops, mflops])
+
+    dfmk.insert_rows(TABLE_NAME, rows)
+    dfmk.fill_table_from_row_pool(TABLE_NAME)
+
+    outfile = f"zout/zreport/{now_str()}_model_flops_results.csv"
+    dfmk[TABLE_NAME].to_csv(outfile, sep=";", index=False)
+    csvfile.fn_display_df(dfmk[TABLE_NAME])
+
+    if args.o:
+        os.system(f"start {outfile}")
+
+
+# ---------------------------------------------------------------------
+# Script Entry
+# ---------------------------------------------------------------------
+if __name__ == "__main__":
+    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+    main()

halib/research/mics.py

@@ -1,5 +1,9 @@
+from ..common import *
+from ..filetype import csvfile
+import pandas as pd
 import platform
 
+
 PC_NAME_TO_ABBR = {
     "DESKTOP-JQD9K01": "MainPC",
     "DESKTOP-5IRHU87": "MSI_Laptop",
@@ -8,9 +12,57 @@ PC_NAME_TO_ABBR = {
     "DESKTOP-QNS3DNF": "1GPU_SV"
 }
 
+DEFAULT_ABBR_WORKING_DISK = {
+    "MainPC": "E:",
+    "MSI_Laptop": "D:",
+    "4090_SV": "E:",
+    "4GPU_SV": "D:",
+}
+
+def list_PCs(show=True):
+    df = pd.DataFrame(list(PC_NAME_TO_ABBR.items()), columns=["PC Name", "Abbreviation"])
+    if show:
+        csvfile.fn_display_df(df)
+    return df
+
 def get_PC_name():
     return platform.node()
 
 def get_PC_abbr_name():
     pc_name = get_PC_name()
     return PC_NAME_TO_ABBR.get(pc_name, "Unknown")
+
+# ! This funcction search for full paths in the obj and normalize them according to the current platform and working disk
+# ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "windows" => "D:/zdataset/DFire"
+# ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "linux" => "/mnt/d/zdataset/DFire"
+def normalize_paths(obj, working_disk, current_platform):
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            obj[key] = normalize_paths(value, working_disk, current_platform)
+        return obj
+    elif isinstance(obj, list):
+        for i, item in enumerate(obj):
+            obj[i] = normalize_paths(item, working_disk, current_platform)
+        return obj
+    elif isinstance(obj, str):
+        # Normalize backslashes to forward slashes for consistency
+        obj = obj.replace("\\", "/")
+        # Regex for Windows-style path: e.g., "E:/zdataset/DFire"
+        win_match = re.match(r"^([A-Z]):/(.*)$", obj)
+        # Regex for Linux-style path: e.g., "/mnt/e/zdataset/DFire"
+        lin_match = re.match(r"^/mnt/([a-z])/(.*)$", obj)
+        if win_match or lin_match:
+            rest = win_match.group(2) if win_match else lin_match.group(2)
+            if current_platform == "windows":
+                # working_disk is like "D:", so "D:/" + rest
+                new_path = working_disk + "/" + rest
+            elif current_platform == "linux":
+                # Extract drive letter from working_disk (e.g., "D:" -> "d")
+                drive_letter = working_disk[0].lower()
+                new_path = "/mnt/" + drive_letter + "/" + rest
+            else:
+                # Unknown platform, return original
+                return obj
+            return new_path
+    # For non-strings or non-path strings, return as is
+    return obj

halib/research/perfcalc.py

@@ -227,9 +227,9 @@ class PerfCalc(ABC):  # Abstract base class for performance calculation
             ), "No metric columns found in the DataFrame. Ensure that the CSV files contain metric columns starting with 'metric_'."
             final_cols = sticky_cols + metric_cols
             df = df[final_cols]
-            # !hahv debug
-            pprint("------ Final DataFrame Columns ------")
-            csvfile.fn_display_df(df)
+            # # !hahv debug
+            # pprint("------ Final DataFrame Columns ------")
+            # csvfile.fn_display_df(df)
             # ! validate all rows in df before returning
             # make sure all rows will have at least values for REQUIRED_COLS and at least one metric column
             for index, row in df.iterrows():

halib/research/perftb.py

@@ -308,7 +308,8 @@ class PerfTB:
         if save_path:
             export_success = False
             try:
-                fig.write_image(save_path, engine="kaleido")
+                # fig.write_image(save_path, engine="kaleido")
+                fig.write_image(save_path, engine="kaleido", width=width, height=height * len(metric_list))
                 export_success = True
             # pprint(f"Saved: {os.path.abspath(save_path)}")
             except Exception as e: