halib 0.1.95__tar.gz → 0.1.96__tar.gz

{halib-0.1.95 → halib-0.1.96}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.95
+Version: 0.1.96
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -52,7 +52,7 @@ Dynamic: summary
 
 # Helper package for coding and automation
 
-**Version 0.1.95**
+**Version 0.1.96**
 + `research/plot': add `PlotHelper` class to plot train history + plot grid of images (e.g., image samples from dataset or model outputs)
 
 

{halib-0.1.95 → halib-0.1.96}/README.md

@@ -1,6 +1,6 @@
 # Helper package for coding and automation
 
-**Version 0.1.95**
+**Version 0.1.96**
 + `research/plot': add `PlotHelper` class to plot train history + plot grid of images (e.g., image samples from dataset or model outputs)
 
 

halib-0.1.96/halib/research/flop_csv.py

@@ -0,0 +1,34 @@
+from halib import *
+from flops import _calculate_flops_for_model
+
+from halib import *
+from argparse import ArgumentParser
+
+
+def main():
+    csv_file = "./results-imagenet.csv"
+    df = pd.read_csv(csv_file)
+    # make param_count column as float
+    # df['param_count'] = df['param_count'].astype(float)
+    df['param_count'] = pd.to_numeric(df['param_count'], errors='coerce').fillna(99999).astype(float)
+    df = df[df['param_count'] < 5.0]  # filter models with param_count < 20M
+
+    dict_ls = []
+
+    for index, row in tqdm(df.iterrows()):
+        console.rule(f"Row {index+1}/{len(df)}")
+        model = row['model']
+        num_class = 2
+        _, _, mflops = _calculate_flops_for_model(model, num_class)
+        dict_ls.append({'model': model, 'param_count': row['param_count'], 'mflops': mflops})
+
+    # Create a DataFrame from the list of dictionaries
+    result_df = pd.DataFrame(dict_ls)
+
+    final_df = pd.merge(df, result_df, on=['model', 'param_count'])
+    final_df.sort_values(by='mflops', inplace=True, ascending=True)
+    csvfile.fn_display_df(final_df)
+
+
+if __name__ == "__main__":
+    main()

halib-0.1.96/halib/research/flops.py

@@ -0,0 +1,156 @@
+import os
+import sys
+import torch
+import timm
+from argparse import ArgumentParser
+from fvcore.nn import FlopCountAnalysis
+from halib import *
+from halib.filetype import csvfile
+from curriculum.utils.config import *
+from curriculum.utils.model_helper import *
+
+
+# ---------------------------------------------------------------------
+# Argument Parser
+# ---------------------------------------------------------------------
+def parse_args():
+    parser = ArgumentParser(description="Calculate FLOPs for TIMM or trained models")
+
+    # Option 1: Direct TIMM model
+    parser.add_argument(
+        "--model_name", type=str, help="TIMM model name (e.g., efficientnet_b0)"
+    )
+    parser.add_argument(
+        "--num_classes", type=int, default=1000, help="Number of output classes"
+    )
+
+    # Option 2: Experiment directory
+    parser.add_argument(
+        "--indir",
+        type=str,
+        default=None,
+        help="Directory containing trained experiment (with .yaml and .pth)",
+    )
+    parser.add_argument(
+        "-o", "--o", action="store_true", help="Open output CSV after saving"
+    )
+    return parser.parse_args()
+
+
+# ---------------------------------------------------------------------
+# Helper Functions
+# ---------------------------------------------------------------------
+def _get_list_of_proc_dirs(indir):
+    assert os.path.exists(indir), f"Input directory {indir} does not exist."
+    pth_files = [f for f in os.listdir(indir) if f.endswith(".pth")]
+    if len(pth_files) > 0:
+        return [indir]
+    return [
+        os.path.join(indir, f)
+        for f in os.listdir(indir)
+        if os.path.isdir(os.path.join(indir, f))
+    ]
+
+
+def _calculate_flops_for_model(model_name, num_classes):
+    """Calculate FLOPs for a plain TIMM model."""
+    try:
+        model = timm.create_model(model_name, pretrained=False, num_classes=num_classes)
+        input_size = timm.data.resolve_data_config(model.default_cfg)["input_size"]
+        dummy_input = torch.randn(1, *input_size)
+        model.eval() # ! set to eval mode to avoid some warnings or errors
+        flops = FlopCountAnalysis(model, dummy_input)
+        gflops = flops.total() / 1e9
+        mflops = flops.total() / 1e6
+        print(f"\nModel: **{model_name}**, Classes: {num_classes}")
+        print(f"Input size: {input_size}, FLOPs: **{gflops:.3f} GFLOPs**, **{mflops:.3f} MFLOPs**\n")
+        return model_name, gflops, mflops
+    except Exception as e:
+        print(f"[Error] Could not calculate FLOPs for {model_name}: {e}")
+        return model_name, -1, -1
+
+
+def _calculate_flops_for_experiment(exp_dir):
+    """Calculate FLOPs for a trained experiment directory."""
+    yaml_files = [f for f in os.listdir(exp_dir) if f.endswith(".yaml")]
+    pth_files = [f for f in os.listdir(exp_dir) if f.endswith(".pth")]
+
+    assert (
+        len(yaml_files) == 1
+    ), f"Expected 1 YAML file in {exp_dir}, found {len(yaml_files)}"
+    assert (
+        len(pth_files) == 1
+    ), f"Expected 1 PTH file in {exp_dir}, found {len(pth_files)}"
+
+    exp_cfg_yaml = os.path.join(exp_dir, yaml_files[0])
+    cfg = ExpConfig.from_yaml(exp_cfg_yaml)
+    ds_label_list = cfg.dataset.get_label_list()
+
+    try:
+        model = build_model(
+            cfg.model.name, num_classes=len(ds_label_list), pretrained=True
+        )
+        model_weights_path = os.path.join(exp_dir, pth_files[0])
+        model.load_state_dict(torch.load(model_weights_path, map_location="cpu"))
+        model.eval()
+
+        input_size = timm.data.resolve_data_config(model.default_cfg)["input_size"]
+        dummy_input = torch.randn(1, *input_size)
+        flops = FlopCountAnalysis(model, dummy_input)
+        gflops = flops.total() / 1e9
+        mflops = flops.total() / 1e6
+
+        return str(cfg), cfg.model.name, gflops, mflops
+    except Exception as e:
+        console.print(f"[red] Error processing {exp_dir}: {e}[/red]")
+        return str(cfg), cfg.model.name, -1, -1
+
+
+# ---------------------------------------------------------------------
+# Main Entry
+# ---------------------------------------------------------------------
+def main():
+    args = parse_args()
+
+    # Case 1: Direct TIMM model input
+    if args.model_name:
+        _calculate_flops_for_model(args.model_name, args.num_classes)
+        return
+
+    # Case 2: Experiment directory input
+    if args.indir is None:
+        print("[Error] Either --model_name or --indir must be specified.")
+        return
+
+    proc_dirs = _get_list_of_proc_dirs(args.indir)
+    pprint(proc_dirs)
+
+    dfmk = csvfile.DFCreator()
+    TABLE_NAME = "model_flops_results"
+    dfmk.create_table(TABLE_NAME, ["exp_name", "model_name", "gflops", "mflops"])
+
+    console.rule(f"Calculating FLOPs for models in {len(proc_dirs)} dir(s)...")
+    rows = []
+    for exp_dir in tqdm(proc_dirs):
+        dir_name = os.path.basename(exp_dir)
+        console.rule(f"{dir_name}")
+        exp_name, model_name, gflops, mflops = _calculate_flops_for_experiment(exp_dir)
+        rows.append([exp_name, model_name, gflops, mflops])
+
+    dfmk.insert_rows(TABLE_NAME, rows)
+    dfmk.fill_table_from_row_pool(TABLE_NAME)
+
+    outfile = f"zout/zreport/{now_str()}_model_flops_results.csv"
+    dfmk[TABLE_NAME].to_csv(outfile, sep=";", index=False)
+    csvfile.fn_display_df(dfmk[TABLE_NAME])
+
+    if args.o:
+        os.system(f"start {outfile}")
+
+
+# ---------------------------------------------------------------------
+# Script Entry
+# ---------------------------------------------------------------------
+if __name__ == "__main__":
+    sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+    main()

{halib-0.1.95 → halib-0.1.96}/halib/research/mics.py

@@ -19,9 +19,11 @@ DEFAULT_ABBR_WORKING_DISK = {
     "4GPU_SV": "D:",
 }
 
-def list_PCs():
+def list_PCs(show=True):
     df = pd.DataFrame(list(PC_NAME_TO_ABBR.items()), columns=["PC Name", "Abbreviation"])
-    csvfile.showdf(df)
+    if show:
+        csvfile.fn_display_df(df)
+    return df
 
 def get_PC_name():
     return platform.node()

{halib-0.1.95 → halib-0.1.96}/halib/research/plot.py

@@ -281,7 +281,7 @@ class PlotHelper:
         # --- Convert to wide "multi-list" format ---
         df = pd.DataFrame(data)
         row_col = df.columns[0]       # first col = row labels
-        col_cols = df.columns[1:]     # the rest = groupable cols
+        # col_cols = df.columns[1:]     # the rest = groupable cols
 
         df = (
             df.melt(id_vars=[row_col], var_name="col", value_name="path")
@@ -356,244 +356,9 @@ class PlotHelper:
     @staticmethod
     def plot_image_grid(
         indir_or_csvf_or_df: Union[str, pd.DataFrame],
-        img_width: int = 300,
-        img_height: int = 300,
-        img_stack_direction: str = "horizontal",  # "horizontal" or "vertical"
-        img_stack_padding_px: int = 5,
-        img_scale_mode: str = "fit",  # "fit" or "fill"
-        format_row_label_func: Optional[Callable[[str], str]] = None,
-        format_col_label_func: Optional[Callable[[str], str]] = None,
-        title: str = "",
-        tickfont=dict(size=16, family="Arial", color="black"),  # <-- bigger labels
-        fig_margin: dict = dict(l=50, r=50, t=50, b=50),
-        outline_color: str = "",
-        outline_size: int = 1,
-        cell_margin_px: int = 10,  # spacing between cells
-        row_line_size: int = 0,  # if >0, draw horizontal dotted lines
-        col_line_size: int = 0   # if >0, draw vertical dotted lines
-    ):
-        """
-            Plot a grid of images using Plotly.
-
-            - Accepts DataFrame where each cell is either:
-                * a Python list object,
-                * a string representation of a Python list (e.g. "['a','b']"),
-                * a JSON list string, or
-                * a single path string.
-            - For each cell, stack the images into a single composite that exactly fits
-            (img_width, img_height) is the target size for each individual image in the stack.
-            The final cell size will depend on the number of images and stacking direction.
-        """
-
-        def process_image_for_slot(path: str, target_size: Tuple[int, int], scale_mode: str, outline: str, outline_size: int) -> Image.Image:
-            try:
-                img = Image.open(path).convert("RGB")
-            except Exception:
-                return Image.new("RGB", target_size, (255, 255, 255))
-
-            if scale_mode == "fit":
-                img_ratio = img.width / img.height
-                target_ratio = target_size[0] / target_size[1]
-
-                if img_ratio > target_ratio:
-                    new_height = target_size[1]
-                    new_width = max(1, int(new_height * img_ratio))
-                else:
-                    new_width = target_size[0]
-                    new_height = max(1, int(new_width / img_ratio))
-
-                img = img.resize((new_width, new_height), Image.Resampling.LANCZOS)
-                left = (new_width - target_size[0]) // 2
-                top = (new_height - target_size[1]) // 2
-                right = left + target_size[0]
-                bottom = top + target_size[1]
-
-                if len(outline) == 7 and outline.startswith("#"):
-                    border_px = outline_size
-                    bordered = Image.new("RGB", (target_size[0] + 2*border_px, target_size[1] + 2*border_px), outline)
-                    bordered.paste(img.crop((left, top, right, bottom)), (border_px, border_px))
-                    return bordered
-                return img.crop((left, top, right, bottom))
-
-            elif scale_mode == "fill":
-                if len(outline) == 7 and outline.startswith("#"):
-                    border_px = outline_size
-                    bordered = Image.new("RGB", (target_size[0] + 2*border_px, target_size[1] + 2*border_px), outline)
-                    img = img.resize(target_size, Image.Resampling.LANCZOS)
-                    bordered.paste(img, (border_px, border_px))
-                    return bordered
-                return img.resize(target_size, Image.Resampling.LANCZOS)
-            else:
-                raise ValueError("img_scale_mode must be 'fit' or 'fill'.")
-
-        def stack_images_base64(image_paths: List[str], direction: str, single_img_size: Tuple[int,int], outline: str, outline_size: int, padding: int) -> Tuple[str, Tuple[int,int]]:
-            image_paths = [p for p in image_paths if p is not None and str(p).strip() != ""]
-            n = len(image_paths)
-            if n == 0:
-                blank = Image.new("RGB", single_img_size, (255,255,255))
-                buf = BytesIO()
-                blank.save(buf, format="PNG")
-                return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode(), single_img_size
-
-            processed = [process_image_for_slot(p, single_img_size, img_scale_mode, outline, outline_size) for p in image_paths]
-            pad_total = padding * (n-1)
-
-            if direction == "horizontal":
-                total_w = sum(im.width for im in processed) + pad_total
-                total_h = max(im.height for im in processed)
-                stacked = Image.new("RGB", (total_w, total_h), (255,255,255))
-                x = 0
-                for im in processed:
-                    stacked.paste(im, (x,0))
-                    x += im.width + padding
-            elif direction == "vertical":
-                total_w = max(im.width for im in processed)
-                total_h = sum(im.height for im in processed) + pad_total
-                stacked = Image.new("RGB", (total_w, total_h), (255,255,255))
-                y = 0
-                for im in processed:
-                    stacked.paste(im, (0,y))
-                    y += im.height + padding
-            else:
-                raise ValueError("img_stack_direction must be 'horizontal' or 'vertical'.")
-
-            buf = BytesIO()
-            stacked.save(buf, format="PNG")
-            encoded = base64.b64encode(buf.getvalue()).decode()
-            return f"data:image/png;base64,{encoded}", (total_w, total_h)
-
-        # --- Load DataFrame ---
-        if isinstance(indir_or_csvf_or_df, str):
-            fname, ext = os.path.splitext(indir_or_csvf_or_df)
-            if ext.lower() == ".csv":
-                df = pd.read_csv(indir_or_csvf_or_df)
-            elif os.path.isdir(indir_or_csvf_or_df):
-                df = PlotHelper.img_grid_indir_1(indir_or_csvf_or_df, log=False)
-            else:
-                raise ValueError("Input string must be a valid CSV file or directory path")
-        elif isinstance(indir_or_csvf_or_df, pd.DataFrame):
-            df = indir_or_csvf_or_df.copy()
-        else:
-            raise ValueError("Input must be CSV file path, DataFrame, or directory path")
-
-        rows = df.iloc[:,0].astype(str).tolist()
-        columns = list(df.columns[1:])
-        n_rows, n_cols = len(rows), len(columns)
-
-        fig = go.Figure()
-        col_widths = [0]*n_cols
-        row_heights = [0]*n_rows
-
-        cell_imgs = [[None]*n_cols for _ in range(n_rows)]
-        for i in range(n_rows):
-            for j, col_label in enumerate(columns):
-                raw_cell = df.iloc[i, j+1]
-                image_paths = PlotHelper._parse_cell_to_list(raw_cell)
-                image_paths = [str(p).strip() for p in image_paths if str(p).strip() != ""]
-
-                img_src, (cell_w_actual, cell_h_actual) = stack_images_base64(
-                    image_paths, img_stack_direction, (img_width, img_height),
-                    outline=outline_color, outline_size=outline_size,
-                    padding=img_stack_padding_px
-                )
-
-                col_widths[j] = max(col_widths[j], cell_w_actual)
-                row_heights[i] = max(row_heights[i], cell_h_actual)
-                cell_imgs[i][j] = img_src
-
-        # Compute x/y positions including cell_margin
-        x_positions = []
-        cum_w = 0
-        for w in col_widths:
-            x_positions.append(cum_w)
-            cum_w += w + cell_margin_px
-
-        y_positions = []
-        cum_h = 0
-        for h in row_heights:
-            y_positions.append(-cum_h)
-            cum_h += h + cell_margin_px
-
-        # Add images to figure
-        for i in range(n_rows):
-            for j in range(n_cols):
-                fig.add_layout_image(
-                    dict(
-                        source=cell_imgs[i][j],
-                        x=x_positions[j],
-                        y=y_positions[i],
-                        xref="x",
-                        yref="y",
-                        sizex=col_widths[j],
-                        sizey=row_heights[i],
-                        xanchor="left",
-                        yanchor="top",
-                        layer="above",
-                    )
-                )
-        # ! Optional grid lines
-        # Add horizontal grid lines if row_line_size > 0
-        if row_line_size > 0:
-            for i in range(1, n_rows):
-                # Place line in the middle of the gap between rows
-                y = (
-                    y_positions[i - 1] - row_heights[i - 1] - y_positions[i]
-                ) / 2 + y_positions[i]
-                fig.add_shape(
-                    type="line",
-                    x0=-cell_margin_px,
-                    x1=cum_w - cell_margin_px,
-                    y0=y,
-                    y1=y,
-                    line=dict(width=row_line_size, color="black", dash="dot"),
-                )
-
-        # Add vertical grid lines if col_line_size > 0
-        if col_line_size > 0:
-            for j in range(1, n_cols):
-                x = x_positions[j] - cell_margin_px / 2
-                fig.add_shape(
-                    type="line",
-                    x0=x,
-                    x1=x,
-                    y0=cell_margin_px,
-                    y1=-cum_h + cell_margin_px,
-                    line=dict(width=col_line_size, color="black", dash="dot"),
-                )
-        # Axis labels
-        col_labels = [format_col_label_func(c) if format_col_label_func else c for c in columns]
-        row_labels = [format_row_label_func(r) if format_row_label_func else r for r in rows]
-
-        fig.update_xaxes(
-            tickvals=[x_positions[j] + col_widths[j]/2 for j in range(n_cols)],
-            ticktext=col_labels,
-            range=[-cell_margin_px, cum_w - cell_margin_px],
-            showgrid=False,
-            zeroline=False,
-            tickfont=tickfont  # <-- apply bigger font here
-        )
-        fig.update_yaxes(
-            tickvals=[y_positions[i] - row_heights[i]/2 for i in range(n_rows)],
-            ticktext=row_labels,
-            range=[-cum_h + cell_margin_px, cell_margin_px],
-            showgrid=False,
-            zeroline=False,
-            tickfont=tickfont  # <-- apply bigger font here
-        )
-
-        fig.update_layout(
-            width=cum_w + 100,
-            height=cum_h + 100,
-            title=title,
-            title_x=0.5,
-            margin=fig_margin,
-        )
-
-        fig.show()
-
-    @staticmethod
-    def plot_image_grid1(
-        indir_or_csvf_or_df: Union[str, pd.DataFrame],
+        save_path: str = None,
+        dpi: int = 300, # DPI for saving raster images or PDF
+        show: bool = True, # whether to show the plot in an interactive window
         img_width: int = 300,
         img_height: int = 300,
         img_stack_direction: str = "horizontal",  # "horizontal" or "vertical"
@@ -609,7 +374,7 @@ class PlotHelper:
         cell_margin_px: int = 10,  # padding (top, left, right, bottom) inside each cell
         row_line_size: int = 0,  # if >0, draw horizontal dotted lines
         col_line_size: int = 0,  # if >0, draw vertical dotted lines
-    ):
+    ) -> go.Figure:
         """
         Plot a grid of images using Plotly.
 
@@ -931,7 +696,21 @@ class PlotHelper:
             margin=fig_margin,
         )
 
-        fig.show()
+        # === EXPORT IF save_path IS GIVEN ===
+        if save_path:
+            import kaleido  # lazy import – only needed when saving
+            import os
+
+            ext = os.path.splitext(save_path)[1].lower()
+            if ext in [".png", ".jpg", ".jpeg"]:
+                fig.write_image(save_path, scale=dpi / 96)  # scale = dpi / base 96
+            elif ext in [".pdf", ".svg"]:
+                fig.write_image(save_path)  # PDF/SVG are vector → dpi ignored
+            else:
+                raise ValueError("save_path must end with .png, .jpg, .pdf, or .svg")
+        if show:
+            fig.show()
+        return fig
 
 
 @click.command()

{halib-0.1.95 → halib-0.1.96}/halib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.1.95
+Version: 0.1.96
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -52,7 +52,7 @@ Dynamic: summary
 
 # Helper package for coding and automation
 
-**Version 0.1.95**
+**Version 0.1.96**
 + `research/plot': add `PlotHelper` class to plot train history + plot grid of images (e.g., image samples from dataset or model outputs)
 
 

{halib-0.1.95 → halib-0.1.96}/halib.egg-info/SOURCES.txt

@@ -29,6 +29,8 @@ halib/research/__init__.py
 halib/research/base_config.py
 halib/research/base_exp.py
 halib/research/dataset.py
+halib/research/flop_csv.py
+halib/research/flops.py
 halib/research/metrics.py
 halib/research/mics.py
 halib/research/params_gen.py

{halib-0.1.95 → halib-0.1.96}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt") as f:
 
 setuptools.setup(
     name="halib",
-    version="0.1.95",
+    version="0.1.96",
     author="Hoang Van Ha",
     author_email="hoangvanhauit@gmail.com",
     description="Small library for common tasks",