halib 0.1.49__py3-none-any.whl → 0.1.51__py3-none-any.whl

halib/__init__.py

@@ -21,7 +21,9 @@ __all__ = [
     "plt",
     "pprint",
     "pprint_box",
+    "pprint_local_path",
     "px",
+    "pprint_local_path",
     "rcolor_all_str",
     "rcolor_palette_all",
     "rcolor_palette",
@@ -36,6 +38,7 @@ __all__ = [
     "time",
 ]
 import warnings
+
 warnings.filterwarnings("ignore", message="Unable to import Axes3D")
 
 # common libraries
@@ -54,7 +57,15 @@ from .system import cmd
 from .system import filesys as fs
 from .filetype import csvfile
 from .cuda import tcuda
-from .common import console, console_log, ConsoleLog, now_str, norm_str, pprint_box
+from .common import (
+    console,
+    console_log,
+    ConsoleLog,
+    now_str,
+    norm_str,
+    pprint_box,
+    pprint_local_path,
+)
 
 # for log
 from loguru import logger

halib/common.py

@@ -1,13 +1,17 @@
-import arrow
 import re
+import rich
+import arrow
+import pathlib
+from pathlib import Path
+import urllib.parse
+
+from rich import print
+from rich.panel import Panel
 from rich.console import Console
 from rich.pretty import pprint, Pretty
-from rich.panel import Panel
-import rich
 
 console = Console()
 
-
 def seed_everything(seed=42):
     import random
     import numpy as np
@@ -54,7 +58,6 @@ def pprint_box(obj, title="", border_style="green"):
         Panel(Pretty(obj, expand_all=True), title=title, border_style=border_style)
     )
 
-
 def console_rule(msg, do_norm_msg=True, is_end_tag=False):
     msg = norm_str(msg) if do_norm_msg else msg
     if is_end_tag:
@@ -86,3 +89,20 @@ class ConsoleLog:
         if exc_type is not None:
             print(f"An exception of type {exc_type} occurred.")
             print(f"Exception message: {exc_value}")
+
+
+def pprint_local_path(local_path, tag=""):
+    # Create a file URI
+    file_path = Path(local_path).resolve()
+    is_file = file_path.is_file()
+    is_dir = file_path.is_dir()
+    type_str = "📄" if is_file else "📁" if is_dir else "❓"
+    file_uri = file_path.as_uri()
+    content_str = f"{type_str} [link={file_uri}]{file_uri}[/link]"
+    if isinstance(tag, str) and len(tag) > 0:
+        with ConsoleLog(tag):
+            console.print(content_str)
+    else:
+        # If tag is not provided, just print the link
+        console.print(f"{content_str}")
+    return file_uri

halib/research/perftb.py

@@ -0,0 +1,757 @@
+import warnings
+warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+import random
+import itertools
+import plotly.express as px
+import plotly.graph_objects as go
+from collections import defaultdict
+from plotly.subplots import make_subplots
+from typing import Dict, List, Union, Optional
+
+from rich.pretty import pprint
+import pandas as pd
+
+# from halib import *
+# internal imports
+from ..filetype import csvfile
+from ..common import ConsoleLog
+
+
+class DatasetMetrics:
+    """Class to store metrics definitions for a specific dataset."""
+
+    def __init__(self, dataset_name: str, metric_names: List[str]):
+        self.dataset_name = dataset_name
+        self.metric_names = set(metric_names)  # Unique metric names
+        self.experiment_results: Dict[str, Dict[str, Union[float, int, None]]] = (
+            defaultdict(dict)
+        )
+
+    def add_experiment_result(
+        self, experiment_name: str, metrics: Dict[str, Union[float, int]]
+    ) -> None:
+        """Add experiment results for this dataset, only for defined metrics."""
+        # normalize metric names to lowercase
+        metrics = {k.lower(): v for k, v in metrics.items()}
+        # make sure every metric in metrics is defined for this dataset
+        for metric in metrics:
+            assert metric in self.metric_names, (
+                f"Metric <<{metric}>> not defined for dataset <<{self.dataset_name}>>. "
+                f"Available metrics: {self.metric_names}"
+            )
+        for metric in self.metric_names:
+            self.experiment_results[experiment_name][metric] = metrics.get(metric)
+
+    def get_metrics(self, experiment_name: str) -> Dict[str, Union[float, int, None]]:
+        """Retrieve metrics for a specific experiment."""
+        return self.experiment_results.get(
+            experiment_name, {metric: None for metric in self.metric_names}
+        )
+
+    def __str__(self) -> str:
+        return f"Dataset: {self.dataset_name}, Metrics: {', '.join(self.metric_names)}"
+
+
+class PerfTB:
+    """Class to manage performance table data with datasets as primary structure."""
+
+    def __init__(self):
+        # Dictionary of dataset_name -> DatasetMetrics
+        self.datasets: Dict[str, DatasetMetrics] = {}
+        self.experiments: set = set()
+
+    def add_dataset(self, dataset_name: str, metric_names: List[str]) -> None:
+        """
+        Add a new dataset with its associated metrics.
+
+        Args:
+            dataset_name: Name of the dataset
+            metric_names: List of metric names for this dataset
+        """
+        # normalize metric names to lowercase
+        metric_names = [metric.lower() for metric in metric_names]
+        self.datasets[dataset_name] = DatasetMetrics(dataset_name, metric_names)
+
+    def table_meta(self):
+        """
+        Return metadata about the performance table.
+        """
+        return {
+            "num_datasets": len(self.datasets),
+            "num_experiments": len(self.experiments),
+            "datasets_metrics": {
+                dataset_name: dataset.metric_names
+                for dataset_name, dataset in self.datasets.items()
+            }
+        }
+
+    def add_experiment(
+        self,
+        experiment_name: str,
+        dataset_name: str,
+        metrics: Dict[str, Union[float, int]],
+    ) -> None:
+        """
+        Add experiment results for a specific dataset.
+
+        Args:
+            experiment_name: Name or identifier of the experiment
+            dataset_name: Name of the dataset
+            metrics: Dictionary of metric names and their values
+        """
+        # normalize metric names to lowercase
+        metrics = {k.lower(): v for k, v in metrics.items()}
+        if dataset_name not in self.datasets:
+            raise ValueError(
+                f"Dataset <<{dataset_name}>> not defined. Add dataset first."
+            )
+        self.experiments.add(experiment_name)
+        self.datasets[dataset_name].add_experiment_result(experiment_name, metrics)
+
+    def get_metrics_for_dataset(
+        self, experiment_name: str, dataset_name: str
+    ) -> Optional[Dict[str, Union[float, int, None]]]:
+        """
+        Retrieve performance metrics for a specific dataset and experiment.
+
+        Args:
+            experiment_name: Name or identifier of the experiment
+            dataset_name: Name of the dataset
+
+        Returns:
+            Dictionary of metrics or None if dataset not found
+        """
+        dataset = self.datasets.get(dataset_name)
+        if dataset:
+            return dataset.get_metrics(experiment_name)
+        return None
+
+    def get_all_experiments(self) -> List[str]:
+        """Return list of all experiment names."""
+        return sorted(self.experiments)
+
+    def get_all_datasets(self) -> List[str]:
+        """Return list of all dataset names."""
+        return sorted(self.datasets.keys())
+
+    def to_dataframe(self) -> pd.DataFrame:
+        """
+        Convert the performance table to a pandas DataFrame with MultiIndex columns.
+        Level 1: Datasets
+        Level 2: Metrics
+
+        Returns:
+            pandas DataFrame with experiments as rows and (dataset, metric) as columns
+        """
+        # Create MultiIndex for columns (dataset, metric)
+        columns = []
+        for dataset_name in self.get_all_datasets():
+            for metric in sorted(self.datasets[dataset_name].metric_names):
+                columns.append((dataset_name, metric))
+        columns = pd.MultiIndex.from_tuples(columns, names=["Dataset", "Metric"])
+
+        # Initialize DataFrame with experiments as index
+        df = pd.DataFrame(index=sorted(self.experiments), columns=columns)
+
+        # Populate DataFrame
+        for exp in self.experiments:
+            for dataset_name in self.datasets:
+                metrics = self.datasets[dataset_name].get_metrics(exp)
+                for metric, value in metrics.items():
+                    df.loc[exp, (dataset_name, metric)] = value
+
+        return df
+
+    def plot(
+        self,
+        save_path: str,
+        title: Optional[str] = None,
+        custom_highlight_method_fn: Optional[callable] = None,
+        custom_sort_exp_fn: Optional[
+            callable
+        ] = None,  # Function to sort experiments; should accept a list of experiment names and return a sorted list
+        open_plot: bool = False,
+        show_raw_df: bool = False,
+        experiment_names: Optional[List[str]] = None,
+        datasets: Optional[List[str]] = None,
+        height: int = 400,
+        width: int = 700,
+    ) -> None:
+        """
+        Plot comparison of experiments across datasets and their metrics using Plotly.
+        Splits plots if metrics have significantly different value ranges.
+
+        Args:
+            save_path: Base file path to save the figure(s) (extension optional)
+            open_plot: If True, attempts to open the saved image file(s) (Windows only)
+            experiment_names: List of experiments to compare (default: all)
+            datasets: List of datasets to include (default: all)
+            height: Base height of the plot (scaled by # of facet rows)
+            width: Width of the plot
+            range_diff_threshold: Range threshold to split metrics across different axes
+        """
+        experiment_names = experiment_names or self.get_all_experiments()
+        datasets = datasets or self.get_all_datasets()
+
+        records = []
+
+        for dataset in datasets:
+            if dataset not in self.datasets:
+                print(f"Warning: Dataset '{dataset}' not found. Skipping...")
+                continue
+
+            metric_names = sorted(self.datasets[dataset].metric_names)
+            for exp in experiment_names:
+                metric_values = self.get_metrics_for_dataset(exp, dataset)
+                if not metric_values:
+                    continue
+                for metric in metric_names:
+                    value = metric_values.get(metric)
+                    if value is not None:
+                        records.append(
+                            {
+                                "Experiment": exp,
+                                "Dataset": dataset,
+                                "Metric": metric,
+                                "Value": value,
+                            }
+                        )
+
+        if not records:
+            print("No data found to plot.")
+            return
+
+        df = pd.DataFrame(records)
+        if show_raw_df:
+            with ConsoleLog("PerfTB DF"):
+                csvfile.fn_display_df(df)
+
+        metric_list = df["Metric"].unique()
+        fig = make_subplots(
+            rows=len(metric_list),
+            cols=1,
+            shared_xaxes=False,
+            subplot_titles=metric_list,
+            vertical_spacing=0.1,
+        )
+
+        unique_experiments = df["Experiment"].unique()
+        color_cycle = itertools.cycle(px.colors.qualitative.Plotly)
+
+        color_map = {
+            exp: color
+            for exp, color in zip(unique_experiments, color_cycle)
+        }
+
+        pattern_shapes = ["x", "-", "/", "\\", "|", "+", "."]
+        pattern_color = "black"  # Color for patterns
+
+        current_our_method = -1  # Start with -1 to avoid index error
+        exp_pattern_dict = {}
+        for row_idx, metric in enumerate(metric_list, start=1):
+            metric_df = df[df["Metric"] == metric]
+            list_exp = list(metric_df["Experiment"].unique())
+            if custom_sort_exp_fn:
+                list_exp = custom_sort_exp_fn(list_exp)
+            for exp in list_exp:
+                should_highlight = (
+                    custom_highlight_method_fn is not None and custom_highlight_method_fn(exp)
+                )
+                pattern_shape = ""  # default no pattern
+                if should_highlight and exp not in exp_pattern_dict:
+                    current_our_method += 1
+                    pattern_shape = pattern_shapes[
+                        current_our_method % len(pattern_shapes)
+                    ]
+                    exp_pattern_dict[exp] = pattern_shape
+                elif exp in exp_pattern_dict:
+                    pattern_shape = exp_pattern_dict[exp]
+                exp_df = metric_df[metric_df["Experiment"] == exp]
+                fig.add_trace(
+                    go.Bar(
+                        x=exp_df["Dataset"],
+                        y=exp_df["Value"],
+                        name=f"{exp}",
+                        legendgroup=exp,
+                        showlegend=(row_idx == 1),  # Show legend only for the first row
+                        marker=dict(
+                            color=color_map[exp],
+                            pattern=(
+                                dict(shape=pattern_shape, fgcolor=pattern_color)
+                                if pattern_shape
+                                else None
+                            ),
+                        ),
+                        text=[f"{v:.5f}" for v in exp_df["Value"]],
+                        textposition="auto",  # <- position them automatically
+                    ),
+                    row=row_idx,
+                    col=1,
+                )
+
+        # Manage layout
+        if title is None:
+            title = "Experiment Comparison by Metric Groups"
+        fig.update_layout(
+            height=height * len(metric_list),
+            width=width,
+            title_text=title,
+            barmode="group",
+            showlegend=True,
+        )
+
+        # Save and open plot
+        if save_path:
+            fig.write_image(save_path, engine="kaleido")
+            print(f"Saved: {os.path.abspath(save_path)}")
+
+            if open_plot and os.name == "nt":  # Windows
+                os.system(f'start "" "{os.path.abspath(save_path)}"')
+        return fig
+
+    def to_csv(self, outfile: str, sep=";", condensed_multiindex: bool = True) -> None:
+        """
+        Save the performance table to a CSV file.
+
+        Args:
+            outfile: Path to the output CSV file
+        """
+        df = self.to_dataframe()
+        if condensed_multiindex:
+            # Extract levels
+            level0 = df.columns.get_level_values(0)
+            level1 = df.columns.get_level_values(1)
+
+            # Build new level0 with blanks after first appearance
+            new_level0 = []
+            prev = None
+            for val in level0:
+                if val == prev:
+                    new_level0.append("")
+                else:
+                    new_level0.append(val)
+                    prev = val
+
+            # Write to CSV
+            df.columns = pd.MultiIndex.from_arrays([new_level0, level1])
+        df.to_csv(outfile, index=True, sep=sep)
+
+    def display(self) -> None:
+        """
+        Display the performance table as a DataFrame.
+        """
+        df = self.to_dataframe()
+        csvfile.fn_display_df(df)
+
+    @classmethod
+    def _read_condensed_multiindex_csv(cls, filepath: str, sep=";", col_exclude_fn: Optional[callable] = None) -> pd.DataFrame:
+        # Read first two header rows
+        df = pd.read_csv(filepath, header=[0, 1], sep=sep)
+        # Extract levels
+        level0 = df.columns.get_level_values(0)
+        level1 = df.columns.get_level_values(1)
+        # pprint(f'{level0=}')
+        # pprint(f'{level1=}')
+        # if blank values in level0, fill them after first appearance
+        # for level0, we need to fill in blanks after first appearance
+        new_level0 = []
+        last_non_blank = level0[0]  # Start with the first value
+        assert last_non_blank != "", (
+            "First level0 value should not be blank. "
+            "Check the CSV file format."
+        )
+        for val in level0:
+            if val == "" or "Unnamed: " in val:
+                new_level0.append(last_non_blank)
+            else:
+                new_level0.append(val)
+                last_non_blank = val
+        # pprint(new_level0)
+        # Rebuild MultiIndex
+        excluded_indices = []
+        if col_exclude_fn:
+            excluded_indices = []
+            for idx, val in enumerate(new_level0):
+                if col_exclude_fn(val):
+                    excluded_indices.append(idx)
+            for idx, val in enumerate(level1):
+                if col_exclude_fn(val):
+                    excluded_indices.append(idx)
+            excluded_indices = list(set(excluded_indices))
+
+        num_prev_cols = len(new_level0)
+        # Remove excluded indices from both levels
+        new_level0 = [
+            val for idx, val in enumerate(new_level0) if idx not in excluded_indices
+        ]
+        new_level1 = [
+            val for idx, val in enumerate(level1) if idx not in excluded_indices
+        ]
+        num_after_cols = len(new_level0)
+        if num_prev_cols != num_after_cols:
+            # get df with only the new level0 index
+            df = df.iloc[:, [i for i in range(len(df.columns)) if i not in excluded_indices]]
+
+        df.columns = pd.MultiIndex.from_arrays([new_level0, new_level1])
+        return df
+
+    @classmethod
+    def from_dataframe(
+        cls,
+        df: pd.DataFrame
+    ) -> "PerfTB":
+        """
+        Load performance table from a DataFrame.
+
+        Args:
+            df: Input DataFrame
+            method_col_name: Column name for methods
+        """
+        # console.log('--- PerfTB.from_dataframe ---')
+        # csvfile.fn_display_df(df)
+        cls_instance = cls()
+        # first loop through MultiIndex columns and extract datasets with their metrics
+        dataset_metrics = {}
+        for (dataset_name, metric_name) in df.columns[1:]:
+            if dataset_name not in dataset_metrics:
+                dataset_metrics[dataset_name] = []
+            dataset_metrics[dataset_name].append(metric_name)
+        for dataset_name, metric_names in dataset_metrics.items():
+            cls_instance.add_dataset(dataset_name, metric_names)
+
+        def safe_cast(val):
+            try:
+                return float(val)
+            except (ValueError, TypeError):
+                return None
+        for _, row in df.iterrows():
+            # Extract experiment name by first column
+            experiment_name = row.iloc[0]
+            # Iterate over MultiIndex columns (except first column)
+            metrics = {}
+            for dataset_name in dataset_metrics.keys():
+                for metric_name in dataset_metrics[dataset_name]:
+                    # Get the value for this dataset and metric
+                    value = row[(dataset_name, metric_name)]
+                    # Cast to float or None if not applicable
+                    metrics[metric_name] = safe_cast(value)
+
+                cls_instance.add_experiment(
+                    experiment_name=experiment_name,
+                    dataset_name=dataset_name,
+                    metrics=metrics,
+                )
+
+        return cls_instance
+
+    @classmethod
+    def from_csv(
+        cls,
+        csv_file: str,
+        sep: str = ";",
+        col_exclude_fn: Optional[callable] = None,
+    ) -> "PerfTB":
+        """
+        Load performance table from a CSV file.
+
+        Args:
+            csv_file: Path to the CSV file
+            sep: Separator used in the CSV file
+        """
+        df = cls._read_condensed_multiindex_csv(csv_file, sep=sep, col_exclude_fn=col_exclude_fn)
+        return cls.from_dataframe(df)
+
+    def filter_index_info(
+        self):
+        """
+        Filter the index information of the performance table.
+        """
+        datasets_metrics = {
+            dataset_name: dataset.metric_names
+            for dataset_name, dataset in self.datasets.items()
+        }
+        meta_dict = {}
+        for i, (dataset_name, metrics) in enumerate(datasets_metrics.items()):
+            sorted_metrics = sorted(metrics) # make sure output should be same
+            meta_dict[dataset_name] = {
+                "index": i,
+                "metrics": sorted(
+                    list(zip(sorted_metrics, range(len(sorted_metrics))))
+                ), # (metric_name, index)
+            }
+        return meta_dict
+
+    def filter(
+        self,
+        dataset_list: List[Union[str, int]] = None,  # list of strings or integers
+        metrics_list: List[Union[list, str]] = None,
+        experiment_list: List[str] = None,
+    ) -> "PerfTB":
+        """
+        Filter the performance table by datasets and experiments.
+        Returns a new PerfTB instance with filtered data.
+        Args:
+            dataset_list: List of dataset names or indices to filter (optional)
+            metrics_list: List of metric names to filter (optional). Note that can be pass a list of list (of metric names) to filter by different set of metrics for each dataset. If using a single list, it will be applied to all datasets.
+            experiment_list: List of experiment NAMES (string) to filter (optional). Indices are not supported.
+        """
+        meta_filter_dict = self.filter_index_info()
+
+        if experiment_list is None:
+            experiment_list = self.get_all_experiments()
+        else:
+            # make sure all experiments are found in the performance table
+            for exp in experiment_list:
+                if exp not in self.experiments:
+                    raise ValueError(
+                        f"Experiment <<{exp}>> not found in the performance table. Available experiments: {self.get_all_experiments()}"
+                    )
+        # pprint(f"Filtering experiments: {experiment_list}")
+        # get dataset list
+        if dataset_list is not None:
+            # if all item in dataset_list are integers, convert them to dataset names
+            if all(isinstance(item, int) and 0 <= item < len(meta_filter_dict) for item in dataset_list):
+                dataset_list = [
+                    list(meta_filter_dict.keys())[item] for item in dataset_list
+                ]
+            elif all(isinstance(item, str) for item in dataset_list):
+                # if all items are strings, use them as dataset names
+                dataset_list = [
+                    item for item in dataset_list if item in meta_filter_dict
+                ]
+            else:
+                raise ValueError(
+                    f"dataset_list should be a list of strings (dataset names) or integers (indices, should be <= {len(meta_filter_dict) - 1}). Got: {dataset_list}"
+                )
+        else:
+            dataset_list = self.get_all_datasets()
+
+        filter_metrics_ls = [] # [list_metric_db_A, list_metric_db_B, ...]
+        all_ds_metrics = []
+        for dataset_name in dataset_list:
+            ds_meta = meta_filter_dict.get(dataset_name, None)
+            if ds_meta:
+                ds_metrics = ds_meta["metrics"]
+                all_ds_metrics.append([metric[0] for metric in ds_metrics])
+
+        if metrics_list is None:
+            filter_metrics_ls = all_ds_metrics
+        elif isinstance(metrics_list, list):
+            all_string = all(isinstance(item, str) for item in metrics_list)
+            if all_string:
+                # normalize metrics_list to lowercase
+                metrics_list = [metric.lower() for metric in metrics_list]
+                filter_metrics_ls = [metrics_list] * len(dataset_list)
+            else:
+                all_list = all(isinstance(item, list) for item in metrics_list)
+                pprint(f'{all_list=}')
+                if all_list:
+                    print('b')
+                    if len(metrics_list) != len(dataset_list):
+                        raise ValueError(
+                            f"metrics_list should be a list of strings (metric names) or a list of lists of metric names for each dataset. Got: {len(metrics_list)} metrics for {len(dataset_list)} datasets."
+                        )
+                    # normalize each list of metrics to lowercase
+                    filter_metrics_ls = [
+                        [metric.lower() for metric in item] for item in metrics_list
+                    ]
+
+        else:
+            raise ValueError(
+                f"metrics_list should be a list of strings (metric names) or a list of lists of metric names for each dataset. Got: {metrics_list}"
+            )
+
+        # make sure that all metrics in filtered_metrics_list are valid for the datasets
+        final_metrics_list = []
+        for idx, dataset_name in enumerate(dataset_list):
+            valid_metrics_list = all_ds_metrics[idx]
+            current_metrics = filter_metrics_ls[idx]
+            new_valid_ds_metrics = []
+            for metric in current_metrics:
+                if metric in valid_metrics_list:
+                    new_valid_ds_metrics.append(metric)
+            assert len(new_valid_ds_metrics) > 0, (
+                f"No valid metrics found for dataset <<{dataset_name}>>. "
+                f"Available metrics: {valid_metrics_list}. "
+                f"Filtered metrics: {current_metrics}"
+            )
+            final_metrics_list.append(new_valid_ds_metrics)
+
+        assert len(experiment_list) > 0, "No experiments to filter."
+        assert len(dataset_list) > 0, "No datasets to filter."
+        assert len(final_metrics_list) > 0, "No metrics to filter."
+        filtered_tb = PerfTB()
+        for db, metrics in zip(dataset_list, final_metrics_list):
+            # add dataset with its metrics
+            filtered_tb.add_dataset(db, metrics)
+
+        # now add experiments with their metrics
+        for exp in experiment_list:
+            for db, metrics in zip(dataset_list, final_metrics_list):
+                # get metrics for this experiment and dataset
+                metrics_dict = self.get_metrics_for_dataset(exp, db)
+                if metrics_dict:
+                    # filter metrics to only those defined for this dataset
+                    filtered_metrics = {k: v for k, v in metrics_dict.items() if k in metrics}
+                    if filtered_metrics:
+                        filtered_tb.add_experiment(exp, db, filtered_metrics)
+
+        return filtered_tb
+
+
+def test_perftb_create() -> PerfTB:
+    # Create a performance table
+    perf_table = PerfTB()
+
+    # Define datasets and their metrics first
+    perf_table.add_dataset("dataset1", ["accuracy", "f1_score"])
+    perf_table.add_dataset("dataset2", ["accuracy", "f1_score", "precision"])
+
+    # Add experiment results
+    perf_table.add_experiment(
+        experiment_name="our_method1",
+        dataset_name="dataset1",
+        metrics={"accuracy": 100, "f1_score": 0.93},
+    )
+    perf_table.add_experiment(
+        experiment_name="our_method2",
+        dataset_name="dataset2",
+        metrics={"accuracy": 100, "precision": 0.87},  # Missing precision will be None
+    )
+    perf_table.add_experiment(
+        experiment_name="our_method2",
+        dataset_name="dataset1",
+        metrics={"accuracy": 90, "f1_score": 0.85},
+    )
+    method_list = [f"method{idx}" for idx in range(3, 7)]
+    # add random values for methods 3-6
+    for method in method_list:
+        perf_table.add_experiment(
+            experiment_name=method,
+            dataset_name="dataset1",
+            metrics={
+                "accuracy": random.randint(80, 100),
+                "f1_score": random.uniform(0.7, 0.95),
+            },
+        )
+        perf_table.add_experiment(
+            experiment_name=method,
+            dataset_name="dataset2",
+            metrics={
+                "accuracy": random.randint(80, 100),
+                "precision": random.uniform(0.7, 0.95),
+                "f1_score": random.uniform(0.7, 0.95),
+            },
+        )
+
+    # Get metrics for a specific dataset
+    metrics = perf_table.get_metrics_for_dataset("model1", "f1_score")
+    if metrics:
+        print(f"\nMetrics for model1 on dataset1: {metrics}")
+
+    return perf_table
+
+def test_perftb_dataframe() -> None:
+    # Create a performance table
+    perf_table = test_perftb_create()
+
+    # Convert to DataFrame
+    df = perf_table.to_dataframe()
+    print("\nPerformance Table as DataFrame:")
+    csvfile.fn_display_df(df)
+
+    # Save to CSV
+    perf_table.to_csv("zout/perf_tb.csv", sep=";")
+
+def test_perftb_plot() -> None:
+    # Create a performance table
+    perf_table = test_perftb_create()
+
+    # Plot the performance table
+    perf_table.plot(
+        save_path="zout/perf_tb.svg",
+        title="Performance Comparison",
+        custom_highlight_method_fn=lambda exp: exp.startswith("our_method"),
+        custom_sort_exp_fn=lambda exps: sorted(exps, reverse=True),
+        open_plot=False,
+        show_raw_df=False,
+    )
+
+def test_load_perftb() -> None:
+    # Load performance table from CSV
+    def col_exclude_fn(col_name: str) -> bool:
+        # Exclude columns that are not metrics (e.g., "Unnamed" columns)
+        return col_name in ["Year", "data split", "test procedure", "code?"]
+
+    perf_table = PerfTB.from_csv("test/bench.csv", sep=";", col_exclude_fn=col_exclude_fn)
+    # print("\nLoaded Performance Table:")
+    # perf_table.display()
+    perf_table.to_csv("zout/loaded_perf_tb.csv", sep=";")
+
+    # Plot loaded performance table
+    perf_table.plot(
+        save_path="zout/loaded_perf_plot.svg",
+        title="Loaded Performance Comparison",
+        custom_highlight_method_fn=lambda exp: exp.startswith("Ours"),
+        custom_sort_exp_fn=lambda exps: sorted(exps, reverse=True),
+        open_plot=False,
+        show_raw_df=False,
+    )
+    return perf_table
+
+def test_filtered_perftb() -> None:
+    perf_table_item = test_load_perftb()
+    # pprint(perf_table_item.meta())
+    pprint(perf_table_item.filter_index_info())
+    perf_table_item.filter(
+        dataset_list=[0, 2],  # Use indices of datasets
+        # dataset_list=[
+        #     "BOWFire_dataset_chino2015bowfire (small)",
+        #     "FD-Dataset_li2020efficient (large)",
+        # ],
+        metrics_list=[
+            "acc",
+            "f1",
+        ],  # [["acc"], ["f1"]],  # Use a single list of metrics for all datasets or a list of lists for different metrics per dataset
+        # experiment_list=["ADFireNet_yar2023effective"],
+    ).plot(
+        save_path="zout/filtered_perf_tb.svg",
+        chk_highlight_method_fn=lambda exp: exp.startswith("Ours"),
+        custom_sort_exp_fn=lambda exps: sorted(exps, reverse=True),
+        title="Filtered Performance Comparison",
+    )
+
+def test_mics() -> None:
+    # Test reading a CSV with MultiIndex columns
+    perf_table = test_perftb_create()
+    perf_table.display()
+    perf_table.plot(
+        save_path="zout/test1.svg",
+        title="Performance Comparison",
+        custom_highlight_method_fn=lambda exp: exp.startswith("our_"),
+        custom_sort_exp_fn=lambda exps: sorted(exps, reverse=True),
+        open_plot=False,
+    )
+    perf_table.to_csv("zout/perf_tb1.csv", sep=";")
+    tb = PerfTB.from_csv("./zout/perf_tb1.csv", sep=";")
+    tb.display()
+    ftb = tb.filter(
+        dataset_list=[1],
+        metrics_list=["precision"],
+        experiment_list=["method3", "method6"],
+    )
+    ftb.display()
+
+    ftb.plot(
+        save_path="zout/perf_tb11_plot.svg",
+        title="Performance Comparison",
+        custom_highlight_method_fn=lambda exp: exp.startswith("our_"),
+        custom_sort_exp_fn=lambda exps: sorted(exps, reverse=True),
+        open_plot=True,
+    )
+
+
+# Example usage
+if __name__ == "__main__":
+    test_mics()

{halib-0.1.49.dist-info → halib-0.1.51.dist-info}/METADATA

@@ -1,156 +1,166 @@
-Metadata-Version: 2.1
-Name: halib
-Version: 0.1.49
-Summary: Small library for common tasks
-Author: Hoang Van Ha
-Author-email: hoangvanhauit@gmail.com
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE.txt
-Requires-Dist: arrow
-Requires-Dist: click
-Requires-Dist: enlighten
-Requires-Dist: kaleido ==0.1.*
-Requires-Dist: loguru
-Requires-Dist: more-itertools
-Requires-Dist: moviepy
-Requires-Dist: networkx
-Requires-Dist: numpy
-Requires-Dist: omegaconf
-Requires-Dist: opencv-python
-Requires-Dist: pandas
-Requires-Dist: Pillow
-Requires-Dist: Pyarrow
-Requires-Dist: pycurl
-Requires-Dist: python-telegram-bot
-Requires-Dist: requests
-Requires-Dist: rich
-Requires-Dist: scikit-learn
-Requires-Dist: matplotlib
-Requires-Dist: seaborn
-Requires-Dist: plotly
-Requires-Dist: pygwalker
-Requires-Dist: tabulate
-Requires-Dist: itables
-Requires-Dist: timebudget
-Requires-Dist: tqdm
-Requires-Dist: tube-dl
-Requires-Dist: wandb
-
-Helper package for coding and automation
-
-**Version 0.1.49**
-
-+ add `research` package to help with research tasks, including `benchquery` for benchmarking queries from dataframe
-+ add `wandb` module to allow easy sync offline data to Weights & Biases (wandb) and batch clear wandb runs.
-
-**Version 0.1.47**
-+ add `pprint_box` to print object/string in a box frame (like in `inspect`)
-
-**Version 0.1.46**
-+ filter the warning message of `UserWarning: Unable to import Axes3D.`
-+ auto_wrap_text for `fn_display_df` to avoid long text in the table
-
-**Version 0.1.42**
-+ add <rich_color.py>: add basic color list (for easy usage)
-
-**Version 0.1.41**
-+ add <rich_color.py> to display rich color information in <rich> python package (rcolor_str, rcolor_pallet_all, etc.)
-
-**Version 0.1.40**
-
-+  update <csvfile.py> to use `itables` and `pygwalker` to display dataframe in jupyter notebook.
-
-**Version 0.1.38**
-
-+  add <torchloader.py> to search for best cfg for torch dataloader (num_workers, batch_size, pin_memory, et.)
-
-**Version 0.1.37**
-
-+  add <dataset.py> to help split classification dataset into train/val(test)
----
-**Version 0.1.33**
-
-+ add `plot.py` module to plot DL model training history (with columlns: epoch, train_accuracy, val_accuracy, train_loss, val_loss) using `seaborn` and `matplotlib`
----
-**Version 0.1.29**
-
-+ for `tele_noti` module, `kaleido==0.1.*` is required for plotly since `kaleido 0.2.*` is not working (taking for ever to generate image)
----
-**Version 0.1.24**
-
-+ rename `sys` to `system` to avoid conflict with built-in `sys` module
-+ add `tele_noti` module to send notification to telegram after a specific interval for training progress monitoring
----
-**Version 0.1.22**
-
-+ add `cuda.py` module to check CUDA availability (for both pytorch and tensorflow)
----
-**Version 0.1.21**
-
-+ using `networkx` and `omegaconf` to allow yaml file inheritance and override
----
-**Version 0.1.15**
-
-+ `__init__.py`: add common logging library; also `console_log` decorator to log function (start and end)
-
----
-
-**Version 0.1.10**
-
-+ filesys: fix typo on "is_exit" to "is_exist"
-+ gdrive: now support uploading file to folder and return direct link (shareable link)
-
-**Version 0.1.9**
-
-+ add dependencies requirement.txt
-
-**Version 0.1.8**
-
-Fix bugs:
-
-+ [performance] instead of inserting directly new rows into table dataframe, first insert it into in-memory `row_pool_dict`, that fill data in that dict into the actual dataframe when needed.
-
----
-
-**Version 0.1.7**
-
-Fix bugs:
-
-+ fix insert into table so slow by allowing insert multiple rows at once
-
----
-
-**Version 0.1.6**
-
-New features:
-
-+ add DFCreator for manipulating table (DataFrame) - create, insert row, display, write to file
-
----
-
-**Version 0.1.5**
-
-New Features
-
-+ add cmd module
-+ new package structure
-
----
-
-**Version 0.1.4**
-
-New Features
-
-+ add support to create Bitbucket Project from template
-
----
-
-**Version 0.1.2**
-
-New Features
-
-+ add support to upload local to google drive.
+Metadata-Version: 2.1
+Name: halib
+Version: 0.1.51
+Summary: Small library for common tasks
+Author: Hoang Van Ha
+Author-email: hoangvanhauit@gmail.com
+License: UNKNOWN
+Platform: UNKNOWN
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: arrow
+Requires-Dist: click
+Requires-Dist: enlighten
+Requires-Dist: kaleido (==0.1.*)
+Requires-Dist: loguru
+Requires-Dist: more-itertools
+Requires-Dist: moviepy
+Requires-Dist: networkx
+Requires-Dist: numpy
+Requires-Dist: omegaconf
+Requires-Dist: opencv-python
+Requires-Dist: pandas
+Requires-Dist: Pillow
+Requires-Dist: Pyarrow
+Requires-Dist: pycurl
+Requires-Dist: python-telegram-bot
+Requires-Dist: requests
+Requires-Dist: rich
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: plotly
+Requires-Dist: pygwalker
+Requires-Dist: tabulate
+Requires-Dist: itables
+Requires-Dist: timebudget
+Requires-Dist: tqdm
+Requires-Dist: tube-dl
+Requires-Dist: wandb
+
+Helper package for coding and automation
+
+**Version 0.1.51**
+
++ add `research/perftb` module to allow creating and managing performance tables for experiments, including filtering by datasets, metrics, and experiments.
+
+**Version 0.1.50**
+
++ add `pprint_local_path` to print local path (file/directory) in clickable link (as file URI)
+
++ add `research` package to help with research tasks, including `benchquery` for benchmarking queries from dataframe
++ add `wandb` module to allow easy sync offline data to Weights & Biases (wandb) and batch clear wandb runs.
+
+**Version 0.1.47**
++ add `pprint_box` to print object/string in a box frame (like in `inspect`)
+
+**Version 0.1.46**
++ filter the warning message of `UserWarning: Unable to import Axes3D.`
++ auto_wrap_text for `fn_display_df` to avoid long text in the table
+
+**Version 0.1.42**
++ add <rich_color.py>: add basic color list (for easy usage)
+
+**Version 0.1.41**
++ add <rich_color.py> to display rich color information in <rich> python package (rcolor_str, rcolor_pallet_all, etc.)
+
+**Version 0.1.40**
+
++  update <csvfile.py> to use `itables` and `pygwalker` to display dataframe in jupyter notebook.
+
+**Version 0.1.38**
+
++  add <torchloader.py> to search for best cfg for torch dataloader (num_workers, batch_size, pin_memory, et.)
+
+**Version 0.1.37**
+
++  add <dataset.py> to help split classification dataset into train/val(test)
+---
+**Version 0.1.33**
+
++ add `plot.py` module to plot DL model training history (with columlns: epoch, train_accuracy, val_accuracy, train_loss, val_loss) using `seaborn` and `matplotlib`
+---
+**Version 0.1.29**
+
++ for `tele_noti` module, `kaleido==0.1.*` is required for plotly since `kaleido 0.2.*` is not working (taking for ever to generate image)
+---
+**Version 0.1.24**
+
++ rename `sys` to `system` to avoid conflict with built-in `sys` module
++ add `tele_noti` module to send notification to telegram after a specific interval for training progress monitoring
+---
+**Version 0.1.22**
+
++ add `cuda.py` module to check CUDA availability (for both pytorch and tensorflow)
+---
+**Version 0.1.21**
+
++ using `networkx` and `omegaconf` to allow yaml file inheritance and override
+---
+**Version 0.1.15**
+
++ `__init__.py`: add common logging library; also `console_log` decorator to log function (start and end)
+
+---
+
+**Version 0.1.10**
+
++ filesys: fix typo on "is_exit" to "is_exist"
++ gdrive: now support uploading file to folder and return direct link (shareable link)
+
+**Version 0.1.9**
+
++ add dependencies requirement.txt
+
+**Version 0.1.8**
+
+Fix bugs:
+
++ [performance] instead of inserting directly new rows into table dataframe, first insert it into in-memory `row_pool_dict`, that fill data in that dict into the actual dataframe when needed.
+
+---
+
+**Version 0.1.7**
+
+Fix bugs:
+
++ fix insert into table so slow by allowing insert multiple rows at once
+
+---
+
+**Version 0.1.6**
+
+New features:
+
++ add DFCreator for manipulating table (DataFrame) - create, insert row, display, write to file
+
+---
+
+**Version 0.1.5**
+
+New Features
+
++ add cmd module
++ new package structure
+
+---
+
+**Version 0.1.4**
+
+New Features
+
++ add support to create Bitbucket Project from template
+
+---
+
+**Version 0.1.2**
+
+New Features
+
++ add support to upload local to google drive.
+
+

{halib-0.1.49.dist-info → halib-0.1.51.dist-info}/RECORD

@@ -1,5 +1,5 @@
-halib/__init__.py,sha256=r3xoKNX_4QxzC7MsdSXRZSge29xBI5LdMwNouzf3xv8,1543
-halib/common.py,sha256=bR0MPbUmsBhLdu5LMRr-dnf6XG8iyjLEJsVxT_q-qKw,2315
+halib/__init__.py,sha256=dHuxmYnZjeFT2VXuZU5yKX7c81XqcXX5tsKFtcaxkqI,1657
+halib/common.py,sha256=ZWKT46IBnXygJIJyy2Mi1PNK_f9GfuppNxYYOrqFAx8,3001
 halib/csvfile.py,sha256=Eoeni0NIbNG3mB5ESWAvNwhJxOjmCaPd1qqYRHImbvk,1567
 halib/cuda.py,sha256=1bvtBY8QvTWdLaxalzK9wqXPl0Ft3AfhcrebupxGzEA,1010
 halib/dataset.py,sha256=QU0Hr5QFb8_XlvnOMgC9QJGIpwXAZ9lDd0RdQi_QRec,6743
@@ -30,6 +30,7 @@ halib/online/projectmake.py,sha256=Zrs96WgXvO4nIrwxnCOletL4aTBge-EoF0r7hpKO1w8,4
 halib/research/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/research/benchquery.py,sha256=FuKnbWQtCEoRRtJAfN-zaN-jPiO_EzsakmTOMiqi7GQ,4626
 halib/research/dataset.py,sha256=QU0Hr5QFb8_XlvnOMgC9QJGIpwXAZ9lDd0RdQi_QRec,6743
+halib/research/perftb.py,sha256=hptHgCMLWsGKSeGz8g7oB3XdMy9eK5DzASewn0RALOg,29997
 halib/research/plot.py,sha256=-pDUk4z3C_GnyJ5zWmf-mGMdT4gaipVJWzIgcpIPiRk,9448
 halib/research/torchloader.py,sha256=yqUjcSiME6H5W210363HyRUrOi3ISpUFAFkTr1w4DCw,6503
 halib/research/wandb_op.py,sha256=YzLEqME5kIRxi3VvjFkW83wnFrsn92oYeqYuNwtYRkY,4188
@@ -42,8 +43,8 @@ halib/system/filesys.py,sha256=ERpnELLDKJoTIIKf-AajgkY62nID4qmqmX5TkE95APU,2931
 halib/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/utils/listop.py,sha256=Vpa8_2fI0wySpB2-8sfTBkyi_A4FhoFVVvFiuvW8N64,339
 halib/utils/tele_noti.py,sha256=-4WXZelCA4W9BroapkRyIdUu9cUVrcJJhegnMs_WpGU,5928
-halib-0.1.49.dist-info/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.1.49.dist-info/METADATA,sha256=G3MPTsD8-W3o1MptmTpzCvslPuQg6uVISpkSwRoLJqw,4208
-halib-0.1.49.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
-halib-0.1.49.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.1.49.dist-info/RECORD,,
+halib-0.1.51.dist-info/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.1.51.dist-info/METADATA,sha256=V6qfi8fSUhVYiRhXS8wlG1F3f91zzmZBVnKZiwD3sAU,4365
+halib-0.1.51.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+halib-0.1.51.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.1.51.dist-info/RECORD,,

{halib-0.1.49.dist-info → halib-0.1.51.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (70.2.0)
+Generator: bdist_wheel (0.37.1)
 Root-Is-Purelib: true
 Tag: py3-none-any