halib 0.2.32__py3-none-any.whl → 0.2.34__py3-none-any.whl

halib/utils/plotly_op.py

@@ -0,0 +1,234 @@
+import os
+import pandas as pd
+from typing import List, Optional, Callable, Union
+from rich.pretty import pprint
+import plotly.express as px
+from ..common.common import pprint_local_path
+
+
+class PlotlyUtils:
+    @staticmethod
+    # Extract experiment IDs from complex naming convention
+    def exp_id_extractor(df: pd.DataFrame) -> pd.Series:
+        # MainPC__ds_UFireIndoor2__mt_temp_method_motion_block__9896ed2e67e7__20260202.133758
+        exp_names = df["Name"].to_list()
+        exp_ids = []
+        for name in exp_names:
+            parts = name.split("__")
+            exp_id = parts[-2] if len(parts) >= 2 else name
+            exp_ids.append(exp_id)
+        return pd.Series(exp_ids)
+
+    @staticmethod
+    def exp_id_formatter(exp_id: str) -> str:
+        if len(exp_id) <= 6:
+            return exp_id
+        return f"{exp_id[:6]}"
+
+    @staticmethod
+    def parallel_plot(
+        df_or_csv_file: Union[pd.DataFrame, str],
+        dimensions: List[str] = [],
+        exclude_dims: List[str] = [],
+        exp_col_or_func: Union[
+            str, Callable[[pd.DataFrame], pd.Series]
+        ] = exp_id_extractor,
+        exp_id_formatter: Optional[Callable[[str], str]] = exp_id_formatter,
+        color: Optional[str] = None,
+        csv_separator: str = ";",
+        plot_bar_height: int = 1200,
+        plot_width: int = 1500,
+        title: str = "Parallel Coordinates Plot",
+        template: str = "plotly_white",
+        outdir: str = ".",
+        outfile: str = "zresults_with_table.html",
+    ):
+        # 1. Unified Data Loading
+        if isinstance(df_or_csv_file, str):
+            df = pd.read_csv(df_or_csv_file, sep=csv_separator, encoding="utf-8")
+        else:
+            df = df_or_csv_file.copy()
+
+        # 2. Extract Experiment IDs
+        if callable(exp_col_or_func):
+            df["exp_id"] = exp_col_or_func(df)
+        elif isinstance(exp_col_or_func, str):
+            if exp_col_or_func not in df.columns:
+                raise ValueError(f"Column '{exp_col_or_func}' not found.")
+            df["exp_id"] = df[exp_col_or_func].copy()
+            exclude_dims.append(exp_col_or_func)
+        else:
+            raise ValueError("exp_col_or_func must be a column name or a callable.")
+
+        # 3. Setup Plot Dimensions
+        # Priority: explicit dimensions -> all columns minus exclusions
+        if not dimensions:
+            dimensions = [
+                c for c in df.columns if c not in exclude_dims and c != "exp_id"
+            ]
+
+        df["numeric_id"] = range(len(df))
+        # Ensure numeric_id is the first axis for the labels to work
+        final_plot_dims = ["numeric_id"] + [
+            d for d in dimensions if d != "numeric_id" and d in df.columns
+        ]
+
+        pprint(f"Generating plot: {title}")
+
+        # 4. Create and Configure Plotly Figure
+        fig = px.parallel_coordinates(
+            df,
+            dimensions=final_plot_dims,
+            color=color,
+            title=title,
+            template=template,
+        )
+
+        # Map string IDs to the numeric axis
+        fig.data[0].dimensions[0].tickvals = list(df["numeric_id"])
+        fig.data[0].dimensions[0].ticktext = (
+            [exp_id_formatter(i) for i in df["exp_id"]]
+            if exp_id_formatter
+            else df["exp_id"]
+        )
+        fig.data[0].dimensions[0].label = "Exp IDs"
+
+        fig.update_layout(
+            title={
+                "text": title,
+                "y": 0.98,
+                "x": 0.5,
+                "xanchor": "center",
+                "yanchor": "top",
+            },
+            width=plot_width,
+            height=plot_bar_height,
+            margin=dict(l=150, r=50, t=150, b=50),
+        )
+
+        # 5. Prepare Table Display
+        # Clean up columns: Move exp_id to front, drop internal numeric_id
+        cols = ["exp_id"] + [c for c in df.columns if c not in ["exp_id", "numeric_id"]]
+        df_display = df[cols].copy()
+        df_display.insert(0, "Selection", '<button class="select-btn">Select</button>')
+
+        chart_html = fig.to_html(full_html=False, include_plotlyjs="cdn")
+
+        # Create a display copy with a 'Select' button column
+        df_display = df.copy()
+        df_display_cols = df_display.columns.tolist()
+        # move 'exp_id' to the front for better visibility
+        if "exp_id" in df_display_cols:
+            df_display_cols.insert(
+                0, df_display_cols.pop(df_display_cols.index("exp_id"))
+            )
+            df_display = df_display[df_display_cols]
+        # remove 'numeric_id' from display
+        if "numeric_id" in df_display.columns:
+            df_display = df_display.drop(columns=["numeric_id"])
+        df_display.insert(0, "Selection", '<button class="select-btn">Select</button>')
+
+        table_html = df_display.to_html(
+            classes="display nowrap", table_id="exp_table", index=False, escape=False
+        )
+
+        final_html = f"""
+        <!DOCTYPE html>
+        <html>
+        <head>
+            <title>Experiment Dashboard</title>
+            <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/1.11.5/css/jquery.dataTables.css">
+            <script type="text/javascript" charset="utf8" src="https://code.jquery.com/jquery-3.5.1.js"></script>
+            <script type="text/javascript" charset="utf8" src="https://cdn.datatables.net/1.11.5/js/jquery.dataTables.js"></script>
+            <style>
+                body {{ font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; margin: 20px; background-color: #f0f2f5; }}
+                .container {{ background: white; padding: 30px; border-radius: 12px; box-shadow: 0 4px 20px rgba(0,0,0,0.08); }}
+                #exp_table tbody tr:nth-child(even), #selected_table tbody tr:nth-child(even) {{ background-color: #f2f2f2; }}
+                #exp_table tbody tr:nth-child(odd), #selected_table tbody tr:nth-child(odd) {{ background-color: #ffffff; }}
+                #exp_table tbody tr:hover, #selected_table tbody tr:hover {{ background-color: #e0e0e0; }}
+                table.dataTable thead th {{ background-color: #333; color: white; padding: 12px; }}
+                h2 {{ color: #2c3e50; border-bottom: 2px solid #eee; padding-bottom: 10px; margin-top: 40px; }}
+                .select-btn {{ background-color: #28a745; color: white; border: none; padding: 5px 10px; cursor: pointer; border-radius: 4px; }}
+                .remove-btn {{ background-color: #dc3545; color: white; border: none; padding: 5px 10px; cursor: pointer; border-radius: 4px; }}
+                .clear-btn {{ background-color: #6c757d; color: white; border: none; padding: 10px 15px; cursor: pointer; border-radius: 4px; margin-bottom: 10px; }}
+            </style>
+        </head>
+        <body>
+            <div class="container">
+                {chart_html}
+
+                <h2>Selected Experiments</h2>
+                <button id="clear_all" class="clear-btn">Clear All</button>
+                <div style="overflow-x:auto; margin-bottom: 40px;">
+                    <table id="selected_table" class="display nowrap">
+                        <thead>{df_display.iloc[:0].to_html(index=False, escape=False).split("<thead>")[1].split("</thead>")[0]}</thead>
+                        <tbody></tbody>
+                    </table>
+                </div>
+
+                <hr>
+
+                <h2>Full Experiment Raw Data</h2>
+                <div style="overflow-x:auto;">
+                    {table_html}
+                </div>
+            </div>
+
+            <script>
+                $(document).ready( function () {{
+                    var mainTable = $('#exp_table').DataTable({{
+                        "pageLength": 25,
+                        "order": [[ 5, "desc" ]],
+                        "stripeClasses": []
+                    }});
+
+                    var selectedTable = $('#selected_table').DataTable({{
+                        "paging": false,
+                        "searching": false,
+                        "info": false,
+                        "stripeClasses": []
+                    }});
+
+                    // Handle Select button
+                    $('#exp_table tbody').on('click', '.select-btn', function () {{
+                        var data = mainTable.row($(this).parents('tr')).data();
+                        var rowNode = $(this).parents('tr');
+
+                        // Change button to remove in the new table
+                        var newData = [...data];
+                        newData[0] = '<button class="remove-btn">Remove</button>';
+
+                        // Check if already in selectedTable (optional but good)
+                        var alreadyExists = false;
+                        selectedTable.rows().every(function() {{
+                            if(this.data()[1] === data[1]) {{ alreadyExists = true; }}
+                        }});
+
+                        if(!alreadyExists) {{
+                            selectedTable.row.add(newData).draw();
+                        }}
+                    }});
+
+                    // Handle Remove button
+                    $('#selected_table tbody').on('click', '.remove-btn', function () {{
+                        selectedTable.row($(this).parents('tr')).remove().draw();
+                    }});
+
+                    // Handle Clear All
+                    $('#clear_all').on('click', function() {{
+                        selectedTable.clear().draw();
+                    }});
+                }});
+            </script>
+        </body>
+        </html>
+        """
+
+        final_html_path = os.path.join(outdir, outfile)
+        with open(final_html_path, "w") as f:
+            f.write(final_html)
+
+        pprint_local_path(
+            final_html_path, get_wins_path=True, tag="PlotlyUtils.parallel_plot"
+        )
+        return fig

halib/utils/slack_op.py

@@ -0,0 +1,86 @@
+import time
+from slack_sdk import WebClient
+from slack_sdk.errors import SlackApiError
+from rich.pretty import pprint
+
+"""
+Utilities for interacting with Slack for experiment notification via Wandb Logger.
+"""
+class SlackUtils:
+    _instance = None
+
+    def __new__(cls, token=None):
+        """
+        Singleton __new__ method.
+        Ensures only one instance of SlackUtils exists.
+        """
+        if cls._instance is None:
+            if token is None:
+                raise ValueError(
+                    "A Slack Token is required for the first initialization."
+                )
+
+            # Create the instance
+            cls._instance = super(SlackUtils, cls).__new__(cls)
+
+            # Initialize the WebClient only once
+            cls._instance.client = WebClient(token=token)
+            cls._instance.token = token
+
+        return cls._instance
+
+    def clear_channel(self, channel_id, sleep_interval=1.0):
+        """
+        Fetches and deletes all messages in a specified channel.
+        """
+        cursor = None
+        deleted_count = 0
+
+        pprint(f"--- Starting cleanup for Channel ID: {channel_id} ---")
+
+        while True:
+            try:
+                # Fetch history in batches of 100
+                response = self.client.conversations_history(  # ty:ignore[unresolved-attribute]
+                    channel=channel_id, cursor=cursor, limit=100
+                )
+
+                messages = response.get("messages", [])
+
+                if not messages:
+                    pprint("No more messages found to delete.")
+                    break
+
+                for msg in messages:
+                    ts = msg.get("ts")
+
+                    try:
+                        # Attempt delete
+                        self.client.chat_delete(  # ty:ignore[unresolved-attribute]
+                            channel=channel_id, ts=ts
+                        )
+                        pprint(f"Deleted: {ts}")
+                        deleted_count += 1
+
+                        # Rate limit protection (Tier 3 limit)
+                        time.sleep(sleep_interval)
+
+                    except SlackApiError as e:
+                        error_code = e.response["error"]
+                        if error_code == "cant_delete_message":
+                            pprint(f"Skipped (Permission denied): {ts}")
+                        elif error_code == "message_not_found":
+                            pprint(f"Skipped (Already deleted): {ts}")
+                        else:
+                            pprint(f"Error deleting {ts}: {error_code}")
+                # Check for pagination
+                if response["has_more"]:
+                    cursor = response["response_metadata"]["next_cursor"]
+                else:
+                    break
+
+            except SlackApiError as e:
+                print(f"Critical API Error fetching history: {e.response['error']}")
+                break
+
+        print(f"--- Completed. Total messages deleted: {deleted_count} ---")

halib/utils/wandb_op.py

@@ -0,0 +1,140 @@
+import glob
+import wandb
+import subprocess
+from pathlib import Path
+from typing import Optional, Literal
+
+from tap import Tap
+from tqdm import tqdm
+
+from rich.console import Console
+import yaml
+
+wandb_console = Console()
+
+
+# --- Argument Parser ---
+class WandbArgs(Tap):
+    # --- Auth ---
+    api_key_yaml: Path = Path(
+        "E:/Dev/__halib/.env.yaml"
+    )  # Path to YAML file containing W&B API key
+
+    # --- Operations ---
+    op: Literal["sync", "delete"] = "delete"  # Operation to perform
+
+    # --- Project & Filtering ---
+    project: str = "paper2_main"  # W&B project name
+    pattern: Optional[str] = None  # Run name pattern to match for deletion
+
+    # --- Paths ---
+    outdir: Path = Path("./zout/zruns")  # Directory containing runs to sync
+
+    def configure(self):
+        self.add_argument("-prj", "--project")
+        self.add_argument("-pt", "--pattern")
+        self.add_argument("-o", "--outdir")
+        self.add_argument("-k", "--api_key_yaml")
+
+    def validate(self):
+        if self.op == "sync" and not self.outdir.exists():
+            raise ValueError(f"Output directory {self.outdir} does not exist.")
+        if self.op == "delete" and not self.project.strip():
+            raise ValueError("Project name must be a non-empty string.")
+
+
+def login_wandb(api_key: Optional[str]):
+    """Handles authentication with Weights & Biases."""
+    try:
+        if api_key:
+            # Explicit login with key
+            wandb.login(key=api_key)
+            wandb_console.print(
+                "[green]Successfully logged into W&B using provided API key.[/green]"
+            )
+        else:
+            # Attempts to find key in environment variables or netrc file
+            if wandb.login():
+                wandb_console.print(
+                    "[blue]Logged into W&B using existing credentials.[/blue]"
+                )
+            else:
+                wandb_console.print(
+                    "[red]Authentication failed. Please provide an API key with -k.[/red]"
+                )
+                exit(1)
+    except Exception as e:
+        wandb_console.print(f"[bold red]Login Error:[/bold red] {e}")
+        exit(1)
+
+
+# --- Logic Functions ---
+
+
+def sync_runs(outdir: Path):
+    outdir_path = outdir.absolute()
+    sub_dirs = [d for d in outdir_path.iterdir() if d.is_dir()]
+
+    if not sub_dirs:
+        wandb_console.print(f"[red]No subdirectories found in {outdir_path}.[/red]")
+        return
+
+    wandb_console.rule(f"Syncing from {outdir_path}")
+
+    wandb_dirs = []
+    for exp_dir in sub_dirs:
+        wandb_dirs.extend(glob.glob(str(exp_dir / "wandb" / "*run-*")))
+
+    if not wandb_dirs:
+        wandb_console.print("No wandb runs found.")
+        return
+
+    for i, wandb_dir in enumerate(wandb_dirs):
+        wandb_console.print(f"[{i+1}/{len(wandb_dirs)}] Syncing: {wandb_dir}")
+
+        # Note: 'wandb sync' command uses the credentials from wandb.login()
+        process = subprocess.Popen(
+            ["wandb", "sync", wandb_dir],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+        for line in process.stdout:  # ty:ignore[not-iterable]
+            if "ERROR" in line:
+                wandb_console.print(f"[red]{line.strip()}[/red]")
+        process.wait()
+
+
+def delete_runs(project: str, pattern: Optional[str]):
+    api = wandb.Api()
+    runs = api.runs(project)
+
+    deleted = 0
+    for run in tqdm(runs, desc="Processing runs"):
+        if pattern is None or pattern in run.name:
+            run.delete()
+            deleted += 1
+    wandb_console.print(f"[green]Total runs deleted: {deleted}[/green]")
+
+
+def main():
+    args = WandbArgs().parse_args()
+    cfg_dict = {}
+    with open(args.api_key_yaml, "r") as f:
+        cfg_dict = yaml.safe_load(f)
+    wandb_api_key = cfg_dict.get("WANDB_API_KEY")
+    assert (
+        wandb_api_key is not None
+    ), "W&B API key not found in the specified YAML file."
+    # 1. Login first
+    login_wandb(wandb_api_key)
+
+    # 2. Execute operation
+    if args.op == "sync":
+        sync_runs(args.outdir)
+    elif args.op == "delete":
+        delete_runs(args.project, args.pattern)
+
+
+if __name__ == "__main__":
+    main()

{halib-0.2.32.dist-info → halib-0.2.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.2.32
+Version: 0.2.34
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -57,7 +57,12 @@ Dynamic: summary
 
 ## v0.2.x (Experiment & Core Updates)
 
-### **v0.2.32**
+### **v0.2.34**
+
+- ✨ **New Feature:**: introduce `utils.PlotlyUtils` with parallel coordinates plot and data table support
+
+- 🚀 **Improvement:**: move `wandb_op.py` to `utils` and add `scripts` folder
+
 
 - ✨ **New Feature:**: add `common.common.log_func` as decorator to log function entry, exit, with execution time and arguments.
 

{halib-0.2.32.dist-info → halib-0.2.34.dist-info}/RECORD

@@ -100,11 +100,14 @@ halib/utils/dict_op.py,sha256=wYE6Iw-_CnCWdMg9tpJ2Y2-e2ESkW9FxmdBkZkbUh80,299
 halib/utils/gpu_mon.py,sha256=vD41_ZnmPLKguuq9X44SB_vwd9JrblO4BDzHLXZhhFY,2233
 halib/utils/list.py,sha256=bbey9_0IaMXnHx1pudv3C3_WU9uFQEQ5qHPklSN-7o0,498
 halib/utils/listop.py,sha256=Vpa8_2fI0wySpB2-8sfTBkyi_A4FhoFVVvFiuvW8N64,339
+halib/utils/plotly_op.py,sha256=RzN7dHzNzJUfSgzKh6Yh60xR709YQUw0IXEvui5EBvw,9799
 halib/utils/slack.py,sha256=2ugWE_eJ0s479ObACJbx7iEu3kjMPD4Rt2hEwuMpuNQ,3099
+halib/utils/slack_op.py,sha256=2ugWE_eJ0s479ObACJbx7iEu3kjMPD4Rt2hEwuMpuNQ,3099
 halib/utils/tele_noti.py,sha256=-4WXZelCA4W9BroapkRyIdUu9cUVrcJJhegnMs_WpGU,5928
 halib/utils/video.py,sha256=zLoj5EHk4SmP9OnoHjO8mLbzPdtq6gQPzTQisOEDdO8,3261
-halib-0.2.32.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.2.32.dist-info/METADATA,sha256=nNZlX11Q-7O3QWNeUooBUbYCicmwJnJC-4ue44mVYA4,8183
-halib-0.2.32.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-halib-0.2.32.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.2.32.dist-info/RECORD,,
+halib/utils/wandb_op.py,sha256=qqDdTMW4J07bzuJTTg2HoLAPs21nVEbwt2-Aa5ZKiVk,4336
+halib-0.2.34.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.2.34.dist-info/METADATA,sha256=C5Pu5me0s-Giwn_xdybYhRxCoMu0rR6cFzkIOdfuejw,8379
+halib-0.2.34.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+halib-0.2.34.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.2.34.dist-info/RECORD,,