halib 0.2.32__tar.gz → 0.2.33__tar.gz

{halib-0.2.32 → halib-0.2.33}/.gitignore

@@ -53,4 +53,7 @@ dist
 
 venv*/
 
-test/test_plot/output
+test/test_plot/output
+
+# api keys and configurations
+.env.yaml

halib-0.2.33/MANIFEST.in

@@ -0,0 +1,6 @@
+prune _archived
+prune .env.yaml
+prune test
+prune zout
+exclude *.toml
+include halib/system/_list_pc.csv

{halib-0.2.32 → halib-0.2.33}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.2.32
+Version: 0.2.33
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -57,7 +57,10 @@ Dynamic: summary
 
 ## v0.2.x (Experiment & Core Updates)
 
-### **v0.2.32**
+### **v0.2.33**
+
+- 🚀 **Improvement:**: move `wandb_op.py` to `utils` and add `scripts` folder
+
 
 - ✨ **New Feature:**: add `common.common.log_func` as decorator to log function entry, exit, with execution time and arguments.
 

{halib-0.2.32 → halib-0.2.33}/README.md

@@ -2,7 +2,10 @@
 
 ## v0.2.x (Experiment & Core Updates)
 
-### **v0.2.32**
+### **v0.2.33**
+
+- 🚀 **Improvement:**: move `wandb_op.py` to `utils` and add `scripts` folder
+
 
 - ✨ **New Feature:**: add `common.common.log_func` as decorator to log function entry, exit, with execution time and arguments.
 

halib-0.2.33/halib/utils/wandb_op.py

@@ -0,0 +1,140 @@
+import glob
+import wandb
+import subprocess
+from pathlib import Path
+from typing import Optional, Literal
+
+from tap import Tap
+from tqdm import tqdm
+
+from rich.console import Console
+import yaml
+
+wandb_console = Console()
+
+
+# --- Argument Parser ---
+class WandbArgs(Tap):
+    # --- Auth ---
+    api_key_yaml: Path = Path(
+        "E:/Dev/__halib/.env.yaml"
+    )  # Path to YAML file containing W&B API key
+
+    # --- Operations ---
+    op: Literal["sync", "delete"] = "delete"  # Operation to perform
+
+    # --- Project & Filtering ---
+    project: str = "paper2_main"  # W&B project name
+    pattern: Optional[str] = None  # Run name pattern to match for deletion
+
+    # --- Paths ---
+    outdir: Path = Path("./zout/zruns")  # Directory containing runs to sync
+
+    def configure(self):
+        self.add_argument("-prj", "--project")
+        self.add_argument("-pt", "--pattern")
+        self.add_argument("-o", "--outdir")
+        self.add_argument("-k", "--api_key_yaml")
+
+    def validate(self):
+        if self.op == "sync" and not self.outdir.exists():
+            raise ValueError(f"Output directory {self.outdir} does not exist.")
+        if self.op == "delete" and not self.project.strip():
+            raise ValueError("Project name must be a non-empty string.")
+
+
+def login_wandb(api_key: Optional[str]):
+    """Handles authentication with Weights & Biases."""
+    try:
+        if api_key:
+            # Explicit login with key
+            wandb.login(key=api_key)
+            wandb_console.print(
+                "[green]Successfully logged into W&B using provided API key.[/green]"
+            )
+        else:
+            # Attempts to find key in environment variables or netrc file
+            if wandb.login():
+                wandb_console.print(
+                    "[blue]Logged into W&B using existing credentials.[/blue]"
+                )
+            else:
+                wandb_console.print(
+                    "[red]Authentication failed. Please provide an API key with -k.[/red]"
+                )
+                exit(1)
+    except Exception as e:
+        wandb_console.print(f"[bold red]Login Error:[/bold red] {e}")
+        exit(1)
+
+
+# --- Logic Functions ---
+
+
+def sync_runs(outdir: Path):
+    outdir_path = outdir.absolute()
+    sub_dirs = [d for d in outdir_path.iterdir() if d.is_dir()]
+
+    if not sub_dirs:
+        wandb_console.print(f"[red]No subdirectories found in {outdir_path}.[/red]")
+        return
+
+    wandb_console.rule(f"Syncing from {outdir_path}")
+
+    wandb_dirs = []
+    for exp_dir in sub_dirs:
+        wandb_dirs.extend(glob.glob(str(exp_dir / "wandb" / "*run-*")))
+
+    if not wandb_dirs:
+        wandb_console.print("No wandb runs found.")
+        return
+
+    for i, wandb_dir in enumerate(wandb_dirs):
+        wandb_console.print(f"[{i+1}/{len(wandb_dirs)}] Syncing: {wandb_dir}")
+
+        # Note: 'wandb sync' command uses the credentials from wandb.login()
+        process = subprocess.Popen(
+            ["wandb", "sync", wandb_dir],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+        for line in process.stdout:  # ty:ignore[not-iterable]
+            if "ERROR" in line:
+                wandb_console.print(f"[red]{line.strip()}[/red]")
+        process.wait()
+
+
+def delete_runs(project: str, pattern: Optional[str]):
+    api = wandb.Api()
+    runs = api.runs(project)
+
+    deleted = 0
+    for run in tqdm(runs, desc="Processing runs"):
+        if pattern is None or pattern in run.name:
+            run.delete()
+            deleted += 1
+    wandb_console.print(f"[green]Total runs deleted: {deleted}[/green]")
+
+
+def main():
+    args = WandbArgs().parse_args()
+    cfg_dict = {}
+    with open(args.api_key_yaml, "r") as f:
+        cfg_dict = yaml.safe_load(f)
+    wandb_api_key = cfg_dict.get("WANDB_API_KEY")
+    assert (
+        wandb_api_key is not None
+    ), "W&B API key not found in the specified YAML file."
+    # 1. Login first
+    login_wandb(wandb_api_key)
+
+    # 2. Execute operation
+    if args.op == "sync":
+        sync_runs(args.outdir)
+    elif args.op == "delete":
+        delete_runs(args.project, args.pattern)
+
+
+if __name__ == "__main__":
+    main()

{halib-0.2.32 → halib-0.2.33}/halib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.2.32
+Version: 0.2.33
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -57,7 +57,10 @@ Dynamic: summary
 
 ## v0.2.x (Experiment & Core Updates)
 
-### **v0.2.32**
+### **v0.2.33**
+
+- 🚀 **Improvement:**: move `wandb_op.py` to `utils` and add `scripts` folder
+
 
 - ✨ **New Feature:**: add `common.common.log_func` as decorator to log function entry, exit, with execution time and arguments.
 

{halib-0.2.32 → halib-0.2.33}/halib.egg-info/SOURCES.txt

@@ -18,7 +18,6 @@ halib/exp/core/__init__.py
 halib/exp/core/base_config.py
 halib/exp/core/base_exp.py
 halib/exp/core/param_gen.py
-halib/exp/core/wandb_op.py
 halib/exp/data/__init__.py
 halib/exp/data/dataclass_util.py
 halib/exp/data/dataset.py
@@ -52,4 +51,5 @@ halib/system/path.py
 halib/utils/__init__.py
 halib/utils/dict.py
 halib/utils/list.py
-halib/utils/slack.py
+halib/utils/slack.py
+halib/utils/wandb_op.py

{halib-0.2.32 → halib-0.2.33}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt", "r", encoding="utf-8") as f:
 
 setuptools.setup(
     name="halib",
-    version="0.2.32",
+    version="0.2.33",
     author="Hoang Van Ha",
     author_email="hoangvanhauit@gmail.com",
     description="Small library for common tasks",

halib-0.2.32/MANIFEST.in

@@ -1,5 +0,0 @@
-prune _archived
-prune test
-prune zout
-exclude *.toml
-include halib/system/_list_pc.csv

halib-0.2.32/halib/exp/core/wandb_op.py

@@ -1,117 +0,0 @@
-import os
-import glob
-import wandb
-import argparse
-import subprocess
-
-from tqdm import tqdm
-from rich.console import Console
-
-console = Console()
-
-def sync_runs(outdir):
-    outdir = os.path.abspath(outdir)
-    assert os.path.exists(outdir), f"Output directory {outdir} does not exist."
-    sub_dirs = [name for name in os.listdir(outdir) if os.path.isdir(os.path.join(outdir, name))]
-    assert len(sub_dirs) > 0, f"No subdirectories found in {outdir}."
-    console.rule("Parent Directory")
-    console.print(f"[yellow]{outdir}[/yellow]")
-
-    exp_dirs = [os.path.join(outdir, sub_dir) for sub_dir in sub_dirs]
-    wandb_dirs = []
-    for exp_dir in exp_dirs:
-        wandb_dirs.extend(glob.glob(f"{exp_dir}/wandb/*run-*"))
-    if len(wandb_dirs) == 0:
-        console.print(f"No wandb runs found in {outdir}.")
-        return
-    else:
-        console.print(f"Found [bold]{len(wandb_dirs)}[/bold] wandb runs in {outdir}.")
-        for i, wandb_dir in enumerate(wandb_dirs):
-            console.rule(f"Syncing wandb run {i + 1}/{len(wandb_dirs)}")
-            console.print(f"Syncing: {wandb_dir}")
-            process = subprocess.Popen(
-                ["wandb", "sync", wandb_dir],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,
-                text=True,
-            )
-
-            for line in process.stdout:
-                console.print(line.strip())
-                if " ERROR Error while calling W&B API" in line:
-                    break
-            process.stdout.close()
-            process.wait()
-            if process.returncode != 0:
-                console.print(f"[red]Error syncing {wandb_dir}. Return code: {process.returncode}[/red]")
-            else:
-                console.print(f"Successfully synced {wandb_dir}.")
-
-def delete_runs(project, pattern=None):
-    console.rule("Delete W&B Runs")
-    confirm_msg = f"Are you sure you want to delete all runs in"
-    confirm_msg += f" \n\tproject: [red]{project}[/red]"
-    if pattern:
-        confirm_msg += f"\n\tpattern: [blue]{pattern}[/blue]"
-
-    console.print(confirm_msg)
-    confirmation = input(f"This action cannot be undone. [y/N]: ").strip().lower()
-    if confirmation != "y":
-        print("Cancelled.")
-        return
-
-    print("Confirmed. Proceeding...")
-    api = wandb.Api()
-    runs = api.runs(project)
-
-    deleted = 0
-    console.rule("Deleting W&B Runs")
-    if len(runs) == 0:
-        print("No runs found in the project.")
-        return
-    for run in tqdm(runs):
-        if pattern is None or pattern in run.name:
-            run.delete()
-            console.print(f"Deleted run: [red]{run.name}[/red]")
-            deleted += 1
-
-    console.print(f"Total runs deleted: {deleted}")
-
-
-def valid_argument(args):
-    if args.op == "sync":
-        assert os.path.exists(args.outdir), f"Output directory {args.outdir} does not exist."
-    elif args.op == "delete":
-        assert isinstance(args.project, str) and len(args.project.strip()) > 0, "Project name must be a non-empty string."
-    else:
-        raise ValueError(f"Unknown operation: {args.op}")
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Operations on W&B runs")
-    parser.add_argument("-op", "--op", type=str, help="Operation to perform", default="sync", choices=["delete", "sync"])
-    parser.add_argument("-prj", "--project", type=str, default="fire-paper2-2025", help="W&B project name")
-    parser.add_argument("-outdir", "--outdir", type=str, help="arg1 description", default="./zout/train")
-    parser.add_argument("-pt", "--pattern",
-        type=str,
-        default=None,
-        help="Run name pattern to match for deletion",
-    )
-
-    return parser.parse_args()
-
-
-def main():
-    args = parse_args()
-    # Validate arguments, stop if invalid
-    valid_argument(args)
-
-    op = args.op
-    if op == "sync":
-        sync_runs(args.outdir)
-    elif op == "delete":
-        delete_runs(args.project, args.pattern)
-    else:
-        raise ValueError(f"Unknown operation: {op}")
-
-if __name__ == "__main__":
-    main()