halib 0.1.48__py3-none-any.whl → 0.1.49__py3-none-any.whl

halib/research/wandb_op.py

@@ -0,0 +1,116 @@
+import glob
+from rich.pretty import pprint
+import os
+import subprocess
+import argparse
+import wandb
+from tqdm import tqdm
+from rich.console import Console
+console = Console()
+
+def sync_runs(outdir):
+    outdir = os.path.abspath(outdir)
+    assert os.path.exists(outdir), f"Output directory {outdir} does not exist."
+    sub_dirs = [name for name in os.listdir(outdir) if os.path.isdir(os.path.join(outdir, name))]
+    assert len(sub_dirs) > 0, f"No subdirectories found in {outdir}."
+    console.rule("Parent Directory")
+    console.print(f"[yellow]{outdir}[/yellow]")
+
+    exp_dirs = [os.path.join(outdir, sub_dir) for sub_dir in sub_dirs]
+    wandb_dirs = []
+    for exp_dir in exp_dirs:
+        wandb_dirs.extend(glob.glob(f"{exp_dir}/wandb/*run-*"))
+    if len(wandb_dirs) == 0:
+        console.print(f"No wandb runs found in {outdir}.")
+        return
+    else:
+        console.print(f"Found [bold]{len(wandb_dirs)}[/bold] wandb runs in {outdir}.")
+        for i, wandb_dir in enumerate(wandb_dirs):
+            console.rule(f"Syncing wandb run {i + 1}/{len(wandb_dirs)}")
+            console.print(f"Syncing: {wandb_dir}")
+            process = subprocess.Popen(
+                ["wandb", "sync", wandb_dir],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+            )
+
+            for line in process.stdout:
+                console.print(line.strip())
+                if " ERROR Error while calling W&B API" in line:
+                    break
+            process.stdout.close()
+            process.wait()
+            if process.returncode != 0:
+                console.print(f"[red]Error syncing {wandb_dir}. Return code: {process.returncode}[/red]")
+            else:
+                console.print(f"Successfully synced {wandb_dir}.")
+
+def delete_runs(project, pattern=None):
+    console.rule("Delete W&B Runs")
+    confirm_msg = f"Are you sure you want to delete all runs in"
+    confirm_msg += f" \n\tproject: [red]{project}[/red]"
+    if pattern:
+        confirm_msg += f"\n\tpattern: [blue]{pattern}[/blue]"
+
+    console.print(confirm_msg)
+    confirmation = input(f"This action cannot be undone. [y/N]: ").strip().lower()
+    if confirmation != "y":
+        print("Cancelled.")
+        return
+
+    print("Confirmed. Proceeding...")
+    api = wandb.Api()
+    runs = api.runs(project)
+
+    deleted = 0
+    console.rule("Deleting W&B Runs")
+    if len(runs) == 0:
+        print("No runs found in the project.")
+        return
+    for run in tqdm(runs):
+        if pattern is None or pattern in run.name:
+            run.delete()
+            console.print(f"Deleted run: [red]{run.name}[/red]")
+            deleted += 1
+
+    console.print(f"Total runs deleted: {deleted}")
+
+
+def valid_argument(args):
+    if args.op == "sync":
+        assert os.path.exists(args.outdir), f"Output directory {args.outdir} does not exist."
+    elif args.op == "delete":
+        assert isinstance(args.project, str) and len(args.project.strip()) > 0, "Project name must be a non-empty string."
+    else:
+        raise ValueError(f"Unknown operation: {args.op}")
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Operations on W&B runs")
+    parser.add_argument("-op", "--op", type=str, help="Operation to perform", default="sync", choices=["delete", "sync"])
+    parser.add_argument("-prj", "--project", type=str, default="fire-paper2-2025", help="W&B project name")
+    parser.add_argument("-outdir", "--outdir", type=str, help="arg1 description", default="./zout/train")
+    parser.add_argument("-pt", "--pattern",
+        type=str,
+        default=None,
+        help="Run name pattern to match for deletion",
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    # Validate arguments, stop if invalid
+    valid_argument(args)
+
+    op = args.op
+    if op == "sync":
+        sync_runs(args.outdir)
+    elif op == "delete":
+        delete_runs(args.project, args.pattern)
+    else:
+        raise ValueError(f"Unknown operation: {op}")
+
+if __name__ == "__main__":
+    main()

{halib-0.1.48.dist-info → halib-0.1.49.dist-info}/METADATA

@@ -1,158 +1,156 @@
-Metadata-Version: 2.1
-Name: halib
-Version: 0.1.48
-Summary: Small library for common tasks
-Author: Hoang Van Ha
-Author-email: hoangvanhauit@gmail.com
-License: UNKNOWN
-Platform: UNKNOWN
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE.txt
-Requires-Dist: arrow
-Requires-Dist: click
-Requires-Dist: enlighten
-Requires-Dist: kaleido (==0.1.*)
-Requires-Dist: loguru
-Requires-Dist: more-itertools
-Requires-Dist: moviepy
-Requires-Dist: networkx
-Requires-Dist: numpy
-Requires-Dist: omegaconf
-Requires-Dist: opencv-python
-Requires-Dist: pandas
-Requires-Dist: Pillow
-Requires-Dist: Pyarrow
-Requires-Dist: pycurl
-Requires-Dist: python-telegram-bot
-Requires-Dist: requests
-Requires-Dist: rich
-Requires-Dist: scikit-learn
-Requires-Dist: matplotlib
-Requires-Dist: seaborn
-Requires-Dist: plotly
-Requires-Dist: pygwalker
-Requires-Dist: tabulate
-Requires-Dist: itables
-Requires-Dist: timebudget
-Requires-Dist: tqdm
-Requires-Dist: tube-dl
-
-Helper package for coding and automation
-
-**Version 0.1.48**
-
-+ add `research` module to help with research tasks, including `benchquery` for benchmarking queries from dataframe
-
-**Version 0.1.47**
-+ add `pprint_box` to print object/string in a box frame (like in `inspect`)
-
-**Version 0.1.46**
-+ filter the warning message of `UserWarning: Unable to import Axes3D.`
-+ auto_wrap_text for `fn_display_df` to avoid long text in the table
-
-**Version 0.1.42**
-+ add <rich_color.py>: add basic color list (for easy usage)
-
-**Version 0.1.41**
-+ add <rich_color.py> to display rich color information in <rich> python package (rcolor_str, rcolor_pallet_all, etc.)
-
-**Version 0.1.40**
-
-+  update <csvfile.py> to use `itables` and `pygwalker` to display dataframe in jupyter notebook.
-
-**Version 0.1.38**
-
-+  add <torchloader.py> to search for best cfg for torch dataloader (num_workers, batch_size, pin_memory, et.)
-
-**Version 0.1.37**
-
-+  add <dataset.py> to help split classification dataset into train/val(test)
----
-**Version 0.1.33**
-
-+ add `plot.py` module to plot DL model training history (with columlns: epoch, train_accuracy, val_accuracy, train_loss, val_loss) using `seaborn` and `matplotlib`
----
-**Version 0.1.29**
-
-+ for `tele_noti` module, `kaleido==0.1.*` is required for plotly since `kaleido 0.2.*` is not working (taking for ever to generate image)
----
-**Version 0.1.24**
-
-+ rename `sys` to `system` to avoid conflict with built-in `sys` module
-+ add `tele_noti` module to send notification to telegram after a specific interval for training progress monitoring
----
-**Version 0.1.22**
-
-+ add `cuda.py` module to check CUDA availability (for both pytorch and tensorflow)
----
-**Version 0.1.21**
-
-+ using `networkx` and `omegaconf` to allow yaml file inheritance and override
----
-**Version 0.1.15**
-
-+ `__init__.py`: add common logging library; also `console_log` decorator to log function (start and end)
-
----
-
-**Version 0.1.10**
-
-+ filesys: fix typo on "is_exit" to "is_exist"
-+ gdrive: now support uploading file to folder and return direct link (shareable link)
-
-**Version 0.1.9**
-
-+ add dependencies requirement.txt
-
-**Version 0.1.8**
-
-Fix bugs:
-
-+ [performance] instead of inserting directly new rows into table dataframe, first insert it into in-memory `row_pool_dict`, that fill data in that dict into the actual dataframe when needed.
-
----
-
-**Version 0.1.7**
-
-Fix bugs:
-
-+ fix insert into table so slow by allowing insert multiple rows at once
-
----
-
-**Version 0.1.6**
-
-New features:
-
-+ add DFCreator for manipulating table (DataFrame) - create, insert row, display, write to file
-
----
-
-**Version 0.1.5**
-
-New Features
-
-+ add cmd module
-+ new package structure
-
----
-
-**Version 0.1.4**
-
-New Features
-
-+ add support to create Bitbucket Project from template
-
----
-
-**Version 0.1.2**
-
-New Features
-
-+ add support to upload local to google drive.
-
-
+Metadata-Version: 2.1
+Name: halib
+Version: 0.1.49
+Summary: Small library for common tasks
+Author: Hoang Van Ha
+Author-email: hoangvanhauit@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: arrow
+Requires-Dist: click
+Requires-Dist: enlighten
+Requires-Dist: kaleido ==0.1.*
+Requires-Dist: loguru
+Requires-Dist: more-itertools
+Requires-Dist: moviepy
+Requires-Dist: networkx
+Requires-Dist: numpy
+Requires-Dist: omegaconf
+Requires-Dist: opencv-python
+Requires-Dist: pandas
+Requires-Dist: Pillow
+Requires-Dist: Pyarrow
+Requires-Dist: pycurl
+Requires-Dist: python-telegram-bot
+Requires-Dist: requests
+Requires-Dist: rich
+Requires-Dist: scikit-learn
+Requires-Dist: matplotlib
+Requires-Dist: seaborn
+Requires-Dist: plotly
+Requires-Dist: pygwalker
+Requires-Dist: tabulate
+Requires-Dist: itables
+Requires-Dist: timebudget
+Requires-Dist: tqdm
+Requires-Dist: tube-dl
+Requires-Dist: wandb
+
+Helper package for coding and automation
+
+**Version 0.1.49**
+
++ add `research` package to help with research tasks, including `benchquery` for benchmarking queries from dataframe
++ add `wandb` module to allow easy sync offline data to Weights & Biases (wandb) and batch clear wandb runs.
+
+**Version 0.1.47**
++ add `pprint_box` to print object/string in a box frame (like in `inspect`)
+
+**Version 0.1.46**
++ filter the warning message of `UserWarning: Unable to import Axes3D.`
++ auto_wrap_text for `fn_display_df` to avoid long text in the table
+
+**Version 0.1.42**
++ add <rich_color.py>: add basic color list (for easy usage)
+
+**Version 0.1.41**
++ add <rich_color.py> to display rich color information in <rich> python package (rcolor_str, rcolor_pallet_all, etc.)
+
+**Version 0.1.40**
+
++  update <csvfile.py> to use `itables` and `pygwalker` to display dataframe in jupyter notebook.
+
+**Version 0.1.38**
+
++  add <torchloader.py> to search for best cfg for torch dataloader (num_workers, batch_size, pin_memory, et.)
+
+**Version 0.1.37**
+
++  add <dataset.py> to help split classification dataset into train/val(test)
+---
+**Version 0.1.33**
+
++ add `plot.py` module to plot DL model training history (with columlns: epoch, train_accuracy, val_accuracy, train_loss, val_loss) using `seaborn` and `matplotlib`
+---
+**Version 0.1.29**
+
++ for `tele_noti` module, `kaleido==0.1.*` is required for plotly since `kaleido 0.2.*` is not working (taking for ever to generate image)
+---
+**Version 0.1.24**
+
++ rename `sys` to `system` to avoid conflict with built-in `sys` module
++ add `tele_noti` module to send notification to telegram after a specific interval for training progress monitoring
+---
+**Version 0.1.22**
+
++ add `cuda.py` module to check CUDA availability (for both pytorch and tensorflow)
+---
+**Version 0.1.21**
+
++ using `networkx` and `omegaconf` to allow yaml file inheritance and override
+---
+**Version 0.1.15**
+
++ `__init__.py`: add common logging library; also `console_log` decorator to log function (start and end)
+
+---
+
+**Version 0.1.10**
+
++ filesys: fix typo on "is_exit" to "is_exist"
++ gdrive: now support uploading file to folder and return direct link (shareable link)
+
+**Version 0.1.9**
+
++ add dependencies requirement.txt
+
+**Version 0.1.8**
+
+Fix bugs:
+
++ [performance] instead of inserting directly new rows into table dataframe, first insert it into in-memory `row_pool_dict`, that fill data in that dict into the actual dataframe when needed.
+
+---
+
+**Version 0.1.7**
+
+Fix bugs:
+
++ fix insert into table so slow by allowing insert multiple rows at once
+
+---
+
+**Version 0.1.6**
+
+New features:
+
++ add DFCreator for manipulating table (DataFrame) - create, insert row, display, write to file
+
+---
+
+**Version 0.1.5**
+
+New Features
+
++ add cmd module
++ new package structure
+
+---
+
+**Version 0.1.4**
+
+New Features
+
++ add support to create Bitbucket Project from template
+
+---
+
+**Version 0.1.2**
+
+New Features
+
++ add support to upload local to google drive.

{halib-0.1.48.dist-info → halib-0.1.49.dist-info}/RECORD

@@ -32,6 +32,7 @@ halib/research/benchquery.py,sha256=FuKnbWQtCEoRRtJAfN-zaN-jPiO_EzsakmTOMiqi7GQ,
 halib/research/dataset.py,sha256=QU0Hr5QFb8_XlvnOMgC9QJGIpwXAZ9lDd0RdQi_QRec,6743
 halib/research/plot.py,sha256=-pDUk4z3C_GnyJ5zWmf-mGMdT4gaipVJWzIgcpIPiRk,9448
 halib/research/torchloader.py,sha256=yqUjcSiME6H5W210363HyRUrOi3ISpUFAFkTr1w4DCw,6503
+halib/research/wandb_op.py,sha256=YzLEqME5kIRxi3VvjFkW83wnFrsn92oYeqYuNwtYRkY,4188
 halib/sys/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/sys/cmd.py,sha256=b2x7JPcNnFjLGheIESVYvqAb-w2UwBM1PAwYxMZ5YjA,228
 halib/sys/filesys.py,sha256=ERpnELLDKJoTIIKf-AajgkY62nID4qmqmX5TkE95APU,2931
@@ -41,8 +42,8 @@ halib/system/filesys.py,sha256=ERpnELLDKJoTIIKf-AajgkY62nID4qmqmX5TkE95APU,2931
 halib/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/utils/listop.py,sha256=Vpa8_2fI0wySpB2-8sfTBkyi_A4FhoFVVvFiuvW8N64,339
 halib/utils/tele_noti.py,sha256=-4WXZelCA4W9BroapkRyIdUu9cUVrcJJhegnMs_WpGU,5928
-halib-0.1.48.dist-info/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.1.48.dist-info/METADATA,sha256=iaGDSmQyhQWr6hLkyRpK6ZpkW6tuoAoGOrPNuK3CQp8,3960
-halib-0.1.48.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-halib-0.1.48.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.1.48.dist-info/RECORD,,
+halib-0.1.49.dist-info/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.1.49.dist-info/METADATA,sha256=G3MPTsD8-W3o1MptmTpzCvslPuQg6uVISpkSwRoLJqw,4208
+halib-0.1.49.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
+halib-0.1.49.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.1.49.dist-info/RECORD,,

{halib-0.1.48.dist-info → halib-0.1.49.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.1)
+Generator: setuptools (70.2.0)
 Root-Is-Purelib: true
 Tag: py3-none-any