halib 0.1.93__tar.gz → 0.1.95__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {halib-0.1.93 → halib-0.1.95}/PKG-INFO +2 -2
- {halib-0.1.93 → halib-0.1.95}/README.md +1 -1
- {halib-0.1.93 → halib-0.1.95}/halib/common.py +53 -10
- {halib-0.1.93 → halib-0.1.95}/halib/filetype/yamlfile.py +23 -0
- halib-0.1.95/halib/research/mics.py +66 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/perfcalc.py +3 -3
- {halib-0.1.93 → halib-0.1.95}/halib/research/perftb.py +2 -1
- {halib-0.1.93 → halib-0.1.95}/halib.egg-info/PKG-INFO +2 -2
- {halib-0.1.93 → halib-0.1.95}/setup.py +1 -1
- halib-0.1.93/halib/research/mics.py +0 -16
- {halib-0.1.93 → halib-0.1.95}/.gitignore +0 -0
- {halib-0.1.93 → halib-0.1.95}/GDriveFolder.txt +0 -0
- {halib-0.1.93 → halib-0.1.95}/LICENSE.txt +0 -0
- {halib-0.1.93 → halib-0.1.95}/MANIFEST.in +0 -0
- {halib-0.1.93 → halib-0.1.95}/guide_publish_pip.pdf +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/__init__.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/cuda.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/filetype/__init__.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/filetype/csvfile.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/filetype/jsonfile.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/filetype/textfile.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/filetype/videofile.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/online/__init__.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/online/gdrive.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/online/gdrive_mkdir.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/online/gdrive_test.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/online/projectmake.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/__init__.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/base_config.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/base_exp.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/dataset.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/metrics.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/params_gen.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/plot.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/profiler.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/torchloader.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/research/wandb_op.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/rich_color.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/system/__init__.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/system/cmd.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/system/filesys.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/utils/__init__.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/utils/dataclass_util.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/utils/dict_op.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/utils/gpu_mon.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/utils/listop.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/utils/tele_noti.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib/utils/video.py +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib.egg-info/SOURCES.txt +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib.egg-info/dependency_links.txt +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib.egg-info/requires.txt +0 -0
- {halib-0.1.93 → halib-0.1.95}/halib.egg-info/top_level.txt +0 -0
- {halib-0.1.93 → halib-0.1.95}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: halib
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.95
|
|
4
4
|
Summary: Small library for common tasks
|
|
5
5
|
Author: Hoang Van Ha
|
|
6
6
|
Author-email: hoangvanhauit@gmail.com
|
|
@@ -52,7 +52,7 @@ Dynamic: summary
|
|
|
52
52
|
|
|
53
53
|
# Helper package for coding and automation
|
|
54
54
|
|
|
55
|
-
**Version 0.1.
|
|
55
|
+
**Version 0.1.95**
|
|
56
56
|
+ `research/plot': add `PlotHelper` class to plot train history + plot grid of images (e.g., image samples from dataset or model outputs)
|
|
57
57
|
|
|
58
58
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import os
|
|
1
2
|
import re
|
|
2
3
|
import rich
|
|
3
4
|
import arrow
|
|
@@ -9,6 +10,8 @@ from rich import print
|
|
|
9
10
|
from rich.panel import Panel
|
|
10
11
|
from rich.console import Console
|
|
11
12
|
from rich.pretty import pprint, Pretty
|
|
13
|
+
from pathlib import PureWindowsPath
|
|
14
|
+
|
|
12
15
|
|
|
13
16
|
console = Console()
|
|
14
17
|
|
|
@@ -91,18 +94,58 @@ class ConsoleLog:
|
|
|
91
94
|
print(f"Exception message: {exc_value}")
|
|
92
95
|
|
|
93
96
|
|
|
94
|
-
def
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
97
|
+
def linux_to_wins_path(path: str) -> str:
|
|
98
|
+
"""
|
|
99
|
+
Convert a Linux-style WSL path (/mnt/c/... or /mnt/d/...) to a Windows-style path (C:\...).
|
|
100
|
+
"""
|
|
101
|
+
# Handle only /mnt/<drive>/... style
|
|
102
|
+
if (
|
|
103
|
+
path.startswith("/mnt/")
|
|
104
|
+
and len(path) > 6
|
|
105
|
+
and path[5].isalpha()
|
|
106
|
+
and path[6] == "/"
|
|
107
|
+
):
|
|
108
|
+
drive = path[5].upper() # Extract drive letter
|
|
109
|
+
win_path = f"{drive}:{path[6:]}" # Replace "/mnt/c/" with "C:/"
|
|
110
|
+
else:
|
|
111
|
+
win_path = path # Return unchanged if not a WSL-style path
|
|
112
|
+
# Normalize to Windows-style backslashes
|
|
113
|
+
return str(PureWindowsPath(win_path))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def pprint_local_path(
|
|
117
|
+
local_path: str, get_wins_path: bool = False, tag: str = ""
|
|
118
|
+
) -> str:
|
|
119
|
+
"""
|
|
120
|
+
Pretty-print a local path with emoji and clickable file:// URI.
|
|
121
|
+
|
|
122
|
+
Args:
|
|
123
|
+
local_path: Path to file or directory (Linux or Windows style).
|
|
124
|
+
get_wins_path: If True on Linux, convert WSL-style path to Windows style before printing.
|
|
125
|
+
tag: Optional console log tag.
|
|
126
|
+
|
|
127
|
+
Returns:
|
|
128
|
+
The file URI string.
|
|
129
|
+
"""
|
|
130
|
+
p = Path(local_path).resolve()
|
|
131
|
+
type_str = "📄" if p.is_file() else "📁" if p.is_dir() else "❓"
|
|
132
|
+
|
|
133
|
+
if get_wins_path and os.name == "posix":
|
|
134
|
+
# Try WSL → Windows conversion
|
|
135
|
+
converted = linux_to_wins_path(str(p))
|
|
136
|
+
if converted != str(p): # Conversion happened
|
|
137
|
+
file_uri = str(PureWindowsPath(converted).as_uri())
|
|
138
|
+
else:
|
|
139
|
+
file_uri = p.as_uri()
|
|
140
|
+
else:
|
|
141
|
+
file_uri = p.as_uri()
|
|
142
|
+
|
|
101
143
|
content_str = f"{type_str} [link={file_uri}]{file_uri}[/link]"
|
|
102
|
-
|
|
144
|
+
|
|
145
|
+
if tag:
|
|
103
146
|
with ConsoleLog(tag):
|
|
104
147
|
console.print(content_str)
|
|
105
148
|
else:
|
|
106
|
-
|
|
107
|
-
|
|
149
|
+
console.print(content_str)
|
|
150
|
+
|
|
108
151
|
return file_uri
|
|
@@ -6,6 +6,8 @@ from omegaconf import OmegaConf
|
|
|
6
6
|
from rich.console import Console
|
|
7
7
|
from argparse import ArgumentParser
|
|
8
8
|
|
|
9
|
+
from ..research.mics import *
|
|
10
|
+
|
|
9
11
|
console = Console()
|
|
10
12
|
|
|
11
13
|
|
|
@@ -51,6 +53,27 @@ def load_yaml(yaml_file, to_dict=False, log_info=False):
|
|
|
51
53
|
else:
|
|
52
54
|
return omgconf
|
|
53
55
|
|
|
56
|
+
def load_yaml_with_PC_abbr(
|
|
57
|
+
yaml_file, pc_abbr_to_working_disk=DEFAULT_ABBR_WORKING_DISK
|
|
58
|
+
):
|
|
59
|
+
# current PC abbreviation
|
|
60
|
+
pc_abbr = get_PC_abbr_name()
|
|
61
|
+
|
|
62
|
+
# current plaftform: windows or linux
|
|
63
|
+
current_platform = platform.system().lower()
|
|
64
|
+
|
|
65
|
+
assert pc_abbr in pc_abbr_to_working_disk, f"The is no mapping for {pc_abbr} to <working_disk>"
|
|
66
|
+
|
|
67
|
+
# working disk
|
|
68
|
+
working_disk = pc_abbr_to_working_disk.get(pc_abbr)
|
|
69
|
+
|
|
70
|
+
# load yaml file
|
|
71
|
+
data_dict = load_yaml(yaml_file=yaml_file, to_dict=True)
|
|
72
|
+
|
|
73
|
+
# Normalize paths in the loaded data
|
|
74
|
+
data_dict = normalize_paths(data_dict, working_disk, current_platform)
|
|
75
|
+
return data_dict
|
|
76
|
+
|
|
54
77
|
|
|
55
78
|
def parse_args():
|
|
56
79
|
parser = ArgumentParser(description="desc text")
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
from ..common import *
|
|
2
|
+
from ..filetype import csvfile
|
|
3
|
+
import pandas as pd
|
|
4
|
+
import platform
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
PC_NAME_TO_ABBR = {
|
|
8
|
+
"DESKTOP-JQD9K01": "MainPC",
|
|
9
|
+
"DESKTOP-5IRHU87": "MSI_Laptop",
|
|
10
|
+
"DESKTOP-96HQCNO": "4090_SV",
|
|
11
|
+
"DESKTOP-Q2IKLC0": "4GPU_SV",
|
|
12
|
+
"DESKTOP-QNS3DNF": "1GPU_SV"
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
DEFAULT_ABBR_WORKING_DISK = {
|
|
16
|
+
"MainPC": "E:",
|
|
17
|
+
"MSI_Laptop": "D:",
|
|
18
|
+
"4090_SV": "E:",
|
|
19
|
+
"4GPU_SV": "D:",
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
def list_PCs():
|
|
23
|
+
df = pd.DataFrame(list(PC_NAME_TO_ABBR.items()), columns=["PC Name", "Abbreviation"])
|
|
24
|
+
csvfile.showdf(df)
|
|
25
|
+
|
|
26
|
+
def get_PC_name():
|
|
27
|
+
return platform.node()
|
|
28
|
+
|
|
29
|
+
def get_PC_abbr_name():
|
|
30
|
+
pc_name = get_PC_name()
|
|
31
|
+
return PC_NAME_TO_ABBR.get(pc_name, "Unknown")
|
|
32
|
+
|
|
33
|
+
# ! This funcction search for full paths in the obj and normalize them according to the current platform and working disk
|
|
34
|
+
# ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "windows" => "D:/zdataset/DFire"
|
|
35
|
+
# ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "linux" => "/mnt/d/zdataset/DFire"
|
|
36
|
+
def normalize_paths(obj, working_disk, current_platform):
|
|
37
|
+
if isinstance(obj, dict):
|
|
38
|
+
for key, value in obj.items():
|
|
39
|
+
obj[key] = normalize_paths(value, working_disk, current_platform)
|
|
40
|
+
return obj
|
|
41
|
+
elif isinstance(obj, list):
|
|
42
|
+
for i, item in enumerate(obj):
|
|
43
|
+
obj[i] = normalize_paths(item, working_disk, current_platform)
|
|
44
|
+
return obj
|
|
45
|
+
elif isinstance(obj, str):
|
|
46
|
+
# Normalize backslashes to forward slashes for consistency
|
|
47
|
+
obj = obj.replace("\\", "/")
|
|
48
|
+
# Regex for Windows-style path: e.g., "E:/zdataset/DFire"
|
|
49
|
+
win_match = re.match(r"^([A-Z]):/(.*)$", obj)
|
|
50
|
+
# Regex for Linux-style path: e.g., "/mnt/e/zdataset/DFire"
|
|
51
|
+
lin_match = re.match(r"^/mnt/([a-z])/(.*)$", obj)
|
|
52
|
+
if win_match or lin_match:
|
|
53
|
+
rest = win_match.group(2) if win_match else lin_match.group(2)
|
|
54
|
+
if current_platform == "windows":
|
|
55
|
+
# working_disk is like "D:", so "D:/" + rest
|
|
56
|
+
new_path = working_disk + "/" + rest
|
|
57
|
+
elif current_platform == "linux":
|
|
58
|
+
# Extract drive letter from working_disk (e.g., "D:" -> "d")
|
|
59
|
+
drive_letter = working_disk[0].lower()
|
|
60
|
+
new_path = "/mnt/" + drive_letter + "/" + rest
|
|
61
|
+
else:
|
|
62
|
+
# Unknown platform, return original
|
|
63
|
+
return obj
|
|
64
|
+
return new_path
|
|
65
|
+
# For non-strings or non-path strings, return as is
|
|
66
|
+
return obj
|
|
@@ -227,9 +227,9 @@ class PerfCalc(ABC): # Abstract base class for performance calculation
|
|
|
227
227
|
), "No metric columns found in the DataFrame. Ensure that the CSV files contain metric columns starting with 'metric_'."
|
|
228
228
|
final_cols = sticky_cols + metric_cols
|
|
229
229
|
df = df[final_cols]
|
|
230
|
-
# !hahv debug
|
|
231
|
-
pprint("------ Final DataFrame Columns ------")
|
|
232
|
-
csvfile.fn_display_df(df)
|
|
230
|
+
# # !hahv debug
|
|
231
|
+
# pprint("------ Final DataFrame Columns ------")
|
|
232
|
+
# csvfile.fn_display_df(df)
|
|
233
233
|
# ! validate all rows in df before returning
|
|
234
234
|
# make sure all rows will have at least values for REQUIRED_COLS and at least one metric column
|
|
235
235
|
for index, row in df.iterrows():
|
|
@@ -308,7 +308,8 @@ class PerfTB:
|
|
|
308
308
|
if save_path:
|
|
309
309
|
export_success = False
|
|
310
310
|
try:
|
|
311
|
-
fig.write_image(save_path, engine="kaleido")
|
|
311
|
+
# fig.write_image(save_path, engine="kaleido")
|
|
312
|
+
fig.write_image(save_path, engine="kaleido", width=width, height=height * len(metric_list))
|
|
312
313
|
export_success = True
|
|
313
314
|
# pprint(f"Saved: {os.path.abspath(save_path)}")
|
|
314
315
|
except Exception as e:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: halib
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.95
|
|
4
4
|
Summary: Small library for common tasks
|
|
5
5
|
Author: Hoang Van Ha
|
|
6
6
|
Author-email: hoangvanhauit@gmail.com
|
|
@@ -52,7 +52,7 @@ Dynamic: summary
|
|
|
52
52
|
|
|
53
53
|
# Helper package for coding and automation
|
|
54
54
|
|
|
55
|
-
**Version 0.1.
|
|
55
|
+
**Version 0.1.95**
|
|
56
56
|
+ `research/plot': add `PlotHelper` class to plot train history + plot grid of images (e.g., image samples from dataset or model outputs)
|
|
57
57
|
|
|
58
58
|
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import platform
|
|
2
|
-
|
|
3
|
-
PC_NAME_TO_ABBR = {
|
|
4
|
-
"DESKTOP-JQD9K01": "MainPC",
|
|
5
|
-
"DESKTOP-5IRHU87": "MSI_Laptop",
|
|
6
|
-
"DESKTOP-96HQCNO": "4090_SV",
|
|
7
|
-
"DESKTOP-Q2IKLC0": "4GPU_SV",
|
|
8
|
-
"DESKTOP-QNS3DNF": "1GPU_SV"
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
def get_PC_name():
|
|
12
|
-
return platform.node()
|
|
13
|
-
|
|
14
|
-
def get_PC_abbr_name():
|
|
15
|
-
pc_name = get_PC_name()
|
|
16
|
-
return PC_NAME_TO_ABBR.get(pc_name, "Unknown")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|