halib 0.2.30__py3-none-any.whl

halib/system/filesys.py

@@ -0,0 +1,164 @@
+import glob
+import os
+import shutil
+from distutils.dir_util import copy_tree
+from concurrent.futures import ThreadPoolExecutor
+
+COMMON_IMG_EXT = ["jpg", "jpeg", "png", "bmp", "tiff", "gif"]
+COMMON_VIDEO_EXT = ["mp4", "avi", "mov", "mkv", "flv", "wmv"]
+
+def is_exist(path):
+    return os.path.exists(path)
+
+
+def is_dir(path):
+    return os.path.isdir(path)
+
+
+def get_current_dir():
+    return os.getcwd()
+
+
+def change_current_dir(new_dir):
+    if is_dir(new_dir):
+        os.chdir(new_dir)
+
+
+def get_dir_name(directory):
+    return os.path.basename(os.path.normpath(directory))
+
+
+def get_parent_dir(directory, return_full_path=False):
+    if not return_full_path:
+        return os.path.basename(os.path.dirname(directory))
+    else:
+        return os.path.dirname(directory)
+
+
+def make_dir(directory):
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+
+def copy_dir(src_dir, dst_dir, dirs_exist_ok=True, ignore_patterns=None):
+    shutil.copytree(
+        src_dir, dst_dir, dirs_exist_ok=dirs_exist_ok, ignore=ignore_patterns
+    )
+
+
+def delete_dir(directory):
+    shutil.rmtree(directory)
+
+
+def list_dirs(directory):
+    folders = list(
+        filter(
+            lambda x: os.path.isdir(os.path.join(directory, x)), os.listdir(directory)
+        )
+    )
+    return folders
+
+
+def list_files(directory):
+    files = list(
+        filter(
+            lambda x: os.path.isfile(os.path.join(directory, x)), os.listdir(directory)
+        )
+    )
+    return files
+
+def filter_files_by_extension(directory, ext=None, recursive=True, num_workers=0):
+    """
+    Filters files using glob and multithreading.
+    If ext is None, returns ALL files.
+
+    Args:
+        directory (str): Path to search.
+        ext (str, list, or None): Extension(s) to find. If None, return all files.
+        recursive (bool): Whether to search subdirectories.
+        num_workers (int): Number of threads for checking file existence.
+    """
+    assert os.path.exists(directory) and os.path.isdir(
+        directory
+    ), "Directory does not exist"
+
+    # 1. Normalize extensions to a tuple (only if ext is provided)
+    extensions = None
+    if ext is not None:
+        if isinstance(ext, list):
+            extensions = tuple(ext)
+        else:
+            extensions = (ext,)
+
+    # 2. Define pattern
+    pattern = (
+        os.path.join(directory, "**", "*")
+        if recursive
+        else os.path.join(directory, "*")
+    )
+
+    # 3. Helper function for the thread workers
+    def validate_file(path):
+        if os.path.isfile(path):
+            return path
+        return None
+
+    result_files = []
+    if num_workers <= 0:
+        num_workers = os.cpu_count() or 4
+    # 4. Initialize ThreadPool with user-defined workers
+    with ThreadPoolExecutor(max_workers=num_workers) as executor:
+        # Step A: Get the iterator (lazy evaluation)
+        all_paths = glob.iglob(pattern, recursive=recursive)
+
+        # Step B: Apply Filter
+        if extensions is None:
+            # If ext is None, we skip the endswith check and pass everything
+            candidate_paths = all_paths
+        else:
+            # Filter by extension string FIRST (Fast CPU op)
+            candidate_paths = (p for p in all_paths if p.endswith(extensions))
+
+        # Step C: Parallelize the disk check (Slow I/O op)
+        for result in executor.map(validate_file, candidate_paths):
+            if result:
+                result_files.append(result)
+
+    return result_files
+
+
+def is_file(path):
+    return os.path.isfile(path)
+
+
+def get_file_name(file_path, split_file_ext=False):
+    if is_file(file_path):
+        if split_file_ext:
+            filename, file_extension = os.path.splitext(os.path.basename(file_path))
+            return filename, file_extension
+        else:
+            return os.path.basename(file_path)
+    else:
+        raise OSError("Not a file")
+
+
+def get_absolute_path(file_path):
+    return os.path.abspath(file_path)
+
+
+# dest can be a directory
+def copy_file(source, dest):
+    shutil.copy2(source, dest)
+
+
+def delete_file(path):
+    if is_file(path):
+        os.remove(path)
+
+
+def rename_dir_or_file(old, new):
+    os.renames(old, new)
+
+
+def move_dir_or_file(source, destination):
+    shutil.move(source, destination)

halib/system/path.py

@@ -0,0 +1,106 @@
+from ..common.common import *
+from ..filetype import csvfile
+import pandas as pd
+import platform
+import re  # <--- [FIX 1] Added missing import
+import csv
+from importlib import resources
+
+PC_TO_ABBR = {}
+ABBR_DISK_MAP = {}
+pc_df = None
+cPlatform = platform.system().lower()
+
+
+def load_pc_meta_info():
+    # 1. Define the package where the file lives (dotted notation)
+    #    Since the file is in 'halib/system/', the package is 'halib.system'
+    package_name = "halib.system"
+    file_name = "_list_pc.csv"
+
+    # 2. Locate the file
+    csv_path = resources.files(package_name).joinpath(file_name)
+    global PC_TO_ABBR, ABBR_DISK_MAP, pc_df
+    pc_df = pd.read_csv(csv_path, sep=';', encoding='utf-8')  # ty:ignore[no-matching-overload]
+    PC_TO_ABBR = dict(zip(pc_df['pc_name'], pc_df['abbr']))
+    ABBR_DISK_MAP = dict(zip(pc_df['abbr'], pc_df['working_disk']))
+    # pprint("Loaded PC meta info:")
+    # pprint(ABBR_DISK_MAP)
+    # pprint(PC_TO_ABBR)
+# ! must be called at the module load time
+load_pc_meta_info()
+
+
+def list_PCs(show=True):
+    global pc_df
+    if show:
+        csvfile.fn_display_df(pc_df)
+    return pc_df
+
+
+def get_PC_name():
+    return platform.node()
+
+
+def get_PC_abbr_name():
+    pc_name = get_PC_name()
+    return PC_TO_ABBR.get(pc_name, "Unknown")
+
+
+def get_os_platform():
+    return platform.system().lower()
+
+
+def get_working_disk(abbr_disk_map=ABBR_DISK_MAP):
+    pc_abbr = get_PC_abbr_name()
+    return abbr_disk_map.get(pc_abbr, None)
+
+cDisk = get_working_disk()
+
+# ! This function search for full paths in the obj and normalize them according to the current platform and working disk
+# ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "windows" => "D:/zdataset/DFire"
+# ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "linux" => "/mnt/d/zdataset/DFire"
+def normalize_paths(obj, working_disk=cDisk, current_platform=cPlatform):
+    # [FIX 3] Resolve defaults inside function to be safer/cleaner
+    if working_disk is None:
+        working_disk = get_working_disk()
+    if current_platform is None:
+        current_platform = get_os_platform()
+
+    # [FIX 2] If PC is unknown, working_disk is None. Return early to avoid crash.
+    if working_disk is None:
+        return obj
+
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            obj[key] = normalize_paths(value, working_disk, current_platform)
+        return obj
+    elif isinstance(obj, list):
+        for i, item in enumerate(obj):
+            obj[i] = normalize_paths(item, working_disk, current_platform)
+        return obj
+    elif isinstance(obj, str):
+        # Normalize backslashes to forward slashes for consistency
+        obj = obj.replace("\\", "/")
+
+        # Regex for Windows-style path: e.g., "E:/zdataset/DFire"
+        win_match = re.match(r"^([A-Z]):/(.*)$", obj)
+        # Regex for Linux-style path: e.g., "/mnt/e/zdataset/DFire"
+        lin_match = re.match(r"^/mnt/([a-z])/(.*)$", obj)
+
+        if win_match or lin_match:
+            rest = win_match.group(2) if win_match else lin_match.group(2)
+
+            if current_platform == "windows":
+                # working_disk is like "D:", so "D:/" + rest
+                new_path = f"{working_disk}/{rest}"
+            elif current_platform == "linux":
+                # Extract drive letter from working_disk (e.g., "D:" -> "d")
+                drive_letter = working_disk[0].lower()
+                new_path = f"/mnt/{drive_letter}/{rest}"
+            else:
+                return obj
+            return new_path
+
+    # For non-strings or non-path strings, return as is
+    return obj

halib/tele_noti.py

@@ -0,0 +1,166 @@
+# Watch a log file and send a telegram message when train reaches a certain epoch or end
+
+import os
+import yaml
+import asyncio
+import telegram
+import pandas as pd
+
+from rich.pretty import pprint
+from rich.console import Console
+import plotly.graph_objects as go
+
+from .system import filesys as fs
+from .filetype import textfile, csvfile
+
+from argparse import ArgumentParser
+
+tele_console = Console()
+
+
+def parse_args():
+    parser = ArgumentParser(description="desc text")
+    parser.add_argument(
+        "-cfg",
+        "--cfg",
+        type=str,
+        help="yaml file for tele",
+        default=r"E:\Dev\halib\cfg_tele_noti.yaml",
+    )
+
+    return parser.parse_args()
+
+
+def get_watcher_message_df(target_file, num_last_lines):
+    file_ext = fs.get_file_name(target_file, split_file_ext=True)[1]
+    supported_ext = [".txt", ".log", ".csv"]
+    assert (
+        file_ext in supported_ext
+    ), f"File extension {file_ext} not supported. Supported extensions are {supported_ext}"
+    last_lines_df = None
+    if file_ext in [".txt", ".log"]:
+        lines = textfile.read_line_by_line(target_file)
+        if num_last_lines > len(lines):
+            num_last_lines = len(lines)
+        last_line_arr = lines[-num_last_lines:]
+        # add a line start with word "epoch"
+        epoch_info_list = "Epoch: n/a"
+        for line in reversed(lines):
+            if "epoch" in line.lower():
+                epoch_info_list = line
+                break
+        last_line_arr.insert(0, epoch_info_list)  # insert at the beginning
+        dfCreator = csvfile.DFCreator()
+        dfCreator.create_table("last_lines", ["line"])
+        last_line_arr = [[line] for line in last_line_arr]
+        dfCreator.insert_rows("last_lines", last_line_arr)
+        dfCreator.fill_table_from_row_pool("last_lines")
+        last_lines_df = dfCreator["last_lines"].copy()
+    else:
+        df = pd.read_csv(target_file)
+        num_rows = len(df)
+        if num_last_lines > num_rows:
+            num_last_lines = num_rows
+        last_lines_df = df.tail(num_last_lines)
+    return last_lines_df
+
+
+def df2img(df: pd.DataFrame, output_img_dir, decimal_places, out_img_scale):
+    df = df.round(decimal_places)
+    fig = go.Figure(
+        data=[
+            go.Table(
+                header=dict(values=list(df.columns), align="center"),
+                cells=dict(
+                    values=df.values.transpose(),
+                    fill_color=[["white", "lightgrey"] * df.shape[0]],
+                    align="center",
+                ),
+            )
+        ]
+    )
+    if not os.path.exists(output_img_dir):
+        os.makedirs(output_img_dir)
+    img_path = os.path.normpath(os.path.join(output_img_dir, "last_lines.png"))
+    fig.write_image(img_path, scale=out_img_scale)
+    return img_path
+
+
+def compose_message_and_img_path(
+    target_file, project, num_last_lines, decimal_places, out_img_scale, output_img_dir
+):
+    context_msg = f">> Project: {project} \n>> File: {target_file} \n>> Last {num_last_lines} lines:"
+    msg_df = get_watcher_message_df(target_file, num_last_lines)
+    try:
+        img_path = df2img(msg_df, output_img_dir, decimal_places, out_img_scale)
+    except Exception as e:
+        pprint(f"Error: {e}")
+        img_path = None
+    return context_msg, img_path
+
+
+async def send_to_telegram(cfg_dict, interval_in_sec):
+    # pprint(cfg_dict)
+    token = cfg_dict["telegram"]["token"]
+    chat_id = cfg_dict["telegram"]["chat_id"]
+
+    noti_settings = cfg_dict["noti_settings"]
+    project = noti_settings["project"]
+    target_file = noti_settings["target_file"]
+    num_last_lines = noti_settings["num_last_lines"]
+    output_img_dir = noti_settings["output_img_dir"]
+    decimal_places = noti_settings["decimal_places"]
+    out_img_scale = noti_settings["out_img_scale"]
+
+    bot = telegram.Bot(token=token)
+    async with bot:
+        try:
+            context_msg, img_path = compose_message_and_img_path(
+                target_file,
+                project,
+                num_last_lines,
+                decimal_places,
+                out_img_scale,
+                output_img_dir,
+            )
+            time_now = next_time = pd.Timestamp.now().strftime("%Y-%m-%d %H:%M:%S")
+            sep_line = "-" * 50
+            context_msg = f"{sep_line}\n>> Time: {time_now}\n{context_msg}"
+            # calculate the next time to send message
+            next_time = pd.Timestamp.now() + pd.Timedelta(seconds=interval_in_sec)
+            next_time = next_time.strftime("%Y-%m-%d %H:%M:%S")
+            next_time_info = f"Next msg: {next_time}"
+            tele_console.rule()
+            tele_console.print("[green] Send message to telegram [/green]")
+            tele_console.print(
+                f"[red] Next message will be sent at <{next_time}> [/red]"
+            )
+            await bot.send_message(text=context_msg, chat_id=chat_id)
+            if img_path:
+                await bot.send_photo(chat_id=chat_id, photo=open(img_path, "rb"))
+            await bot.send_message(text=next_time_info, chat_id=chat_id)
+        except Exception as e:
+            pprint(f"Error: {e}")
+            pprint("Message not sent to telegram")
+
+
+async def run_forever(cfg_path):
+    cfg_dict = yaml.safe_load(open(cfg_path, "r"))
+    noti_settings = cfg_dict["noti_settings"]
+    interval_in_min = noti_settings["interval_in_min"]
+    interval_in_sec = int(interval_in_min * 60)
+    pprint(
+        f"Message will be sent every {interval_in_min} minutes or {interval_in_sec} seconds"
+    )
+    while True:
+        await send_to_telegram(cfg_dict, interval_in_sec)
+        await asyncio.sleep(interval_in_sec)
+
+
+async def main():
+    args = parse_args()
+    await run_forever(args.cfg)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

halib/textfile.py

@@ -0,0 +1,13 @@
+def read_line_by_line(file_path):
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+        lines = [line.rstrip() for line in lines]
+        return lines
+
+
+def write(lines, outfile, append=False):
+    mode = 'a' if append else 'w'
+    with open(outfile, mode, encoding='utf-8') as f:
+        for line in lines:
+            f.write(line)
+            f.write('\n')

halib/torchloader.py

@@ -0,0 +1,162 @@
+"""
+ * @author Hoang Van-Ha
+ * @email hoangvanhauit@gmail.com
+ * @create date 2024-03-27 15:40:22
+ * @modify date 2024-03-27 15:40:22
+ * @desc this module works as a utility tools for finding the best configuration for dataloader (num_workers, batch_size, pin_menory, etc.) that fits your hardware.
+"""
+from argparse import ArgumentParser
+from .common import *
+from .filetype import csvfile
+from .filetype.yamlfile import load_yaml
+from rich import inspect
+from torch.utils.data import DataLoader
+from torchvision import datasets, transforms
+from tqdm import tqdm
+from typing import Union
+import itertools as it  # for cartesian product
+import os
+import time
+import traceback
+
+
+def parse_args():
+    parser = ArgumentParser(description="desc text")
+    parser.add_argument("-cfg", "--cfg", type=str, help="cfg file for searching")
+    return parser.parse_args()
+
+
+def get_test_range(cfg: dict, search_item="num_workers"):
+    item_search_cfg = cfg["search_space"].get(search_item, None)
+    if item_search_cfg is None:
+        raise ValueError(f"search_item: {search_item} not found in cfg")
+    if isinstance(item_search_cfg, list):
+        return item_search_cfg
+    elif isinstance(item_search_cfg, dict):
+        if "mode" in item_search_cfg:
+            mode = item_search_cfg["mode"]
+            assert mode in ["range", "list"], f"mode: {mode} not supported"
+            value_in_mode = item_search_cfg.get(mode, None)
+            if value_in_mode is None:
+                raise ValueError(f"mode<{mode}>: data not found in <{search_item}>")
+            if mode == "range":
+                assert len(value_in_mode) == 3, f"range must have 3 values: start, stop, step"
+                start = value_in_mode[0]
+                stop = value_in_mode[1]
+                step = value_in_mode[2]
+                return list(range(start, stop, step))
+            elif mode == "list":
+                return item_search_cfg["list"]
+    else:
+        return [item_search_cfg]  # for int, float, str, bool, etc.
+
+
+def load_an_batch(loader_iter):
+    start = time.time()
+    next(loader_iter)
+    end = time.time()
+    return end - start
+
+
+def test_dataloader_with_cfg(origin_dataloader: DataLoader, cfg: Union[dict, str]):
+    try:
+        if isinstance(cfg, str):
+            cfg = load_yaml(cfg, to_dict=True)
+        dfmk = csvfile.DFCreator()
+        search_items = ["batch_size", "num_workers", "persistent_workers", "pin_memory"]
+        batch_limit = cfg["general"]["batch_limit"]
+        csv_cfg = cfg["general"]["to_csv"]
+        log_batch_info = cfg["general"]["log_batch_info"]
+
+        save_to_csv = csv_cfg["enabled"]
+        log_dir = csv_cfg["log_dir"]
+        filename = csv_cfg["filename"]
+        filename = f"{now_str()}_{filename}.csv"
+        outfile = os.path.join(log_dir, filename)
+
+        dfmk.create_table(
+            "cfg_search",
+            (search_items + ["avg_time_taken"]),
+        )
+        ls_range_test = []
+        for item in search_items:
+            range_test = get_test_range(cfg, search_item=item)
+            range_test = [(item, i) for i in range_test]
+            ls_range_test.append(range_test)
+
+        all_combinations = list(it.product(*ls_range_test))
+
+        rows = []
+        for cfg_idx, combine in enumerate(all_combinations):
+            console.rule(f"Testing cfg {cfg_idx+1}/{len(all_combinations)}")
+            inspect(combine)
+            batch_size = combine[search_items.index("batch_size")][1]
+            num_workers = combine[search_items.index("num_workers")][1]
+            persistent_workers = combine[search_items.index("persistent_workers")][1]
+            pin_memory = combine[search_items.index("pin_memory")][1]
+
+            test_dataloader = DataLoader(origin_dataloader.dataset, batch_size=batch_size, num_workers=num_workers, persistent_workers=persistent_workers, pin_memory=pin_memory, shuffle=True)
+            row = [
+                batch_size,
+                num_workers,
+                persistent_workers,
+                pin_memory,
+                0.0,
+            ]
+
+            # calculate the avg time taken to load the data for <batch_limit> batches
+            trainiter = iter(test_dataloader)
+            time_elapsed = 0
+            pprint('Start testing...')
+            for i in tqdm(range(batch_limit)):
+                single_batch_time = load_an_batch(trainiter)
+                if log_batch_info:
+                    pprint(f"Batch {i+1} took {single_batch_time:.4f} seconds to load")
+                time_elapsed += single_batch_time
+            row[-1] = time_elapsed / batch_limit
+            rows.append(row)
+        dfmk.insert_rows('cfg_search', rows)
+        dfmk.fill_table_from_row_pool('cfg_search')
+        with ConsoleLog("results"):
+            csvfile.fn_display_df(dfmk['cfg_search'])
+            if save_to_csv:
+                dfmk["cfg_search"].to_csv(outfile, index=False)
+                console.print(f"[red] Data saved to <{outfile}> [/red]")
+
+    except Exception as e:
+        traceback.print_exc()
+        print(e)
+        # get current directory of this python file
+        current_dir = os.path.dirname(os.path.realpath(__file__))
+        standar_cfg_path = os.path.join(current_dir, "torchloader_search.yaml")
+        pprint(
+            f"Make sure you get the  right <cfg.yaml> file. An example of <cfg.yaml> file can be found at this path: {standar_cfg_path}"
+        )
+        return
+
+def main():
+    args = parse_args()
+    cfg_yaml = args.cfg
+    cfg_dict = load_yaml(cfg_yaml, to_dict=True)
+
+    # Define transforms for data augmentation and normalization
+    transform = transforms.Compose(
+        [
+            transforms.RandomHorizontalFlip(),  # Randomly flip images horizontally
+            transforms.RandomRotation(10),  # Randomly rotate images by 10 degrees
+            transforms.ToTensor(),  # Convert images to PyTorch tensors
+            transforms.Normalize(
+                (0.5, 0.5, 0.5), (0.5, 0.5, 0.5)
+            ),  # Normalize pixel values to [-1, 1]
+        ]
+    )
+    test_dataset = datasets.CIFAR10(
+        root="./data", train=False, download=True, transform=transform
+    )
+    batch_size = 64
+    train_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)
+    test_dataloader_with_cfg(train_loader, cfg_dict)
+
+
+if __name__ == "__main__":
+    main()

halib/utils/dataclass_util.py

@@ -0,0 +1,40 @@
+import yaml
+from typing import Any
+from rich.pretty import pprint
+from ..filetype import yamlfile
+# from halib.filetype import yamlfile
+from dataclasses import make_dataclass
+
+def dict_to_dataclass(name: str, data: dict):
+    fields = []
+    values = {}
+
+    for key, value in data.items():
+        if isinstance(value, dict):
+            sub_dc = dict_to_dataclass(key.capitalize(), value)
+            fields.append((key, type(sub_dc)))
+            values[key] = sub_dc
+        else:
+            field_type = type(value) if value is not None else Any
+            fields.append((key, field_type))
+            values[key] = value
+
+    DC = make_dataclass(name.capitalize(), fields)
+    return DC(**values)
+
+def yaml_to_dataclass(name: str, yaml_str: str):
+    data = yaml.safe_load(yaml_str)
+    return dict_to_dataclass(name, data)
+
+
+def yamlfile_to_dataclass(name: str, file_path: str):
+    data_dict = yamlfile.load_yaml(file_path, to_dict=True)
+    if "__base__" in data_dict:
+        del data_dict["__base__"]
+    return dict_to_dataclass(name, data_dict)
+
+if __name__ == "__main__":
+    cfg = yamlfile_to_dataclass("Config", "test/dataclass_util_test_cfg.yaml")
+
+    # ! NOTICE: after print out this dataclass, we can copy the output and paste it into CHATGPT to generate a list of needed dataclass classes using `from dataclass_wizard import YAMLWizard`
+    pprint(cfg)