halib 0.1.7__py3-none-any.whl → 0.1.99__py3-none-any.whl

halib/rich_color.py

@@ -0,0 +1,285 @@
+from rich.console import Console
+from rich.pretty import pprint
+from rich.table import Table
+from rich.text import Text
+from rich.panel import Panel
+
+# List of colors
+# ! https://rich.readthedocs.io/en/stable/appendix/colors.html
+all_colors = [
+    "black",
+    "red",
+    "green",
+    "yellow",
+    "blue",
+    "magenta",
+    "cyan",
+    "white",
+    "bright_black",
+    "bright_red",
+    "bright_green",
+    "bright_yellow",
+    "bright_blue",
+    "bright_magenta",
+    "bright_cyan",
+    "bright_white",
+    "grey0",
+    "navy_blue",
+    "dark_blue",
+    "blue3",
+    "blue1",
+    "dark_green",
+    "deep_sky_blue4",
+    "dodger_blue3",
+    "dodger_blue2",
+    "green4",
+    "spring_green4",
+    "turquoise4",
+    "deep_sky_blue3",
+    "dodger_blue1",
+    "dark_cyan",
+    "light_sea_green",
+    "deep_sky_blue2",
+    "deep_sky_blue1",
+    "green3",
+    "spring_green3",
+    "cyan3",
+    "dark_turquoise",
+    "turquoise2",
+    "green1",
+    "spring_green2",
+    "spring_green1",
+    "medium_spring_green",
+    "cyan2",
+    "cyan1",
+    "purple4",
+    "purple3",
+    "blue_violet",
+    "grey37",
+    "medium_purple4",
+    "slate_blue3",
+    "royal_blue1",
+    "chartreuse4",
+    "pale_turquoise4",
+    "steel_blue",
+    "steel_blue3",
+    "cornflower_blue",
+    "dark_sea_green4",
+    "cadet_blue",
+    "sky_blue3",
+    "chartreuse3",
+    "sea_green3",
+    "aquamarine3",
+    "medium_turquoise",
+    "steel_blue1",
+    "sea_green2",
+    "sea_green1",
+    "dark_slate_gray2",
+    "dark_red",
+    "dark_magenta",
+    "orange4",
+    "light_pink4",
+    "plum4",
+    "medium_purple3",
+    "slate_blue1",
+    "wheat4",
+    "grey53",
+    "light_slate_grey",
+    "medium_purple",
+    "light_slate_blue",
+    "yellow4",
+    "dark_sea_green",
+    "light_sky_blue3",
+    "sky_blue2",
+    "chartreuse2",
+    "pale_green3",
+    "dark_slate_gray3",
+    "sky_blue1",
+    "chartreuse1",
+    "light_green",
+    "aquamarine1",
+    "dark_slate_gray1",
+    "deep_pink4",
+    "medium_violet_red",
+    "dark_violet",
+    "purple",
+    "medium_orchid3",
+    "medium_orchid",
+    "dark_goldenrod",
+    "rosy_brown",
+    "grey63",
+    "medium_purple2",
+    "medium_purple1",
+    "dark_khaki",
+    "navajo_white3",
+    "grey69",
+    "light_steel_blue3",
+    "light_steel_blue",
+    "dark_olive_green3",
+    "dark_sea_green3",
+    "light_cyan3",
+    "light_sky_blue1",
+    "green_yellow",
+    "dark_olive_green2",
+    "pale_green1",
+    "dark_sea_green2",
+    "pale_turquoise1",
+    "red3",
+    "deep_pink3",
+    "magenta3",
+    "dark_orange3",
+    "indian_red",
+    "hot_pink3",
+    "hot_pink2",
+    "orchid",
+    "orange3",
+    "light_salmon3",
+    "light_pink3",
+    "pink3",
+    "plum3",
+    "violet",
+    "gold3",
+    "light_goldenrod3",
+    "tan",
+    "misty_rose3",
+    "thistle3",
+    "plum2",
+    "yellow3",
+    "khaki3",
+    "light_yellow3",
+    "grey84",
+    "light_steel_blue1",
+    "yellow2",
+    "dark_olive_green1",
+    "dark_sea_green1",
+    "honeydew2",
+    "light_cyan1",
+    "red1",
+    "deep_pink2",
+    "deep_pink1",
+    "magenta2",
+    "magenta1",
+    "orange_red1",
+    "indian_red1",
+    "hot_pink",
+    "medium_orchid1",
+    "dark_orange",
+    "salmon1",
+    "light_coral",
+    "pale_violet_red1",
+    "orchid2",
+    "orchid1",
+    "orange1",
+    "sandy_brown",
+    "light_salmon1",
+    "light_pink1",
+    "pink1",
+    "plum1",
+    "gold1",
+    "light_goldenrod2",
+    "navajo_white1",
+    "misty_rose1",
+    "thistle1",
+    "yellow1",
+    "light_goldenrod1",
+    "khaki1",
+    "wheat1",
+    "cornsilk1",
+    "grey100",
+    "grey3",
+    "grey7",
+    "grey11",
+    "grey15",
+    "grey19",
+    "grey23",
+    "grey27",
+    "grey30",
+    "grey35",
+    "grey39",
+    "grey42",
+    "grey46",
+    "grey50",
+    "grey54",
+    "grey58",
+    "grey62",
+    "grey66",
+    "grey70",
+    "grey74",
+    "grey78",
+    "grey82",
+    "grey85",
+    "grey89",
+    "grey93",
+]
+
+basic_colors = [
+    "black",
+    "red",
+    "green",
+    "yellow",
+    "blue",
+    "magenta",
+    "cyan",
+    "white",
+    "bright_black",
+    "bright_red",
+    "bright_green",
+    "bright_yellow",
+    "bright_blue",
+    "bright_magenta",
+    "bright_cyan",
+    "bright_white",
+]
+
+def rcolor_all_str():
+    pprint(all_colors)
+
+def rcolor_basic_str():
+    pprint(basic_colors)
+
+def rcolor_str(in_str, color="white"):
+    assert color in all_colors, f"color must be one of {all_colors}"
+    return f"[{color}]{in_str}[/{color}]"
+
+def rcolor_palette(color_list):
+    # make sure all colors are valid (in all_colors)
+    for color in color_list:
+        assert (
+            color in all_colors
+        ), f"color must be a valid color. call <rcolor_all_str()> or <rcolor_palette_all()> to see all valid colors"
+    # Initialize console
+    console = Console()
+
+    # Create a table with horizontal lines and six columns
+    table = Table(show_header=True, header_style="bold magenta", show_lines=True)
+
+    # Define the columns
+    table.add_column("Color Name 1", style="bold")
+    table.add_column("Sample 1", style="bold")
+    table.add_column("Color Name 2", style="bold")
+    table.add_column("Sample 2", style="bold")
+    table.add_column("Color Name 3", style="bold")
+    table.add_column("Sample 3", style="bold")
+
+    # Adjust the number of rows needed for the table
+    num_colors = len(color_list)
+    num_rows = (num_colors + 2) // 3  # Ceiling division to ensure all colors fit
+
+    # Add rows to the table
+    for i in range(num_rows):
+        color1 = color_list[i * 3] if i * 3 < num_colors else ""
+        color2 = color_list[i * 3 + 1] if i * 3 + 1 < num_colors else ""
+        color3 = color_list[i * 3 + 2] if i * 3 + 2 < num_colors else ""
+        filled_rect1 = Text(" " * 10, style=f"on {color1}") if color1 else ""
+        filled_rect2 = Text(" " * 10, style=f"on {color2}") if color2 else ""
+        filled_rect3 = Text(" " * 10, style=f"on {color3}") if color3 else ""
+        table.add_row(color1, filled_rect1, color2, filled_rect2, color3, filled_rect3)
+
+    # Print the table
+    console.print(table)
+
+def rcolor_palette_basic():
+    rcolor_palette(basic_colors)
+
+def rcolor_palette_all():
+    rcolor_palette(all_colors)

halib/sys/filesys.py

@@ -4,7 +4,7 @@ import shutil
 from distutils.dir_util import copy_tree
 
 
-def is_exit(path):
+def is_exist(path):
     return os.path.exists(path)
 
 
@@ -38,8 +38,9 @@ def make_dir(directory):
 
 
 def copy_dir(src_dir, dst_dir, dirs_exist_ok=True, ignore_patterns=None):
-    shutil.copytree(src_dir, dst_dir, dirs_exist_ok=dirs_exist_ok,
-                    ignore=ignore_patterns)
+    shutil.copytree(
+        src_dir, dst_dir, dirs_exist_ok=dirs_exist_ok, ignore=ignore_patterns
+    )
 
 
 def delete_dir(directory):
@@ -47,14 +48,20 @@ def delete_dir(directory):
 
 
 def list_dirs(directory):
-    folders = list(filter(lambda x: os.path.isdir(os.path.join(directory, x)),
-                          os.listdir(directory)))
+    folders = list(
+        filter(
+            lambda x: os.path.isdir(os.path.join(directory, x)), os.listdir(directory)
+        )
+    )
     return folders
 
 
 def list_files(directory):
-    files = list(filter(lambda x: os.path.isfile(os.path.join(directory, x)),
-                        os.listdir(directory)))
+    files = list(
+        filter(
+            lambda x: os.path.isfile(os.path.join(directory, x)), os.listdir(directory)
+        )
+    )
     return files
 
 
@@ -66,12 +73,12 @@ def filter_files_by_extension(directory, ext, recursive=True):
         else:
             ext_list = [ext]
         if not recursive:
-            filter_pattern = f'{directory}/*'
+            filter_pattern = f"{directory}/*"
         else:
-            filter_pattern = f'{directory}/**/*'
+            filter_pattern = f"{directory}/**/*"
 
         for ext_item in ext_list:
-            ext_filter = f'{filter_pattern}.{ext_item}'
+            ext_filter = f"{filter_pattern}.{ext_item}"
             files = glob.glob(filter_pattern, recursive=True)
             files = [f for f in files if is_file(f) and f.endswith(ext_item)]
             result_files.extend(files)

halib/system/cmd.py

@@ -0,0 +1,8 @@
+import subprocess
+
+
+# @link: https://geekflare.com/learn-python-subprocess/
+# subprocess.run(['ls', '-la'])
+def run(args):
+    process = subprocess.run(args, capture_output=True, text=True)
+    return process.returncode

halib/system/filesys.py

@@ -0,0 +1,124 @@
+import glob
+import os
+import shutil
+from distutils.dir_util import copy_tree
+
+
+def is_exist(path):
+    return os.path.exists(path)
+
+
+def is_directory(path):
+    return os.path.isdir(path)
+
+
+def get_current_dir():
+    return os.getcwd()
+
+
+def change_current_dir(new_dir):
+    if is_directory(new_dir):
+        os.chdir(new_dir)
+
+
+def get_dir_name(directory):
+    return os.path.basename(os.path.normpath(directory))
+
+
+def get_parent_dir(directory, return_full_path=False):
+    if not return_full_path:
+        return os.path.basename(os.path.dirname(directory))
+    else:
+        return os.path.dirname(directory)
+
+
+def make_dir(directory):
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+
+def copy_dir(src_dir, dst_dir, dirs_exist_ok=True, ignore_patterns=None):
+    shutil.copytree(
+        src_dir, dst_dir, dirs_exist_ok=dirs_exist_ok, ignore=ignore_patterns
+    )
+
+
+def delete_dir(directory):
+    shutil.rmtree(directory)
+
+
+def list_dirs(directory):
+    folders = list(
+        filter(
+            lambda x: os.path.isdir(os.path.join(directory, x)), os.listdir(directory)
+        )
+    )
+    return folders
+
+
+def list_files(directory):
+    files = list(
+        filter(
+            lambda x: os.path.isfile(os.path.join(directory, x)), os.listdir(directory)
+        )
+    )
+    return files
+
+
+def filter_files_by_extension(directory, ext, recursive=True):
+    if is_directory(directory):
+        result_files = []
+        if isinstance(ext, list):
+            ext_list = ext
+        else:
+            ext_list = [ext]
+        if not recursive:
+            filter_pattern = f"{directory}/*"
+        else:
+            filter_pattern = f"{directory}/**/*"
+
+        for ext_item in ext_list:
+            ext_filter = f"{filter_pattern}.{ext_item}"
+            files = glob.glob(filter_pattern, recursive=True)
+            files = [f for f in files if is_file(f) and f.endswith(ext_item)]
+            result_files.extend(files)
+        return result_files
+    else:
+        raise OSError("Directory not exists")
+
+
+def is_file(path):
+    return os.path.isfile(path)
+
+
+def get_file_name(file_path, split_file_ext=False):
+    if is_file(file_path):
+        if split_file_ext:
+            filename, file_extension = os.path.splitext(os.path.basename(file_path))
+            return filename, file_extension
+        else:
+            return os.path.basename(file_path)
+    else:
+        raise OSError("Not a file")
+
+
+def get_absolute_path(file_path):
+    return os.path.abspath(file_path)
+
+
+# dest can be a directory
+def copy_file(source, dest):
+    shutil.copy2(source, dest)
+
+
+def delete_file(path):
+    if is_file(path):
+        os.remove(path)
+
+
+def rename_dir_or_file(old, new):
+    os.renames(old, new)
+
+
+def move_dir_or_file(source, destination):
+    shutil.move(source, destination)

halib/tele_noti.py

@@ -0,0 +1,166 @@
+# Watch a log file and send a telegram message when train reaches a certain epoch or end
+
+import os
+import yaml
+import asyncio
+import telegram
+import pandas as pd
+
+from rich.pretty import pprint
+from rich.console import Console
+import plotly.graph_objects as go
+
+from .system import filesys as fs
+from .filetype import textfile, csvfile
+
+from argparse import ArgumentParser
+
+tele_console = Console()
+
+
+def parse_args():
+    parser = ArgumentParser(description="desc text")
+    parser.add_argument(
+        "-cfg",
+        "--cfg",
+        type=str,
+        help="yaml file for tele",
+        default=r"E:\Dev\halib\cfg_tele_noti.yaml",
+    )
+
+    return parser.parse_args()
+
+
+def get_watcher_message_df(target_file, num_last_lines):
+    file_ext = fs.get_file_name(target_file, split_file_ext=True)[1]
+    supported_ext = [".txt", ".log", ".csv"]
+    assert (
+        file_ext in supported_ext
+    ), f"File extension {file_ext} not supported. Supported extensions are {supported_ext}"
+    last_lines_df = None
+    if file_ext in [".txt", ".log"]:
+        lines = textfile.read_line_by_line(target_file)
+        if num_last_lines > len(lines):
+            num_last_lines = len(lines)
+        last_line_arr = lines[-num_last_lines:]
+        # add a line start with word "epoch"
+        epoch_info_list = "Epoch: n/a"
+        for line in reversed(lines):
+            if "epoch" in line.lower():
+                epoch_info_list = line
+                break
+        last_line_arr.insert(0, epoch_info_list)  # insert at the beginning
+        dfCreator = csvfile.DFCreator()
+        dfCreator.create_table("last_lines", ["line"])
+        last_line_arr = [[line] for line in last_line_arr]
+        dfCreator.insert_rows("last_lines", last_line_arr)
+        dfCreator.fill_table_from_row_pool("last_lines")
+        last_lines_df = dfCreator["last_lines"].copy()
+    else:
+        df = pd.read_csv(target_file)
+        num_rows = len(df)
+        if num_last_lines > num_rows:
+            num_last_lines = num_rows
+        last_lines_df = df.tail(num_last_lines)
+    return last_lines_df
+
+
+def df2img(df: pd.DataFrame, output_img_dir, decimal_places, out_img_scale):
+    df = df.round(decimal_places)
+    fig = go.Figure(
+        data=[
+            go.Table(
+                header=dict(values=list(df.columns), align="center"),
+                cells=dict(
+                    values=df.values.transpose(),
+                    fill_color=[["white", "lightgrey"] * df.shape[0]],
+                    align="center",
+                ),
+            )
+        ]
+    )
+    if not os.path.exists(output_img_dir):
+        os.makedirs(output_img_dir)
+    img_path = os.path.normpath(os.path.join(output_img_dir, "last_lines.png"))
+    fig.write_image(img_path, scale=out_img_scale)
+    return img_path
+
+
+def compose_message_and_img_path(
+    target_file, project, num_last_lines, decimal_places, out_img_scale, output_img_dir
+):
+    context_msg = f">> Project: {project} \n>> File: {target_file} \n>> Last {num_last_lines} lines:"
+    msg_df = get_watcher_message_df(target_file, num_last_lines)
+    try:
+        img_path = df2img(msg_df, output_img_dir, decimal_places, out_img_scale)
+    except Exception as e:
+        pprint(f"Error: {e}")
+        img_path = None
+    return context_msg, img_path
+
+
+async def send_to_telegram(cfg_dict, interval_in_sec):
+    # pprint(cfg_dict)
+    token = cfg_dict["telegram"]["token"]
+    chat_id = cfg_dict["telegram"]["chat_id"]
+
+    noti_settings = cfg_dict["noti_settings"]
+    project = noti_settings["project"]
+    target_file = noti_settings["target_file"]
+    num_last_lines = noti_settings["num_last_lines"]
+    output_img_dir = noti_settings["output_img_dir"]
+    decimal_places = noti_settings["decimal_places"]
+    out_img_scale = noti_settings["out_img_scale"]
+
+    bot = telegram.Bot(token=token)
+    async with bot:
+        try:
+            context_msg, img_path = compose_message_and_img_path(
+                target_file,
+                project,
+                num_last_lines,
+                decimal_places,
+                out_img_scale,
+                output_img_dir,
+            )
+            time_now = next_time = pd.Timestamp.now().strftime("%Y-%m-%d %H:%M:%S")
+            sep_line = "-" * 50
+            context_msg = f"{sep_line}\n>> Time: {time_now}\n{context_msg}"
+            # calculate the next time to send message
+            next_time = pd.Timestamp.now() + pd.Timedelta(seconds=interval_in_sec)
+            next_time = next_time.strftime("%Y-%m-%d %H:%M:%S")
+            next_time_info = f"Next msg: {next_time}"
+            tele_console.rule()
+            tele_console.print("[green] Send message to telegram [/green]")
+            tele_console.print(
+                f"[red] Next message will be sent at <{next_time}> [/red]"
+            )
+            await bot.send_message(text=context_msg, chat_id=chat_id)
+            if img_path:
+                await bot.send_photo(chat_id=chat_id, photo=open(img_path, "rb"))
+            await bot.send_message(text=next_time_info, chat_id=chat_id)
+        except Exception as e:
+            pprint(f"Error: {e}")
+            pprint("Message not sent to telegram")
+
+
+async def run_forever(cfg_path):
+    cfg_dict = yaml.safe_load(open(cfg_path, "r"))
+    noti_settings = cfg_dict["noti_settings"]
+    interval_in_min = noti_settings["interval_in_min"]
+    interval_in_sec = int(interval_in_min * 60)
+    pprint(
+        f"Message will be sent every {interval_in_min} minutes or {interval_in_sec} seconds"
+    )
+    while True:
+        await send_to_telegram(cfg_dict, interval_in_sec)
+        await asyncio.sleep(interval_in_sec)
+
+
+async def main():
+    args = parse_args()
+    await run_forever(args.cfg)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())