halib 0.2.30__py3-none-any.whl

halib/online/gdrive_test.py

@@ -0,0 +1,50 @@
+from argparse import ArgumentParser
+import gdrive
+
+
+def parse_args():
+    parser = ArgumentParser(description="Upload local folder to Google Drive")
+    parser.add_argument(
+        "-a",
+        "--authFile",
+        type=str,
+        help="authenticate file to Google Drive",
+        default="settings.yaml",
+    )
+    parser.add_argument("-s", "--source", type=str, help="Folder to upload")
+    parser.add_argument(
+        "-d", "--destination", type=str, help="Destination folder ID in Google Drive"
+    )
+    parser.add_argument(
+        "-c",
+        "--contentOnly",
+        type=str,
+        help="Parent Folder in Google Drive",
+        default="True",
+    )
+    parser.add_argument(
+        "-i",
+        "--ignoreFile",
+        type=str,
+        help="file containing files/folders to ignore",
+        default=None,
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    auth_file = args.authFile
+    local_folder = args.source
+    gg_folder_id = args.destination
+    content_only = args.contentOnly.lower() == "true"
+    ignore_file = args.ignoreFile
+    gdrive.get_gg_drive(auth_file)
+    gdrive.upload_folder_to_drive(
+        local_folder, gg_folder_id, content_only=content_only, ignore_file=ignore_file
+    )
+
+
+if __name__ == "__main__":
+    main()

halib/online/projectmake.py

@@ -0,0 +1,131 @@
+# coding=utf-8
+
+import os
+import json
+import pycurl
+import shutil
+import certifi
+import subprocess
+from io import BytesIO
+
+from argparse import ArgumentParser
+
+from ..filetype import jsonfile
+from ..system import filesys
+
+def get_curl(url, user_and_pass, verbose=True):
+    c = pycurl.Curl()
+    c.setopt(pycurl.VERBOSE, verbose)
+    c.setopt(pycurl.CAINFO, certifi.where())
+    c.setopt(pycurl.URL, url)
+    c.setopt(pycurl.USERPWD, user_and_pass)
+    return c
+
+
+def get_user_and_pass(username, appPass):
+    return f"{username}:{appPass}"
+
+
+def create_repo(
+    username, appPass, repo_name, workspace, proj_name, template_repo="py-proj-template"
+):
+    buffer = BytesIO()
+    url = f"https://api.bitbucket.org/2.0/repositories/{workspace}/{repo_name}"
+    data = json.dumps({"scm": "git", "project": {"key": f"{proj_name}"}})
+
+    user_and_pass = get_user_and_pass(username, appPass)
+    c = get_curl(url, user_and_pass)
+    c.setopt(pycurl.WRITEDATA, buffer)
+    c.setopt(pycurl.POST, 1)
+    c.setopt(pycurl.POSTFIELDS, data)
+    c.setopt(pycurl.HTTPHEADER, ["Accept: application/json"])
+    c.perform()
+    RESPOND_CODE = c.getinfo(pycurl.HTTP_CODE)
+    c.close()
+    # log info
+    body = buffer.getvalue()
+    msg = body.decode("iso-8859-1")
+    successful = True if str(RESPOND_CODE) == "200" else False
+
+    if successful and template_repo:
+        template_repo_url = f"https://{username}:{appPass}@bitbucket.org/{workspace}/{template_repo}.git"
+        git_clone(template_repo_url)
+        template_folder = f"./{template_repo}"
+
+        created_repo_url = (
+            f"https://{username}:{appPass}@bitbucket.org/{workspace}/{repo_name}.git"
+        )
+        git_clone(created_repo_url)
+        created_folder = f"./{repo_name}"
+        shutil.copytree(
+            template_folder,
+            created_folder,
+            dirs_exist_ok=True,
+            ignore=shutil.ignore_patterns(".git"),
+        )
+        os.system('rmdir /S /Q "{}"'.format(template_folder))
+        project_folder = "project_name"
+
+        filesys.change_current_dir(created_folder)
+        filesys.rename_dir_or_file(project_folder, repo_name)
+        # push to remote
+        subprocess.check_call(["C:/batch/gitp.bat", "init proj from template"])
+
+    return successful, msg
+
+
+def parse_args():
+    parser = ArgumentParser(description="Upload local folder to Google Drive")
+    parser.add_argument(
+        "-a",
+        "--authFile",
+        type=str,
+        help="authenticate file (json) to Bitbucket",
+        default="bitbucket.json",
+    )
+    parser.add_argument(
+        "-r", "--repoName", type=str, help="Repository name", default="hahv-proj"
+    )
+    parser.add_argument(
+        "-t", "--templateRepo", type=str, help="template repo to fork", default="True"
+    )
+    return parser.parse_args()
+
+
+def git_clone(url):
+    subprocess.check_call(["git", "clone", url])
+
+
+def main():
+    args = parse_args()
+    authFile = args.authFile
+    repo_name = args.repoName
+
+    authInfo = jsonfile.read(authFile)
+    username = authInfo["username"]
+    appPass = authInfo["appPass"]
+    workspace_id = authInfo["workspace_id"]
+    project_id = authInfo["project_id"]
+    use_template = args.templateRepo.lower() == "true"
+    template_repo = authInfo["template_repo"] if use_template else ""
+
+    extra_info = f"[Use template project {template_repo}]" if use_template else ""
+    print(f"[BitBucket] creating {repo_name} Project in Bitbucket {extra_info}")
+
+    successful, msg = create_repo(
+        username,
+        appPass,
+        repo_name,
+        workspace_id,
+        project_id,
+        template_repo=template_repo,
+    )
+    if successful:
+        print(f"[Bitbucket] {repo_name} created successfully.{extra_info}")
+    else:
+        formatted_msg = jsonfile.beautify(msg)
+        print(f"[Bitbucket] {repo_name} created failed. Details:\n{formatted_msg}")
+
+
+if __name__ == "__main__":
+    main()

halib/online/tele_noti.py

@@ -0,0 +1,165 @@
+# Watch a log file and send a telegram message when train reaches a certain epoch or end
+
+import os
+import yaml
+import asyncio
+import telegram
+import pandas as pd
+
+from rich.pretty import pprint
+from rich.console import Console
+import plotly.graph_objects as go
+
+from ..system import filesys as fs
+from ..filetype import textfile, csvfile
+
+from argparse import ArgumentParser
+
+tele_console = Console()
+
+
+def parse_args():
+    parser = ArgumentParser(description="desc text")
+    parser.add_argument(
+        "-cfg",
+        "--cfg",
+        type=str,
+        help="yaml file for tele",
+        default=r"E:\Dev\__halib\halib\online\tele_noti_cfg.yaml",
+    )
+
+    return parser.parse_args()
+
+def get_watcher_message_df(target_file, num_last_lines):
+    file_ext = fs.get_file_name(target_file, split_file_ext=True)[1]
+    supported_ext = [".txt", ".log", ".csv"]
+    assert (
+        file_ext in supported_ext
+    ), f"File extension {file_ext} not supported. Supported extensions are {supported_ext}"
+    last_lines_df = None
+    if file_ext in [".txt", ".log"]:
+        lines = textfile.read_line_by_line(target_file)
+        if num_last_lines > len(lines):
+            num_last_lines = len(lines)
+        last_line_arr = lines[-num_last_lines:]
+        # add a line start with word "epoch"
+        epoch_info_list = "Epoch: n/a"
+        for line in reversed(lines):
+            if "epoch" in line.lower():
+                epoch_info_list = line
+                break
+        last_line_arr.insert(0, epoch_info_list)  # insert at the beginning
+        dfCreator = csvfile.DFCreator()
+        dfCreator.create_table("last_lines", ["line"])
+        last_line_arr = [[line] for line in last_line_arr]
+        dfCreator.insert_rows("last_lines", last_line_arr)
+        dfCreator.fill_table_from_row_pool("last_lines")
+        last_lines_df = dfCreator["last_lines"].copy()
+    else:
+        df = pd.read_csv(target_file)
+        num_rows = len(df)
+        if num_last_lines > num_rows:
+            num_last_lines = num_rows
+        last_lines_df = df.tail(num_last_lines)
+    return last_lines_df
+
+
+def df2img(df: pd.DataFrame, output_img_dir, decimal_places, out_img_scale):
+    df = df.round(decimal_places)
+    fig = go.Figure(
+        data=[
+            go.Table(
+                header=dict(values=list(df.columns), align="center"),
+                cells=dict(
+                    values=df.values.transpose(),
+                    fill_color=[["white", "lightgrey"] * df.shape[0]],
+                    align="center",
+                ),
+            )
+        ]
+    )
+    if not os.path.exists(output_img_dir):
+        os.makedirs(output_img_dir)
+    img_path = os.path.normpath(os.path.join(output_img_dir, "last_lines.png"))
+    fig.write_image(img_path, scale=out_img_scale)
+    return img_path
+
+
+def compose_message_and_img_path(
+    target_file, project, num_last_lines, decimal_places, out_img_scale, output_img_dir
+):
+    context_msg = f">> Project: {project} \n>> File: {target_file} \n>> Last {num_last_lines} lines:"
+    msg_df = get_watcher_message_df(target_file, num_last_lines)
+    try:
+        img_path = df2img(msg_df, output_img_dir, decimal_places, out_img_scale)
+    except Exception as e:
+        pprint(f"Error: {e}")
+        img_path = None
+    return context_msg, img_path
+
+
+async def send_to_telegram(cfg_dict, interval_in_sec):
+    # pprint(cfg_dict)
+    token = cfg_dict["telegram"]["token"]
+    chat_id = cfg_dict["telegram"]["chat_id"]
+
+    noti_settings = cfg_dict["noti_settings"]
+    project = noti_settings["project"]
+    target_file = noti_settings["target_file"]
+    num_last_lines = noti_settings["num_last_lines"]
+    output_img_dir = noti_settings["output_img_dir"]
+    decimal_places = noti_settings["decimal_places"]
+    out_img_scale = noti_settings["out_img_scale"]
+
+    bot = telegram.Bot(token=token)
+    async with bot:
+        try:
+            context_msg, img_path = compose_message_and_img_path(
+                target_file,
+                project,
+                num_last_lines,
+                decimal_places,
+                out_img_scale,
+                output_img_dir,
+            )
+            time_now = next_time = pd.Timestamp.now().strftime("%Y-%m-%d %H:%M:%S")
+            sep_line = "-" * 50
+            context_msg = f"{sep_line}\n>> Time: {time_now}\n{context_msg}"
+            # calculate the next time to send message
+            next_time = pd.Timestamp.now() + pd.Timedelta(seconds=interval_in_sec)
+            next_time = next_time.strftime("%Y-%m-%d %H:%M:%S")
+            next_time_info = f"Next msg: {next_time}"
+            tele_console.rule()
+            tele_console.print("[green] Send message to telegram [/green]")
+            tele_console.print(
+                f"[red] Next message will be sent at <{next_time}> [/red]"
+            )
+            await bot.send_message(text=context_msg, chat_id=chat_id)
+            if img_path:
+                await bot.send_photo(chat_id=chat_id, photo=open(img_path, "rb"))
+            await bot.send_message(text=next_time_info, chat_id=chat_id)
+        except Exception as e:
+            pprint(f"Error: {e}")
+            pprint("Message not sent to telegram")
+
+
+async def run_forever(cfg_path):
+    cfg_dict = yaml.safe_load(open(cfg_path, "r"))
+    noti_settings = cfg_dict["noti_settings"]
+    interval_in_min = noti_settings["interval_in_min"]
+    interval_in_sec = int(interval_in_min * 60)
+    pprint(
+        f"Message will be sent every {interval_in_min} minutes or {interval_in_sec} seconds"
+    )
+    while True:
+        await send_to_telegram(cfg_dict, interval_in_sec)
+        await asyncio.sleep(interval_in_sec)
+
+
+async def main():
+    args = parse_args()
+    await run_forever(args.cfg)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

halib/plot.py

@@ -0,0 +1,301 @@
+from .common import now_str, norm_str, ConsoleLog
+from .filetype import csvfile
+from .system import filesys as fs
+from functools import partial
+from rich.console import Console
+from rich.pretty import pprint
+import click
+import csv
+import matplotlib
+import matplotlib.pyplot as plt
+import numpy as np
+import os
+import pandas as pd
+import seaborn as sns
+
+
+console = Console()
+desktop_path = os.path.expanduser("~/Desktop")
+REQUIRED_COLUMNS = ["epoch", "train_loss", "val_loss", "train_acc", "val_acc"]
+
+import csv
+
+
+def get_delimiter(file_path, bytes=4096):
+    sniffer = csv.Sniffer()
+    data = open(file_path, "r").read(bytes)
+    delimiter = sniffer.sniff(data).delimiter
+    return delimiter
+
+
+# Function to verify that the DataFrame has the required columns, and only the required columns
+def verify_csv(csv_file, required_columns=REQUIRED_COLUMNS):
+    delimiter = get_delimiter(csv_file)
+    df = pd.read_csv(csv_file, sep=delimiter)
+    # change the column names to lower case
+    df.columns = [col.lower() for col in df.columns]
+    for col in required_columns:
+        if col not in df.columns:
+            raise ValueError(
+                f"Required columns are: {REQUIRED_COLUMNS}, but found {df.columns}"
+            )
+    df = df[required_columns].copy()
+    return df
+
+
+def get_valid_tags(csv_files, tags):
+    if tags is not None and len(tags) > 0:
+        assert all(
+            isinstance(tag, str) for tag in tags
+        ), "tags must be a list of strings"
+        assert all(
+            len(tag) > 0 for tag in tags
+        ), "tags must be a list of non-empty strings"
+        valid_tags = tags
+    else:
+        valid_tags = []
+        for csv_file in csv_files:
+            file_name = fs.get_file_name(csv_file, split_file_ext=True)[0]
+            tag = norm_str(file_name)
+            valid_tags.append(tag)
+    return valid_tags
+
+
+def plot_ax(df, ax, metric="loss", tag=""):
+    pprint(locals())
+    # reset plt
+    assert metric in ["loss", "acc"], "metric must be either 'loss' or 'acc'"
+    part = ["train", "val"]
+    for p in part:
+        label = f"{tag}_{p}_{metric}"
+        ax.plot(df["epoch"], df[f"{p}_{metric}"], label=label)
+    return ax
+
+
+def actual_plot_seaborn(frame, csv_files, axes, tags, log):
+    # clear the axes
+    for ax in axes:
+        ax.clear()
+    ls_df = []
+    valid_tags = get_valid_tags(csv_files, tags)
+    for csv_file in csv_files:
+        df = verify_csv(csv_file)
+        if log:
+            with ConsoleLog(f"plotting {csv_file}"):
+                csvfile.fn_display_df(df)
+        ls_df.append(df)
+
+    ls_metrics = ["loss", "acc"]
+    for df_item, tag in zip(ls_df, valid_tags):
+        # add tag to columns,excpet epoch
+        df_item.columns = [
+            f"{tag}_{col}" if col != "epoch" else col for col in df_item.columns
+        ]
+    # merge the dataframes on the epoch column
+    df_combined = ls_df[0]
+    for df_item in ls_df[1:]:
+        df_combined = pd.merge(df_combined, df_item, on="epoch", how="outer")
+    # csvfile.fn_display_df(df_combined)
+
+    for i, metric in enumerate(ls_metrics):
+        tags_str = "+".join(valid_tags) if len(valid_tags) > 1 else valid_tags[0]
+        title = f"{tags_str}_{metric}-by-epoch"
+        cols = [col for col in df_combined.columns if col != "epoch" and metric in col]
+        cols = sorted(cols)
+        # pprint(cols)
+        plot_data = df_combined[cols]
+
+        # line from same csv file (same tag) should have the same marker
+        all_markers = [
+            marker for marker in plt.Line2D.markers if marker and marker != " "
+        ]
+        tag2marker = {tag: marker for tag, marker in zip(valid_tags, all_markers)}
+        plot_markers = []
+        for col in cols:
+            # find the tag:
+            tag = None
+            for valid_tag in valid_tags:
+                if valid_tag in col:
+                    tag = valid_tag
+                    break
+            plot_markers.append(tag2marker[tag])
+        # pprint(list(zip(cols, plot_markers)))
+
+        # create color
+        sequential_palettes = [
+            "Reds",
+            "Greens",
+            "Blues",
+            "Oranges",
+            "Purples",
+            "Greys",
+            "BuGn",
+            "BuPu",
+            "GnBu",
+            "OrRd",
+            "PuBu",
+            "PuRd",
+            "RdPu",
+            "YlGn",
+            "PuBuGn",
+            "YlGnBu",
+            "YlOrBr",
+            "YlOrRd",
+        ]
+        # each csvfile (tag) should have a unique color
+        tag2palette = {
+            tag: palette for tag, palette in zip(valid_tags, sequential_palettes)
+        }
+        plot_colors = []
+        for tag in valid_tags:
+            palette = tag2palette[tag]
+            total_colors = 10
+            ls_colors = sns.color_palette(palette, total_colors).as_hex()
+            num_part = len(ls_metrics)
+            subarr = np.array_split(np.arange(total_colors), num_part)
+            for idx, col in enumerate(cols):
+                if tag in col:
+                    chosen_color = ls_colors[
+                        subarr[int(idx % num_part)].mean().astype(int)
+                    ]
+                    plot_colors.append(chosen_color)
+
+        # pprint(list(zip(cols, plot_colors)))
+        sns.lineplot(
+            data=plot_data,
+            markers=plot_markers,
+            palette=plot_colors,
+            ax=axes[i],
+            dashes=False,
+        )
+        axes[i].set(xlabel="epoch", ylabel=metric, title=title)
+        axes[i].legend()
+        axes[i].grid()
+
+
+def actual_plot(frame, csv_files, axes, tags, log):
+    ls_df = []
+    valid_tags = get_valid_tags(csv_files, tags)
+    for csv_file in csv_files:
+        df = verify_csv(csv_file)
+        if log:
+            with ConsoleLog(f"plotting {csv_file}"):
+                csvfile.fn_display_df(df)
+        ls_df.append(df)
+
+    metric_values = ["loss", "acc"]
+    for i, metric in enumerate(metric_values):
+        for df_item, tag in zip(ls_df, valid_tags):
+            metric_ax = plot_ax(df_item, axes[i], metric, tag)
+
+        # set the title, xlabel, ylabel, legend, and grid
+        tags_str = "+".join(valid_tags) if len(valid_tags) > 1 else valid_tags[0]
+        metric_ax.set(
+            xlabel="epoch", ylabel=metric, title=f"{tags_str}_{metric}-by-epoch"
+        )
+        metric_ax.legend()
+        metric_ax.grid()
+
+
+def plot_csv_files(
+    csv_files,
+    outdir="./out/plot",
+    tags=None,
+    log=False,
+    save_fig=False,
+    update_in_min=1,
+):
+    # if csv_files is a string, convert it to a list
+    if isinstance(csv_files, str):
+        csv_files = [csv_files]
+    # if tags is a string, convert it to a list
+    if isinstance(tags, str):
+        tags = [tags]
+    valid_tags = get_valid_tags(csv_files, tags)
+    assert len(valid_tags) == len(
+        csv_files
+    ), "Unable to determine tags for each csv file"
+    live_update_in_ms = int(update_in_min * 60 * 1000)
+    fig, axes = plt.subplots(2, 1, figsize=(10, 17))
+    if live_update_in_ms:  # live update in min should be > 0
+        from matplotlib.animation import FuncAnimation
+
+        anim = FuncAnimation(
+            fig,
+            partial(
+                actual_plot_seaborn, csv_files=csv_files, axes=axes, tags=tags, log=log
+            ),
+            interval=live_update_in_ms,
+            blit=False,
+            cache_frame_data=False,
+        )
+        plt.show()
+    else:
+        actual_plot_seaborn(None, csv_files, axes, tags, log)
+        plt.show()
+
+    if save_fig:
+        os.makedirs(outdir, exist_ok=True)
+        tags_str = "+".join(valid_tags) if len(valid_tags) > 1 else valid_tags[0]
+        tag = f"{now_str()}_{tags_str}"
+        fig.savefig(f"{outdir}/{tag}_plot.png")
+        enable_plot_pgf()
+        fig.savefig(f"{outdir}/{tag}_plot.pdf")
+    if live_update_in_ms:
+        return anim
+
+
+def enable_plot_pgf():
+    matplotlib.use("pdf")
+    matplotlib.rcParams.update(
+        {
+            "pgf.texsystem": "pdflatex",
+            "font.family": "serif",
+            "text.usetex": True,
+            "pgf.rcfonts": False,
+        }
+    )
+
+
+def save_fig_latex_pgf(filename, directory="."):
+    enable_plot_pgf()
+    if ".pgf" not in filename:
+        filename = f"{directory}/{filename}.pgf"
+    plt.savefig(filename)
+
+
+# https: // click.palletsprojects.com/en/8.1.x/api/
+@click.command()
+@click.option("--csvfiles", "-f", multiple=True, type=str, help="csv files to plot")
+@click.option(
+    "--outdir",
+    "-o",
+    type=str,
+    help="output directory for the plot",
+    default=str(desktop_path),
+)
+@click.option(
+    "--tags", "-t", multiple=True, type=str, help="tags for the csv files", default=[]
+)
+@click.option("--log", "-l", is_flag=True, help="log the csv files")
+@click.option("--save_fig", "-s", is_flag=True, help="save the plot as a file")
+@click.option(
+    "--update_in_min",
+    "-u",
+    type=float,
+    help="update the plot every x minutes",
+    default=0.0,
+)
+def main(
+    csvfiles,
+    outdir,
+    tags,
+    log,
+    save_fig,
+    update_in_min,
+):
+    plot_csv_files(list(csvfiles), outdir, list(tags), log, save_fig, update_in_min)
+
+
+if __name__ == "__main__":
+    main()