halib 0.1.23__tar.gz → 0.1.25__tar.gz

{halib-0.1.23/halib.egg-info → halib-0.1.25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: halib
-Version: 0.1.23
+Version: 0.1.25
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -15,6 +15,11 @@ License-File: LICENSE.txt
 
 Helper package for coding and automation
 
+**Version 0.1.24**
+
++ rename `sys` to `system` to avoid conflict with built-in `sys` module
++ add `tele_noti` module to send notification to telegram after a specific interval for training progress monitoring
+---
 **Version 0.1.22**
 
 + add `cuda.py` module to check CUDA availability (for both pytorch and tensorflow)

{halib-0.1.23 → halib-0.1.25}/README.md

@@ -1,5 +1,10 @@
 Helper package for coding and automation
 
+**Version 0.1.24**
+
++ rename `sys` to `system` to avoid conflict with built-in `sys` module
++ add `tele_noti` module to send notification to telegram after a specific interval for training progress monitoring
+---
 **Version 0.1.22**
 
 + add `cuda.py` module to check CUDA availability (for both pytorch and tensorflow)

{halib-0.1.23 → halib-0.1.25}/halib/__init__.py

@@ -28,8 +28,8 @@ import numpy as np
 import pandas as pd
 from .filetype import *
 from .filetype.yamlfile import load_yaml
-from .sys import cmd
-from .sys import filesys as fs
+from .system import cmd
+from .system import filesys as fs
 from .cuda import tcuda
 
 # for log

{halib-0.1.23 → halib-0.1.25}/halib/filetype/videofile.py

@@ -1,12 +1,10 @@
-from enum import Enum
-
 import cv2
+import textfile
 import enlighten
-from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
+from enum import Enum
+from ..system import filesys
 from tube_dl import Youtube, Playlist
-
-from halib.sys import filesys
-from halib.filetype import textfile
+from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
 
 
 class VideoResolution(Enum):

{halib-0.1.23 → halib-0.1.25}/halib/online/gdrive.py

@@ -15,9 +15,8 @@ from pydrive.auth import GoogleAuth
 from pydrive.drive import GoogleDrive
 from pydrive.files import GoogleDriveFileList
 
-
-from halib.sys import filesys
-from halib.filetype import textfile
+from ..system import filesys
+from ..filetype import textfile
 
 # Import general libraries
 

{halib-0.1.23 → halib-0.1.25}/halib/online/gdrive_mkdir.py

@@ -1,8 +1,8 @@
 from argparse import ArgumentParser
 from datetime import datetime
 
-from halib.online import gdrive
-from halib.filetype import textfile
+import gdrive
+from ..filetype import textfile
 
 
 def parse_args():

{halib-0.1.23 → halib-0.1.25}/halib/online/gdrive_test.py

@@ -1,8 +1,5 @@
 from argparse import ArgumentParser
-
-from halib.online import gdrive
-
-
+import gdrive
 def parse_args():
     parser = ArgumentParser(description="Upload local folder to Google Drive")
     parser.add_argument(

{halib-0.1.23 → halib-0.1.25}/halib/online/projectmake.py

@@ -8,10 +8,8 @@ import subprocess
 
 import certifi
 import pycurl
-
-from halib.filetype import jsonfile
-from halib.sys import filesys
-
+from ..filetype import jsonfile
+from ..system import filesys
 
 def get_curl(url, user_and_pass, verbose=True):
     c = pycurl.Curl()

halib-0.1.25/halib/tele_noti.py

@@ -0,0 +1,123 @@
+# Watch a log file and send a telegram message when train reaches a certain epoch or end
+
+import os
+import yaml
+import asyncio
+import telegram
+import pandas as pd
+
+from rich.pretty import pprint
+import plotly.graph_objects as go
+
+from system import filesys as fs
+from filetype import textfile, csvfile
+
+from argparse import ArgumentParser
+
+def parse_args():
+    parser = ArgumentParser(
+        description="desc text")
+    parser.add_argument(
+        "-cfg",
+        "--cfg",
+        type=str,
+        help="yaml file for tele",
+        default=r"E:\Dev\halib\cfg_tele_noti.yaml",
+    )
+
+    return parser.parse_args()
+
+def get_watcher_message_df(target_file, num_last_lines):
+    file_ext = fs.get_file_name(target_file, split_file_ext=True)[1]
+    supported_ext = ['.txt', '.log', '.csv']
+    assert file_ext in supported_ext, f"File extension {file_ext} not supported. Supported extensions are {supported_ext}"
+    last_lines_df = None
+    if file_ext in ['.txt', '.log']:
+        lines = textfile.read_line_by_line(target_file)
+        if num_last_lines > len(lines):
+            num_last_lines = len(lines)
+        last_line_arr = lines[-num_last_lines:]
+        dfCreator = csvfile.DFCreator()
+        dfCreator.create_table('last_lines', ['line'])
+        last_line_arr = [[line] for line in last_line_arr]
+        dfCreator.insert_rows('last_lines', last_line_arr)
+        dfCreator.fill_table_from_row_pool('last_lines')
+        last_lines_df = dfCreator['last_lines'].copy()
+    else:
+        df = pd.read_csv(target_file)
+        num_rows = len(df)
+        if num_last_lines > num_rows:
+            num_last_lines = num_rows
+        last_lines_df = df.tail(num_last_lines)
+    return last_lines_df
+
+
+def df2img(df: pd.DataFrame, output_img_dir, decimal_places, out_img_scale):
+    df = df.round(decimal_places)
+    fig = go.Figure(data=[
+                     go.Table(
+                        header=dict(values=list(df.columns),align='center'),
+                        cells=dict(values=df.values.transpose(),
+                                   fill_color = [["white","lightgrey"]*df.shape[0]],
+                                   align='center'
+                                  )
+                            )
+                       ])
+    img_path = os.path.normpath(os.path.join(output_img_dir, "last_lines.png"))
+    fig.write_image(img_path, scale=out_img_scale)
+    return img_path
+
+
+def compose_message_and_img_path(
+    target_file, project, num_last_lines, decimal_places, out_img_scale, output_img_dir
+):
+    context_msg = f">> Project: {project} \n>> File: {target_file} \n>> Last {num_last_lines} lines:"
+    msg_df = get_watcher_message_df(target_file, num_last_lines)
+    img_path = df2img(msg_df, output_img_dir, decimal_places, out_img_scale)
+    return context_msg, img_path
+
+
+async def send_to_telegram(cfg_dict):
+    # pprint(cfg_dict)
+    token = cfg_dict["telegram"]["token"]
+    chat_id = cfg_dict["telegram"]["chat_id"]
+
+    noti_settings = cfg_dict["noti_settings"]
+    project = noti_settings["project"]
+    target_file = noti_settings["target_file"]
+    num_last_lines = noti_settings["num_last_lines"]
+    output_img_dir = noti_settings["output_img_dir"]
+    decimal_places = noti_settings["decimal_places"]
+    out_img_scale = noti_settings["out_img_scale"]
+
+    bot = telegram.Bot(token=token)
+    async with bot:
+        context_msg, img_path = compose_message_and_img_path(
+            target_file,
+            project,
+            num_last_lines,
+            decimal_places,
+            out_img_scale,
+            output_img_dir,
+        )
+        pprint('Sending message to telegram...')
+        await bot.send_message(text=context_msg, chat_id=chat_id)
+        await bot.send_photo(chat_id=chat_id, photo=open(img_path, "rb"))
+
+
+async def run_forever(cfg_path):
+    cfg_dict = yaml.safe_load(open(cfg_path, "r"))
+    noti_settings = cfg_dict["noti_settings"]
+    interval_in_min = noti_settings["interval_in_min"]
+    interval_in_sec = int(interval_in_min * 60)
+    pprint(f'Message will be sent every {interval_in_min} minutes or {interval_in_sec} seconds')
+    while True:
+        await send_to_telegram(cfg_dict)
+        await asyncio.sleep(interval_in_sec)
+
+async def main():
+    args = parse_args()
+    await run_forever(args.cfg)
+
+if __name__ == "__main__":
+    asyncio.run(main())

{halib-0.1.23 → halib-0.1.25/halib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: halib
-Version: 0.1.23
+Version: 0.1.25
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -15,6 +15,11 @@ License-File: LICENSE.txt
 
 Helper package for coding and automation
 
+**Version 0.1.24**
+
++ rename `sys` to `system` to avoid conflict with built-in `sys` module
++ add `tele_noti` module to send notification to telegram after a specific interval for training progress monitoring
+---
 **Version 0.1.22**
 
 + add `cuda.py` module to check CUDA availability (for both pytorch and tensorflow)

{halib-0.1.23 → halib-0.1.25}/halib.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ halib/__init__.py
 halib/cuda.py
 halib/listop.py
 halib/plot.py
+halib/tele_noti.py
 halib.egg-info/PKG-INFO
 halib.egg-info/SOURCES.txt
 halib.egg-info/dependency_links.txt
@@ -24,6 +25,6 @@ halib/online/gdrive.py
 halib/online/gdrive_mkdir.py
 halib/online/gdrive_test.py
 halib/online/projectmake.py
-halib/sys/__init__.py
-halib/sys/cmd.py
-halib/sys/filesys.py
+halib/system/__init__.py
+halib/system/cmd.py
+halib/system/filesys.py

halib-0.1.25/halib.egg-info/requires.txt

@@ -0,0 +1,13 @@
+rich
+tqdm
+tabulate
+loguru
+omegaconf
+networkx
+click
+python-telegram-bot
+plotly
+moviepy
+tube_dl
+enlighten
+pycurl

{halib-0.1.23 → halib-0.1.25}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt") as f:
 
 setuptools.setup(
     name="halib",
-    version="0.1.23",
+    version="0.1.25",
     author="Hoang Van Ha",
     author_email="hoangvanhauit@gmail.com",
     description="Small library for common tasks",

halib-0.1.23/halib.egg-info/requires.txt

@@ -1,7 +0,0 @@
-rich
-tqdm
-tabulate
-loguru
-omegaconf
-networkx
-click