halib 0.1.23__py3-none-any.whl → 0.1.25__py3-none-any.whl

halib/__init__.py

@@ -28,8 +28,8 @@ import numpy as np
 import pandas as pd
 from .filetype import *
 from .filetype.yamlfile import load_yaml
-from .sys import cmd
-from .sys import filesys as fs
+from .system import cmd
+from .system import filesys as fs
 from .cuda import tcuda
 
 # for log

halib/filetype/videofile.py

@@ -1,12 +1,10 @@
-from enum import Enum
-
 import cv2
+import textfile
 import enlighten
-from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
+from enum import Enum
+from ..system import filesys
 from tube_dl import Youtube, Playlist
-
-from halib.sys import filesys
-from halib.filetype import textfile
+from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
 
 
 class VideoResolution(Enum):

halib/gdrive.py

@@ -12,7 +12,7 @@ from pydrive.auth import GoogleAuth
 from pydrive.drive import GoogleDrive
 from pydrive.files import GoogleDriveFileList
 
-from halib.sys import filesys
+from halib.system import filesys
 from halib.filetype import textfile
 
 # Import general libraries

halib/online/gdrive.py

@@ -15,9 +15,8 @@ from pydrive.auth import GoogleAuth
 from pydrive.drive import GoogleDrive
 from pydrive.files import GoogleDriveFileList
 
-
-from halib.sys import filesys
-from halib.filetype import textfile
+from ..system import filesys
+from ..filetype import textfile
 
 # Import general libraries
 

halib/online/gdrive_mkdir.py

@@ -1,8 +1,8 @@
 from argparse import ArgumentParser
 from datetime import datetime
 
-from halib.online import gdrive
-from halib.filetype import textfile
+import gdrive
+from ..filetype import textfile
 
 
 def parse_args():

halib/online/gdrive_test.py

@@ -1,8 +1,5 @@
 from argparse import ArgumentParser
-
-from halib.online import gdrive
-
-
+import gdrive
 def parse_args():
     parser = ArgumentParser(description="Upload local folder to Google Drive")
     parser.add_argument(

halib/online/projectmake.py

@@ -8,10 +8,8 @@ import subprocess
 
 import certifi
 import pycurl
-
-from halib.filetype import jsonfile
-from halib.sys import filesys
-
+from ..filetype import jsonfile
+from ..system import filesys
 
 def get_curl(url, user_and_pass, verbose=True):
     c = pycurl.Curl()

halib/projectmake.py

@@ -10,7 +10,7 @@ import certifi
 import pycurl
 
 from halib.filetype import jsonfile
-from halib.sys import filesys
+from halib.system import filesys
 
 
 def get_curl(url, user_and_pass, verbose=True):

halib/system/cmd.py

@@ -0,0 +1,8 @@
+import subprocess
+
+
+# @link: https://geekflare.com/learn-python-subprocess/
+# subprocess.run(['ls', '-la'])
+def run(args):
+    process = subprocess.run(args, capture_output=True, text=True)
+    return process.returncode

halib/system/filesys.py

@@ -0,0 +1,124 @@
+import glob
+import os
+import shutil
+from distutils.dir_util import copy_tree
+
+
+def is_exist(path):
+    return os.path.exists(path)
+
+
+def is_directory(path):
+    return os.path.isdir(path)
+
+
+def get_current_dir():
+    return os.getcwd()
+
+
+def change_current_dir(new_dir):
+    if is_directory(new_dir):
+        os.chdir(new_dir)
+
+
+def get_dir_name(directory):
+    return os.path.basename(os.path.normpath(directory))
+
+
+def get_parent_dir(directory, return_full_path=False):
+    if not return_full_path:
+        return os.path.basename(os.path.dirname(directory))
+    else:
+        return os.path.dirname(directory)
+
+
+def make_dir(directory):
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+
+def copy_dir(src_dir, dst_dir, dirs_exist_ok=True, ignore_patterns=None):
+    shutil.copytree(
+        src_dir, dst_dir, dirs_exist_ok=dirs_exist_ok, ignore=ignore_patterns
+    )
+
+
+def delete_dir(directory):
+    shutil.rmtree(directory)
+
+
+def list_dirs(directory):
+    folders = list(
+        filter(
+            lambda x: os.path.isdir(os.path.join(directory, x)), os.listdir(directory)
+        )
+    )
+    return folders
+
+
+def list_files(directory):
+    files = list(
+        filter(
+            lambda x: os.path.isfile(os.path.join(directory, x)), os.listdir(directory)
+        )
+    )
+    return files
+
+
+def filter_files_by_extension(directory, ext, recursive=True):
+    if is_directory(directory):
+        result_files = []
+        if isinstance(ext, list):
+            ext_list = ext
+        else:
+            ext_list = [ext]
+        if not recursive:
+            filter_pattern = f"{directory}/*"
+        else:
+            filter_pattern = f"{directory}/**/*"
+
+        for ext_item in ext_list:
+            ext_filter = f"{filter_pattern}.{ext_item}"
+            files = glob.glob(filter_pattern, recursive=True)
+            files = [f for f in files if is_file(f) and f.endswith(ext_item)]
+            result_files.extend(files)
+        return result_files
+    else:
+        raise OSError("Directory not exists")
+
+
+def is_file(path):
+    return os.path.isfile(path)
+
+
+def get_file_name(file_path, split_file_ext=False):
+    if is_file(file_path):
+        if split_file_ext:
+            filename, file_extension = os.path.splitext(os.path.basename(file_path))
+            return filename, file_extension
+        else:
+            return os.path.basename(file_path)
+    else:
+        raise OSError("Not a file")
+
+
+def get_absolute_path(file_path):
+    return os.path.abspath(file_path)
+
+
+# dest can be a directory
+def copy_file(source, dest):
+    shutil.copy2(source, dest)
+
+
+def delete_file(path):
+    if is_file(path):
+        os.remove(path)
+
+
+def rename_dir_or_file(old, new):
+    os.renames(old, new)
+
+
+def move_dir_or_file(source, destination):
+    shutil.move(source, destination)

halib/tele_noti.py

@@ -0,0 +1,123 @@
+# Watch a log file and send a telegram message when train reaches a certain epoch or end
+
+import os
+import yaml
+import asyncio
+import telegram
+import pandas as pd
+
+from rich.pretty import pprint
+import plotly.graph_objects as go
+
+from system import filesys as fs
+from filetype import textfile, csvfile
+
+from argparse import ArgumentParser
+
+def parse_args():
+    parser = ArgumentParser(
+        description="desc text")
+    parser.add_argument(
+        "-cfg",
+        "--cfg",
+        type=str,
+        help="yaml file for tele",
+        default=r"E:\Dev\halib\cfg_tele_noti.yaml",
+    )
+
+    return parser.parse_args()
+
+def get_watcher_message_df(target_file, num_last_lines):
+    file_ext = fs.get_file_name(target_file, split_file_ext=True)[1]
+    supported_ext = ['.txt', '.log', '.csv']
+    assert file_ext in supported_ext, f"File extension {file_ext} not supported. Supported extensions are {supported_ext}"
+    last_lines_df = None
+    if file_ext in ['.txt', '.log']:
+        lines = textfile.read_line_by_line(target_file)
+        if num_last_lines > len(lines):
+            num_last_lines = len(lines)
+        last_line_arr = lines[-num_last_lines:]
+        dfCreator = csvfile.DFCreator()
+        dfCreator.create_table('last_lines', ['line'])
+        last_line_arr = [[line] for line in last_line_arr]
+        dfCreator.insert_rows('last_lines', last_line_arr)
+        dfCreator.fill_table_from_row_pool('last_lines')
+        last_lines_df = dfCreator['last_lines'].copy()
+    else:
+        df = pd.read_csv(target_file)
+        num_rows = len(df)
+        if num_last_lines > num_rows:
+            num_last_lines = num_rows
+        last_lines_df = df.tail(num_last_lines)
+    return last_lines_df
+
+
+def df2img(df: pd.DataFrame, output_img_dir, decimal_places, out_img_scale):
+    df = df.round(decimal_places)
+    fig = go.Figure(data=[
+                     go.Table(
+                        header=dict(values=list(df.columns),align='center'),
+                        cells=dict(values=df.values.transpose(),
+                                   fill_color = [["white","lightgrey"]*df.shape[0]],
+                                   align='center'
+                                  )
+                            )
+                       ])
+    img_path = os.path.normpath(os.path.join(output_img_dir, "last_lines.png"))
+    fig.write_image(img_path, scale=out_img_scale)
+    return img_path
+
+
+def compose_message_and_img_path(
+    target_file, project, num_last_lines, decimal_places, out_img_scale, output_img_dir
+):
+    context_msg = f">> Project: {project} \n>> File: {target_file} \n>> Last {num_last_lines} lines:"
+    msg_df = get_watcher_message_df(target_file, num_last_lines)
+    img_path = df2img(msg_df, output_img_dir, decimal_places, out_img_scale)
+    return context_msg, img_path
+
+
+async def send_to_telegram(cfg_dict):
+    # pprint(cfg_dict)
+    token = cfg_dict["telegram"]["token"]
+    chat_id = cfg_dict["telegram"]["chat_id"]
+
+    noti_settings = cfg_dict["noti_settings"]
+    project = noti_settings["project"]
+    target_file = noti_settings["target_file"]
+    num_last_lines = noti_settings["num_last_lines"]
+    output_img_dir = noti_settings["output_img_dir"]
+    decimal_places = noti_settings["decimal_places"]
+    out_img_scale = noti_settings["out_img_scale"]
+
+    bot = telegram.Bot(token=token)
+    async with bot:
+        context_msg, img_path = compose_message_and_img_path(
+            target_file,
+            project,
+            num_last_lines,
+            decimal_places,
+            out_img_scale,
+            output_img_dir,
+        )
+        pprint('Sending message to telegram...')
+        await bot.send_message(text=context_msg, chat_id=chat_id)
+        await bot.send_photo(chat_id=chat_id, photo=open(img_path, "rb"))
+
+
+async def run_forever(cfg_path):
+    cfg_dict = yaml.safe_load(open(cfg_path, "r"))
+    noti_settings = cfg_dict["noti_settings"]
+    interval_in_min = noti_settings["interval_in_min"]
+    interval_in_sec = int(interval_in_min * 60)
+    pprint(f'Message will be sent every {interval_in_min} minutes or {interval_in_sec} seconds')
+    while True:
+        await send_to_telegram(cfg_dict)
+        await asyncio.sleep(interval_in_sec)
+
+async def main():
+    args = parse_args()
+    await run_forever(args.cfg)
+
+if __name__ == "__main__":
+    asyncio.run(main())

halib/videofile.py

@@ -5,7 +5,7 @@ import enlighten
 from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip
 from tube_dl import Youtube, Playlist
 
-from halib.sys import filesys
+from halib.system import filesys
 from halib.filetype import textfile
 
 

{halib-0.1.23.dist-info → halib-0.1.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: halib
-Version: 0.1.23
+Version: 0.1.25
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -19,9 +19,20 @@ Requires-Dist: loguru
 Requires-Dist: omegaconf
 Requires-Dist: networkx
 Requires-Dist: click
+Requires-Dist: python-telegram-bot
+Requires-Dist: plotly
+Requires-Dist: moviepy
+Requires-Dist: tube-dl
+Requires-Dist: enlighten
+Requires-Dist: pycurl
 
 Helper package for coding and automation
 
+**Version 0.1.24**
+
++ rename `sys` to `system` to avoid conflict with built-in `sys` module
++ add `tele_noti` module to send notification to telegram after a specific interval for training progress monitoring
+---
 **Version 0.1.22**
 
 + add `cuda.py` module to check CUDA availability (for both pytorch and tensorflow)

{halib-0.1.23.dist-info → halib-0.1.25.dist-info}/RECORD

@@ -1,32 +1,36 @@
-halib/__init__.py,sha256=x7N0Wlt_VvG9-4CygKTXUNibbHUIOBnik8sZ9EydX9I,2364
+halib/__init__.py,sha256=AHuaFyMGLG-jkc082gvnPmx5TtARRMeEDcZcGDkM1Tg,2370
 halib/csvfile.py,sha256=Eoeni0NIbNG3mB5ESWAvNwhJxOjmCaPd1qqYRHImbvk,1567
 halib/cuda.py,sha256=gLuQ0l_sm8UhTPWEoAvM8J4Ta91mSvfuSaJ3tzEOWnU,916
 halib/filesys.py,sha256=r1SftGKM7nyw6QbY5UmcueZLkXEIleSzhui7dQsosPw,2907
-halib/gdrive.py,sha256=-dx8hbknor1stIXhAzCnCfOHAPWm9a9z7L0epsOBHjA,6274
+halib/gdrive.py,sha256=p302rTv1H4gFsAlmmiUL0iKrbg09_3_dEhtm-N4WGvg,6277
 halib/gdrive_mkdir.py,sha256=0Gq65i2YaWaGMdJqXX7zthhb0qZnNwrhdoHK6IBIVv8,1456
 halib/gdrive_test.py,sha256=e8yOYB5MZhdHbeLzjvTcOXbJNKk_DKNWURIRkKxHBMs,1378
 halib/jsonfile.py,sha256=9XfdFS1wcTdBRmpAGzVu2dVCcJp8RCpsSY16f6KUNts,480
 halib/listop.py,sha256=Vpa8_2fI0wySpB2-8sfTBkyi_A4FhoFVVvFiuvW8N64,339
 halib/plot.py,sha256=drLaHL_JVnIeFxsgYYXepJHIECzOmRImX40cWPJJshs,475
-halib/projectmake.py,sha256=EOj_CADQBUfQX35JNRdF3nZepnB_ruJFoQeXmwfhh0w,4045
+halib/projectmake.py,sha256=8dNTxB2jgQlwvovaOnJcCKgHe5vM0nBwzm46Yc8INEI,4048
+halib/tele_noti.py,sha256=emmh8sHisdniU0g6bzqIXnTKbDkC-CTzM4yTOzwUk1M,4459
 halib/textfile.py,sha256=EhVFrit-nRBJx18e6rtIqcE1cSbgsLnMXe_kdhi1EPI,399
-halib/videofile.py,sha256=HVRijfyPN_VJ4HnichUHNLGPWS-jhCHBhA2LHB5Xgjo,4734
+halib/videofile.py,sha256=NTLTZ-j6YD47duw2LN2p-lDQDglYFP1LpEU_0gzHLdI,4737
 halib/filetype/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/filetype/csvfile.py,sha256=mXqJYcEKyRDLbdkgy68yceJEPhfLV-L2bz5ljrOKwSM,4181
 halib/filetype/jsonfile.py,sha256=9LBdM7LV9QgJA1bzJRkq69qpWOP22HDXPGirqXTgSCw,480
 halib/filetype/textfile.py,sha256=QtuI5PdLxu4hAqSeafr3S8vCXwtvgipWV4Nkl7AzDYM,399
-halib/filetype/videofile.py,sha256=n4lRKhQH8uRxVrhvdw_NUfrnChocflv3LfGlGya0WUs,4761
+halib/filetype/videofile.py,sha256=4nfVAYYtoT76y8P4WYyxNna4Iv1o2iV6xaMcUzNPC4s,4736
 halib/filetype/yamlfile.py,sha256=CxiOz2pgFBkP_V8gmPpFWeXyJ3uug-na42m4ydrWtYU,1993
 halib/online/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-halib/online/gdrive.py,sha256=tiVCJpJKci0wQ7H9iKti0K3AxsTp_q8e9uBSxPOBB6s,7794
-halib/online/gdrive_mkdir.py,sha256=Ur9-J1uzaM9qgpWF-d6md3gjkrFdPiMNLmbJtpQjXDI,1496
-halib/online/gdrive_test.py,sha256=njzCzd__exYp-0yPyGsznG0Sm3GfPWLicxgSmm4V37o,1362
-halib/online/projectmake.py,sha256=r9vPID23o_GG7TJy32PIUMvjtl2f-M9nlYJq63GL6ZA,4041
+halib/online/gdrive.py,sha256=RmF4y6UPxektkKIctmfT-pKWZsBM9FVUeld6zZmJkp0,7787
+halib/online/gdrive_mkdir.py,sha256=wSJkQMJCDuS1gxQ2lHQHq_IrJ4xR_SEoPSo9n_2WNFU,1474
+halib/online/gdrive_test.py,sha256=guwj8e08Nb7nkXXhWRTONNPZKdzmmDhRWOtg0HDtW6Q,1338
+halib/online/projectmake.py,sha256=R9MgcpF4nrqJTUPpelj338n8IBAFI1ZiYw65szEOZn4,4032
 halib/sys/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/sys/cmd.py,sha256=b2x7JPcNnFjLGheIESVYvqAb-w2UwBM1PAwYxMZ5YjA,228
 halib/sys/filesys.py,sha256=ERpnELLDKJoTIIKf-AajgkY62nID4qmqmX5TkE95APU,2931
-halib-0.1.23.dist-info/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.1.23.dist-info/METADATA,sha256=FDYPO0PaIZf4en5PmwaIxOxN6uAdyLLSyoewMT20U2s,1901
-halib-0.1.23.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-halib-0.1.23.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.1.23.dist-info/RECORD,,
+halib/system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+halib/system/cmd.py,sha256=b2x7JPcNnFjLGheIESVYvqAb-w2UwBM1PAwYxMZ5YjA,228
+halib/system/filesys.py,sha256=ERpnELLDKJoTIIKf-AajgkY62nID4qmqmX5TkE95APU,2931
+halib-0.1.25.dist-info/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.1.25.dist-info/METADATA,sha256=jMf6PaHFEbIcqdYF8v0an7SzhWmI-wxpRdH5_v4Ps_0,2264
+halib-0.1.25.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+halib-0.1.25.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.1.25.dist-info/RECORD,,