halib 0.2.1__py3-none-any.whl → 0.2.2__py3-none-any.whl

halib/system/filesys.py

@@ -2,13 +2,16 @@ import glob
 import os
 import shutil
 from distutils.dir_util import copy_tree
+from concurrent.futures import ThreadPoolExecutor
 
+COMMON_IMG_EXT = ["jpg", "jpeg", "png", "bmp", "tiff", "gif"]
+COMMON_VIDEO_EXT = ["mp4", "avi", "mov", "mkv", "flv", "wmv"]
 
 def is_exist(path):
     return os.path.exists(path)
 
 
-def is_directory(path):
+def is_dir(path):
     return os.path.isdir(path)
 
 
@@ -17,7 +20,7 @@ def get_current_dir():
 
 
 def change_current_dir(new_dir):
-    if is_directory(new_dir):
+    if is_dir(new_dir):
         os.chdir(new_dir)
 
 
@@ -64,27 +67,64 @@ def list_files(directory):
     )
     return files
 
-
-def filter_files_by_extension(directory, ext, recursive=True):
-    if is_directory(directory):
-        result_files = []
+def filter_files_by_extension(directory, ext=None, recursive=True, num_workers=0):
+    """
+    Filters files using glob and multithreading.
+    If ext is None, returns ALL files.
+
+    Args:
+        directory (str): Path to search.
+        ext (str, list, or None): Extension(s) to find. If None, return all files.
+        recursive (bool): Whether to search subdirectories.
+        num_workers (int): Number of threads for checking file existence.
+    """
+    assert os.path.exists(directory) and os.path.isdir(
+        directory
+    ), "Directory does not exist"
+
+    # 1. Normalize extensions to a tuple (only if ext is provided)
+    extensions = None
+    if ext is not None:
         if isinstance(ext, list):
-            ext_list = ext
+            extensions = tuple(ext)
         else:
-            ext_list = [ext]
-        if not recursive:
-            filter_pattern = f"{directory}/*"
+            extensions = (ext,)
+
+    # 2. Define pattern
+    pattern = (
+        os.path.join(directory, "**", "*")
+        if recursive
+        else os.path.join(directory, "*")
+    )
+
+    # 3. Helper function for the thread workers
+    def validate_file(path):
+        if os.path.isfile(path):
+            return path
+        return None
+
+    result_files = []
+    if num_workers <= 0:
+        num_workers = os.cpu_count() or 4
+    # 4. Initialize ThreadPool with user-defined workers
+    with ThreadPoolExecutor(max_workers=num_workers) as executor:
+        # Step A: Get the iterator (lazy evaluation)
+        all_paths = glob.iglob(pattern, recursive=recursive)
+
+        # Step B: Apply Filter
+        if extensions is None:
+            # If ext is None, we skip the endswith check and pass everything
+            candidate_paths = all_paths
         else:
-            filter_pattern = f"{directory}/**/*"
-
-        for ext_item in ext_list:
-            ext_filter = f"{filter_pattern}.{ext_item}"
-            files = glob.glob(filter_pattern, recursive=True)
-            files = [f for f in files if is_file(f) and f.endswith(ext_item)]
-            result_files.extend(files)
-        return result_files
-    else:
-        raise OSError("Directory not exists")
+            # Filter by extension string FIRST (Fast CPU op)
+            candidate_paths = (p for p in all_paths if p.endswith(extensions))
+
+        # Step C: Parallelize the disk check (Slow I/O op)
+        for result in executor.map(validate_file, candidate_paths):
+            if result:
+                result_files.append(result)
+
+    return result_files
 
 
 def is_file(path):

halib/system/path.py

@@ -0,0 +1,73 @@
+from ..common.common import *
+from ..filetype import csvfile
+import pandas as pd
+import platform
+
+
+PC_NAME_TO_ABBR = {
+    "DESKTOP-JQD9K01": "MainPC",
+    "DESKTOP-5IRHU87": "MSI_Laptop",
+    "DESKTOP-96HQCNO": "4090_SV",
+    "DESKTOP-Q2IKLC0": "4GPU_SV",
+    "DESKTOP-QNS3DNF": "1GPU_SV",
+}
+
+DEFAULT_ABBR_WORKING_DISK = {
+    "MainPC": "E:",
+    "MSI_Laptop": "D:",
+    "4090_SV": "E:",
+    "4GPU_SV": "D:",
+}
+
+def list_PCs(show=True):
+    df = pd.DataFrame(
+        list(PC_NAME_TO_ABBR.items()), columns=["PC Name", "Abbreviation"]
+    )
+    if show:
+        csvfile.fn_display_df(df)
+    return df
+
+
+def get_PC_name():
+    return platform.node()
+
+
+def get_PC_abbr_name():
+    pc_name = get_PC_name()
+    return PC_NAME_TO_ABBR.get(pc_name, "Unknown")
+
+
+# ! This funcction search for full paths in the obj and normalize them according to the current platform and working disk
+# ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "windows" => "D:/zdataset/DFire"
+# ! E.g: "E:/zdataset/DFire", but working_disk: "D:", current_platform: "linux" => "/mnt/d/zdataset/DFire"
+def normalize_paths(obj, working_disk, current_platform):
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            obj[key] = normalize_paths(value, working_disk, current_platform)
+        return obj
+    elif isinstance(obj, list):
+        for i, item in enumerate(obj):
+            obj[i] = normalize_paths(item, working_disk, current_platform)
+        return obj
+    elif isinstance(obj, str):
+        # Normalize backslashes to forward slashes for consistency
+        obj = obj.replace("\\", "/")
+        # Regex for Windows-style path: e.g., "E:/zdataset/DFire"
+        win_match = re.match(r"^([A-Z]):/(.*)$", obj)
+        # Regex for Linux-style path: e.g., "/mnt/e/zdataset/DFire"
+        lin_match = re.match(r"^/mnt/([a-z])/(.*)$", obj)
+        if win_match or lin_match:
+            rest = win_match.group(2) if win_match else lin_match.group(2)
+            if current_platform == "windows":
+                # working_disk is like "D:", so "D:/" + rest
+                new_path = working_disk + "/" + rest
+            elif current_platform == "linux":
+                # Extract drive letter from working_disk (e.g., "D:" -> "d")
+                drive_letter = working_disk[0].lower()
+                new_path = "/mnt/" + drive_letter + "/" + rest
+            else:
+                # Unknown platform, return original
+                return obj
+            return new_path
+    # For non-strings or non-path strings, return as is
+    return obj

halib/utils/dict.py

@@ -0,0 +1,9 @@
+def flatten_dict(d, parent_key="", sep="."):
+    items = {}
+    for k, v in d.items():
+        key = f"{parent_key}{sep}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.update(flatten_dict(v, key, sep=sep))
+        else:
+            items[key] = v
+    return items

halib/utils/list.py

@@ -0,0 +1,12 @@
+def subtract(list_a, list_b):
+    return [item for item in list_a if item not in list_b]
+
+
+def union(list_a, list_b, no_duplicate=False):
+    if no_duplicate:
+        return list(set(list_a) | set(list_b))
+    else:
+        return list_a + list_b
+
+def intersection(list_a, list_b):
+    return list(set(list_a) & set(list_b))

{halib-0.2.1.dist-info → halib-0.2.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: halib
-Version: 0.2.1
+Version: 0.2.2
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -53,6 +53,9 @@ Dynamic: summary
 
 # Helper package for coding and automation
 
+**Version 0.2.2**
++ reorganize packages with most changes in `research` package
+
 **Version 0.2.01**
 + `research/base_exp`: add `eval_exp` method to evaluate experiment (e.g., model evaluation on test set) after experiment running is done.
 

{halib-0.2.1.dist-info → halib-0.2.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-halib/__init__.py,sha256=dHuxmYnZjeFT2VXuZU5yKX7c81XqcXX5tsKFtcaxkqI,1657
+halib/__init__.py,sha256=ZPa8Dvj-NfbwlwzO0PiqnKtLvu7uKGNb3tRpw7S4ICM,1657
 halib/common.py,sha256=9hn-IXOlGZODoBHy8U2A0aLgmPEnTeQjbzAVGwXAjwo,4242
 halib/csvfile.py,sha256=Eoeni0NIbNG3mB5ESWAvNwhJxOjmCaPd1qqYRHImbvk,1567
 halib/cuda.py,sha256=1bvtBY8QvTWdLaxalzK9wqXPl0Ft3AfhcrebupxGzEA,1010
@@ -16,18 +16,22 @@ halib/tele_noti.py,sha256=xL8f889VFR65uVJTj1NNT3qSi3lJ7s_iDjcObwZ0-V0,5926
 halib/textfile.py,sha256=EhVFrit-nRBJx18e6rtIqcE1cSbgsLnMXe_kdhi1EPI,399
 halib/torchloader.py,sha256=-q9YE-AoHZE1xQX2dgNxdqtucEXYs4sQ22WXdl6EGfI,6500
 halib/videofile.py,sha256=NTLTZ-j6YD47duw2LN2p-lDQDglYFP1LpEU_0gzHLdI,4737
+halib/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+halib/common/common.py,sha256=olkeXdFdojOkySP5aurzxKlehngRwBHdNBw5JfE4_fQ,5038
+halib/common/rich_color.py,sha256=tyK5fl3Dtv1tKsfFzt_5Rco4Fj72QliA-w5aGXaVuqQ,6392
 halib/filetype/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-halib/filetype/csvfile.py,sha256=4Klf8YNzY1MaCD3o5Wp5GG3KMfQIBOEVzHV_7DO5XBo,6604
-halib/filetype/ipynb.py,sha256=hd1IxQmSdNyCT1WwteSmy6e_xFVpdEPC1jM0drr6ag4,1663
-halib/filetype/jsonfile.py,sha256=9LBdM7LV9QgJA1bzJRkq69qpWOP22HDXPGirqXTgSCw,480
-halib/filetype/textfile.py,sha256=QtuI5PdLxu4hAqSeafr3S8vCXwtvgipWV4Nkl7AzDYM,399
-halib/filetype/videofile.py,sha256=4nfVAYYtoT76y8P4WYyxNna4Iv1o2iV6xaMcUzNPC4s,4736
-halib/filetype/yamlfile.py,sha256=59P9cdqTx655XXeQtkmAJoR_UhhVN4L8Tro-kd8Ri5g,2741
+halib/filetype/csvfile.py,sha256=dwWM1LglQr1SaSWq2KDl93fYzQn1638JGo_l1GbBhbk,6460
+halib/filetype/ipynb.py,sha256=pd2LgmPa7ZbF0YlQJbeQZEsl6jHQUSoyVtkCT7WhU5Q,1657
+halib/filetype/jsonfile.py,sha256=2HcBqXYjLNvqFok3PHOgH59vlhDCZLZpt7ezvgx1TFM,474
+halib/filetype/textfile.py,sha256=3koEFyVme1SEHdud7TnjArHndoiqfMGfMdYY3NIFegM,397
+halib/filetype/videofile.py,sha256=wDyZp7Dh0ZuNgQUvt8gLTpy3Flx1jDr-QsO4-jzriGE,8104
+halib/filetype/yamlfile.py,sha256=ZV8-kL8MWVYC-HGjo6nv0gkJ4joauo43uqez5dqTOdk,2739
 halib/online/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/online/gdrive.py,sha256=RmF4y6UPxektkKIctmfT-pKWZsBM9FVUeld6zZmJkp0,7787
 halib/online/gdrive_mkdir.py,sha256=wSJkQMJCDuS1gxQ2lHQHq_IrJ4xR_SEoPSo9n_2WNFU,1474
 halib/online/gdrive_test.py,sha256=hMWzz4RqZwETHp4GG4WwVNFfYvFQhp2Boz5t-DqwMo0,1342
-halib/online/projectmake.py,sha256=Zrs96WgXvO4nIrwxnCOletL4aTBge-EoF0r7hpKO1w8,4034
+halib/online/projectmake.py,sha256=-WhJNmM6RpAfnWcQeinDBFjRKig165l7uqBk-XQwcbA,4036
+halib/online/tele_noti.py,sha256=ErR5ys8gHmdjjDfSfVJxBIbPxgH7QD0oNRQgZweGi4s,5941
 halib/research/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/research/base_config.py,sha256=AqZHZ0NNQ3WmUOfRzs36lf3o0FrehSdVLbdmgNpbV7A,2833
 halib/research/base_exp.py,sha256=mC2bcG3M-dIvAkLw3d4O4Nu0HyvfrmvHvHX4iW8qurY,5749
@@ -44,21 +48,42 @@ halib/research/plot.py,sha256=GBCXP1QnzRlNqjAl9UvGvW3I9II61DBStJNQThrLy38,28578
 halib/research/profiler.py,sha256=GRAewTo0jGkOputjmRwtYVfJYBze_ivsOnrW9exWkPQ,11772
 halib/research/torchloader.py,sha256=yqUjcSiME6H5W210363HyRUrOi3ISpUFAFkTr1w4DCw,6503
 halib/research/wandb_op.py,sha256=YzLEqME5kIRxi3VvjFkW83wnFrsn92oYeqYuNwtYRkY,4188
+halib/research/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+halib/research/core/base_config.py,sha256=FXYFFGKry-HwNrjlPlTeTB-ROhM1Bow5Xxems2fdW5k,4204
+halib/research/core/base_exp.py,sha256=6CpuTP43NA7gE5JUN4V_-UAKHrlzLPJNI8piCBh3p1s,5734
+halib/research/core/param_gen.py,sha256=I9JHrDCaep4CjvApDoX0QzFuw38zMC2PsDFueuA7pjM,4271
+halib/research/core/wandb_op.py,sha256=powL2QyLBqF-6PUGAOqd60s1npHLLKJxPns3S4hKeNo,4160
+halib/research/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+halib/research/data/dataclass_util.py,sha256=OPZzqmug0be4JEq0hJ68pKjnyl0PRYQMVJGhKw1kvyk,1382
+halib/research/data/dataset.py,sha256=hm7CDTmudo_KV21RF0TQknHsw44PKhlMoBs9i8WIq6k,6743
+halib/research/data/torchloader.py,sha256=oWUplXlGd1IB6CqdRd-mGe-DfMjjZxz9hQ7SWONb-0s,6519
+halib/research/perf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+halib/research/perf/flop_calc.py,sha256=Kb3Gwqc7QtGALZzfyYXBA_9SioReJpTJdUX84kqj-Aw,6907
+halib/research/perf/gpu_mon.py,sha256=vD41_ZnmPLKguuq9X44SB_vwd9JrblO4BDzHLXZhhFY,2233
+halib/research/perf/perfcalc.py,sha256=FSWDStz9f94dObyAHYHRtq4fuo0dIw7l9JH_x5Wd7cQ,16225
+halib/research/perf/perfmetrics.py,sha256=qRiNiCKGUSTLY7gPMVMuVHGAAyeosfGWup2eM4490aw,5485
+halib/research/perf/perftb.py,sha256=IWElg3OB5dmhfxnY8pMZvkL2y_EnvLmEx3gJlpUR1Fs,31066
+halib/research/perf/profiler.py,sha256=5ZjES8kAqEsSV1mC3Yr_1ivFLwQDc_yv4HY7dKt_AS0,11782
+halib/research/viz/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+halib/research/viz/plot.py,sha256=51FhD1mH4nthTrY3K4mrO5pxI5AzvHXpZy5_ToqkYHs,28580
 halib/sys/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/sys/cmd.py,sha256=b2x7JPcNnFjLGheIESVYvqAb-w2UwBM1PAwYxMZ5YjA,228
 halib/sys/filesys.py,sha256=ERpnELLDKJoTIIKf-AajgkY62nID4qmqmX5TkE95APU,2931
 halib/system/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/system/cmd.py,sha256=b2x7JPcNnFjLGheIESVYvqAb-w2UwBM1PAwYxMZ5YjA,228
-halib/system/filesys.py,sha256=ERpnELLDKJoTIIKf-AajgkY62nID4qmqmX5TkE95APU,2931
+halib/system/filesys.py,sha256=102J2fkQhmH1_-HQVy2FQ4NOU8LTjMWV3hToT_APtq8,4401
+halib/system/path.py,sha256=E1inJqRWqUYknP8Cdx0T5nUV-rLG9WYy2DiQmBxaQJc,2642
 halib/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/utils/dataclass_util.py,sha256=rj2IMLlUzbm2OlF5_B2dRTk9njZOaF7tTjYkOsq8uLY,1416
+halib/utils/dict.py,sha256=wYE6Iw-_CnCWdMg9tpJ2Y2-e2ESkW9FxmdBkZkbUh80,299
 halib/utils/dict_op.py,sha256=wYE6Iw-_CnCWdMg9tpJ2Y2-e2ESkW9FxmdBkZkbUh80,299
 halib/utils/gpu_mon.py,sha256=vD41_ZnmPLKguuq9X44SB_vwd9JrblO4BDzHLXZhhFY,2233
+halib/utils/list.py,sha256=BM-8sRhYyqF7bh4p7TQtV7P_gnFruUCA6DTUOombaZg,337
 halib/utils/listop.py,sha256=Vpa8_2fI0wySpB2-8sfTBkyi_A4FhoFVVvFiuvW8N64,339
 halib/utils/tele_noti.py,sha256=-4WXZelCA4W9BroapkRyIdUu9cUVrcJJhegnMs_WpGU,5928
 halib/utils/video.py,sha256=zLoj5EHk4SmP9OnoHjO8mLbzPdtq6gQPzTQisOEDdO8,3261
-halib-0.2.1.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.2.1.dist-info/METADATA,sha256=VZ-msDI3P56gyQR12cQm1Ea6HZ7A6c1tXAHtuSzadJU,6548
-halib-0.2.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-halib-0.2.1.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.2.1.dist-info/RECORD,,
+halib-0.2.2.dist-info/licenses/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.2.2.dist-info/METADATA,sha256=d0kciUOqcsw8mMago2IYt6gH4X3MHExycZt4RqEX61M,6632
+halib-0.2.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+halib-0.2.2.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.2.2.dist-info/RECORD,,