halib 0.1.19__py3-none-any.whl → 0.1.21__py3-none-any.whl

halib/__init__.py

@@ -15,12 +15,18 @@ __all__ = [
     "console",
     "console_log",
     "ConsoleLog",
-    "re"
+    "re",
+    "now_str",
+    "omegaconf",
+    "OmegaConf",
+    "DictConfig",
+    "load_yaml",
 ]
 
 import numpy as np
 import pandas as pd
 from .filetype import *
+from .filetype.yamlfile import load_yaml
 from .sys import cmd
 from .sys import filesys as fs
 
@@ -35,14 +41,23 @@ from tqdm import tqdm
 import matplotlib.pyplot as plt
 import re
 import arrow
+import omegaconf
+from omegaconf import OmegaConf
+from omegaconf.dictconfig import DictConfig
 
 console = Console()
 
+
 def now_str(sep_date_time="."):
-    assert sep_date_time in [".", "_", "-"], "sep_date_time must be one of '.', '_', or '-'"
+    assert sep_date_time in [
+        ".",
+        "_",
+        "-",
+    ], "sep_date_time must be one of '.', '_', or '-'"
     now_string = arrow.now().format(f"YYYYMMDD{sep_date_time}HHmmss")
     return now_string
 
+
 def norm_str(in_str):
     # Replace one or more whitespace characters with a single underscore
     norm_string = re.sub(r"\s+", "_", in_str)
@@ -50,6 +65,7 @@ def norm_str(in_str):
     norm_string = norm_string.strip()
     return norm_string
 
+
 def console_rule(msg, do_norm_msg=True, is_end_tag=False):
     msg = norm_str(msg) if do_norm_msg else msg
     if is_end_tag:
@@ -57,6 +73,7 @@ def console_rule(msg, do_norm_msg=True, is_end_tag=False):
     else:
         console.rule(f"<{msg}>")
 
+
 def console_log(func):
     def wrapper(*args, **kwargs):
         console_rule(func.__name__)
@@ -66,6 +83,7 @@ def console_log(func):
 
     return wrapper
 
+
 class ConsoleLog:
     def __init__(self, message):
         self.message = message

halib/filetype/yamlfile.py

@@ -0,0 +1,65 @@
+import time
+import networkx as nx
+from rich import inspect
+from rich.pretty import pprint
+from omegaconf import OmegaConf
+from rich.console import Console
+from argparse import ArgumentParser
+
+console = Console()
+
+def _load_yaml_recursively(
+    yaml_file, yaml_files=[], share_nx_graph=nx.DiGraph(), log_info=False
+):
+    conf = OmegaConf.load(yaml_file)
+    yaml_files.append(yaml_file)
+    if "__base__" in conf:
+        parent = conf["__base__"]
+        if isinstance(parent, str):
+            parent = [parent]
+        for p in parent:
+            edge = (yaml_file, p)
+            share_nx_graph.add_edge(*edge)
+            for cycle in nx.simple_cycles(share_nx_graph):
+                assert False, f"Cyclic dependency detected: {cycle}"
+            # update conf with parent; BY loading parent and merging with conf (the child)
+            conf = OmegaConf.merge(
+                _load_yaml_recursively(p, yaml_files, share_nx_graph), conf
+            )
+    if log_info:
+        console.rule()
+        console.print(f"current yaml_file: {yaml_file}")
+        inspect(yaml_files)
+        pprint(OmegaConf.to_container(conf, resolve=True))
+        time.sleep(1)
+    return conf
+
+
+def load_yaml(yaml_file, to_dict=False, log_info=False):
+    yaml_files = []
+    share_nx_graph = nx.DiGraph()
+    omgconf = _load_yaml_recursively(yaml_file, yaml_files=yaml_files, share_nx_graph=share_nx_graph, log_info=log_info)
+
+    if to_dict:
+        return OmegaConf.to_container(omgconf, resolve=True)
+    else:
+        return omgconf
+
+
+def parse_args():
+    parser = ArgumentParser(
+        description="desc text")
+    parser.add_argument(
+        "-cfg", "--cfg", type=str, help="cfg file", default="cfg__default.yaml")
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    cfg_file = args.cfg
+    cfg = load_yaml(cfg_file, to_dict=True)
+    console.rule()
+    pprint(cfg)
+
+if __name__ == "__main__":
+    main()

{halib-0.1.19.dist-info → halib-0.1.21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: halib
-Version: 0.1.19
+Version: 0.1.21
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -16,9 +16,15 @@ Requires-Dist: rich
 Requires-Dist: tqdm
 Requires-Dist: tabulate
 Requires-Dist: loguru
+Requires-Dist: omegaconf
+Requires-Dist: networkx
 
 Helper package for coding and automation
 
+**Version 0.1.21**
+
++ using `networkx` and `omegaconf` to allow yaml file inheritance and override
+---
 **Version 0.1.15**
 
 + `__init__.py`: add common logging library; also `console_log` decorator to log function (start and end)

{halib-0.1.19.dist-info → halib-0.1.21.dist-info}/RECORD

@@ -1,4 +1,4 @@
-halib/__init__.py,sha256=r6lmaJf9c7Yyb7bc7WfxYecz0mEMB20SF7huwOEgt4g,2050
+halib/__init__.py,sha256=6E1oqNe2lqLUSGcWZXMZskNtPxZk42FwCOc4c2q4Ou4,2323
 halib/csvfile.py,sha256=Eoeni0NIbNG3mB5ESWAvNwhJxOjmCaPd1qqYRHImbvk,1567
 halib/filesys.py,sha256=r1SftGKM7nyw6QbY5UmcueZLkXEIleSzhui7dQsosPw,2907
 halib/gdrive.py,sha256=-dx8hbknor1stIXhAzCnCfOHAPWm9a9z7L0epsOBHjA,6274
@@ -15,6 +15,7 @@ halib/filetype/csvfile.py,sha256=mXqJYcEKyRDLbdkgy68yceJEPhfLV-L2bz5ljrOKwSM,418
 halib/filetype/jsonfile.py,sha256=9LBdM7LV9QgJA1bzJRkq69qpWOP22HDXPGirqXTgSCw,480
 halib/filetype/textfile.py,sha256=QtuI5PdLxu4hAqSeafr3S8vCXwtvgipWV4Nkl7AzDYM,399
 halib/filetype/videofile.py,sha256=n4lRKhQH8uRxVrhvdw_NUfrnChocflv3LfGlGya0WUs,4761
+halib/filetype/yamlfile.py,sha256=CxiOz2pgFBkP_V8gmPpFWeXyJ3uug-na42m4ydrWtYU,1993
 halib/online/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/online/gdrive.py,sha256=tiVCJpJKci0wQ7H9iKti0K3AxsTp_q8e9uBSxPOBB6s,7794
 halib/online/gdrive_mkdir.py,sha256=Ur9-J1uzaM9qgpWF-d6md3gjkrFdPiMNLmbJtpQjXDI,1496
@@ -23,8 +24,8 @@ halib/online/projectmake.py,sha256=r9vPID23o_GG7TJy32PIUMvjtl2f-M9nlYJq63GL6ZA,4
 halib/sys/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 halib/sys/cmd.py,sha256=b2x7JPcNnFjLGheIESVYvqAb-w2UwBM1PAwYxMZ5YjA,228
 halib/sys/filesys.py,sha256=ERpnELLDKJoTIIKf-AajgkY62nID4qmqmX5TkE95APU,2931
-halib-0.1.19.dist-info/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
-halib-0.1.19.dist-info/METADATA,sha256=XNYBNT58HTKUCBL4RH0sw3zzj86Yr3AqaVkAuutUuEM,1620
-halib-0.1.19.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-halib-0.1.19.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
-halib-0.1.19.dist-info/RECORD,,
+halib-0.1.21.dist-info/LICENSE.txt,sha256=qZssdna4aETiR8znYsShUjidu-U4jUT9Q-EWNlZ9yBQ,1100
+halib-0.1.21.dist-info/METADATA,sha256=cFTD9w4Ai5PDO8RQSrnpyvPcHX3Yju25TZtf2vSLpdI,1772
+halib-0.1.21.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+halib-0.1.21.dist-info/top_level.txt,sha256=7AD6PLaQTreE0Fn44mdZsoHBe_Zdd7GUmjsWPyQ7I-k,6
+halib-0.1.21.dist-info/RECORD,,