halib 0.1.19__tar.gz → 0.1.21__tar.gz

{halib-0.1.19/halib.egg-info → halib-0.1.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: halib
-Version: 0.1.19
+Version: 0.1.21
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -15,6 +15,10 @@ License-File: LICENSE.txt
 
 Helper package for coding and automation
 
+**Version 0.1.21**
+
++ using `networkx` and `omegaconf` to allow yaml file inheritance and override
+---
 **Version 0.1.15**
 
 + `__init__.py`: add common logging library; also `console_log` decorator to log function (start and end)

{halib-0.1.19 → halib-0.1.21}/README.md

@@ -1,5 +1,9 @@
 Helper package for coding and automation
 
+**Version 0.1.21**
+
++ using `networkx` and `omegaconf` to allow yaml file inheritance and override
+---
 **Version 0.1.15**
 
 + `__init__.py`: add common logging library; also `console_log` decorator to log function (start and end)

{halib-0.1.19 → halib-0.1.21}/halib/__init__.py

@@ -15,12 +15,18 @@ __all__ = [
     "console",
     "console_log",
     "ConsoleLog",
-    "re"
+    "re",
+    "now_str",
+    "omegaconf",
+    "OmegaConf",
+    "DictConfig",
+    "load_yaml",
 ]
 
 import numpy as np
 import pandas as pd
 from .filetype import *
+from .filetype.yamlfile import load_yaml
 from .sys import cmd
 from .sys import filesys as fs
 
@@ -35,14 +41,23 @@ from tqdm import tqdm
 import matplotlib.pyplot as plt
 import re
 import arrow
+import omegaconf
+from omegaconf import OmegaConf
+from omegaconf.dictconfig import DictConfig
 
 console = Console()
 
+
 def now_str(sep_date_time="."):
-    assert sep_date_time in [".", "_", "-"], "sep_date_time must be one of '.', '_', or '-'"
+    assert sep_date_time in [
+        ".",
+        "_",
+        "-",
+    ], "sep_date_time must be one of '.', '_', or '-'"
     now_string = arrow.now().format(f"YYYYMMDD{sep_date_time}HHmmss")
     return now_string
 
+
 def norm_str(in_str):
     # Replace one or more whitespace characters with a single underscore
     norm_string = re.sub(r"\s+", "_", in_str)
@@ -50,6 +65,7 @@ def norm_str(in_str):
     norm_string = norm_string.strip()
     return norm_string
 
+
 def console_rule(msg, do_norm_msg=True, is_end_tag=False):
     msg = norm_str(msg) if do_norm_msg else msg
     if is_end_tag:
@@ -57,6 +73,7 @@ def console_rule(msg, do_norm_msg=True, is_end_tag=False):
     else:
         console.rule(f"<{msg}>")
 
+
 def console_log(func):
     def wrapper(*args, **kwargs):
         console_rule(func.__name__)
@@ -66,6 +83,7 @@ def console_log(func):
 
     return wrapper
 
+
 class ConsoleLog:
     def __init__(self, message):
         self.message = message

halib-0.1.21/halib/filetype/yamlfile.py

@@ -0,0 +1,65 @@
+import time
+import networkx as nx
+from rich import inspect
+from rich.pretty import pprint
+from omegaconf import OmegaConf
+from rich.console import Console
+from argparse import ArgumentParser
+
+console = Console()
+
+def _load_yaml_recursively(
+    yaml_file, yaml_files=[], share_nx_graph=nx.DiGraph(), log_info=False
+):
+    conf = OmegaConf.load(yaml_file)
+    yaml_files.append(yaml_file)
+    if "__base__" in conf:
+        parent = conf["__base__"]
+        if isinstance(parent, str):
+            parent = [parent]
+        for p in parent:
+            edge = (yaml_file, p)
+            share_nx_graph.add_edge(*edge)
+            for cycle in nx.simple_cycles(share_nx_graph):
+                assert False, f"Cyclic dependency detected: {cycle}"
+            # update conf with parent; BY loading parent and merging with conf (the child)
+            conf = OmegaConf.merge(
+                _load_yaml_recursively(p, yaml_files, share_nx_graph), conf
+            )
+    if log_info:
+        console.rule()
+        console.print(f"current yaml_file: {yaml_file}")
+        inspect(yaml_files)
+        pprint(OmegaConf.to_container(conf, resolve=True))
+        time.sleep(1)
+    return conf
+
+
+def load_yaml(yaml_file, to_dict=False, log_info=False):
+    yaml_files = []
+    share_nx_graph = nx.DiGraph()
+    omgconf = _load_yaml_recursively(yaml_file, yaml_files=yaml_files, share_nx_graph=share_nx_graph, log_info=log_info)
+
+    if to_dict:
+        return OmegaConf.to_container(omgconf, resolve=True)
+    else:
+        return omgconf
+
+
+def parse_args():
+    parser = ArgumentParser(
+        description="desc text")
+    parser.add_argument(
+        "-cfg", "--cfg", type=str, help="cfg file", default="cfg__default.yaml")
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    cfg_file = args.cfg
+    cfg = load_yaml(cfg_file, to_dict=True)
+    console.rule()
+    pprint(cfg)
+
+if __name__ == "__main__":
+    main()

{halib-0.1.19 → halib-0.1.21/halib.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: halib
-Version: 0.1.19
+Version: 0.1.21
 Summary: Small library for common tasks
 Author: Hoang Van Ha
 Author-email: hoangvanhauit@gmail.com
@@ -15,6 +15,10 @@ License-File: LICENSE.txt
 
 Helper package for coding and automation
 
+**Version 0.1.21**
+
++ using `networkx` and `omegaconf` to allow yaml file inheritance and override
+---
 **Version 0.1.15**
 
 + `__init__.py`: add common logging library; also `console_log` decorator to log function (start and end)

{halib-0.1.19 → halib-0.1.21}/halib.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ halib/filetype/csvfile.py
 halib/filetype/jsonfile.py
 halib/filetype/textfile.py
 halib/filetype/videofile.py
+halib/filetype/yamlfile.py
 halib/online/__init__.py
 halib/online/gdrive.py
 halib/online/gdrive_mkdir.py

{halib-0.1.19 → halib-0.1.21}/halib.egg-info/requires.txt

@@ -2,3 +2,5 @@ rich
 tqdm
 tabulate
 loguru
+omegaconf
+networkx

{halib-0.1.19 → halib-0.1.21}/setup.py

@@ -8,7 +8,7 @@ with open("requirements.txt") as f:
 
 setuptools.setup(
     name="halib",
-    version="0.1.19",
+    version="0.1.21",
     author="Hoang Van Ha",
     author_email="hoangvanhauit@gmail.com",
     description="Small library for common tasks",