holado 0.13.22__py3-none-any.whl

holado_helper/script/script.py

@@ -0,0 +1,110 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+# This file is a helper to build scripts using HolAdo solution, with CSV files as input.
+# Usually, it is copied and adapted for specific usage.
+
+
+import logging
+import os
+from holado_helper.script.initialize_script import get_logging_argument_parser
+from holado_core.common.tools.tools import Tools
+
+logger = logging.getLogger(__name__)
+
+class Script(object):
+    
+    def __init__(self, with_args=True):
+        self.__with_args = with_args
+        
+        self.__args = None
+        self.__cwd = None
+        self.__user_work_dir=None
+    
+    @property
+    def args(self):
+        return self.__args
+    
+    @property
+    def cwd(self):
+        """ Current working directory
+        A script can change its working directory by calling 'change_working_dir' method.
+        """
+        if self.__cwd is not None:
+            return self.__cwd
+        else:
+            return os.getcwd()
+    
+    @property
+    def user_working_dir(self):
+        """ User working directory
+        Although the user working directory is persisted.
+        """
+        if self.__user_work_dir is not None:
+            return self.__user_work_dir
+        else:
+            return os.getcwd()
+    
+    @user_working_dir.setter
+    def user_working_dir(self, wd):
+        self.__user_work_dir = wd
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"Changed user working dir to '{self.__user_work_dir}'")
+    
+    def parse_args(self):
+        arg_parser = self.new_argument_parser()
+        self.__args = arg_parser.parse_args()
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"Parsed args: {self.__args}")
+    
+    def new_argument_parser(self):
+        raise NotImplementedError
+    
+    def _new_argument_parser(self, description=None, additional_arguments=None, add_help=True):
+        parser = get_logging_argument_parser(description=description, additional_arguments=additional_arguments, add_help=add_help)
+        return parser
+    
+    def change_working_dir(self, work_dir_path):
+        from holado_helper.script.initialize_script import change_working_dir as init_cd    # @UnresolvedImport
+        
+        # If user working dir is not yet defined, save current working dir as user working dir
+        if self.__user_work_dir is None:
+            self.user_working_dir = os.getcwd()
+        
+        # Change working dir
+        new_cwd = init_cd(work_dir_path)
+        self.__cwd = new_cwd
+        if Tools.do_log(logger, logging.DEBUG):
+            logger.debug(f"Changed script working dir to '{self.__cwd}'")
+    
+    def initialize(self):
+        if self.args is not None and hasattr(self.args, 'log_level') and hasattr(self.args, 'log_in_file'):
+            from holado_helper.script.initialize_script import change_logging_config # @UnresolvedImport
+            change_logging_config(log_level=self.args.log_level, log_in_file=self.args.log_in_file)
+    
+    def run(self):
+        self.parse_args()
+        self.initialize()
+        self.run_script()
+        
+    def run_script(self):
+        raise NotImplementedError
+    
+    def _build_path(self, filename, cwd):
+        if os.path.isabs(filename):
+            return filename
+        else:
+            return os.path.join(cwd, filename)
+
+

holado_json/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+
+

holado_json/filesystem/json_file.py

@@ -0,0 +1,94 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+from holado.common.handlers.object import DeleteableObject
+from holado_system.system.filesystem.file import File
+import json
+from holado.common.context.session_context import SessionContext
+
+
+def __get_path_manager():
+    return SessionContext.instance().path_manager
+
+
+class JSONFile(DeleteableObject):
+    """
+    Manage a JSON file
+    """
+
+    def __init__(self, path, auto_flush=True, do_open=False, **open_kwargs):
+        super().__init__(f"JSON file '{path}'")
+
+        self.__file = File(path, auto_flush=auto_flush, do_open=do_open, **open_kwargs)
+
+    def __enter__(self):
+        self.__file.__enter__()
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.__file.__exit__(exc_type, exc_val, exc_tb)
+    
+    @property
+    def path(self):
+        return self.__file.path
+
+    @property
+    def internal_file(self):
+        return self.__file
+    
+    @property
+    def is_open(self):
+        return self.__file.is_open
+
+    def open(self, **kwargs):
+        self.__file.open(**kwargs)
+
+    def close(self):
+        self.__file.close()
+
+    def read_as_json_object(self, **loads_kwargs):
+        with self.__file as fin:
+            content = fin.read()
+        return json.loads(content, **loads_kwargs)
+
+    def write_json_object(self, data, indent=4, **dumps_kwargs):
+        dumps_kwargs['indent'] = indent
+        
+        content = json.dumps(data, **dumps_kwargs)
+        with self.__file as fout:
+            fout.write(content)
+
+
+
+    @classmethod
+    def read_json_file_content_as_json_object(cls, path, open_kwargs=None, loads_kwargs=None):
+        open_kwargs = open_kwargs if open_kwargs is not None else {}
+        loads_kwargs = loads_kwargs if loads_kwargs is not None else {}
+        
+        file = JSONFile(path, **open_kwargs)
+        return file.read_as_json_object(**loads_kwargs)
+
+    @classmethod
+    def write_json_file_with_json_object(cls, path, data, open_kwargs=None, dumps_kwargs=None):
+        open_kwargs = open_kwargs if open_kwargs is not None else {}
+        dumps_kwargs = dumps_kwargs if dumps_kwargs is not None else {}
+        
+        __get_path_manager().makedirs(path)
+        
+        file = JSONFile(path, **open_kwargs)
+        file.write_json_object(data, **dumps_kwargs)
+        return file
+
+
+
+

holado_json/filesystem/stream_json_file.py

@@ -0,0 +1,117 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+from holado.common.handlers.object import DeleteableObject
+from holado_system.system.filesystem.file import File
+import json
+from holado.common.context.session_context import SessionContext
+
+
+def __get_path_manager():
+    return SessionContext.instance().path_manager
+
+
+class StreamJSONFile(DeleteableObject):
+    """
+    Manage a stream JSON file.
+    
+    A stream JSON file is a file with many root elements that are valid JSON.
+    It is useful for example to process workflows of data formatted in JSON.
+    An example in HolAdo is the mechanism saving docker container logs in a JSON file, 
+    which adds in a single file a log by line in JSON format.
+    
+    Note: Current implementation assumes that file has a line per JSON object
+    """
+
+    def __init__(self, path, auto_flush=True, do_open=False, **open_kwargs):
+        super().__init__(f"stream JSON file '{path}'")
+
+        self.__file = File(path, auto_flush=auto_flush, do_open=do_open, **open_kwargs)
+        self.__current_line = None
+        self.__current_pos = None
+
+    def __enter__(self):
+        self.__file.__enter__()
+        return self
+    
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.__file.__exit__(exc_type, exc_val, exc_tb)
+    
+    @property
+    def path(self):
+        return self.__file.path
+
+    @property
+    def internal_file(self):
+        return self.__file
+    
+    @property
+    def is_open(self):
+        return self.__file.is_open
+
+    def open(self, **kwargs):
+        self.__file.open(**kwargs)
+
+    def close(self):
+        self.__file.close()
+
+    def read_element_as_json_object(self, **loads_kwargs):
+        line = self.__file.readline()
+        if not line or len(line.strip()) == 0:
+            return None
+        
+        return json.loads(line, **loads_kwargs)
+    
+    def read_elements_as_json_object_list(self, **loads_kwargs):
+        res = []
+        while True:
+            element = self.read_element_as_json_object(**loads_kwargs)
+            if element is not None:
+                res.append(element)
+            else:
+                break
+        return res
+    
+    def write_element_json_object(self, data, **dumps_kwargs):
+        dumps_kwargs['indent'] = None
+        
+        content = json.dumps(data, **dumps_kwargs)
+        self.__file.writelines([content])
+    
+    def write_elements_json_object_list(self, data_list, **dumps_kwargs):
+        for data in data_list:
+            self.write_element_json_object(data, **dumps_kwargs)
+
+
+    @classmethod
+    def read_stream_json_file_content_as_json_object_list(cls, path, open_kwargs=None, loads_kwargs=None):
+        open_kwargs = open_kwargs if open_kwargs is not None else {}
+        loads_kwargs = loads_kwargs if loads_kwargs is not None else {}
+        
+        file = StreamJSONFile(path, **open_kwargs)
+        return file.read_elements_as_json_object_list(**loads_kwargs)
+
+    @classmethod
+    def write_stream_xml_file_with_dict_list(cls, path, data_list, open_kwargs=None, dumps_kwargs=None):
+        open_kwargs = open_kwargs if open_kwargs is not None else {}
+        dumps_kwargs = dumps_kwargs if dumps_kwargs is not None else {}
+        
+        __get_path_manager().makedirs(path)
+        
+        file = StreamJSONFile(path, **open_kwargs)
+        file.write_elements_json_object_list(data_list, **dumps_kwargs)
+        return file
+
+
+
+

holado_json/ipc/json.py

@@ -0,0 +1,146 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+from holado_core.common.tables.table_with_header import TableWithHeader
+from holado_core.common.exceptions.technical_exception import TechnicalException
+from holado_python.standard_library.typing import Typing
+from holado_core.common.exceptions.functional_exception import FunctionalException
+
+logger = logging.getLogger(__name__)
+
+
+def create_name_value_table_from_json(json_value, recursive=False, uncollapse_list=False):
+    from holado_json.ipc.json_converter import JsonConverter
+    
+    res = TableWithHeader()
+    res.header.add_cells_from_contents(["Name", "Value"])
+    
+    converter = JsonConverter()
+    json_value = converter.to_json(json_value)
+    
+    name_values = convert_json_value_2_name_values(json_value, recursive=recursive, uncollapse_list=uncollapse_list)
+    for name, value in name_values:
+        res.add_row(cells_content=(name, value))
+    
+    return res
+    
+def create_table_with_header_from_json(json_value, recursive=False, uncollapse_list=False):
+    from holado_json.ipc.json_converter import JsonConverter
+    
+    res = TableWithHeader()
+    
+    converter = JsonConverter()
+    json_value = converter.to_json(json_value)
+    
+    if isinstance(json_value, list):
+        json_list = json_value
+    else:
+        json_list = [json_value]
+    
+    for json_val in json_list:
+        json_val = converter.to_json(json_val)
+        
+        name_values = convert_json_value_2_name_values(json_val, recursive=recursive, uncollapse_list=uncollapse_list)
+        
+        # Add new columns if needed
+        for name, value in name_values:
+            if not res.has_column(name, raise_exception=False):
+                res.add_column(name=name, cells_content = [None]*res.nb_rows)
+        
+        # Fill table
+        values_by_name = {name: value for name, value in name_values}
+        res.add_row(contents_by_colname = values_by_name)
+        
+    # Order columns
+    if res.nb_columns > 0:
+        res.order_columns(names = sorted(res.get_column_names()))
+        
+    return res
+    
+def convert_json_value_2_name_values(json_value, recursive=False, uncollapse_list=False, as_dict=False, sorted_=False):
+    """Convert json object as a list of (name, value).
+    If as_dict is True, the result is a dict of {name:value}
+    """
+    res = []
+    __fill_name_values_with_json_value(res, None, json_value, recursive=recursive, uncollapse_list=uncollapse_list)
+    
+    if sorted_:
+        res = sorted(res)
+    if as_dict:
+        res = {e[0]:e[1] for e in res}
+    return res
+    
+def __fill_name_values_with_json_value(res, name_prefix, json_value, recursive=False, uncollapse_list=False):
+    if isinstance(json_value, dict):
+        __fill_name_values_with_json_dict(res, name_prefix, json_value, recursive=recursive, uncollapse_list=uncollapse_list)
+    elif isinstance(json_value, list):
+        __fill_name_values_with_json_list(res, name_prefix, json_value, recursive=recursive, uncollapse_list=uncollapse_list)
+    else:
+        res.append((name_prefix, json_value))
+    
+def __fill_name_values_with_json_dict(res, name_prefix, json_value, recursive=False, uncollapse_list=False):
+    if not isinstance(json_value, dict):
+        raise TechnicalException(f"json value is expected to be a dict (obtained: {Typing.get_object_class_fullname(json_value)}")
+    
+    sorted_dict = dict(sorted(json_value.items()))
+    for name, value in sorted_dict.items():
+        new_prefix = f"{name_prefix}.{name}" if name_prefix is not None and len(name_prefix) > 0 else name
+        if recursive:
+            __fill_name_values_with_json_value(res, new_prefix, value, recursive=recursive, uncollapse_list=uncollapse_list)
+        else:
+            res.append((new_prefix, value))
+    
+def __fill_name_values_with_json_list(res, name_prefix, json_value, recursive=False, uncollapse_list=False):
+    if not isinstance(json_value, list):
+        raise TechnicalException(f"json value is expected to be a list (obtained: {Typing.get_object_class_fullname(json_value)}")
+    
+    if not uncollapse_list:
+        res.append((name_prefix, json_value))
+        return
+        
+    if len(json_value) == 0:
+        res.append((name_prefix+"[]", None))
+        return
+        
+    for index, value in enumerate(json_value):
+        new_prefix = f"{name_prefix}[{index}]" if name_prefix is not None else f"[{index}]"
+        __fill_name_values_with_json_value(res, new_prefix, value, recursive=recursive, uncollapse_list=uncollapse_list)
+
+
+def set_object_attributes_with_json_dict(obj, json_value):
+    if not isinstance(json_value, dict):
+        raise TechnicalException(f"json value is expected to be a dict (obtained: {Typing.get_object_class_fullname(json_value)}")
+    
+    for attr_name, attr_value_json in json_value.items():
+        if hasattr(obj, attr_name):
+            if isinstance(attr_value_json, dict) and not isinstance(getattr(obj, attr_name), dict):
+                set_object_attributes_with_json_dict(getattr(obj, attr_name), attr_value_json)
+            else:
+                setattr(obj, attr_name, attr_value_json)
+        else:
+            raise FunctionalException(f"Object doesn't have attribute '{attr_name}' (object type: {Typing.get_object_class_fullname(obj)})")
+
+
+def update_json_object_with_missing_attributes(json_obj, json_ref, recursive=False):
+    if not isinstance(json_obj, dict):
+        raise TechnicalException(f"json object is expected to be a dict (obtained: {Typing.get_object_class_fullname(json_obj)}")
+    if not isinstance(json_ref, dict):
+        raise TechnicalException(f"json reference is expected to be a dict (obtained: {Typing.get_object_class_fullname(json_ref)}")
+
+    for key, value in json_ref.items():
+        if key not in json_obj:
+            json_obj[key] = value
+        elif recursive and isinstance(json_obj[key], dict) and isinstance(value, dict):
+            update_json_object_with_missing_attributes(json_obj[key], value, recursive=recursive)
+    

holado_json/ipc/json_converter.py

@@ -0,0 +1,69 @@
+
+#################################################
+# HolAdo (Holistic Automation do)
+#
+# (C) Copyright 2021-2025 by Eric Klumpp
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+# The Software is provided “as is”, without warranty of any kind, express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall the authors or copyright holders be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the software or the use or other dealings in the Software.
+#################################################
+
+import logging
+from holado_core.common.exceptions.technical_exception import TechnicalException
+from holado_json.ipc.json_types import JsonTypes
+from holado_core.common.handlers.features.resource_by_type import FeatureObjectAndClassResourceByType
+from holado_python.standard_library.typing import Typing
+
+logger = logging.getLogger(__name__)
+
+class JsonConverter(FeatureObjectAndClassResourceByType):
+    
+    def __init__(self):
+        super().__init__()
+        self.import_registered_resources_in_class(class_=JsonTypes)
+    
+    @classmethod
+    def _get_class_resource_description(cls, plural=False):
+        if plural:
+            return 'class json types'
+        else:
+            return 'class json type'
+    
+    def _get_resource_description(self, plural=False):
+        if plural:
+            return 'json types'
+        else:
+            return 'json type'
+    
+    def to_json(self, obj):
+        resource = self.get_resource_for_type(obj=obj)
+        if resource[1] is not None:
+            return resource[1](obj)
+        else:
+            raise TechnicalException(f"Resource 'to_json' is not defined for object of type {Typing.get_object_class_fullname(obj)}")
+    
+    def from_json_to(self, obj_json, class_, _parents=None):
+        if logger.isEnabledFor(logging.TRACE):  # @UndefinedVariable
+            logger.trace(f"[{Typing.get_object_class_fullname(self)}] from_json_to({obj_json}, {Typing.get_class_fullname(class_)})")
+        # Find resource associated to class_, else find the resource associated to json object
+        resource = None
+        if class_ is not None:
+            resource = self.get_resource_for_type(class_=class_, raise_if_not_found=False)
+        if resource is None:
+            resource = self.get_resource_for_type(obj=obj_json)
+        
+        # If defined, use complex json conversion method, else use simple method
+        if resource[2] is not None:
+            return resource[2](obj_json, class_, _parents=_parents, converter=self)
+        elif resource[0] is not None:
+            return resource[0](obj_json)
+        else:
+            raise TechnicalException(f"Resource 'from_json_to' is not defined for type {class_}")
+        
+    
+    
+    
+