PySerials 0.0.0.dev61__tar.gz → 0.0.0.dev63__tar.gz

{pyserials-0.0.0.dev61 → pyserials-0.0.0.dev63}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PySerials
-Version: 0.0.0.dev61
+Version: 0.0.0.dev63
 Requires-Python: >=3.10
 Requires-Dist: jsonschema<5,>=4.21.0
 Requires-Dist: referencing>=0.35.1
@@ -8,6 +8,6 @@ Requires-Dist: jsonpath-ng<2,>=1.6.1
 Requires-Dist: ruamel.yaml>=0.18
 Requires-Dist: ruamel.yaml.string<1,>=0.1.1
 Requires-Dist: tomlkit<0.14,>=0.11.8
-Requires-Dist: MDit==0.0.0.dev58
-Requires-Dist: ExceptionMan==0.0.0.dev58
+Requires-Dist: MDit==0.0.0.dev60
+Requires-Dist: ExceptionMan==0.0.0.dev60
 Requires-Dist: ProtocolMan==0.0.0.dev2

{pyserials-0.0.0.dev61 → pyserials-0.0.0.dev63}/pyproject.toml

@@ -17,7 +17,7 @@ namespaces = true
 # ----------------------------------------- Project Metadata -------------------------------------
 #
 [project]
-version = "0.0.0.dev61"
+version = "0.0.0.dev63"
 name = "PySerials"
 dependencies = [
     "jsonschema >= 4.21.0, < 5",
@@ -26,8 +26,8 @@ dependencies = [
     "ruamel.yaml >= 0.18", # https://yaml.readthedocs.io/en/stable/
     "ruamel.yaml.string >= 0.1.1, < 1",
     "tomlkit >= 0.11.8, < 0.14", # https://tomlkit.readthedocs.io/en/stable/,
-    "MDit == 0.0.0.dev58",
-    "ExceptionMan == 0.0.0.dev58",
+    "MDit == 0.0.0.dev60",
+    "ExceptionMan == 0.0.0.dev60",
     "ProtocolMan == 0.0.0.dev2",
 ]
 requires-python = ">=3.10"

{pyserials-0.0.0.dev61 → pyserials-0.0.0.dev63}/src/PySerials.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: PySerials
-Version: 0.0.0.dev61
+Version: 0.0.0.dev63
 Requires-Python: >=3.10
 Requires-Dist: jsonschema<5,>=4.21.0
 Requires-Dist: referencing>=0.35.1
@@ -8,6 +8,6 @@ Requires-Dist: jsonpath-ng<2,>=1.6.1
 Requires-Dist: ruamel.yaml>=0.18
 Requires-Dist: ruamel.yaml.string<1,>=0.1.1
 Requires-Dist: tomlkit<0.14,>=0.11.8
-Requires-Dist: MDit==0.0.0.dev58
-Requires-Dist: ExceptionMan==0.0.0.dev58
+Requires-Dist: MDit==0.0.0.dev60
+Requires-Dist: ExceptionMan==0.0.0.dev60
 Requires-Dist: ProtocolMan==0.0.0.dev2

{pyserials-0.0.0.dev61 → pyserials-0.0.0.dev63}/src/PySerials.egg-info/requires.txt

@@ -4,6 +4,6 @@ jsonpath-ng<2,>=1.6.1
 ruamel.yaml>=0.18
 ruamel.yaml.string<1,>=0.1.1
 tomlkit<0.14,>=0.11.8
-MDit==0.0.0.dev58
-ExceptionMan==0.0.0.dev58
+MDit==0.0.0.dev60
+ExceptionMan==0.0.0.dev60
 ProtocolMan==0.0.0.dev2

{pyserials-0.0.0.dev61 → pyserials-0.0.0.dev63}/src/pyserials/exception/update.py

@@ -49,8 +49,8 @@ class PySerialsUpdateException(_base.PySerialsException):
         return
 
 
-class PySerialsUpdateDictFromAddonError(PySerialsUpdateException):
-    """Base class for all exceptions raised by `pyserials.update.dict_from_addon`.
+class PySerialsUpdateRecursiveDataError(PySerialsUpdateException):
+    """Base class for all exceptions raised by `pyserials.update.recursive_update`.
 
     Attributes
     ----------

{pyserials-0.0.0.dev61 → pyserials-0.0.0.dev63}/src/pyserials/nested_dict.py

@@ -36,6 +36,7 @@ class NestedDict:
         relative_key_key: str | None = None,
         implicit_root: bool = True,
         getter_function_name: str = "get",
+        skip_key_func: Callable[[list[str]], bool] | None = None,
     ):
         self._data = data or {}
         self._templater = _ps.update.TemplateFiller(
@@ -62,6 +63,7 @@ class NestedDict:
             relative_key_key=relative_key_key,
             implicit_root=implicit_root,
             getter_function_name=getter_function_name,
+            skip_key_func=skip_key_func,
         )
         return
 

{pyserials-0.0.0.dev61 → pyserials-0.0.0.dev63}/src/pyserials/update.py

@@ -1,6 +1,5 @@
 from __future__ import annotations as _annotations
 
-import re
 from typing import TYPE_CHECKING as _TYPE_CHECKING
 import re as _re
 from functools import partial as _partial
@@ -11,69 +10,261 @@ from jsonpath_ng import exceptions as _jsonpath_exceptions
 import pyserials.exception as _exception
 
 if _TYPE_CHECKING:
-    from typing import Literal, Sequence, Any, Callable
+    from typing import Literal, Sequence, Any, Callable, Iterable
+    UPDATE_OPTIONS = Literal["skip", "write", "raise"] | Callable[
+        [tuple[Any] | tuple[Any, Any]], None | tuple[Any, str]
+    ]
+    FUNC_ITEMS = Callable[[Any], Iterable[tuple[Any, Any]]]
+    FUNC_CONTAINS = Callable[[Any, Any], bool]
+    FUNC_GET = Callable[[Any, Any], Any]
+    FUNC_SET = Callable[[Any, Any, Any], None]
+    FUNC_CONSTRUCT = Callable[[], Any]
+    RECURSIVE_DTYPE_FUNCS = tuple[FUNC_ITEMS, FUNC_CONTAINS, FUNC_GET, FUNC_SET, FUNC_CONSTRUCT]
+
+
+def recursive_update(
+    source: Any,
+    addon: Any,
+    recursive_types: dict[type | tuple[type, ...], RECURSIVE_DTYPE_FUNCS] | None = None,
+    types: dict[type | tuple[type, ...], UPDATE_OPTIONS | tuple[UPDATE_OPTIONS, UPDATE_OPTIONS]] | None = None,
+    paths: dict[str, UPDATE_OPTIONS] | None = None,
+    constructor: Callable[[], Any] | None = None,
+    undefined_new: UPDATE_OPTIONS = "write",
+    undefined_existing: UPDATE_OPTIONS = "skip",
+    type_mismatch: UPDATE_OPTIONS = "raise",
+) -> dict[str, list[str]]:
+    """Recursively update a complex data structure using another data structure.
+
+    Parameters
+    ----------
+    source
+        Data structure to update in place.
+        The type of this object must be
+        defined in the `recursive_types` argument.
+    addon
+        Data structure containing additional data to update `source` with.
+    recursive_types
+        Definition of recursive data types.
+        Each key is a type (or tuple of types) used
+        to identify data types with the `isinstance` function.
+        Each value is a tuple of three functions:
+        1. Function to extract items from the data. It must accept an instance
+           of the type and return an iterable of key-value pairs.
+        2. Function to check if a key is in the data. It must accept an instance
+           of the type and a key, and return a boolean.
+        3. Function to get a value from the data. It must accept an instance
+           of the type and a key, and return the value.
+        4. Function to set a key-value pair in the data. It must accept an instance
+           of the type, a key, and a value, respectively.
+        5. Function to construct a new instance of the type. It must accept no arguments.
+
+        By default, `dict` is defined as follows:
+        ```python
+        recursive_types = {
+            dict: (
+                lambda dic: dic.items(),
+                lambda dic, key: key in dic,
+                lambda dic, key: dic[key],
+                lambda dic, key, value: dic.update({key: value}),
+                lambda: dict(),
+            )
+        }
+        ```
+        This default argument will be updated/overwritten with any custom types provided.
+        For example, to add support for a custom class `MyClass`, you can use:
+        ```python
+        recursive_types = {
+            MyClass: (
+                lambda obj: vars(object).items(),
+                lambda obj, key: hasattr(obj, key),
+                lambda obj, key: getattr(obj, key),
+                lambda obj, key, value: setattr(obj, key, value),
+                lambda: MyClass(),
+            )
+        }
+        ```
+    types
+        Update behavior for specific data types.
+        Each key is a type (or tuple of types) used
+        to identify data types with the `isinstance` function.
+        Each value is can be single update option for all cases,
+        or a tuple of two update options for when the corresponding
+        key/attribute does not exist in the source data, and when it does, respectively.
+        Each update option can be either a keyword specifying a predefined behavior,
+        or a function for custom behavior. The available keywords are:
+        - "skip": Ignore the key/attribute; do not change anything.
+        - "write": Write the key/attribute in source data with the value from the addon, overwriting if it exists.
+
+        A custom function must accept a single argument, a tuple of either one or two values.
+        If it is two values, the first value is the current value in the source data,
+        and the second value is the value from the addon.
+        If it is one value, the value is the value from the addon,
+        meaning the key/attribute does not exist in the source data.
+        The function must either return `None` (for when nothing must be changes in the source)
+        or a tuple of two values:
+        1. The new value to write in the source data.
+        2. A string specifying the change type.
+
+        By default, the following behavior is defined for basic types:
+        ```python
+        types = {list: ("write", lambda data: (data[0] + data[1], "append"))}
+        ```
+
+        The default behavior for any data type not specified in this argument
+        is determined by the `undefined_new` and `undefined_existing` arguments.
+    paths
+        Update behavior for specific keys using JSONPath expressions.
+        This is the same as the `types` argument, but targeting specific keys
+        instead of data types.
+        Everything is the same as in the `types` argument, except that the keys
+        are JSONPath expressions as strings.
+    constructor
+        Custom constructor for creating new instances of the source data type.
+        This is used when the addon data contains a recursive key/attribute
+        that is not present in the source data. If not provided,
+        the type of the addon value will be used to create a new instance.
+    undefined_new
+        Behavior for when a non-recursive data with no defined behavior
+        in the `types` argument is found in the addon data
+        but not in the source data.
+    undefined_existing
+        Behavior for when a non-recursive data with no defined behavior
+        in the `types` argument is found in the addon data
+        and in the source data.
+    type_mismatch
+        Behavior for when a key/attribute in the source data
+        is not a recursive type, but the corresponding key/attribute
+        in the addon data is a recursive type.
+    """
+    def get_funcs(data: Any) -> RECURSIVE_DTYPE_FUNCS | None:
+        for typ, funcs in recursive_types.items():
+            if isinstance(data, typ):
+                return funcs
+        return None
 
+    def recursive(
+        src: Any,
+        add: Any,
+        src_funcs: RECURSIVE_DTYPE_FUNCS,
+        add_funcs: RECURSIVE_DTYPE_FUNCS,
+        path: str,
+    ):
 
-def dict_from_addon(
-    data: dict,
-    addon: dict,
-    append_list: bool = True,
-    append_dict: bool = True,
-    raise_duplicates: bool = False,
-    raise_type_mismatch: bool = True,
-) -> dict[str, list[str]]:
-    """Recursively update a dictionary from another dictionary."""
-    def recursive(source: dict, add: dict, path: str, log: dict):
+        def apply(behavior: UPDATE_OPTIONS | tuple[UPDATE_OPTIONS, UPDATE_OPTIONS]):
+            action = behavior[int(key_exists_in_src)] if isinstance(behavior, tuple) else behavior
+            if action == "raise":
+                raise_error(typ="duplicate")
+            elif action == "skip":
+                change_type = "skip"
+            elif action == "write":
+                change_type = "write"
+                fn_src_set(src, key, value)
+            elif not isinstance(action, str):
+                out = action((source_value, value) if key_exists_in_src else (value,))
+                if out:
+                    new_value, change_type = out
+                    fn_src_set(src, key, new_value)
+                else:
+                    change_type = "skip"
+            else:
+                raise ValueError(f"Invalid update behavior '{action}' for key '{key}' at path '{path}'.")
+            log[fullpath] = (type(source_value) if key_exists_in_src else None, type(value), change_type)
+            return
 
         def raise_error(typ: Literal["duplicate", "type_mismatch"]):
-            raise _exception.update.PySerialsUpdateDictFromAddonError(
+            raise _exception.update.PySerialsUpdateRecursiveDataError(
                 problem_type=typ,
                 path=fullpath,
-                data=source[key],
-                data_full=data,
+                data=src[key],
+                data_full=src,
                 data_addon=value,
                 data_addon_full=addon,
             )
 
-        for key, value in add.items():
-            fullpath = f"{path}.{key}"
-            if key not in source:
-                log["added"].append(fullpath)
-                source[key] = value
-                continue
-            if type(source[key]) is not type(value):
-                if raise_type_mismatch:
-                    raise_error(typ="type_mismatch")
-                continue
-            if not isinstance(value, (list, dict)):
-                if raise_duplicates:
-                    raise_error(typ="duplicate")
-                log["skipped"].append(fullpath)
-            elif isinstance(value, list):
-                if append_list:
-                    appended = False
-                    for elem in value:
-                        if elem not in source[key]:
-                            source[key].append(elem)
-                            appended = True
-                    if appended:
-                        log["list_appended"].append(fullpath)
-                elif raise_duplicates:
-                    raise_error(typ="duplicate")
-                else:
-                    log["skipped"].append(fullpath)
+        _, fn_src_contains, fn_src_get, fn_src_set, _ = src_funcs
+        fn_add_items, _, _, _, fn_add_construct = add_funcs
+
+        for key, value in fn_add_items(add):
+            fullpath = f"{path}.'{key}'"
+            try:
+                full_jpath = _jsonpath.parse(fullpath)  # quote to avoid JSONPath syntax errors
+            except Exception as e:
+                print(fullpath)
+                raise e
+            key_exists_in_src = fn_src_contains(src, key)
+            source_value = fn_src_get(src, key) if key_exists_in_src else None
+            for jpath_str, matches in jsonpath_match.items():
+                if full_jpath in matches:
+                    apply(paths[jpath_str])
+                    break
             else:
-                if append_dict:
-                    recursive(source=source[key], add=value, path=f"{fullpath}.", log=log)
-                elif raise_duplicates:
-                    raise_error(typ="duplicate")
+                for typ, action in type_to_arg.items():
+                    if isinstance(value, typ):
+                        apply(action)
+                        break
                 else:
-                    log["skipped"].append(fullpath)
-        return log
-    full_log = recursive(
-        source=data, add=addon, path="$", log={"added": [], "skipped": [], "list_appended": []}
+                    funcs_value = get_funcs(value)
+                    if funcs_value:
+                        # Value is a recursive type
+                        if key_exists_in_src:
+                            funcs_src_value = get_funcs(source_value)
+                            if not funcs_src_value:
+                                # Source value is not a recursive type
+                                apply(type_mismatch)
+                            else:
+                                recursive(
+                                    src=fn_src_get(src, key),
+                                    add=value,
+                                    path=fullpath,
+                                    src_funcs=src_funcs,
+                                    add_funcs=funcs_value,
+                                )
+                        else:
+                            # Source value does not exist; create a new instance
+                            new_instance = constructor() if constructor else fn_add_construct()
+                            fn_src_set(src, key, new_instance)
+                            funcs_src_value = get_funcs(new_instance)
+                            recursive(
+                                src=new_instance,
+                                add=value,
+                                path=fullpath,
+                                src_funcs=funcs_src_value,
+                                add_funcs=funcs_value,
+                            )
+                    else:
+                        # addon value is of a non-recursive type that does not have any defined behavior;
+                        # Apply the default behavior for of ("write", "skip") for the key.
+                        apply(undefined_existing if key_exists_in_src else undefined_new)
+        return
+
+    type_to_arg = {list: ("write", lambda data: (data[0] + data[1], "append"))} | (types or {})
+    recursive_types = {
+        dict: (
+            lambda dic: dic.items(),
+            lambda dic, key: key in dic,
+            lambda dic, key: dic[key],
+            lambda dic, key, value: dic.update({key: value}),
+            lambda: dict(),
+        )
+    } | (recursive_types or {})
+    jsonpath_match = {
+        jpath_str: [match.full_path for match in _jsonpath.parse(jpath_str).find(addon)]
+        for jpath_str in (paths or {}).keys()
+    }
+    log = {}
+    funcs_src = get_funcs(source)
+    funcs_add = get_funcs(addon)
+    for funcs, param_name, data in ((funcs_src, "source", source), (funcs_add, "addon", addon)):
+        if not funcs:
+            raise ValueError(f"Data type '{type(data)}' of '{param_name}' is not provided in 'recursive_types'.")
+    recursive(
+        src=source,
+        add=addon,
+        path="$",
+        src_funcs=funcs_src,
+        add_funcs=funcs_add,
     )
-    return full_log
+    return log
 
 
 def data_from_jsonschema(data: dict | list, schema: dict) -> None:
@@ -129,6 +320,7 @@ class TemplateFiller:
         relative_key_key: str | None = None,
         implicit_root: bool = True,
         getter_function_name: str = "get",
+        skip_key_func: Callable[[list[str]], bool] | None = None,
     ):
         self._marker_start_value = marker_start_value
         self._marker_end_value = marker_end_value
@@ -151,6 +343,7 @@ class TemplateFiller:
         self._template_keys = relative_template_keys or []
         self._relative_key_key = relative_key_key
         self._getter_function_name = getter_function_name
+        self._skip_func = skip_key_func
 
         self._pattern_value: dict[int, _RegexPattern] = {}
         self._data = None
@@ -190,7 +383,11 @@ class TemplateFiller:
             code_lines = ["def __inline_code__():"]
             code_lines.extend([f"    {line}" for line in code_str.strip("\n").splitlines()])
             code_str_full = "\n".join(code_lines)
-            global_context = self._code_context.copy() | {self._getter_function_name: getter_function}
+            global_context = self._code_context.copy() | {
+                self._getter_function_name: getter_function,
+                "__current_path__": current_path,
+                "__relative_path_anchor__": relative_path_anchor
+            }
             for name, partial_func_data in self._code_context_partial.items():
                 if isinstance(partial_func_data, tuple):
                     func, arg_name = partial_func_data
@@ -250,6 +447,7 @@ class TemplateFiller:
                         return output, True
                     return output
                 path_expr = self._concat_json_paths(root_path_expr, path_expr)
+            # print("IN GET ADD VAL", path_expr)
             cached_result = self._visited_paths.get(path_expr)
             if cached_result:
                 value, matched = cached_result
@@ -266,7 +464,7 @@ class TemplateFiller:
             return self._no_match_value
 
         def get_value(jsonpath, return_all_matches: bool, from_code: bool) -> tuple[Any, bool]:
-            matches = _rec_match(jsonpath)
+            matches = recursive_match(jsonpath)
             if not matches:
                 if from_code:
                     return None, False
@@ -294,14 +492,14 @@ class TemplateFiller:
                 current_chain=current_chain + (jsonpath,),
             ), True
 
-        def _rec_match(expr) -> list:
+        def recursive_match(expr) -> list:
             matches = expr.find(self._data)
             if matches:
                 return matches
             if isinstance(expr.left, _jsonpath.Root):
                 return []
             whole_matches = []
-            left_matches = _rec_match(expr.left)
+            left_matches = recursive_match(expr.left)
             for left_match in left_matches:
                 left_match_filled = self._recursive_subst(
                     templ=left_match.value,
@@ -357,6 +555,11 @@ class TemplateFiller:
                 template_end=self._marker_end_value,
             ) from exception
 
+        # print("IN MAIN", self._extract_fields(current_path))
+
+        if self._skip_func and self._skip_func(self._extract_fields(current_path)):
+            return templ
+
         if current_path in self._visited_paths:
             return self._visited_paths[current_path][0]
 
@@ -432,6 +635,7 @@ class TemplateFiller:
 
         if isinstance(templ, dict):
             new_dict = {}
+            addons = []
             for key, val in templ.items():
                 key_filled = self._recursive_subst(
                     templ=key,
@@ -442,7 +646,7 @@ class TemplateFiller:
                     is_key=True,
                 )
                 if isinstance(key, str) and self._pattern_unpack.fullmatch(key):
-                    new_dict.update(key_filled)
+                    addons.append((key_filled, val))
                     continue
                 if key_filled in self._template_keys:
                     new_dict[key_filled] = val
@@ -455,6 +659,15 @@ class TemplateFiller:
                     level=0,
                     current_chain=current_chain + (new_path,),
                 )
+            for addon_dict, addon_settings in sorted(
+                addons, key=lambda addon: addon[1].get("priority", 0) if addon[1] else 0
+            ):
+                addon_settings = {k: v for k, v in (addon_settings or {}).items() if k not in ("priority",)}
+                recursive_update(
+                    source=new_dict,
+                    addon=addon_dict,
+                    **addon_settings,
+                )
             if not is_relative_template:
                 self._visited_paths[current_path] = (new_dict, True)
             return new_dict
@@ -534,7 +747,7 @@ class _RegexPattern:
     def __init__(self, start: str, end: str):
         start_esc = _re.escape(start)
         end_esc = _re.escape(end)
-        self.pattern = _re.compile(rf"{start_esc}(.*?)(?={end_esc}){end_esc}", re.DOTALL)
+        self.pattern = _re.compile(rf"{start_esc}(.*?)(?={end_esc}){end_esc}", _re.DOTALL)
         return
 
     def fullmatch(self, string: str) -> _re.Match | None:
@@ -547,3 +760,4 @@ class _RegexPattern:
 
     def sub(self, repl, string: str) -> str:
         return self.pattern.sub(repl, string)
+

{pyserials-0.0.0.dev61 → pyserials-0.0.0.dev63}/src/pyserials/write.py

@@ -15,13 +15,27 @@ def to_string(
     data_type: _Literal["json", "yaml", "toml"],
     sort_keys: bool = False,
     indent: int | None = None,
+    default: Callable[[Any], Any] | None = None,
     end_of_file_newline: bool = True,
+    indent_mapping: int = 2,
+    indent_sequence: int = 4,
+    indent_sequence_offset: int = 2,
+    multiline_string_to_block: bool = True,
+    remove_top_level_indent: bool = True
 ):
     if data_type == "json":
-        return to_json_string(data, sort_keys=sort_keys, indent=indent)
+        return to_json_string(data, sort_keys=sort_keys, indent=indent, default=default, end_of_file_newline=end_of_file_newline)
     if data_type == "yaml":
-        return to_yaml_string(data, end_of_file_newline=end_of_file_newline)
-    return to_toml_string(data, sort_keys=sort_keys)
+        return to_yaml_string(
+            data, 
+            end_of_file_newline=end_of_file_newline,
+            indent_mapping=indent_mapping,
+            indent_sequence=indent_sequence,
+            indent_sequence_offset=indent_sequence_offset,
+            multiline_string_to_block=multiline_string_to_block,
+            remove_top_level_indent=remove_top_level_indent
+        )
+    return to_toml_string(data, sort_keys=sort_keys, end_of_file_newline=end_of_file_newline)
 
 
 def to_yaml_string(
@@ -49,8 +63,10 @@ def to_yaml_string(
 def to_toml_string(
     data: dict | list | str | int | float | bool | _yaml.CommentedMap | _yaml.CommentedSeq,
     sort_keys: bool = False,
+    end_of_file_newline: bool = True,
 ) -> str:
-    return _tomlkit.dumps(data, sort_keys=sort_keys)
+    string = _tomlkit.dumps(data, sort_keys=sort_keys)
+    return f"{string.rstrip("\n")}\n" if end_of_file_newline else string
 
 
 def to_json_string(
@@ -58,8 +74,10 @@ def to_json_string(
     sort_keys: bool = False,
     indent: int | None = None,
     default: Callable[[Any], Any] | None = None,
+    end_of_file_newline: bool = True,
 ) -> str:
-    return _json.dumps(data, indent=indent, sort_keys=sort_keys, default=default)
+    string = _json.dumps(data, indent=indent, sort_keys=sort_keys, default=default)
+    return f"{string.rstrip("\n")}\n" if end_of_file_newline else string
 
 
 def to_yaml_file(