PySerials 0.0.0.dev30__tar.gz → 0.0.0.dev31__tar.gz

{pyserials-0.0.0.dev30 → pyserials-0.0.0.dev31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PySerials
-Version: 0.0.0.dev30
+Version: 0.0.0.dev31
 Requires-Python: >=3.10
 Requires-Dist: jsonschema<5,>=4.21.0
 Requires-Dist: referencing>=0.35.1
@@ -8,6 +8,6 @@ Requires-Dist: jsonpath-ng<2,>=1.6.1
 Requires-Dist: ruamel.yaml<0.18,>=0.17.32
 Requires-Dist: ruamel.yaml.string<1,>=0.1.1
 Requires-Dist: tomlkit<0.12,>=0.11.8
-Requires-Dist: MDit==0.0.0.dev27
-Requires-Dist: ExceptionMan==0.0.0.dev27
+Requires-Dist: MDit==0.0.0.dev28
+Requires-Dist: ExceptionMan==0.0.0.dev28
 Requires-Dist: ProtocolMan==0.0.0.dev2

{pyserials-0.0.0.dev30 → pyserials-0.0.0.dev31}/pyproject.toml

@@ -17,7 +17,7 @@ namespaces = true
 # ----------------------------------------- Project Metadata -------------------------------------
 #
 [project]
-version = "0.0.0.dev30"
+version = "0.0.0.dev31"
 name = "PySerials"
 dependencies = [
     "jsonschema >= 4.21.0, < 5",
@@ -26,8 +26,8 @@ dependencies = [
     "ruamel.yaml >= 0.17.32, < 0.18", # https://yaml.readthedocs.io/en/stable/
     "ruamel.yaml.string >= 0.1.1, < 1",
     "tomlkit >= 0.11.8, < 0.12", # https://tomlkit.readthedocs.io/en/stable/,
-    "MDit == 0.0.0.dev27",
-    "ExceptionMan == 0.0.0.dev27",
+    "MDit == 0.0.0.dev28",
+    "ExceptionMan == 0.0.0.dev28",
     "ProtocolMan == 0.0.0.dev2",
 ]
 requires-python = ">=3.10"

{pyserials-0.0.0.dev30 → pyserials-0.0.0.dev31}/src/PySerials.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: PySerials
-Version: 0.0.0.dev30
+Version: 0.0.0.dev31
 Requires-Python: >=3.10
 Requires-Dist: jsonschema<5,>=4.21.0
 Requires-Dist: referencing>=0.35.1
@@ -8,6 +8,6 @@ Requires-Dist: jsonpath-ng<2,>=1.6.1
 Requires-Dist: ruamel.yaml<0.18,>=0.17.32
 Requires-Dist: ruamel.yaml.string<1,>=0.1.1
 Requires-Dist: tomlkit<0.12,>=0.11.8
-Requires-Dist: MDit==0.0.0.dev27
-Requires-Dist: ExceptionMan==0.0.0.dev27
+Requires-Dist: MDit==0.0.0.dev28
+Requires-Dist: ExceptionMan==0.0.0.dev28
 Requires-Dist: ProtocolMan==0.0.0.dev2

{pyserials-0.0.0.dev30 → pyserials-0.0.0.dev31}/src/PySerials.egg-info/requires.txt

@@ -4,6 +4,6 @@ jsonpath-ng<2,>=1.6.1
 ruamel.yaml<0.18,>=0.17.32
 ruamel.yaml.string<1,>=0.1.1
 tomlkit<0.12,>=0.11.8
-MDit==0.0.0.dev27
-ExceptionMan==0.0.0.dev27
+MDit==0.0.0.dev28
+ExceptionMan==0.0.0.dev28
 ProtocolMan==0.0.0.dev2

{pyserials-0.0.0.dev30 → pyserials-0.0.0.dev31}/src/pyserials/nested_dict.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING as _TYPE_CHECKING
 import pyserials as _ps
 
 if _TYPE_CHECKING:
-    from typing import Callable
+    from typing import Callable, Any
 
 
 class NestedDict:
@@ -13,31 +13,59 @@ class NestedDict:
     def __init__(
         self,
         data: dict | None = None,
-        template_marker_start: str = "${{",
-        template_marker_end: str = "}}",
-        template_marker_unpack_start: str = "*{{",
-        template_marker_unpack_end: str = "}}",
-        template_implicit_root: bool = True,
-        template_stringer: Callable[[str], str] = str,
+        marker_start_value: str = "$",
+        marker_end_value: str = "$",
+        repeater_start_value: str = "{",
+        repeater_end_value: str = "}",
+        repeater_count_value: int = 2,
+        start_list: str = "$[[",
+        start_unpack: str = "*{{",
+        start_code: str = "#{{",
+        end_list: str = "]]$",
+        end_unpack: str = "}}*",
+        end_code: str = "}}#",
+        recursive: bool = True,
+        raise_no_match: bool = True,
+        leave_no_match: bool = False,
+        no_match_value: Any = None,
+        stringer: Callable[[str], str] = str,
+        unpack_string_joiner: str = ", ",
         relative_template_keys: list[str] | None = None,
+        implicit_root: bool = True,
     ):
         self._data = data or {}
         self._templater = _ps.update.TemplateFiller(
-            marker_start=template_marker_start,
-            marker_end=template_marker_end,
-            marker_unpack_start=template_marker_unpack_start,
-            marker_unpack_end=template_marker_unpack_end,
-            implicit_root=template_implicit_root,
-            stringer=template_stringer,
+            marker_start_value=marker_start_value,
+            marker_end_value=marker_end_value,
+            repeater_start_value=repeater_start_value,
+            repeater_end_value=repeater_end_value,
+            repeater_count_value=repeater_count_value,
+            start_list=start_list,
+            start_unpack=start_unpack,
+            start_code=start_code,
+            end_list=end_list,
+            end_unpack=end_unpack,
+            end_code=end_code,
         )
-        self._relative_template_keys = relative_template_keys
+        self._recursive = recursive
+        self._raise_no_match = raise_no_match
+        self._leave_no_match = leave_no_match
+        self._no_match_value = no_match_value
+        self._stringer = stringer
+        self._unpack_string_joiner = unpack_string_joiner
+        self._relative_template_keys = relative_template_keys or []
+        self._implicit_root = implicit_root
         return
 
     def fill(
         self,
         path: str = "",
-        always_list: bool = False,
-        recursive: bool = True,
+        recursive: bool | None = None,
+        raise_no_match: bool | None = None,
+        leave_no_match: bool | None = None,
+        stringer: Callable[[str], str] | None = None,
+        unpack_string_joiner: str | None = None,
+        level: int = 0,
     ):
         if not path:
             value = self._data
@@ -46,7 +74,14 @@ class NestedDict:
         if not value:
             return
         filled_value = self.fill_data(
-            data=value, current_path=path, always_list=always_list, recursive=recursive,
+            data=value,
+            current_path=path,
+            recursive=recursive,
+            raise_no_match=raise_no_match,
+            leave_no_match=leave_no_match,
+            stringer=stringer,
+            unpack_string_joiner=unpack_string_joiner,
+            level=level,
         )
         if not path:
             self._data = filled_value
@@ -58,16 +93,26 @@ class NestedDict:
         self,
         data,
         current_path: str = "",
-        always_list: bool = False,
-        recursive: bool = True,
+        recursive: bool | None = None,
+        raise_no_match: bool | None = None,
+        leave_no_match: bool | None = None,
+        stringer: Callable[[str], str] | None = None,
+        unpack_string_joiner: str | None = None,
+        level: int = 0,
     ):
         return self._templater.fill(
             templated_data=data,
             source_data=self._data,
             current_path=current_path,
-            always_list=always_list,
-            recursive=recursive,
+            recursive=recursive if recursive is not None else self._recursive,
+            raise_no_match=raise_no_match if raise_no_match is not None else self._raise_no_match,
+            leave_no_match=leave_no_match if leave_no_match is not None else self._leave_no_match,
+            no_match_value=self._no_match_value,
+            stringer=stringer if stringer is not None else self._stringer,
+            unpack_string_joiner=unpack_string_joiner if unpack_string_joiner is not None else self._unpack_string_joiner,
             relative_template_keys=self._relative_template_keys,
+            implicit_root=self._implicit_root,
+            level=level,
         )
 
     def __call__(self):

{pyserials-0.0.0.dev30 → pyserials-0.0.0.dev31}/src/pyserials/update.py

@@ -1,4 +1,6 @@
 from __future__ import annotations as _annotations
+
+import re
 from typing import TYPE_CHECKING as _TYPE_CHECKING
 import re as _re
 
@@ -8,7 +10,7 @@ from jsonpath_ng import exceptions as _jsonpath_exceptions
 import pyserials.exception as _exception
 
 if _TYPE_CHECKING:
-    from typing import Literal, Callable, Sequence
+    from typing import Literal, Sequence, Any, Callable
 
 
 def dict_from_addon(
@@ -103,25 +105,29 @@ class TemplateFiller:
 
     def __init__(
         self,
-        marker_start: str = "${{",
-        marker_end: str = "}}",
-        marker_unpack_start: str = "*{{",
-        marker_unpack_end: str = "}}",
-        implicit_root: bool = True,
-        stringer: Callable[[str], str] = str,
+        marker_start_value: str = "$",
+        marker_end_value: str = "$",
+        repeater_start_value: str = "{",
+        repeater_end_value: str = "}",
+        repeater_count_value: int = 2,
+        start_list: str = "$[[",
+        start_unpack: str = "*{{",
+        start_code: str = "#{{",
+        end_list: str = "]]$",
+        end_unpack: str = "}}*",
+        end_code: str = "}}#",
     ):
-        def make_regex(start, end):
-            start_esc = _re.escape(start)
-            end_esc = _re.escape(end)
-            regex_sub = rf"{start_esc}([^{end_esc}]+){end_esc}"
-            return _re.compile(regex_sub)
-
-        self._marker_start = marker_start
-        self._marker_end = marker_end
-        self._pattern_template = make_regex(marker_start, marker_end)
-        self._pattern_template_unpack = make_regex(marker_unpack_start, marker_unpack_end)
-        self._add_prefix = implicit_root
-        self._stringer = stringer
+        self._marker_start_value = marker_start_value
+        self._marker_end_value = marker_end_value
+        self._repeater_start_value = repeater_start_value
+        self._repeater_end_value = repeater_end_value
+        self._repeater_count_value = repeater_count_value
+        self._pattern_list = _RegexPattern(start=start_list, end=end_list)
+        self._pattern_unpack = _RegexPattern(start=start_unpack, end=end_unpack)
+        self._pattern_code = _RegexPattern(start=start_code, end=end_code)
+        self._add_prefix = True
+
+        self._pattern_value: dict[int, _RegexPattern] = {}
         self._data = None
         self._source = None
         self._recursive = None
@@ -129,22 +135,48 @@ class TemplateFiller:
         self._raise_no_match = None
         self._template_keys = None
         self._ignore_templates = True
+        self._leave_no_match = False
+        self._no_match_value = None
+        self._stringer = str
+        self._unpack_string_joiner = ", "
         return
 
+    def _get_value_regex_pattern(self, level: int = 0) -> _RegexPattern:
+        level_patterns = self._pattern_value.setdefault(level, {})
+        if level in level_patterns:
+            return level_patterns[level]
+        count = self._repeater_count_value + level
+        pattern = _RegexPattern(
+            start=f"{self._marker_start_value}{self._repeater_start_value * count} ",
+            end=f" {self._repeater_end_value * count}{self._marker_end_value}",
+        )
+        level_patterns[level] = pattern
+        return pattern
+
     def fill(
         self,
         templated_data: dict | list | str,
         source_data: dict | list,
         current_path: str = "",
-        always_list: bool = True,
         recursive: bool = True,
         raise_no_match: bool = True,
+        leave_no_match: bool = False,
+        no_match_value: Any = None,
+        stringer: Callable[[str], str] = str,
+        unpack_string_joiner: str = ", ",
         relative_template_keys: list[str] | None = None,
+        implicit_root: bool = True,
+        level: int = 0,
     ):
         self._data = templated_data
         self._source = source_data
         self._recursive = recursive
         self._raise_no_match = raise_no_match
+        self._leave_no_match = leave_no_match
+        self._no_match_value = no_match_value
+        self._stringer = stringer
+        self._unpack_string_joiner = unpack_string_joiner
+        self._add_prefix = implicit_root
         self._template_keys = relative_template_keys or []
         path = (f"$.{current_path}" if self._add_prefix else current_path) if current_path else "$"
         if not relative_template_keys:
@@ -152,15 +184,15 @@ class TemplateFiller:
             return self._recursive_subst(
                 templ=self._data,
                 current_path=path,
-                always_list=always_list,
                 relative_path_anchor=path,
+                level=level,
             )
         self._ignore_templates = True
         first_pass = self._recursive_subst(
             templ=self._data,
             current_path=path,
-            always_list=always_list,
             relative_path_anchor=path,
+            level=level,
         )
         if self._data is self._source:
             self._source = first_pass
@@ -169,53 +201,28 @@ class TemplateFiller:
         return self._recursive_subst(
             templ=self._data,
             current_path=path,
-            always_list=always_list,
             relative_path_anchor=path,
+            level=level,
         )
 
-    def _recursive_subst(self, templ, current_path: str, always_list: bool, relative_path_anchor: str):
-
-        def raise_error(
-            path_invalid: str,
-            description_template: str,
-        ):
-            raise _exception.update.PySerialsUpdateTemplatedDataError(
-                description_template=description_template,
-                path_invalid=path_invalid,
-                path=current_path,
-                data=templ,
-                data_full=self._data,
-                data_source=self._source,
-                template_start=self._marker_start,
-                template_end=self._marker_end,
-            )
-
-        def _rec_match(expr):
+    def _recursive_subst(self, templ, current_path: str, relative_path_anchor: str, level: int):
 
-            def raise_error_path_invalid():
+        def get_code_value(code_str: str):
+            code_lines = ["def __inline_code__():"]
+            code_lines.extend([f"    {line}" for line in code_str.strip("\n").splitlines()])
+            code_str_full = "\n".join(code_lines)
+            global_context = {}
+            local_context = {}
+            try:
+                exec(code_str_full, global_context, local_context)
+                return local_context["__inline_code__"]()
+            except Exception as e:
                 raise_error(
-                    path_invalid=str(expr),
-                    description_template="Path {path_invalid} is missing in the source data.",
+                    description_template=f"Code at {{path_invalid}} raised an exception: {e}\n{code_str_full}",
+                    path_invalid=current_path,
                 )
 
-            matches = expr.find(self._source)
-            if matches:
-                return matches
-            if isinstance(expr.left, _jsonpath.Root):
-                raise_error_path_invalid()
-            whole_matches = []
-            left_matches = _rec_match(expr.left)
-            for left_match in left_matches:
-                left_match_filled = self._recursive_subst(
-                    left_match.value, current_path=str(expr.left), always_list=False, relative_path_anchor=str(expr.left)
-                ) if isinstance(left_match.value, str) else left_match.value
-                right_matches = expr.right.find(left_match_filled)
-                whole_matches.extend(right_matches)
-            if not whole_matches:
-                raise_error_path_invalid()
-            return whole_matches
-
-        def get_address_value(re_match):
+        def get_address_value(re_match, return_all_matches: bool = False):
             path, num_periods = self._remove_leading_periods(re_match.group(1).strip())
             if num_periods == 0:
                 path = f"$.{path}" if self._add_prefix else path
@@ -244,20 +251,26 @@ class TemplateFiller:
                         )
                     root_path_expr = root_path_expr.left
                 path_expr = root_path_expr.child(path_expr)
-            return get_value(path_expr)
+            value, matched = get_value(path_expr, return_all_matches)
+            if matched:
+                return value
+            if self._leave_no_match:
+                return re_match.group()
+            return self._no_match_value
 
-        def get_value(jsonpath):
+        def get_value(jsonpath, return_all_matches: bool) -> tuple[Any, bool]:
             matches = _rec_match(jsonpath)
+            if not matches:
+                if not return_all_matches and self._raise_no_match:
+                    raise_error(
+                        path_invalid=str(jsonpath),
+                        description_template="JSONPath expression {path_invalid} did not match any data.",
+                    )
+                return None, False
             values = [m.value for m in matches]
-            if not values and self._raise_no_match:
-                raise_error(
-                    path_invalid=str(jsonpath),
-                    description_template="JSONPath expression {path_invalid} did not match any data.",
-                )
-            single = len(values) == 1 and not always_list
-            output = values[0] if single else values
+            output = values if return_all_matches or len(values) > 1 else values[0]
             if not self._recursive:
-                return output
+                return output, True
             if relative_path_anchor == current_path:
                 path_fields = self._extract_fields(jsonpath)
                 has_template_key = any(field in self._template_keys for field in path_fields)
@@ -267,45 +280,140 @@ class TemplateFiller:
             return self._recursive_subst(
                 output,
                 current_path=str(jsonpath),
-                always_list=always_list,
-                relative_path_anchor=_rel_path_anchor
-            )
+                relative_path_anchor=_rel_path_anchor,
+                level=0,
+            ), True
+
+        def _rec_match(expr) -> list:
+            matches = expr.find(self._source)
+            if matches:
+                return matches
+            if isinstance(expr.left, _jsonpath.Root):
+                return []
+            whole_matches = []
+            left_matches = _rec_match(expr.left)
+            for left_match in left_matches:
+                left_match_filled = self._recursive_subst(
+                    templ=left_match.value,
+                    current_path=str(expr.left),
+                    relative_path_anchor=str(expr.left),
+                    level=0,
+                ) if isinstance(left_match.value, str) else left_match.value
+                right_matches = expr.right.find(left_match_filled)
+                whole_matches.extend(right_matches)
+            return whole_matches
 
         def get_relative_path(new_path):
             return new_path if current_path == relative_path_anchor else relative_path_anchor
 
+        def raise_error(
+            path_invalid: str,
+            description_template: str,
+        ):
+            raise _exception.update.PySerialsUpdateTemplatedDataError(
+                description_template=description_template,
+                path_invalid=path_invalid,
+                path=current_path,
+                data=templ,
+                data_full=self._data,
+                data_source=self._source,
+                template_start=self._marker_start_value,
+                template_end=self._marker_end_value,
+            )
+
+        def string_filler_unpack(match: _re.Match):
+            match_list = self._pattern_list.fullmatch(match.group(1).strip())
+            if match_list:
+                values = get_address_value(match_list, return_all_matches=True)
+            else:
+                match_code = self._pattern_code.fullmatch(match.group(1).strip())
+                if match_code:
+                    values = get_code_value(match_code.group(1))
+                else:
+                    values = get_address_value(match)
+            return self._unpack_string_joiner.join([self._stringer(val) for val in values])
+
         if isinstance(templ, str):
-            match_whole_str = self._pattern_template.fullmatch(templ) or self._pattern_template_unpack.fullmatch(templ)
-            if match_whole_str:
-                return get_address_value(match_whole_str)
-            return self._pattern_template.sub(
-                lambda x: self._stringer(get_address_value(x)),
+            pattern_nested = self._get_value_regex_pattern(level=level + 1)
+            templ_nested_filled = pattern_nested.sub(
+                lambda x: self._recursive_subst(
+                    templ=x.group(),
+                    current_path=current_path,
+                    relative_path_anchor=get_relative_path(current_path),
+                    level=level+1,
+                ),
                 templ
             )
+            pattern_value = self._get_value_regex_pattern(level=level)
+            whole_match_value = pattern_value.fullmatch(templ_nested_filled)
+            if whole_match_value:
+                return get_address_value(whole_match_value)
+            templ_values_filled = pattern_value.sub(
+                lambda x: str(get_address_value(x)),
+                templ_nested_filled
+            )
+            whole_match_list = self._pattern_list.fullmatch(templ_values_filled.strip())
+            if whole_match_list:
+                return get_address_value(whole_match_list, return_all_matches=True)
+            whole_match_unpack = self._pattern_unpack.fullmatch(templ_values_filled.strip())
+            if whole_match_unpack:
+                submatch_list = self._pattern_list.fullmatch(whole_match_unpack.group(1).strip())
+                if submatch_list:
+                    return get_address_value(submatch_list, return_all_matches=True)
+                submatch_code = self._pattern_code.fullmatch(whole_match_unpack.group(1).strip())
+                if submatch_code:
+                    return get_code_value(submatch_code.group(1))
+                return get_address_value(whole_match_unpack)
+            whole_match_code = self._pattern_code.fullmatch(templ_values_filled.strip())
+            if whole_match_code:
+                templ_list_filled = self._pattern_list.sub(
+                    lambda x: str(get_address_value(x, return_all_matches=True)),
+                    whole_match_code.group(1)
+                )
+                return get_code_value(templ_list_filled)
+            unpacked_filled = self._pattern_unpack.sub(string_filler_unpack, templ_values_filled)
+            return self._pattern_code.sub(
+                lambda x: self._stringer(get_code_value(x.group(1))),
+                unpacked_filled
+            )
+
         if isinstance(templ, list):
             out = []
             for idx, elem in enumerate(templ):
                 new_path = f"{current_path}[{idx}]"
                 elem_filled = self._recursive_subst(
-                    templ=elem, current_path=new_path, always_list=always_list, relative_path_anchor=get_relative_path(new_path),
+                    templ=elem,
+                    current_path=new_path,
+                    relative_path_anchor=get_relative_path(new_path),
+                    level=0,
                 )
-                if isinstance(elem, str) and self._pattern_template_unpack.fullmatch(elem):
+                if isinstance(elem, str) and self._pattern_unpack.fullmatch(elem.strip()):
                     out.extend(elem_filled)
                 else:
                     out.append(elem_filled)
             return out
+
         if isinstance(templ, dict):
             new_dict = {}
             for key, val in templ.items():
                 key_filled = self._recursive_subst(
-                    templ=key, current_path=current_path, always_list=False, relative_path_anchor=relative_path_anchor,
+                    templ=key,
+                    current_path=current_path,
+                    relative_path_anchor=relative_path_anchor,
+                    level=0,
                 )
+                if isinstance(key, str) and self._pattern_unpack.fullmatch(key.strip()):
+                    new_dict.update(key_filled)
+                    continue
                 if key_filled in self._template_keys:
                     new_dict[key_filled] = val
                     continue
                 new_path = f"{current_path}.'{key_filled}'"
                 new_dict[key_filled] = self._recursive_subst(
-                    templ=val, current_path=new_path, always_list=always_list, relative_path_anchor=get_relative_path(new_path),
+                    templ=val,
+                    current_path=new_path,
+                    relative_path_anchor=get_relative_path(new_path),
+                    level=0,
                 )
             return new_dict
         return templ
@@ -334,4 +442,24 @@ class TemplateFiller:
             return
         fields = []
         _recursive_extract(jsonpath)
-        return fields
+        return fields
+
+
+class _RegexPattern:
+
+    def __init__(self, start: str, end: str):
+        start_esc = _re.escape(start)
+        end_esc = _re.escape(end)
+        self.pattern = _re.compile(rf"{start_esc}(.*?)(?={end_esc}){end_esc}", re.DOTALL)
+        return
+
+    def fullmatch(self, string: str) -> _re.Match | None:
+        # Use findall to count occurrences of segments in the text
+        matches = self.pattern.findall(string)
+        if len(matches) == 1:
+            # Verify the match spans the entire string
+            return self.pattern.fullmatch(string)
+        return None
+
+    def sub(self, repl, string: str) -> str:
+        return self.pattern.sub(repl, string)