FourCIPP 1.50.0__py3-none-any.whl

fourcipp/utils/not_set.py

@@ -0,0 +1,77 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2025 FourCIPP Authors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""Not set utils."""
+
+
+class NotSet:
+    """Not set object."""
+
+    def __init__(self, expected: object = object) -> None:
+        """Not set object.
+
+        Args:
+            expected: Expected object to to display
+        """
+        self.expected = expected
+
+    def __str__(self) -> str:  # pragma: no cover
+        """String method."""
+        return f"NotSet({self.expected})"
+
+    def __repr__(self) -> str:  # pragma: no cover
+        """Representation method."""
+        return f"NotSet({self.expected})"
+
+
+NOT_SET = NotSet()
+
+
+def check_if_set(obj: object) -> bool:
+    """Check if object or is NotSet.
+
+    Args:
+        obj: Object to check
+
+    Returns:
+        True if object is set
+    """
+    # Check if object is not of type _NotSet, i.e. it has a value
+    return not isinstance(obj, NotSet)
+
+
+def pop_arguments(key: str, default: object = NOT_SET) -> tuple:
+    """Create arguments for the pop method.
+
+    We need this utility since pop is not implemented using kwargs, instead the default is checked
+     via the number of arguments.
+
+    Args:
+        key: Key to pop the value for
+        default: Default value to return in case of the pop value.
+
+    Returns:
+        Arguments for pop
+    """
+    if check_if_set(default):
+        return (key, default)
+    else:
+        return (key,)

fourcipp/utils/type_hinting.py

@@ -0,0 +1,44 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2025 FourCIPP Authors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""Type hinting utils."""
+
+import pathlib
+from collections.abc import Callable
+from typing import TypeAlias, TypeVar
+
+from fourcipp.utils.not_set import NotSet
+
+# Generic type variable
+T = TypeVar("T")
+
+# For paths we commonly use string or pathlib.path
+Path: TypeAlias = pathlib.Path | str
+
+
+# For casting dicts
+Extractor: TypeAlias = Callable[[str], T]
+LineListExtractor: TypeAlias = Callable[[list[str]], T]
+LineCastingDict: TypeAlias = dict[str, LineListExtractor]
+NestedCastingDict: TypeAlias = dict[str, LineCastingDict | LineListExtractor]
+
+# NotSet
+NotSetAlias: TypeAlias = NotSet | T

fourcipp/utils/validation.py

@@ -0,0 +1,155 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2025 FourCIPP Authors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""Validation utils."""
+
+from __future__ import annotations
+
+import sys
+from collections.abc import Iterable, Sequence
+
+import jsonschema_rs
+
+from fourcipp.utils.yaml_io import dict_to_yaml_string
+
+MAX_INT = 2_147_483_647  # C++ value
+MAX_FLOAT = sys.float_info.max
+
+
+class ValidationError(Exception):
+    """FourCIPP validation error."""
+
+    @staticmethod
+    def path_indexer(path: Sequence[str | int]) -> str:
+        """Create a path representation to walk the dict."""
+        path_for_data = ""
+        for p in path:
+            if isinstance(p, str):
+                p = '"' + p + '"'
+            path_for_data += "[" + str(p) + "]"
+        return path_for_data
+
+    @staticmethod
+    def indent(text: str, n_spaces: int = 4) -> str:
+        """Indent the text."""
+        indent_with_newline = "\n" + " " * n_spaces
+        return indent_with_newline + text.replace("\n", indent_with_newline)
+
+    @classmethod
+    def from_schema_validation_errors(
+        cls, errors: Iterable[jsonschema_rs.ValidationError]
+    ) -> ValidationError:
+        """Create error from multiple errors.
+
+        Args:
+            errors: Errors to raise
+
+        Returns:
+            New error for this case
+        """
+        message = "\nValidation failed, due to the following parameters:"
+
+        for error in errors:
+            message += "\n\n- Parameter in " + cls.path_indexer(error.instance_path)
+            message += cls.indent(cls.indent(dict_to_yaml_string(error.instance), 4))
+            message += cls.indent("Error: " + error.message, 2)
+
+        return cls(message)
+
+    @classmethod
+    def from_overflow_errors(
+        cls, object_paths_with_errors: Iterable[tuple[list[str | int], int | float]]
+    ) -> ValidationError:
+        """Create error from multiple errors.
+
+        Args:
+            errors: Errors to raise
+
+        Returns:
+            New error for this case
+        """
+        message = "\nValidation failed, due to the following parameters:"
+
+        for path, obj in object_paths_with_errors:
+            message += "\n\n- Parameter in " + cls.path_indexer(path)
+            message += cls.indent(cls.indent(str(obj), 4))
+
+            if isinstance(obj, int):
+                error = f"Maximum int value {MAX_INT} exceeded"
+            else:
+                error = f"Maximum float value {MAX_FLOAT} exceeded"
+            message += cls.indent("Error: " + error, 2)
+
+        return cls(message)
+
+
+def find_keys_exceeding_max_value(
+    obj: object, path_for_data: list[str | int] | None = None
+) -> Iterable[tuple[list[str | int], int | float]]:
+    """Find entries exceeding max values.
+
+    Args:
+        obj (object): Object to check
+        path_for_data: Path to the data
+
+    Yields:
+        path for each case where this problem emerges
+    """
+    if path_for_data is None:
+        path_for_data = []
+
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            yield from find_keys_exceeding_max_value(value, path_for_data + [key])
+    elif isinstance(obj, list):
+        for index, item in enumerate(obj):
+            yield from find_keys_exceeding_max_value(item, path_for_data + [index])
+    elif isinstance(obj, int) and abs(obj) > MAX_INT:
+        yield path_for_data, obj
+    elif isinstance(obj, float) and abs(obj) > MAX_FLOAT:
+        yield path_for_data, obj
+
+
+def validate_using_json_schema(data: dict, json_schema: dict) -> bool:
+    """Validate data using a JSON schema.
+
+    Args:
+        data: Data to validate
+        json_schema: Schema for validation
+
+    Returns:
+        True if successful
+    """
+    validator = jsonschema_rs.validator_for(json_schema)
+    try:
+        validator.validate(data)
+    except jsonschema_rs.ValidationError as exception:
+        # Validation failed, look for all errors
+        raise ValidationError.from_schema_validation_errors(
+            validator.iter_errors(data)
+        ) from exception
+    except ValueError as exception:
+        if str(exception).endswith("too big to convert"):
+            raise ValidationError.from_overflow_errors(
+                find_keys_exceeding_max_value(data)
+            ) from exception
+        raise
+    return True

fourcipp/utils/yaml_io.py

@@ -0,0 +1,172 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2025 FourCIPP Authors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+"""YAML io."""
+
+import json
+import pathlib
+from typing import Callable
+
+import regex
+import ryml
+
+from fourcipp.utils.type_hinting import Path
+
+
+def load_yaml(path_to_yaml_file: Path) -> dict:
+    """Load yaml files.
+
+    rapidyaml is the fastest yaml parsing library we could find. Since it returns custom objects we
+    use the library to emit the objects to json and subsequently read it in using the json library.
+    This is still two orders of magnitude faster compared to other yaml libraries.
+
+    Args:
+        path_to_yaml_file: Path to yaml file
+
+    Returns:
+       Loaded data
+    """
+
+    json_str = ryml.emit_json(
+        ryml.parse_in_arena(pathlib.Path(path_to_yaml_file).read_bytes())
+    )
+
+    # Convert `inf` to a string to avoid JSON parsing errors, see https://github.com/biojppm/rapidyaml/issues/312
+    json_str = regex.sub(r":\s*(-?)inf\b", r': "\1inf"', json_str)
+
+    # Convert floats that are missing digits on either side of the decimal point
+    # so .5 to 0.5 and 5. to 5.0
+    json_str = regex.sub(r":\s*(-?)\.([0-9]+)", r": \g<1>0.\2", json_str)
+    json_str = regex.sub(r":\s*(-?)([0-9]+)\.(\D)", r": \1\2.0\3", json_str)
+
+    data = json.loads(json_str)
+
+    return data
+
+
+def dict_to_yaml_string(
+    data: dict,
+    sort_function: Callable[[dict], dict] | None = None,
+    use_fourcipp_yaml_style: bool = True,
+) -> str:
+    """Dump dict as yaml.
+
+    The FourCIPP yaml style sets flow
+    Args:
+        data: Data to dump.
+        sort_function: Function to sort the data.
+        use_fourcipp_yaml_style: If FourCIPP yaml style is to be used
+
+    Returns:
+        YAML string representation of the data
+    """
+
+    if sort_function is not None:
+        data = sort_function(data)
+
+    # Convert dictionary into a ryml tree
+    tree = ryml.parse_in_arena(bytearray(json.dumps(data).encode("utf8")))
+
+    def check_is_vector(tree: ryml.Tree, node_id: int) -> bool:
+        """Check if sequence is of ints, floats or sequence there of.
+
+        In 4C metadata terms, list of strings, bools, etc. could also be vectors.
+        For the sake of simplicity these are omitted.
+
+        Args:
+            tree (ryml.Tree): Tree to check
+            node_id (int): Node id
+
+        Returns:
+            returns if entry is a vector
+        """
+
+        for sub_node, _ in ryml.walk(tree, node_id):
+            # Ignore the root node
+            if sub_node == node_id:
+                continue
+
+            # If sequence contains a dict
+            if tree.is_map(sub_node):
+                return False
+
+            # If sequence contains a sequence
+            elif tree.is_seq(sub_node):
+                if not check_is_vector(tree, sub_node):
+                    return False
+
+            # Else it's a value
+            else:
+                val = tree.val(sub_node).tobytes().decode("ascii")
+                is_not_numeric = (
+                    tree.is_val_quoted(sub_node)  # string
+                    or tree.val_is_null(sub_node)  # null
+                    or val == "true"  # bool
+                    or val == "false"  # bool
+                )
+                if is_not_numeric:
+                    return False
+
+        return True
+
+    # Change style bits to avoid JSON output and format vectors correctly
+    # see https://github.com/biojppm/rapidyaml/issues/520
+    for node_id, depth in ryml.walk(tree):
+        if tree.is_map(node_id):
+            tree.set_container_style(node_id, ryml.NOTYPE)
+        elif tree.is_seq(node_id):
+            if (
+                not use_fourcipp_yaml_style  # do not do special formatting
+                or depth == 1  # is a section
+                or not check_is_vector(tree, node_id)  # is not a vector
+            ):
+                tree.set_container_style(node_id, ryml.NOTYPE)
+
+        if tree.has_key(node_id):
+            tree.set_key_style(node_id, ryml.NOTYPE)
+
+    yaml_string = ryml.emit_yaml(tree)
+
+    if use_fourcipp_yaml_style:
+        # add spaces after commas in vectors
+        yaml_string = regex.sub(r"(?<=\d),(?=\d)|(?<=\]),(?=\[)", ", ", yaml_string)
+
+    return yaml_string
+
+
+def dump_yaml(
+    data: dict,
+    path_to_yaml_file: Path,
+    sort_function: Callable[[dict], dict] | None = None,
+    use_fourcipp_yaml_style: bool = True,
+) -> None:
+    """Dump yaml to file.
+
+    Args:
+        data: Data to dump.
+        path_to_yaml_file: Yaml file path
+        sort_function: Function to sort the data
+        use_fourcipp_yaml_style: If FourCIPP yaml style is to be used
+    """
+    pathlib.Path(path_to_yaml_file).write_text(
+        dict_to_yaml_string(data, sort_function, use_fourcipp_yaml_style),
+        encoding="utf-8",
+    )

fourcipp/version.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '1.50.0'
+__version_tuple__ = version_tuple = (1, 50, 0)
+
+__commit_id__ = commit_id = None