fameio 2.3.1__py3-none-any.whl → 3.1.0__py3-none-any.whl

CHANGELOG.md

@@ -1,8 +1,38 @@
-<!-- SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+<!-- SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 
 SPDX-License-Identifier: CC0-1.0 -->
 
+## [3.1.0](https://gitlab.com/fame-framework/fame-io/-/tags/v3.1.0) - 2025-01-29
+### Changed
+- Speed up of `makeFameRunConfig` for large CSV files #229 (@dlr-cjs, dlr_fn)
+- Improve testing of `tools.py` #227 (@dlr_fn)
+- Reorganize badges in tabular representation in `README.md` #226 (@dlr-cjs, dlr_fn)
+
 # Changelog
+## [3.0.0](https://gitlab.com/fame-framework/fame-io/-/tags/v3.0.0) - 2024-12-02
+### Changed
+- **Breaking**: Update to fameprotobuf v2.0.2 #208, #215 (@dlr-cjs)
+- **Breaking**: Remove section `GeneralProperties.Output` in scenarios - any content there will be ignored #208 (@dlr-cjs)
+- **Breaking**: Set section `JavaPackages` in schema to be mandatory #208 (@dlr-cjs)
+- **Breaking**: Update header of protobuf files to "fameprotobufstreamfilev002    " - disable reading of old files #208, #214 (@dlr-cjs)
+- **Breaking**: Replace subparser from command-line argument `--time-merging` with a threefold argument #212 (@dlr-cjs)
+- **Breaking**: Attribute names "value", "values", and "metadata" are now disallowed as they are reserved for the Metadata implementation #217 (@dlr-cjs)
+- **Breaking**: Refactor package structure #137 (@dlr_fn, @dlr-cjs)
+- **Breaking**: Refactor PathResolver #219 (@dlr-cjs)
+- **Breaking**: Rename all Exceptions to Errors #114 (@dlr-cjs)
+- **Breaking**: Rename all `_KEY` words in packages `scenario` and `schema` removing their underscore in the beginning #222 (@dlr-cjs)
+- Use `Metadata` for `Agent` and `Contract` #209, #224 (@dlr-cjs)
+- Allow `DataItems` to be left out on new mandatory section `JavaPackges` #216 (@dlr-cjs)
+- Complete refactoring of loader.py to improve readability and testability #116, #117, #119, #219, #220 (@dlr-cjs)
+
+### Added
+- Add StringSet writing to protobuf file #208 (@dlr-cjs)
+- Add `Metadata` to `Scenario` and 'Attribute', as well as schema elements `AgentType`, and `AttributeSpecs` #209, #217, #218, (@dlr-cjs, @dlr_fn)
+- Add file UPGRADING.md to describe actions necessary to deal with breaking changes #208 (@dlr-cjs)
+
+### Removed
+- Drop class `Args` in `loader.py` #115 (@dlr-cjs)
+
 ## [2.3.1](https://gitlab.com/fame-framework/fame-io/-/tags/v2.3.1) - 2024-08-26
 ### Fixed
 - Fix ignored default values of `convert_results` for `merge-times` arguments #211 (@dlr-cjs, dlr_fn)

fameio/__init__.py

@@ -1,3 +1,6 @@
-# SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: CC0-1.0
+
+FILE_HEADER_V1 = "famecoreprotobufstreamfilev001"  # noqa
+FILE_HEADER_V2 = "fameprotobufstreamfilev002    "  # noqa

fameio/{source/cli → cli}/__init__.py

@@ -1,3 +1,5 @@
 # SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+
+from fameio.cli.parser import update_default_config

fameio/{source/cli → cli}/convert_results.py

@@ -2,10 +2,10 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 import argparse
-from typing import List, Dict, Any, Optional
+from typing import Any, Optional
 
-from fameio.source.cli.options import Options, ResolveOptions, TimeOptions
-from fameio.source.cli.parser import (
+from fameio.cli.options import Options, ResolveOptions, TimeOptions
+from fameio.cli.parser import (
     add_file_argument,
     add_log_level_argument,
     add_logfile_argument,
@@ -15,7 +15,7 @@ from fameio.source.cli.parser import (
     add_memory_saving_argument,
     add_resolve_complex_argument,
     add_time_argument,
-    add_merge_time_parser,
+    add_merge_time_argument,
     add_inputs_recovery_argument,
     map_namespace_to_options_dict,
 )
@@ -30,7 +30,7 @@ CLI_DEFAULTS = {
     Options.MEMORY_SAVING: False,
     Options.RESOLVE_COMPLEX_FIELD: ResolveOptions.SPLIT,
     Options.TIME: TimeOptions.UTC,
-    Options.TIME_MERGING: {},
+    Options.TIME_MERGING: None,
     Options.INPUT_RECOVERY: False,
 }
 
@@ -38,7 +38,7 @@ _INFILE_PATH_HELP = "Provide path to protobuf file"
 _OUTFILE_PATH_HELP = "Provide path to folder to store output .csv files"
 
 
-def handle_args(args: List[str], defaults: Optional[Dict[Options, Any]] = None) -> Dict[Options, Any]:
+def handle_args(args: list[str], defaults: Optional[dict[Options, Any]] = None) -> dict[Options, Any]:
     """
     Handles command line arguments and returns `run_config` for convert_results script
 
@@ -54,7 +54,7 @@ def handle_args(args: List[str], defaults: Optional[Dict[Options, Any]] = None)
     return map_namespace_to_options_dict(parsed)
 
 
-def _prepare_parser(defaults: Optional[Dict[Options, Any]]) -> argparse.ArgumentParser:
+def _prepare_parser(defaults: Optional[dict[Options, Any]]) -> argparse.ArgumentParser:
     """
     Creates a parser with given defaults to handle `make_config` configuration arguments
 
@@ -73,7 +73,7 @@ def _prepare_parser(defaults: Optional[Dict[Options, Any]]) -> argparse.Argument
     add_memory_saving_argument(parser, _get_default(defaults, Options.MEMORY_SAVING))
     add_resolve_complex_argument(parser, _get_default(defaults, Options.RESOLVE_COMPLEX_FIELD))
     add_time_argument(parser, _get_default(defaults, Options.TIME))
-    add_merge_time_parser(parser, _get_default(defaults, Options.TIME_MERGING))
+    add_merge_time_argument(parser, _get_default(defaults, Options.TIME_MERGING))
     add_inputs_recovery_argument(parser, _get_default(defaults, Options.INPUT_RECOVERY))
 
     return parser

fameio/{source/cli → cli}/make_config.py

@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 from pathlib import Path
-from typing import List, Dict, Any, Optional
+from typing import Any, Optional
 
-from fameio.source.cli.options import Options
-from fameio.source.cli.parser import (
+from fameio.cli.options import Options
+from fameio.cli.parser import (
     add_file_argument,
     add_log_level_argument,
     add_logfile_argument,
@@ -31,7 +31,7 @@ _ENCODING_HELP = (
 )
 
 
-def handle_args(args: List[str], defaults: Optional[Dict[Options, Any]] = None) -> Dict[Options, Any]:
+def handle_args(args: list[str], defaults: Optional[dict[Options, Any]] = None) -> dict[Options, Any]:
     """
     Converts given `arguments` and returns a configuration for the make_config script
 
@@ -47,7 +47,7 @@ def handle_args(args: List[str], defaults: Optional[Dict[Options, Any]] = None)
     return map_namespace_to_options_dict(parsed)
 
 
-def _prepare_parser(defaults: Optional[Dict[Options, Any]]) -> argparse.ArgumentParser:
+def _prepare_parser(defaults: Optional[dict[Options, Any]]) -> argparse.ArgumentParser:
     """
     Creates a parser with given defaults to handle `make_config` configuration arguments
 

fameio/{source/cli → cli}/options.py

@@ -49,11 +49,3 @@ class ResolveOptions(ParsableEnum, Enum):
 
     IGNORE = auto()
     SPLIT = auto()
-
-
-class MergingOptions(Enum):
-    """Specifies options for merging TimeSteps"""
-
-    FOCAL_POINT = auto()
-    STEPS_BEFORE = auto()
-    STEPS_AFTER = auto()

fameio/{source/cli → cli}/parser.py

@@ -5,14 +5,14 @@ import copy
 from argparse import ArgumentParser, ArgumentTypeError, BooleanOptionalAction, Namespace
 from enum import Enum
 from pathlib import Path
-from typing import Optional, Dict, Any, List, Union
+from typing import Optional, Any, Union
 
-from fameio.source.cli.options import MergingOptions, TimeOptions, ResolveOptions, Options
-from fameio.source.logs import LogLevel
+from fameio.cli.options import TimeOptions, ResolveOptions, Options
+from fameio.logs import LogLevel
 
-_ERR_NEGATIVE_INT = "Given value `{}` is not a non-negative int."
+_ERR_INVALID_MERGING_DEFAULT = "Invalid merge-times default: needs list of 3 integers separated by spaces but was: '{}'"
 
-_OPTION_ARGUMENT_NAME: Dict[str, Union[Options, Dict]] = {
+_OPTION_ARGUMENT_NAME: dict[str, Union[Options, dict]] = {
     "file": Options.FILE,
     "log": Options.LOG_LEVEL,
     "logfile": Options.LOG_FILE,
@@ -24,14 +24,7 @@ _OPTION_ARGUMENT_NAME: Dict[str, Union[Options, Dict]] = {
     "time": Options.TIME,
     "input_recovery": Options.INPUT_RECOVERY,
     "complex_column": Options.RESOLVE_COMPLEX_FIELD,
-    "time_merging": {
-        "name": Options.TIME_MERGING,
-        "inner_elements": {
-            "focal_point": MergingOptions.FOCAL_POINT,
-            "steps_before": MergingOptions.STEPS_BEFORE,
-            "steps_after": MergingOptions.STEPS_AFTER,
-        },
-    },
+    "merge_times": Options.TIME_MERGING,
 }
 
 
@@ -51,7 +44,7 @@ def add_file_argument(parser: ArgumentParser, default: Optional[Path], help_text
         parser.add_argument("-f", "--file", type=Path, required=True, help=help_text)
 
 
-def add_select_agents_argument(parser: ArgumentParser, default: List[str]) -> None:
+def add_select_agents_argument(parser: ArgumentParser, default: list[str]) -> None:
     """Adds optional repeatable string argument 'agent' to given `parser`"""
     help_text = "Provide list of agents to extract (default=None)"
     parser.add_argument("-a", "--agents", nargs="*", type=str, default=default, help=help_text)
@@ -71,6 +64,7 @@ def add_output_argument(parser: ArgumentParser, default_value, help_text: str) -
 def add_log_level_argument(parser: ArgumentParser, default_value: str) -> None:
     """Adds optional argument 'log' to given `parser`"""
     help_text = "choose logging level (default: {})".format(default_value)
+    # noinspection PyTypeChecker
     parser.add_argument(
         "-l",
         "--log",
@@ -138,73 +132,22 @@ def add_time_argument(parser: ArgumentParser, default_value: Union[TimeOptions,
     )
 
 
-def add_merge_time_parser(parser: ArgumentParser, defaults: Optional[Dict[MergingOptions, int]]) -> None:
-    """
-    Adds subparser for merging of TimeSteps to given `parser`
-    If at least one valid time merging option is specified in given defaults, calling the subparser becomes mandatory
-    """
-    defaults = defaults if (defaults is not None) and (isinstance(defaults, dict)) else {}
-    if any([option in defaults.keys() for option in MergingOptions]):
-        subparser = parser.add_subparsers(dest="time_merging", required=True, help="Optional merging of TimeSteps")
-    else:
-        subparser = parser.add_subparsers(dest="time_merging", required=False, help="Optional merging of TimeSteps")
-    group_parser = subparser.add_parser("merge-times")
-    add_focal_point_argument(group_parser, defaults.get(MergingOptions.FOCAL_POINT, None))
-    add_steps_before_argument(group_parser, defaults.get(MergingOptions.STEPS_BEFORE, None))
-    add_steps_after_argument(group_parser, defaults.get(MergingOptions.STEPS_AFTER, None))
-
-
-def add_focal_point_argument(parser: ArgumentParser, default_value: Optional[int]) -> None:
-    """Adds `focal-point` argument to given `parser`"""
-    help_text = "TimeStep on which `steps_before` earlier and `steps_after` later TimeSteps are merged on"
-    if default_value is not None:
-        parser.add_argument("-fp", "--focal-point", required=False, type=int, help=help_text, default=default_value)
-    else:
-        parser.add_argument("-fp", "--focal-point", required=True, type=int, help=help_text)
-
-
-def add_steps_before_argument(parser: ArgumentParser, default_value: Optional[int]) -> None:
-    """Adds `steps-before` argument to given `parser`"""
-    help_text = "Range of TimeSteps before the `focal-point` they get merged to"
-    if default_value is not None:
-        parser.add_argument(
-            "-sb", "--steps-before", required=False, type=_non_negative_int, help=help_text, default=default_value
-        )
-    else:
-        parser.add_argument("-sb", "--steps-before", required=True, type=_non_negative_int, help=help_text)
-
-
-def _non_negative_int(value: Any) -> int:
-    """
-    Casts a given ´value` to int and checks it for non-negativity
-
-    Args:
-        value: to check and parse
-
-    Returns:
-        `value` parsed to int if it is a non-negative integer
-
-    Raises:
-        TypeError: if `value` is None
-        ValueError: if `value` cannot be parsed to int
-        argparse.ArgumentTypeError: if `value` is a negative int
-
-    """
-    value = int(value)
-    if value < 0:
-        raise ArgumentTypeError(_ERR_NEGATIVE_INT.format(value))
-    return value
-
-
-def add_steps_after_argument(parser: ArgumentParser, default_value: Optional[int]) -> None:
-    """Adds `steps-after` argument to given `parser`"""
-    help_text = "Range of TimeSteps after the `focal-point` they get merged to"
-    if default_value is not None:
-        parser.add_argument(
-            "-sa", "--steps-after", required=False, type=_non_negative_int, help=help_text, default=default_value
-        )
-    else:
-        parser.add_argument("-sa", "--steps-after", required=True, type=_non_negative_int, help=help_text)
+def add_merge_time_argument(parser: ArgumentParser, defaults: Optional[list[int]] = None) -> None:
+    """Adds optional three-fold argument for merging of TimeSteps to given `parser`"""
+    if defaults is None:
+        defaults = []
+    if (
+        not isinstance(defaults, list)
+        or len(defaults) not in [0, 3]
+        or not all([isinstance(value, int) for value in defaults])
+    ):
+        raise ArgumentTypeError(_ERR_INVALID_MERGING_DEFAULT.format(repr(defaults)))
+
+    help_text = (
+        "Merge multiple time steps to have less lines per output file. "
+        "Provide 3 integers separated by spaces that resemble FocalPoint, StepsBefore, and StepsAfter."
+    )
+    parser.add_argument("-mt", "--merge-times", type=int, nargs=3, default=defaults, help=help_text)
 
 
 def add_inputs_recovery_argument(parser: ArgumentParser, default: bool) -> None:
@@ -227,7 +170,7 @@ def update_default_config(config: Optional[dict], default: dict) -> dict:
     return result
 
 
-def map_namespace_to_options_dict(parsed: Namespace) -> Dict[Options, Any]:
+def map_namespace_to_options_dict(parsed: Namespace) -> dict[Options, Any]:
     """
     Maps given parsing results to their corresponding configuration option
 
@@ -240,7 +183,7 @@ def map_namespace_to_options_dict(parsed: Namespace) -> Dict[Options, Any]:
     return _map_namespace_to_options(parsed, _OPTION_ARGUMENT_NAME)
 
 
-def _map_namespace_to_options(parsed: Namespace, names_to_options: Dict[str, Enum]) -> Dict[Options, Any]:
+def _map_namespace_to_options(parsed: Namespace, names_to_options: dict[str, Enum]) -> dict[Options, Any]:
     """
     Maps given parsing results to their corresponding configuration option; elements that cannot be mapped are ignored.
     If a configuration option has inner elements, these well be also read and added as inner dictionary.

fameio/input/__init__.py

@@ -0,0 +1,27 @@
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+class InputError(Exception):
+    """An error that occurred while parsing any kind of input"""
+
+    pass
+
+
+class SchemaError(InputError):
+    """An error that occurred while parsing a Schema"""
+
+    pass
+
+
+class ScenarioError(InputError):
+    """An error that occurred while parsing a Scenario"""
+
+    pass
+
+
+class YamlLoaderError(InputError):
+    """An error that occurred while parsing a YAML file"""
+
+    pass

fameio/input/loader/__init__.py

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from fameio.input import YamlLoaderError
+from fameio.input.resolver import PathResolver
+from fameio.input.loader.controller import LoaderController
+from fameio.input.loader.loader import FameYamlLoader
+from fameio.logs import log, log_critical_and_raise
+
+ALLOWED_SUFFIXES: tuple[str, ...] = (".yaml", ".yml")
+
+_INFO_LOADING = "Loading YAML file at '{}'."
+_ERR_NO_YAML_SUFFIX = "Only these file suffixes are allowed: {}, but the file suffix was: '{}'."
+
+__CONTROLLERS: list[LoaderController] = [LoaderController()]
+
+
+def _include_callback(own_loader: FameYamlLoader, args: yaml.Node) -> Any:
+    """Uses single instance of _LoaderController to load data whenever an !include-command is found"""
+    return __CONTROLLERS[0].include(own_loader, args)
+
+
+# All FameYamlLoader use the same LoaderController - which can in turn spawn more FameYamlLoader
+FameYamlLoader.add_constructor(FameYamlLoader.INCLUDE_COMMAND, _include_callback)
+
+
+def load_yaml(yaml_file_path: Path, path_resolver: PathResolver = PathResolver(), encoding: str = None) -> dict:
+    """
+    Loads the YAML file from given and returns its content as a dict
+
+    Args:
+        yaml_file_path: Path to the YAML file that is to be read
+        path_resolver: PathResolver to be used to resolve Paths specified within the YAML file
+        encoding: of the YAML file (and all referenced YAML files using !include), platform default is used if omitted
+
+    Returns:
+        Content of the specified YAML file
+
+    Raises:
+        YamlLoaderError: if the YAML file could not be read
+    """
+    log().info(_INFO_LOADING.format(yaml_file_path))
+    _update_current_controller(path_resolver, encoding)
+    return __CONTROLLERS[0].load(yaml_file_path)
+
+
+def _update_current_controller(path_resolver: PathResolver, encoding: str) -> None:
+    """Updates the current LoaderController to use the given `path_resolver` and `encoding`"""
+    __CONTROLLERS[0] = LoaderController(path_resolver, encoding)
+
+
+def validate_yaml_file_suffix(yaml_file: Path) -> None:
+    """
+    Ensures that given file has a file suffix compatible with YAML
+
+    Args:
+        yaml_file: that is to be checked for suffix correctness
+
+    Raises:
+          YamlLoaderError: if given file has no YAML-associated file suffix
+    """
+    if yaml_file.suffix.lower() not in ALLOWED_SUFFIXES:
+        log_critical_and_raise(YamlLoaderError(_ERR_NO_YAML_SUFFIX.format(ALLOWED_SUFFIXES, yaml_file)))

fameio/input/loader/controller.py

@@ -0,0 +1,129 @@
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+from fnmatch import fnmatch
+from pathlib import Path
+from typing import Callable, IO, Any, Final
+
+import yaml
+
+from fameio.input import YamlLoaderError
+from fameio.input.resolver import PathResolver
+from fameio.input.loader.loader import FameYamlLoader
+from fameio.logs import log, log_critical_and_raise
+
+
+class LoaderController:
+    """
+    Controls loading of YAML files by spawning one FameYamlLoader per file.
+    Uses same PathResolver and encoding for all files
+    """
+
+    DISABLING_YAML_FILE_PREFIX: Final[str] = "IGNORE_"
+    NODE_SPLIT_STRING: Final[str] = ":"
+
+    _ERR_NODE_MISSING = "'!include_node [{}, {}]': Cannot find '{}'"
+    _ERR_NOT_LIST = "!include can only combine list-like elements from multiple files!"
+    _WARN_NOTHING_TO_INCLUDE = "Could not find any files matching this '!include' directive '{}'"
+    _INFO_FILE_IGNORED = "Ignoring file '{}' due to prefix '{}'"
+    _DEBUG_SEARCH_NODE = "Searched file '{}' for node '{}'"
+    _DEBUG_JOIN_COMPLETE = "Joined all files '{}' to joined data '{}'"
+    _DEBUG_LOAD_FILE = "Loaded included YAML file '{}'"
+    _DEBUG_FILES_INCLUDED = "!include directive '{}' yielded these files: '{}'"
+
+    def __init__(self, path_resolver: PathResolver = PathResolver(), encoding: str = None) -> None:
+        self._path_resolver = path_resolver
+        self._encoding: str = encoding
+
+    def load(self, yaml_file_path: Path) -> dict:
+        """Spawns a new FameYamlLoader, loads the given `yaml_file_path` and returns its content"""
+        with open(yaml_file_path, "r", encoding=self._encoding) as configfile:
+            data = yaml.load(configfile, self._spawn_loader_builder())
+        return data
+
+    @staticmethod
+    def _spawn_loader_builder() -> Callable[[IO], FameYamlLoader]:
+        """Returns a new Callable that instantiates a new FameYamlLoader with an IO-stream"""
+        return lambda stream: FameYamlLoader(stream)
+
+    def include(self, loader: FameYamlLoader, include_args: yaml.Node) -> Any:
+        """Returns content loaded from the specified `include_args`"""
+        root_path, file_pattern, node_pattern = loader.digest_include(include_args)
+        files = self._resolve_imported_path(root_path, file_pattern)
+        nodes = node_pattern.split(self.NODE_SPLIT_STRING)
+
+        joined_data = None
+        for file_name in files:
+            file_data = self.load(Path(file_name))
+            extracted_node_data = self._extract_node(file_name, file_data, nodes)
+            joined_data = self._join_data(extracted_node_data, joined_data)
+            log().debug(self._DEBUG_LOAD_FILE.format(file_name))
+        log().debug(self._DEBUG_JOIN_COMPLETE.format(files, joined_data))
+        return joined_data
+
+    def _resolve_imported_path(self, root_path: str, include_pattern: str) -> list[str]:
+        """
+        Returns a list of file paths matching the given `include_pattern` relative to the `root_path`.
+        Ignores files starting with the `DISABLING_YAML_FILE_PREFIX`
+        """
+        file_list = self._path_resolver.resolve_file_pattern(root_path, include_pattern)
+        ignore_filter = f"*{self.DISABLING_YAML_FILE_PREFIX}*"
+
+        cleaned_file_list = []
+        for file in file_list:
+            if fnmatch(file, ignore_filter):
+                log().info(self._INFO_FILE_IGNORED.format(file, self.DISABLING_YAML_FILE_PREFIX))
+            else:
+                cleaned_file_list.append(file)
+        if not cleaned_file_list:
+            log().warning(self._WARN_NOTHING_TO_INCLUDE.format(include_pattern))
+        log().debug(self._DEBUG_FILES_INCLUDED.format(include_pattern, cleaned_file_list))
+        return cleaned_file_list
+
+    @staticmethod
+    def _extract_node(file_name: str, data: dict, node_address: list[str]) -> Any:
+        """
+        Returns only the part of the data that is at the specified node address
+
+        Args:
+            file_name: name of the file from which the data were read - used to enrich logging messages
+            data: in which the given node address is searched for; only content below this address is returned
+            node_address: list of nodes to be accessed in data; each node must be an inner element of the previous node
+
+        Returns:
+            Subset of the given data located at the specified node address
+
+        Raises:
+            YamlLoaderError: if any node in the address is not found
+        """
+        for node in node_address:
+            if node:
+                if node not in data.keys():
+                    message = LoaderController._ERR_NODE_MISSING.format(file_name, node_address, node)
+                    log_critical_and_raise(YamlLoaderError(message))
+                data = data[node]
+        log().debug(LoaderController._DEBUG_SEARCH_NODE.format(file_name, node_address))
+        return data
+
+    @staticmethod
+    def _join_data(new_data: list, previous_data: list) -> list:
+        """
+        Joins two lists with data to a larger list
+
+        Args:
+            new_data: list of any data
+            previous_data: list of any data
+
+        Returns:
+            previous data list extended by content of new data, or new data only if no previous data existed
+
+        Raises:
+            YamlLoaderError: if not both elements are lists
+        """
+        if not previous_data:
+            return new_data
+        if isinstance(new_data, list) and isinstance(previous_data, list):
+            previous_data.extend(new_data)
+            return previous_data
+        else:
+            log_critical_and_raise(YamlLoaderError(LoaderController._ERR_NOT_LIST))

fameio/input/loader/loader.py

@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+from os import path
+from typing import IO, Final
+
+import yaml
+
+from fameio.input import YamlLoaderError
+from fameio.logs import log_critical_and_raise, log
+
+
+class FameYamlLoader(yaml.SafeLoader):
+    """Custom YAML Loader for `!include` constructor"""
+
+    INCLUDE_COMMAND: Final[str] = "!include"
+
+    _ERR_ARGUMENT_COUNT = "!include supports only one or two arguments in list but was: '{}'"
+    _ERR_FILE_KEY_MISSING = "Could not find key 'file' on !include statement in mapping format: {}"
+    _ERR_NODE_TYPE = "YAML node type not implemented: {}"
+    _DEBUG_LOADER_INIT = "Initializing custom YAML loader"
+    _DEBUG_SCALAR_NODE = "Found !include in scalar format. File(s) to include: {}"
+    _DEBUG_SEQUENCE_NODE = "Found !include in sequence format. File(s) to include: {}; Restricted to nodes: {}"
+    _DEBUG_MAPPING_NODE = "Found !include in mapping format. File(s) to include: {}; Restricted to nodes: {}"
+
+    def __init__(self, stream: IO) -> None:
+        log().debug(self._DEBUG_LOADER_INIT)
+        self._root_path = path.split(stream.name)[0] if stream.name is not None else path.curdir
+        super().__init__(stream)
+
+    def digest_include(self, node: yaml.Node) -> tuple[str, str, str]:
+        """
+        Reads arguments in an !include statement and returns information which files to include
+
+        Args:
+            node: the current node that is to be deconstructed; could be a file-pattern to load;
+                  or a list of 1-2 arguments, with the first being the file pattern and
+                  the other being a node address string; or a dict that maps "file" to the pattern
+                  and "node" to the node address string
+
+        Returns:
+            Tuple of (`root`, `file_pattern`, `node_pattern`), where
+              `root` is a path to the current file that was read by this FameYamlLoader,
+              `files` is a file pattern,
+              and nodes is an optional address (list of nodes) for name for the node that is to be returned
+        """
+        node_string = ""
+        file_pattern = None
+        if isinstance(node, yaml.nodes.ScalarNode):
+            file_pattern, node_string = self._read_scalar_node(node)
+        elif isinstance(node, yaml.nodes.SequenceNode):
+            file_pattern, node_string = self._read_sequence_node(node)
+        elif isinstance(node, yaml.nodes.MappingNode):
+            file_pattern, node_string = self._read_mapping_node(node)
+        else:
+            log_critical_and_raise(YamlLoaderError(self._ERR_NODE_TYPE.format(node)))
+        return self._root_path, file_pattern, node_string
+
+    def _read_scalar_node(self, args: yaml.nodes.ScalarNode) -> tuple[str, str]:
+        """
+        Reads and returns content of a scalar !include statement; Example: !include "file"
+
+        Args:
+            args: argument assigned to the !include statement
+
+        Returns:
+           given argument converted to string, an empty string since no node-address can be specified in scalar syntax
+        """
+        file_pattern = self.construct_scalar(args)
+        log().debug(self._DEBUG_SCALAR_NODE.format(file_pattern))
+        return str(file_pattern), ""
+
+    def _read_sequence_node(self, args: yaml.nodes.SequenceNode) -> tuple[str, str]:
+        """
+        Reads and returns content of a sequence !include statement; Example: !include ["file", Path:to:Node]
+
+        Args:
+            args: argument assigned to the !include statement
+
+        Returns:
+            first part of argument as file path, the second part of argument as node-address
+        """
+        argument_list = self.construct_sequence(args)
+        if len(argument_list) not in [1, 2]:
+            log_critical_and_raise(YamlLoaderError(self._ERR_ARGUMENT_COUNT.format(str(args))))
+
+        file_pattern = argument_list[0]
+        node_string = argument_list[1] if len(argument_list) == 2 else ""
+        log().debug(self._DEBUG_SEQUENCE_NODE.format(file_pattern, node_string))
+        return file_pattern, node_string
+
+    def _read_mapping_node(self, args: yaml.nodes.MappingNode) -> tuple[str, str]:
+        """
+        Reads and returns content of a mapping !include statement; Example: !include {file="file", node="Path:to:Node"}
+
+        Args:
+            args: argument assigned to the !include statement
+
+        Returns:
+            file argument as file path, node argument as node-address
+        """
+        argument_map = {str(k).lower(): v for k, v in self.construct_mapping(args).items()}
+        if "file" not in argument_map.keys():
+            log_critical_and_raise(YamlLoaderError(self._ERR_FILE_KEY_MISSING.format(str(args))))
+
+        file_pattern = argument_map["file"]
+        node_string = argument_map.get("node", "")
+        log().debug(self._DEBUG_MAPPING_NODE.format(file_pattern, node_string))
+        return file_pattern, node_string