fameio 3.4.0__py3-none-any.whl → 3.5.1__py3-none-any.whl

fameio/output/csv_writer.py

@@ -1,13 +1,15 @@
 # SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+"""Writing of dataframes to CSV files."""
+
 from __future__ import annotations
 
 from pathlib import Path
 
 import pandas as pd
 
-from fameio.logs import log, log_error
+from fameio.logs import log_error
 from fameio.output import OutputError
 from fameio.output.data_transformer import INDEX
 from fameio.series import TimeSeriesManager
@@ -21,49 +23,21 @@ class CsvWriterError(OutputError):
 class CsvWriter:
     """Writes dataframes to different csv files."""
 
-    _ERR_DIR_CREATE = "Could not create directory for output files: '{}'"
     _ERR_FILE_OPEN = "Could not open file for writing: '{}'"
     _ERR_FILE_WRITE = "Could not write to file '{}' due to: {}"
 
-    _INFO_USING_PATH = "Using specified output path: {}"
-    _INFO_USING_DERIVED_PATH = "No output path specified - writing to new local folder: {}"
-
     CSV_FILE_SUFFIX = ".csv"
 
-    def __init__(self, config_output: Path, input_file_path: Path, single_export: bool) -> None:
+    def __init__(self, output_folder: Path, single_export: bool) -> None:
         """Constructs a new CsvWriter.
 
-        Raises:
-             CsvWriterError: if output folder could not be created, logged with level "ERROR"
+        Args:
+            output_folder: to write the output files to
+            single_export: if true, one output file per unique agent is created
         """
         self._single_export = single_export
-        self._output_folder = self._get_output_folder_name(config_output, input_file_path)
+        self._output_folder = output_folder
         self._files: dict[str, Path] = {}
-        self._create_output_folder()
-
-    @staticmethod
-    def _get_output_folder_name(config_output: Path, input_file_path: Path) -> Path:
-        """Returns name of the output folder derived either from the specified `config_output` or `input_file_path`."""
-        if config_output:
-            output_folder_name: str | Path = config_output
-            log().info(CsvWriter._INFO_USING_PATH.format(config_output))
-        else:
-            output_folder_name = input_file_path.stem
-            log().info(CsvWriter._INFO_USING_DERIVED_PATH.format(output_folder_name))
-        return Path(output_folder_name)
-
-    def _create_output_folder(self) -> None:
-        """Creates output folder if not yet present.
-
-        Raises:
-            CsvWriterError: if output folder could not be created, logged with level "ERROR"
-        """
-        log().debug("Creating output folder if required...")
-        if not self._output_folder.is_dir():
-            try:
-                self._output_folder.mkdir(parents=True)
-            except OSError as e:
-                raise log_error(CsvWriterError(self._ERR_DIR_CREATE.format(self._output_folder))) from e
 
     def write_to_files(self, agent_name: str, data: dict[None | str, pd.DataFrame]) -> None:
         """Writes `data` for given `agent_name` to .csv file(s).
@@ -115,7 +89,7 @@ class CsvWriter:
 
     @staticmethod
     def _dataframe_to_csv(data: pd.DataFrame, file: Path, header: bool, index: bool, mode: str) -> None:
-        """Write given data to specified CSV file with specified parameters using semicolon separators.
+        """Write given data to specified CSV file in UTF8 encoding with specified parameters using semicolon separators.
 
         Args:
             data: to be written
@@ -128,7 +102,7 @@ class CsvWriter:
             CsvWriterError: if data could not be written to disk, logged on level "ERROR"
         """
         try:
-            data.to_csv(path_or_buf=file, sep=";", header=header, index=index, mode=mode)
+            data.to_csv(path_or_buf=file, sep=";", header=header, index=index, mode=mode, encoding="UTF-8")
         except OSError as e:
             raise log_error(CsvWriterError(CsvWriter._ERR_FILE_OPEN.format(file))) from e
         except UnicodeError as e:

fameio/output/data_transformer.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+"""Transformation of (complex) time series outputs from agents."""
+
 from __future__ import annotations
 
 from abc import ABC

fameio/output/execution_dao.py

@@ -1,8 +1,11 @@
 # SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+"""Accessing execution content of protobuf messages."""
+
 from __future__ import annotations
 
+from importlib import metadata
 from typing import Any, Final
 
 from fameprotobuf.data_storage_pb2 import DataStorage
@@ -47,6 +50,7 @@ class ExecutionDao:
 
     KEY_COMPILATION: Final[str] = "InputCompilation"
     KEY_RUN: Final[str] = "ModelRun"
+    KEY_EXTRACTION: Final[str] = "OutputExtraction"
     KEY_VERSIONS: Final[str] = "SoftwareVersions"
     KEY_PROCESSES: Final[str] = "ProcessConfiguration"
     KEY_STATISTICS: Final[str] = "Statistics"
@@ -146,6 +150,7 @@ class ExecutionDao:
                 self.KEY_PROCESSES: self._get_dict(self._run_configuration, PROCESS_MAP),
                 self.KEY_STATISTICS: self._get_dict(self._run_simulation, STATISTICS_MAP),
             },
+            self.KEY_EXTRACTION: {self.KEY_VERSIONS: {"FameIo": metadata.version("fameio")}},
         }
         return result
 

fameio/output/files.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+"""Finding output folders and files, creating the output folder."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Final, Optional
+
+from fameio.logs import log, log_error
+from fameio.output import OutputError
+
+_ERR_DIR_CREATE = "Could not create directory for output files: '{}'"
+
+_INFO_USING_PATH = "Using specified output path: '{}'"
+_INFO_USING_DERIVED_PATH = "No output path specified - writing to new local folder: '{}'"
+
+_DEBUG_NEW_FOLDER = "Output folder '{}' not present, trying to create it..."
+_DEBUG_EXISTING_FOLDER = "Output folder '{}' already exists..."
+
+RECOVERED_INPUT_PATH: Final[str] = "./recovered"
+RECOVERED_SCENARIO_PATH: Final[str] = "./recovered/scenario.yaml"
+METADATA_FILE_NAME: Final[str] = "metadata.json"
+
+
+class OutputPathError(OutputError):
+    """An error that occurred during creation of the output path."""
+
+
+def get_output_folder_name(config_output: Optional[Path | str], input_file_path: Path) -> Path:
+    """Returns name of the output folder derived either from the specified `config_output` or `input_file_path`."""
+    if config_output:
+        output_folder_name = config_output
+        log().info(_INFO_USING_PATH.format(config_output))
+    else:
+        output_folder_name = input_file_path.stem
+        log().info(_INFO_USING_DERIVED_PATH.format(output_folder_name))
+    return Path(output_folder_name)
+
+
+def create_output_folder(output_path: Path) -> None:
+    """Creates output folder if not yet present.
+
+    Raises:
+        OutputPathError: if output folder could not be created, logged with level "ERROR"
+    """
+    if not output_path.is_dir():
+        log().debug(_DEBUG_NEW_FOLDER.format(output_path))
+        try:
+            output_path.mkdir(parents=True)
+        except OSError as e:
+            raise log_error(OutputPathError(_ERR_DIR_CREATE.format(output_path))) from e
+    else:
+        log().debug(_DEBUG_EXISTING_FOLDER.format(output_path))

fameio/output/input_dao.py

@@ -1,16 +1,18 @@
 # SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+"""Accessing input content of protobuf messages."""
+
 import ast
-from typing import Any
+from typing import Any, Optional
 
 from fameprotobuf.data_storage_pb2 import DataStorage
 from fameprotobuf.field_pb2 import NestedField
 from fameprotobuf.input_file_pb2 import InputData
 
-from fameio.input.scenario import GeneralProperties, Agent, Contract, Scenario
+from fameio.input.scenario import GeneralProperties, Agent, Contract, Scenario, StringSet, Attribute
 from fameio.input.schema import Schema, AttributeSpecs, AttributeType
-from fameio.logs import log_error
+from fameio.logs import log_error, log
 from fameio.output import OutputError
 from fameio.series import TimeSeriesManager, TimeSeriesError
 
@@ -26,6 +28,8 @@ class InputDao:
     _ERR_MULTIPLE_INPUTS = "File corrupt. More than one input section found on file."
     _ERR_NO_SCHEMA = "No schema found on file - cannot recover inputs."
     _ERR_SERIES_MISSING = "References time series '{}' was not registered on file."
+    _ERR_SCENARIO_METADATA = "Proceeding without metadata for scenario - could not be extracted due to: {}"
+    _ERR_STRING_SET_METADATA = "Proceeding without metadata for string set '{}' - could not be extracted due to: {}"
 
     _FIELD_NAME_MAP: dict = {
         AttributeType.STRING: "string_values",
@@ -61,9 +65,10 @@ class InputDao:
             InputConversionError: if inputs could not be recovered, logged with level "ERROR"
             InputError: if scenario in file is incompatible with this version of fameio, logged with level "ERROR"
         """
-        input_data = self._get_input_data()
+        input_data = self.get_input_data()
         schema = self._get_schema(input_data)
-        scenario = Scenario(schema, self._get_general_properties(input_data))
+        metadata = self._metadata_to_dict(input_data.metadata)
+        scenario = Scenario(schema, self._get_general_properties(input_data), metadata)
         for contract in self._get_contracts(input_data):
             scenario.add_contract(contract)
 
@@ -71,10 +76,13 @@ class InputDao:
         for agent in self._get_agents(input_data, schema):
             scenario.add_agent(agent)
 
+        for name, string_set in self._get_string_sets(input_data).items():
+            scenario.add_string_set(name, string_set)
+
         return self._timeseries_manager, scenario
 
-    def _get_input_data(self) -> InputData:
-        """Check that exactly one previously extracted input data exist, otherwise raises an exception.
+    def get_input_data(self) -> InputData:
+        """Check that exactly one previously extracted input data exist and returns them; otherwise raises an exception.
 
         Returns:
             the previously extracted input data
@@ -93,6 +101,16 @@ class InputDao:
         """Read and return Schema from given `input_data`."""
         return Schema.from_string(input_data.schema)
 
+    @staticmethod
+    def _metadata_to_dict(metadata: Optional[str] = None) -> dict:
+        """Convert given metadata `metadata to dict`, proceeds on error but logs given `message`"""
+        if metadata:
+            try:
+                return ast.literal_eval(metadata)
+            except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
+                log().error(InputDao._ERR_SCENARIO_METADATA.format(e))
+        return {}
+
     @staticmethod
     def _get_general_properties(input_data: InputData) -> GeneralProperties:
         """Read and return GeneralProperties from given `input_data`."""
@@ -119,6 +137,20 @@ class InputDao:
             for contract in input_data.contracts
         ]
 
+    @staticmethod
+    def _get_string_sets(input_data: InputData) -> dict[str, StringSet]:
+        """Read and return StringSets from given `input_data`."""
+        string_sets = {}
+        for dao in input_data.string_sets:
+            values = {
+                entry.name: {StringSet.KEY_METADATA: InputDao._metadata_to_dict(entry.metadata)} for entry in dao.values
+            }
+            metadata = InputDao._metadata_to_dict(dao.metadata)
+            string_sets[dao.name] = StringSet.from_dict(
+                {StringSet.KEY_VALUES: values, StringSet.KEY_METADATA: metadata}
+            )
+        return string_sets
+
     def _init_timeseries(self, input_data: InputData) -> None:
         """Read timeseries from given `input_data` and initialise TimeSeriesManager."""
         self._timeseries_manager.reconstruct_time_series(list(input_data.time_series))
@@ -144,14 +176,14 @@ class InputDao:
                 type_name=agent_dao.class_name,
                 metadata=ast.literal_eval(agent_dao.metadata) if agent_dao.metadata else None,
             )
-            attribute_dict = self._get_attributes(
+            attributes_dict = self._get_attributes_dict(
                 list(agent_dao.fields), schema.agent_types[agent_dao.class_name].attributes
             )
-            agent.init_attributes_from_dict(attribute_dict)
+            agent.init_attributes_from_dict(attributes_dict)
             agents.append(agent)
         return agents
 
-    def _get_attributes(self, fields: list[NestedField], schematics: dict[str, AttributeSpecs]) -> dict[str, Any]:
+    def _get_attributes_dict(self, fields: list[NestedField], schematics: dict[str, AttributeSpecs]) -> dict[str, dict]:
         """Read and return all Attributes as Dictionary from given list of fields.
 
         Args:
@@ -159,14 +191,15 @@ class InputDao:
             schematics: description of the attributes associated by name
 
         Returns:
-            all recovered attributes and their associated values
+            all recovered attributes and their associated values as dictionary
 
         Raises:
             InputConversionError: if attributes could not be reconstructed, logged with level "ERROR"
         """
-        attributes: dict[str, Any] = {}
+        attributes: dict[str, dict[str, Any]] = {}
         for field in fields:
-            attributes[field.field_name] = self._get_field_value(field, schematics[field.field_name])
+            value = self._get_field_value(field, schematics[field.field_name])
+            attributes[field.field_name] = value if not field.metadata else self._get_field_dict(value, field.metadata)
         return attributes
 
     def _get_field_value(self, field: NestedField, schematic: AttributeSpecs) -> Any:
@@ -190,9 +223,20 @@ class InputDao:
                 raise log_error(InputConversionError(self._ERR_SERIES_MISSING.format(field.series_id))) from e
         if attribute_type is AttributeType.BLOCK:
             if schematic.is_list:
-                return [self._get_attributes(list(entry.fields), schematic.nested_attributes) for entry in field.fields]
-            return self._get_attributes(list(field.fields), schematic.nested_attributes)
+                return [
+                    self._get_attributes_dict(list(entry.fields), schematic.nested_attributes) for entry in field.fields
+                ]
+            return self._get_attributes_dict(list(field.fields), schematic.nested_attributes)
         value = getattr(field, self._FIELD_NAME_MAP[attribute_type])
         if schematic.is_list:
             return list(value)
         return list(value)[0]
+
+    def _get_field_dict(self, field_value: Any, metadata: str) -> dict[str, Any]:
+        """Returns dict with metadata and `field_value` associated with either singular or plural key, if is list."""
+        result: dict[str, Any] = {Attribute.KEY_METADATA: self._metadata_to_dict(metadata)}
+        if isinstance(field_value, list):
+            result[Attribute.KEY_VALUES] = field_value
+        else:
+            result[Attribute.KEY_VALUE] = field_value
+        return result

fameio/output/metadata/__init__.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: CC0-1.0
+"""Classes and modules to compile metadata associated with output files."""
+
+from fameio.output import OutputError
+
+
+class MetadataCompilationError(OutputError):
+    """An error occurred while compiling output metadata."""

fameio/output/metadata/compiler.py

@@ -0,0 +1,75 @@
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+"""Compiling metadata files accompanying the output CSV files."""
+
+from __future__ import annotations
+
+import ast
+from typing import Final, Any
+
+from fameprotobuf.input_file_pb2 import InputData
+
+from fameio.input.metadata import Metadata
+from fameio.logs import log_error, log
+from fameio.output.metadata import MetadataCompilationError
+from fameio.output.metadata.locator import Locator
+from fameio.tools import keys_to_lower
+
+
+class MetadataCompiler(Locator):
+    """Compiles metadata for output files based on ExecutionData and InputData."""
+
+    ENTRY_SCHEMA: Final[str] = "Schema".lower()
+    ENTRY_SCENARIO: Final[str] = "Scenario".lower()
+    ENTRY_EXECUTION: Final[str] = "Execution".lower()
+    SEPARATOR: Final[str] = ":"
+
+    _ERR_MALFORMED_DICT_STRING = "Input data reading failed: Malformed string representation of metadata dictionaries."
+    _INFO_NOT_FOUND = "Could not find element at '{}' in input section of provided file."
+
+    def __init__(
+        self, execution_data: dict[str, Any], input_data: InputData, agent_columns: dict[str, list[str]]
+    ) -> None:
+        """Initialises a new MetadataCompiler.
+
+        Args:
+            execution_data: to read execution metadata from
+            input_data: to read schema and scenario metadata from
+            agent_columns: agents and their output columns
+        """
+        super().__init__(agent_columns)
+        try:
+            self._data: dict[str, dict] = {
+                self.ENTRY_SCHEMA: ast.literal_eval(input_data.schema),
+                self.ENTRY_SCENARIO: {
+                    Metadata.KEY_METADATA: ast.literal_eval(input_data.metadata) if input_data.metadata else {}
+                },
+                self.ENTRY_EXECUTION: execution_data,
+            }
+        except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
+            raise log_error(MetadataCompilationError(self._ERR_MALFORMED_DICT_STRING)) from e
+
+    def _replace(self, data_identifier: str) -> Any | None:
+        identifier = data_identifier[1:-1]
+        address = identifier.split(self.SEPARATOR)
+        data_source = address[0].lower()
+        try:
+            if data_source in self._data:
+                return self._get_from(self._data[data_source], address[1:])
+        except KeyError:
+            log().info(self._INFO_NOT_FOUND.format(self.SEPARATOR.join(address)))
+            return None
+        return None
+
+    @staticmethod
+    def _get_from(base: dict, address: list[str]) -> Any:
+        """Returns element in `base` at given `address`.
+
+        Raises:
+            KeyError: if element cannot be found; error not logged
+        """
+        element = base
+        for entry in address:
+            element = keys_to_lower(element)[entry.lower()]
+        return element

fameio/output/metadata/json_writer.py

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+"""Writing of data to JSON files."""
+
+import json
+from pathlib import Path
+
+from fameio.logs import log, log_error
+from fameio.output import OutputError
+from fameio.output.files import METADATA_FILE_NAME
+
+_ERR_OPEN_FILE = "Could not open file for writing: '{}'"
+_INFO_DESTINATION = "Saving JSON to file to {}"
+
+
+class JsonWriterError(OutputError):
+    """An error occurred during writing a JSON file."""
+
+
+def data_to_json_file(data: dict, base_path: Path) -> None:
+    """Save the given data to a JSON file at given path.
+
+    Args:
+        data: to be saved to JSON file
+        base_path: at which the JSON file will be created
+
+    Raises:
+        JsonWriterError: if file could not be opened or written, logged with level "ERROR"
+    """
+    log().info(_INFO_DESTINATION.format(base_path))
+    try:
+        with open(Path(base_path, METADATA_FILE_NAME), "w", encoding="utf-8") as f:
+            json.dump(data, f)
+    except OSError as e:
+        raise log_error(JsonWriterError(_ERR_OPEN_FILE.format(base_path))) from e