fameio 1.8.2__py3-none-any.whl → 2.1.0__py3-none-any.whl

fameio/source/results/data_transformer.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
 
 from abc import ABC
 from builtins import staticmethod
@@ -10,7 +11,7 @@ import pandas as pd
 from fameprotobuf.Services_pb2 import Output
 from pandas import DataFrame
 
-from fameio.source.cli import ResolveOptions
+from fameio.source.cli.options import ResolveOptions
 from fameio.source.results.agent_type import AgentType
 
 INDEX = ("AgentId", "TimeStep")
@@ -21,13 +22,12 @@ class DataTransformer(ABC):
 
     MODES = {
         ResolveOptions.IGNORE: lambda: DataTransformerIgnore(),
-        ResolveOptions.MERGE: lambda: DataTransformerMerge(),
         ResolveOptions.SPLIT: lambda: DataTransformerSplit(),
     }
     SIMPLE_COLUMN_INDEX = -1
 
     @staticmethod
-    def build(complex_column_mode: ResolveOptions) -> "DataTransformer":
+    def build(complex_column_mode: ResolveOptions) -> DataTransformer:
         return DataTransformer.MODES[complex_column_mode]()
 
     def extract_agent_data(
@@ -46,6 +46,7 @@ class DataTransformer(ABC):
             if column_id == DataTransformer.SIMPLE_COLUMN_INDEX:
                 data_frame.rename(columns=self._get_column_map(agent_type), inplace=True)
                 index = INDEX
+                data_frame = data_frame.loc[:, agent_type.get_simple_column_mask()]
             else:
                 data_frame.rename(columns={0: column_name}, inplace=True)
                 index = INDEX + agent_type.get_inner_columns(column_id)
@@ -53,7 +54,6 @@ class DataTransformer(ABC):
             if not data_frame.empty:
                 data_frame.index = pd.MultiIndex.from_tuples(data_frame.index)
                 data_frame.rename_axis(index, inplace=True)
-            data_frame.dropna(how="all", axis=1, inplace=True)
             data_frames[column_name] = data_frame
         return data_frames
 
@@ -81,10 +81,10 @@ class DataTransformer(ABC):
         container: Dict[int, Dict[Tuple, List[Union[float, None, str]]]],
     ) -> None:
         """Adds data from given `series` to specified `container` dict as list"""
-        dummy_list = [None] * len(mask_simple)
+        empty_list = [None] * len(mask_simple)
         for line in series.line:
             index = (series.agentId, line.timeStep)
-            simple_values = dummy_list.copy()
+            simple_values = empty_list.copy()
             for column in line.column:
                 if mask_simple[column.fieldId]:
                     simple_values[column.fieldId] = column.value
@@ -113,22 +113,6 @@ class DataTransformerIgnore(DataTransformer):
     """Ignores complex columns on output"""
 
 
-class DataTransformerMerge(DataTransformer):
-    """Merges complex columns on output into a single column entry"""
-
-    def _get_column_map(self, agent_type: AgentType) -> Dict[int, str]:
-        """Returns mapping of simple (and merged) column IDs to their name (or enhanced name) for given `agent_type`"""
-        return agent_type.get_merged_column_map()
-
-    @staticmethod
-    def _merge_complex_column(column: Output.Series.Line.Column, values: List) -> None:
-        """Merges given complex `column` content and saves it to given `values` list"""
-        result = []
-        for entry in column.entry:
-            result.append((tuple(entry.indexValue), entry.value))
-        values[column.fieldId] = result
-
-
 class DataTransformerSplit(DataTransformer):
     @staticmethod
     def _store_complex_values(column: Output.Series.Line.Column, container: Dict[int, Dict], base_index: Tuple) -> None:

fameio/source/results/input_dao.py

@@ -0,0 +1,163 @@
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+import ast
+from typing import List, Dict, Any, Optional, Tuple
+
+from fameprotobuf.DataStorage_pb2 import DataStorage
+from fameprotobuf.Field_pb2 import NestedField
+from fameprotobuf.InputFile_pb2 import InputData
+
+from fameio.source.logs import log
+from fameio.source.scenario import GeneralProperties, Agent, Contract, Scenario
+from fameio.source.schema import Schema, AttributeSpecs, AttributeType
+from fameio.source.series import TimeSeriesManager
+
+
+class InputConversionException(Exception):
+    """An Exception indication an error during reconstruction of input from its protobuf representation"""
+
+    pass
+
+
+class InputDao:
+    """Data access object for inputs saved in protobuf"""
+
+    _ERR_NO_INPUTS = "No input data found on file."
+    _ERR_MULTIPLE_INPUTS = "File corrupt. More than one input section found on file."
+
+    _FIELD_NAME_MAP: Dict = {
+        AttributeType.STRING: "stringValue",
+        AttributeType.ENUM: "stringValue",
+        AttributeType.INTEGER: "intValue",
+        AttributeType.DOUBLE: "doubleValue",
+        AttributeType.LONG: "longValue",
+        AttributeType.TIME_SERIES: "seriesId",
+        AttributeType.BLOCK: "field",
+    }
+
+    def __init__(self) -> None:
+        self._inputs: List[InputData] = []
+        self._timeseries_manager: TimeSeriesManager = TimeSeriesManager()
+        self._schema: Optional[Schema] = None
+
+    def store_inputs(self, data_storages: List[DataStorage]) -> None:
+        """
+        Extracts and stores Inputs in given DataStorages - if such are present
+
+        Args:
+            data_storages: to be scanned for InputData
+        """
+        self._inputs.extend([data_storage.input for data_storage in data_storages if data_storage.HasField("input")])
+
+    def recover_inputs(self) -> Tuple[TimeSeriesManager, Scenario]:
+        """
+        Recovers inputs to GeneralProperties, Schema, Agents, Contracts, Timeseries
+
+        Return:
+            recovered timeseries and scenario
+
+        Raises:
+            InputConversionException: if inputs could not be recovered
+        """
+        input_data = self._get_input_data()
+        self._schema = self._get_schema(input_data)
+        scenario = Scenario(self._schema, self._get_general_properties(input_data))
+        for contract in self._get_contracts(input_data):
+            scenario.add_contract(contract)
+
+        self._init_timeseries(input_data)
+        for agent in self._get_agents(input_data):
+            scenario.add_agent(agent)
+
+        return self._timeseries_manager, scenario
+
+    def _get_input_data(self) -> InputData:
+        """
+        Check that exactly one previously extracted input data exist, otherwise raises an exception
+
+        Raises:
+            InputConversionException: if no or more than one input is present
+        """
+        if not self._inputs:
+            log().error(self._ERR_NO_INPUTS)
+            raise InputConversionException(self._ERR_NO_INPUTS)
+        if len(self._inputs) > 1:
+            log().error(self._ERR_MULTIPLE_INPUTS)
+            raise InputConversionException(self._ERR_MULTIPLE_INPUTS)
+        return self._inputs[0]
+
+    @staticmethod
+    def _get_schema(input_data: InputData) -> Schema:
+        """Read and return Schema from given `input_data`"""
+        return Schema.from_string(input_data.schema)
+
+    @staticmethod
+    def _get_general_properties(input_data: InputData) -> GeneralProperties:
+        """Read and return GeneralProperties from given `input_data`"""
+        return GeneralProperties(
+            run_id=input_data.runId,
+            simulation_start_time=input_data.simulation.startTime,
+            simulation_stop_time=input_data.simulation.stopTime,
+            simulation_random_seed=input_data.simulation.randomSeed,
+            output_process=input_data.output.process,
+            output_interval=input_data.output.interval,
+        )
+
+    @staticmethod
+    def _get_contracts(input_data: InputData) -> List[Contract]:
+        """Read and return Contracts from given `input_data`"""
+        return [
+            Contract(
+                sender_id=contract.senderId,
+                receiver_id=contract.receiverId,
+                product_name=contract.productName,
+                delivery_interval=contract.deliveryIntervalInSteps,
+                first_delivery_time=contract.firstDeliveryTime,
+                expiration_time=contract.expirationTime,
+                meta_data=ast.literal_eval(contract.metadata),
+            )
+            for contract in input_data.contract
+        ]
+
+    def _init_timeseries(self, input_data: InputData) -> None:
+        """Read timeseries from given `input_data` and initialise TimeSeriesManager"""
+        self._timeseries_manager.reconstruct_time_series(list(input_data.timeSeries))
+
+    def _get_agents(self, input_data: InputData) -> List[Agent]:
+        """Read and return Agents from given `input_data`"""
+        agents = []
+        for agent_dao in input_data.agent:
+            agent = Agent(
+                agent_id=agent_dao.id, type_name=agent_dao.className, meta_data=ast.literal_eval(agent_dao.metadata)
+            )
+            attribute_dict = self._get_attributes(
+                list(agent_dao.field), self._schema.agent_types[agent_dao.className].attributes
+            )
+            agent.init_attributes_from_dict(attribute_dict)
+            agents.append(agent)
+        return agents
+
+    def _get_attributes(self, fields: List[NestedField], schematics: Dict[str, AttributeSpecs]) -> Dict[str, Any]:
+        """Read and return Attributes as Dictionary from given list of fields"""
+        attributes: Dict[str, Any] = {}
+        for field in fields:
+            attributes[field.fieldName] = self._get_field_value(field, schematics[field.fieldName])
+        return attributes
+
+    def _get_field_value(self, field: NestedField, schematic: AttributeSpecs) -> Any:
+        """Extracts and returns value(s) of given `field`"""
+        attribute_type: AttributeType = schematic.attr_type
+        value = field.__getattribute__(self._FIELD_NAME_MAP[attribute_type])
+        if attribute_type is AttributeType.TIME_SERIES:
+            return self._timeseries_manager.get_reconstructed_series_by_id(field.seriesId)
+        elif attribute_type is AttributeType.BLOCK:
+            if schematic.is_list:
+                return [self._get_attributes(list(entry.field), schematic.nested_attributes) for entry in field.field]
+            else:
+                return self._get_attributes(list(field.field), schematic.nested_attributes)
+        else:
+            if schematic.is_list:
+                return list(value)
+            else:
+                return list(value)[0]

fameio/source/results/reader.py

@@ -1,8 +1,8 @@
 # SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
 
-import logging as log
 import struct
 import typing
 from abc import ABC, abstractmethod
@@ -11,6 +11,8 @@ from typing import IO, List
 from fameprotobuf.DataStorage_pb2 import DataStorage
 from google.protobuf.message import DecodeError
 
+from fameio.source.logs import log
+
 
 class Reader(ABC):
     """Abstract base class for protobuf file readers"""
@@ -23,8 +25,8 @@ class Reader(ABC):
     _DEBUG_FILE_END_REACHED = "Reached expected end of file."
 
     _HEADER_LENGTH = 30
-    _HEADER_ENCODING = "utf-8"
-    _BYTES_DEFINING_MESSAGE_LENGTH = 4
+    HEADER_ENCODING = "utf-8"
+    BYTES_DEFINING_MESSAGE_LENGTH = 4
     _READER_HEADERS = {
         "famecoreprotobufstreamfilev001": lambda file, mode: ReaderV1(file, mode),  # noqa
     }
@@ -38,25 +40,34 @@ class Reader(ABC):
         """Reads associated filestream and returns one or multiple DataStorage(s) or empty list"""
 
     @staticmethod
-    def get_reader(file: IO, read_single: bool = False) -> "Reader":
-        """Returns reader matching the given file header - if `read_one` is True, read() gets one messages at a time"""
-        log.debug("Reading file headers...")
+    def get_reader(file: IO, read_single: bool = False) -> Reader:
+        """
+        Returns reader matching the given file header
+
+        Args:
+            file: to be read by the returned Reader
+            read_single: if True, the returned Reader's `read()` method gets one messages at a time
+
+        Returns:
+            Reader that can read the specified file
+        """
+        log().debug("Reading file headers...")
         try:
-            header = file.read(Reader._HEADER_LENGTH).decode(Reader._HEADER_ENCODING)
+            header = file.read(Reader._HEADER_LENGTH).decode(Reader.HEADER_ENCODING)
             return Reader._READER_HEADERS[header](file, read_single)
         except (KeyError, UnicodeDecodeError):
-            log.warning(Reader._WARN_NO_HEADER)
+            log().warning(Reader._WARN_NO_HEADER)
             file.seek(0)
             if read_single:
-                log.error(Reader._ERR_UNSUPPORTED_MODE)
+                log().error(Reader._ERR_UNSUPPORTED_MODE)
             return ReaderV0(file, False)
 
     @typing.final
     def _read_message_length(self) -> int:
         """Returns length of next DataStorage message in file"""
-        message_length_byte = self._file.read(self._BYTES_DEFINING_MESSAGE_LENGTH)
+        message_length_byte = self._file.read(self.BYTES_DEFINING_MESSAGE_LENGTH)
         if not message_length_byte:
-            log.debug(self._DEBUG_FILE_END_REACHED)
+            log().debug(self._DEBUG_FILE_END_REACHED)
             message_length_int = 0
         else:
             message_length_int = struct.unpack(">i", message_length_byte)[0]
@@ -75,7 +86,7 @@ class Reader(ABC):
         else:
             raise IOError(self._ERR_FILE_CORRUPT_NEGATIVE_LENGTH)
         if message_length and len(message) != message_length:
-            log.error(self._ERR_FILE_CORRUPT_MISSING_DATA)
+            log().error(self._ERR_FILE_CORRUPT_MISSING_DATA)
         return self._parse_to_data_storage(message) if message else None
 
     @staticmethod
@@ -96,7 +107,7 @@ class ReaderV0(Reader):
 
     def __init__(self, file: IO, read_single):
         super().__init__(file, read_single)
-        log.warning(self._WARN_DEPRECATED)
+        log().warning(self._WARN_DEPRECATED)
 
     def read(self) -> List[DataStorage]:
         result = self._read_data_storage_message()
@@ -115,5 +126,5 @@ class ReaderV1(Reader):
             messages.append(self._read_data_storage_message(message_length))
             if self._read_single:
                 break
-        log.debug(f"Read {len(messages)} messages from file.")
+        log().debug(f"Read {len(messages)} messages from file.")
         return messages

fameio/source/results/yaml_writer.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+from pathlib import Path
+from typing import Dict
+
+import yaml
+
+from fameio.source.logs import log
+
+ERR_WRITE_EXCEPTION = "Failed to save dictionary to YAML file `{}`"
+INFO_DESTINATION = "Saving scenario to file at {}"
+
+
+def data_to_yaml_file(data: Dict, file_path: Path) -> None:
+    """
+    Save the given data to a YAML file at given path
+
+    Args:
+        data: to be saved to yaml file
+        file_path: at which the file will be created
+    """
+    log().info(INFO_DESTINATION.format(file_path))
+    try:
+        with open(file_path, "w") as f:
+            yaml.dump(data, f, sort_keys=False)
+    except Exception as e:
+        raise RuntimeError(ERR_WRITE_EXCEPTION.format(file_path)) from e

fameio/source/scenario/agent.py

@@ -1,8 +1,10 @@
-# SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
 
-from typing import Any, Dict
+import ast
+from typing import Any, Dict, Optional
 
 from fameio.source.scenario.attribute import Attribute
 from fameio.source.scenario.exception import (
@@ -19,46 +21,71 @@ class Agent:
     _KEY_TYPE = "Type".lower()
     _KEY_ID = "Id".lower()
     _KEY_ATTRIBUTES = "Attributes".lower()
+    _KEY_METADATA = "MetaData".lower()
 
-    _MISSING_KEY = "Agent requires `key` '{}' but is missing it."
-    _MISSING_TYPE = "Agent requires `type` but is missing it."
-    _MISSING_ID = "Agent requires a positive integer `id` but was '{}'."
-    _DOUBLE_ATTRIBUTE = "Cannot add attribute '{}' to agent {} because it already exists."
+    _ERR_MISSING_KEY = "Agent requires `key` '{}' but is missing it."
+    _ERR_MISSING_TYPE = "Agent requires `type` but is missing it."
+    _ERR_MISSING_ID = "Agent requires a positive integer `id` but was '{}'."
+    _ERR_DOUBLE_ATTRIBUTE = "Cannot add attribute '{}' to agent {} because it already exists."
+    _ERR_ATTRIBUTE_OVERWRITE = "Agent's attributes are already set and would be overwritten."
 
-    def __init__(self, agent_id: int, type_name: str) -> None:
+    def __init__(self, agent_id: int, type_name: str, meta_data: Optional[Dict] = None) -> None:
         """Constructs a new Agent"""
-        assert_or_raise(type(agent_id) is int and agent_id >= 0, self._MISSING_ID.format(agent_id))
-        assert_or_raise(type_name and len(type_name.strip()) > 0, self._MISSING_TYPE)
-        self._id = agent_id
-        self._type_name = type_name.strip()
-        self._attributes = {}
+        assert_or_raise(type(agent_id) is int and agent_id >= 0, self._ERR_MISSING_ID.format(agent_id))
+        assert_or_raise(bool(type_name and type_name.strip()), self._ERR_MISSING_TYPE)
+        self._id: int = agent_id
+        self._type_name: str = type_name.strip()
+        self._attributes: Dict = {}
+        self._meta_data: Optional[Dict] = meta_data if meta_data else {}
 
     @classmethod
-    def from_dict(cls, definitions: dict) -> "Agent":
+    def from_dict(cls, definitions: dict) -> Agent:
         """Parses an agent from provided `definitions`"""
         definitions = keys_to_lower(definitions)
-        agent_type = get_or_raise(definitions, Agent._KEY_TYPE, Agent._MISSING_KEY)
-        agent_id = get_or_raise(definitions, Agent._KEY_ID, Agent._MISSING_KEY)
-        result = cls(agent_id, agent_type)
-        attribute_definitions = get_or_default(definitions, Agent._KEY_ATTRIBUTES, dict())
-        result.__init_attributes_from_dict(attribute_definitions)
-        return result
+        agent_type = get_or_raise(definitions, Agent._KEY_TYPE, Agent._ERR_MISSING_TYPE)
+        agent_id = get_or_raise(definitions, Agent._KEY_ID, Agent._ERR_MISSING_ID)
+        agent = cls(agent_id, agent_type)
+        attribute_definitions = get_or_default(definitions, Agent._KEY_ATTRIBUTES, {})
+        agent.init_attributes_from_dict(attribute_definitions)
+        agent._meta_data = get_or_default(definitions, Agent._KEY_METADATA, {})
+        return agent
+
+    def init_attributes_from_dict(self, attributes: Dict[str, Any]) -> None:
+        """Initialize Agent `attributes` from dict; Must only be called when creating a new Agent"""
+        assert_or_raise(not self._attributes, self._ERR_ATTRIBUTE_OVERWRITE)
+        self._attributes = {}
+        for name, value in attributes.items():
+            full_name = f"{self.type_name}({self.id}): {name}"
+            self.add_attribute(name, Attribute(full_name, value))
+
+    def add_attribute(self, name: str, value: Attribute) -> None:
+        """Adds a new attribute to the Agent (raise an error if it already exists)"""
+        if name in self._attributes:
+            raise ValueError(self._ERR_DOUBLE_ATTRIBUTE.format(name, self.display_id))
+        self._attributes[name] = value
+        self._notify_data_changed()
 
     def to_dict(self) -> dict:
         """Serializes the Agent content to a dict"""
-        result = {}
-
-        result[Agent._KEY_TYPE] = self.type_name
-        result[Agent._KEY_ID] = self.id
+        result = {Agent._KEY_TYPE: self.type_name, Agent._KEY_ID: self.id}
 
-        if len(self.attributes) > 0:
+        if self.attributes:
             attributes_dict = {}
             for attr_name, attr_value in self.attributes.items():
                 attributes_dict[attr_name] = attr_value.generic_content
             result[self._KEY_ATTRIBUTES] = attributes_dict
-
+        if self.meta_data:
+            result[self._KEY_METADATA] = self.meta_data
         return result
 
+    def to_string(self) -> str:
+        """Serializes this agent to a string"""
+        return repr(self.to_dict())
+
+    @classmethod
+    def from_string(cls, definitions: str) -> Agent:
+        return cls.from_dict(ast.literal_eval(definitions))
+
     def _notify_data_changed(self):
         """Placeholder method used to signal data changes to derived types"""
         pass
@@ -83,17 +110,7 @@ class Agent:
         """Returns dictionary of all Attributes of this agent"""
         return self._attributes
 
-    def add_attribute(self, name: str, value: Attribute):
-        """Adds a new attribute to the Agent (raise an error if it already exists)"""
-        if name in self._attributes:
-            raise ValueError(self._DOUBLE_ATTRIBUTE.format(name, self.display_id))
-        self._attributes[name] = value
-        self._notify_data_changed()
-
-    def __init_attributes_from_dict(self, attributes: Dict[str, Any]) -> None:
-        """Initialize Agent `attributes` from dict; Must only be called when creating a new Agent"""
-        assert len(self._attributes) == 0
-        self._attributes = {}
-        for name, value in attributes.items():
-            full_name = str(self.type_name) + "(" + str(self.id) + "): " + name
-            self.add_attribute(name, Attribute(full_name, value))
+    @property
+    def meta_data(self) -> dict:
+        """Returns dictionary of all MetaData of this agent"""
+        return self._meta_data

fameio/source/scenario/attribute.py

@@ -1,8 +1,8 @@
 # SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
 
-import logging as log
 from typing import Any, Dict, List
 
 from fameio.source.scenario.exception import log_and_raise
@@ -12,7 +12,6 @@ class Attribute:
     """An Attribute of an agent in a scenario"""
 
     _VALUE_MISSING = "Value not specified for Attribute '{}' - leave out if default shall be used (if defined)."
-    _OVERWRITE = "Value already defined for Attribute '{}' - overwriting value with new one!"
     _LIST_EMPTY = "Attribute '{}' was assigned an empty list - please remove or fill empty assignments."
     _DICT_EMPTY = "Attribute '{}' was assigned an empty dictionary - please remove or fill empty assignments."
     _MIXED_DATA = "Attribute '{}' was assigned a list with mixed complex and simple entries - please fix."
@@ -45,13 +44,11 @@ class Attribute:
         if not definitions:
             log_and_raise(Attribute._DICT_EMPTY.format(name))
 
-        dictionary = {}
+        inner_elements = {}
         for nested_name, value in definitions.items():
             full_name = name + "." + nested_name
-            if nested_name in dictionary:
-                log.warning(Attribute._OVERWRITE.format(full_name))
-            dictionary[nested_name] = Attribute(full_name, value)
-        return dictionary
+            inner_elements[nested_name] = Attribute(full_name, value)
+        return inner_elements
 
     @staticmethod
     def _is_list_of_dict(name: str, definitions: Any) -> bool:
@@ -79,10 +76,10 @@ class Attribute:
         elif self.has_nested_list:
             result = []
             for attr_dict in self.nested_list:
-                sub_dict = {}
+                inner_elements = {}
                 for name, attr in attr_dict.items():
-                    sub_dict[name] = attr.generic_content
-                result.append(sub_dict)
+                    inner_elements[name] = attr.generic_content
+                result.append(inner_elements)
             return result
         elif self.has_nested:
             result = {}
@@ -107,12 +104,12 @@ class Attribute:
         return bool(self._nested)
 
     @property
-    def nested(self) -> Dict[str, "Attribute"]:
+    def nested(self) -> Dict[str, Attribute]:
         """Returns dictionary of all nested Attributes"""
         assert self.has_nested
         return self._nested
 
-    def get_nested_by_name(self, key: str) -> "Attribute":
+    def get_nested_by_name(self, key: str) -> Attribute:
         """Returns nested Attribute by specified name"""
         return self._nested[key]