fameio 3.1.1__py3-none-any.whl → 3.3.0__py3-none-any.whl

fameio/output/input_dao.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 import ast
-from typing import Any, Optional
+from typing import Any
 
 from fameprotobuf.data_storage_pb2 import DataStorage
 from fameprotobuf.field_pb2 import NestedField
@@ -10,19 +10,22 @@ from fameprotobuf.input_file_pb2 import InputData
 
 from fameio.input.scenario import GeneralProperties, Agent, Contract, Scenario
 from fameio.input.schema import Schema, AttributeSpecs, AttributeType
-from fameio.logs import log
-from fameio.series import TimeSeriesManager
+from fameio.logs import log_error
+from fameio.output import OutputError
+from fameio.series import TimeSeriesManager, TimeSeriesError
 
 
-class InputConversionError(Exception):
-    """Indicates an error during reconstruction of input from its protobuf representation"""
+class InputConversionError(OutputError):
+    """Indicates an error during reconstruction of input from its protobuf representation."""
 
 
 class InputDao:
-    """Data access object for inputs saved in protobuf"""
+    """Data access object for inputs saved in protobuf."""
 
     _ERR_NO_INPUTS = "No input data found on file."
     _ERR_MULTIPLE_INPUTS = "File corrupt. More than one input section found on file."
+    _ERR_NO_SCHEMA = "No schema found on file - cannot recover inputs."
+    _ERR_SERIES_MISSING = "References time series '{}' was not registered on file."
 
     _FIELD_NAME_MAP: dict = {
         AttributeType.STRING: "string_values",
@@ -39,11 +42,9 @@ class InputDao:
     def __init__(self) -> None:
         self._inputs: list[InputData] = []
         self._timeseries_manager: TimeSeriesManager = TimeSeriesManager()
-        self._schema: Optional[Schema] = None
 
     def store_inputs(self, data_storages: list[DataStorage]) -> None:
-        """
-        Extracts and stores Inputs in given DataStorages - if such are present
+        """Extracts and stores Inputs in given DataStorages - if such are present.
 
         Args:
             data_storages: to be scanned for InputData
@@ -51,50 +52,50 @@ class InputDao:
         self._inputs.extend([data_storage.input for data_storage in data_storages if data_storage.HasField("input")])
 
     def recover_inputs(self) -> tuple[TimeSeriesManager, Scenario]:
-        """
-        Recovers inputs to GeneralProperties, Schema, Agents, Contracts, Timeseries
+        """Recovers inputs to GeneralProperties, Schema, Agents, Contracts, Timeseries.
 
         Return:
             recovered timeseries and scenario
 
         Raises:
-            InputConversionException: if inputs could not be recovered
+            InputConversionError: if inputs could not be recovered, logged with level "ERROR"
+            InputError: if scenario in file is incompatible with this version of fameio, logged with level "ERROR"
         """
         input_data = self._get_input_data()
-        self._schema = self._get_schema(input_data)
-        scenario = Scenario(self._schema, self._get_general_properties(input_data))
+        schema = self._get_schema(input_data)
+        scenario = Scenario(schema, self._get_general_properties(input_data))
         for contract in self._get_contracts(input_data):
             scenario.add_contract(contract)
 
         self._init_timeseries(input_data)
-        for agent in self._get_agents(input_data):
+        for agent in self._get_agents(input_data, schema):
             scenario.add_agent(agent)
 
         return self._timeseries_manager, scenario
 
     def _get_input_data(self) -> InputData:
-        """
-        Check that exactly one previously extracted input data exist, otherwise raises an exception
+        """Check that exactly one previously extracted input data exist, otherwise raises an exception.
+
+        Returns:
+            the previously extracted input data
 
         Raises:
-            InputConversionException: if no or more than one input is present
+            InputConversionException: if no input, or more than one input is present, logged with level "ERROR"
         """
         if not self._inputs:
-            log().error(self._ERR_NO_INPUTS)
-            raise InputConversionError(self._ERR_NO_INPUTS)
+            raise log_error(InputConversionError(self._ERR_NO_INPUTS))
         if len(self._inputs) > 1:
-            log().error(self._ERR_MULTIPLE_INPUTS)
-            raise InputConversionError(self._ERR_MULTIPLE_INPUTS)
+            raise log_error(InputConversionError(self._ERR_MULTIPLE_INPUTS))
         return self._inputs[0]
 
     @staticmethod
     def _get_schema(input_data: InputData) -> Schema:
-        """Read and return Schema from given `input_data`"""
+        """Read and return Schema from given `input_data`."""
         return Schema.from_string(input_data.schema)
 
     @staticmethod
     def _get_general_properties(input_data: InputData) -> GeneralProperties:
-        """Read and return GeneralProperties from given `input_data`"""
+        """Read and return GeneralProperties from given `input_data`."""
         return GeneralProperties(
             run_id=input_data.run_id,
             simulation_start_time=input_data.simulation.start_time,
@@ -104,7 +105,7 @@ class InputDao:
 
     @staticmethod
     def _get_contracts(input_data: InputData) -> list[Contract]:
-        """Read and return Contracts from given `input_data`"""
+        """Read and return Contracts from given `input_data`."""
         return [
             Contract(
                 sender_id=contract.sender_id,
@@ -119,11 +120,23 @@ class InputDao:
         ]
 
     def _init_timeseries(self, input_data: InputData) -> None:
-        """Read timeseries from given `input_data` and initialise TimeSeriesManager"""
+        """Read timeseries from given `input_data` and initialise TimeSeriesManager."""
         self._timeseries_manager.reconstruct_time_series(list(input_data.time_series))
 
-    def _get_agents(self, input_data: InputData) -> list[Agent]:
-        """Read and return Agents from given `input_data`"""
+    def _get_agents(self, input_data: InputData, schema: Schema) -> list[Agent]:
+        """Read and return Agents from given `input_data`.
+
+        Args:
+            input_data: to read agents from
+            schema: corresponding to the agent definitions
+
+        Returns:
+            all extracted agents
+
+        Raises:
+            InputError: if agents cannot be reconstructed, logged with level "ERROR"
+            InputConversionError: if attributes could not be reconstructed, logged with level "ERROR"
+        """
         agents = []
         for agent_dao in input_data.agents:
             agent = Agent(
@@ -132,29 +145,54 @@ class InputDao:
                 metadata=ast.literal_eval(agent_dao.metadata) if agent_dao.metadata else None,
             )
             attribute_dict = self._get_attributes(
-                list(agent_dao.fields), self._schema.agent_types[agent_dao.class_name].attributes
+                list(agent_dao.fields), schema.agent_types[agent_dao.class_name].attributes
             )
             agent.init_attributes_from_dict(attribute_dict)
             agents.append(agent)
         return agents
 
     def _get_attributes(self, fields: list[NestedField], schematics: dict[str, AttributeSpecs]) -> dict[str, Any]:
-        """Read and return Attributes as Dictionary from given list of fields"""
+        """Read and return all Attributes as Dictionary from given list of fields.
+
+        Args:
+            fields: data fields representing attributes
+            schematics: description of the attributes associated by name
+
+        Returns:
+            all recovered attributes and their associated values
+
+        Raises:
+            InputConversionError: if attributes could not be reconstructed, logged with level "ERROR"
+        """
         attributes: dict[str, Any] = {}
         for field in fields:
             attributes[field.field_name] = self._get_field_value(field, schematics[field.field_name])
         return attributes
 
     def _get_field_value(self, field: NestedField, schematic: AttributeSpecs) -> Any:
-        """Extracts and returns value(s) of given `field`"""
+        """Extracts and returns value(s) of given `field`.
+
+        Args:
+            field: to extract the value(s) from
+            schematic: describing the data type of this field
+
+        Returns:
+            value(s) of provided field
+
+        Raises:
+            InputConversionError: if TimeSeries could not be found, logged with level "ERROR"
+        """
         attribute_type: AttributeType = schematic.attr_type
-        value = getattr(field, self._FIELD_NAME_MAP[attribute_type])
         if attribute_type is AttributeType.TIME_SERIES:
-            return self._timeseries_manager.get_reconstructed_series_by_id(field.series_id)
+            try:
+                return self._timeseries_manager.get_reconstructed_series_by_id(field.series_id)
+            except TimeSeriesError as e:
+                raise log_error(InputConversionError(self._ERR_SERIES_MISSING.format(field.series_id))) from e
         if attribute_type is AttributeType.BLOCK:
             if schematic.is_list:
                 return [self._get_attributes(list(entry.fields), schematic.nested_attributes) for entry in field.fields]
             return self._get_attributes(list(field.fields), schematic.nested_attributes)
+        value = getattr(field, self._FIELD_NAME_MAP[attribute_type])
         if schematic.is_list:
             return list(value)
         return list(value)[0]

fameio/output/output_dao.py

@@ -1,7 +1,9 @@
-# SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
-from typing import Iterable, Optional
+from __future__ import annotations
+
+from typing import Iterable
 
 import pandas as pd
 from fameprotobuf.data_storage_pb2 import DataStorage
@@ -12,9 +14,19 @@ from fameio.output.data_transformer import DataTransformer
 
 
 class OutputDAO:
-    """Grants convenient access to content of Output protobuf messages for given DataStorages"""
+    """Grants convenient access to content of Output protobuf messages for given DataStorages."""
 
     def __init__(self, data_storages: list[DataStorage], agent_type_log: AgentTypeLog) -> None:
+        """
+        Initialise a new OutputDAO
+
+        Args:
+            data_storages: to grant access to by this DAO
+            agent_type_log: new types of agents that might come up in the associated data_storages
+
+        Raises:
+            AgentTypeError: if duplicate agent definitions occur, logged with level "ERROR"
+        """
         self._agent_type_log = agent_type_log
         outputs = self._extract_output_from_data_storages(data_storages)
         self._agent_type_log.update_agents(self._extract_new_agent_types(outputs))
@@ -22,25 +34,25 @@ class OutputDAO:
 
     @staticmethod
     def _extract_output_from_data_storages(data_storages: list[DataStorage]) -> list[Output]:
-        """Returns list of Outputs extracted from given `data_storages`"""
+        """Returns list of Outputs extracted from given `data_storages`."""
         if data_storages is None:
             data_storages = []
         return [data_storage.output for data_storage in data_storages if data_storage.HasField("output")]
 
     @staticmethod
     def _extract_new_agent_types(outputs: list[Output]) -> dict[str, Output.AgentType]:
-        """Returns dict of agent names mapped to its type defined in given `outputs`"""
+        """Returns dict of agent names mapped to its type defined in given `outputs`."""
         list_of_agent_type_lists = [output.agent_types for output in outputs if len(output.agent_types) > 0]
         list_of_agent_types = [item for sublist in list_of_agent_type_lists for item in sublist]
         return {item.class_name: item for item in list_of_agent_types}
 
     @staticmethod
     def _extract_series(outputs: list[Output]) -> dict[str, list[Output.Series]]:
-        """Returns series data from associated `outputs` mapped to the className of its agent"""
+        """Returns series data from associated `outputs` mapped to the className of its agent."""
         list_of_series_lists = [output.series for output in outputs if len(output.series) > 0]
         list_of_series = [series for sublist in list_of_series_lists for series in sublist]
 
-        series_per_class_name = {}
+        series_per_class_name: dict[str, list[Output.Series]] = {}
         for series in list_of_series:
             if series.class_name not in series_per_class_name:
                 series_per_class_name[series.class_name] = []
@@ -48,7 +60,7 @@ class OutputDAO:
         return series_per_class_name
 
     def get_sorted_agents_to_extract(self) -> Iterable[str]:
-        """Returns iterator of requested and available agent names in ascending order by count of series"""
+        """Returns iterator of requested and available agent names in ascending order by count of series."""
         all_series = self._get_agent_names_by_series_count_ascending()
         filtered_series = [agent_name for agent_name in all_series if self._agent_type_log.is_requested(agent_name)]
         return iter(filtered_series)
@@ -59,11 +71,21 @@ class OutputDAO:
         sorted_dict = sorted(length_per_agent_types.items(), key=lambda item: item[1])
         return [agent_name for agent_name, _ in sorted_dict]
 
-    def get_agent_data(self, agent_name: str, data_transformer: DataTransformer) -> dict[Optional[str], pd.DataFrame]:
-        """
-        Returns DataFrame(s) containing all data of given `agent` - data is removed after the first call
+    def get_agent_data(self, agent_name: str, data_transformer: DataTransformer) -> dict[str | None, pd.DataFrame]:
+        """Returns DataFrame(s) containing all data of given `agent` - data is removed after the first call.
+
         Depending on the chosen ResolveOption the dict contains one DataFrame for the simple (and merged columns),
         or, in `SPLIT` mode, additional DataFrames mapped to each complex column's name.
+
+        Args:
+            agent_name: name of agent whose data are to be returned
+            data_transformer: to handle data transformation
+
+        Returns:
+            output data for requested agent: on data frame for all simple columns, any one for each complex column
+
+        Raises:
+            AgentTypeError: if type of agent was not yet registered, logged with level "ERROR"
         """
         agent_series = self._all_series.pop(agent_name) if agent_name in self._all_series else []
         agent_type = self._agent_type_log.get_agent_type(agent_name)

fameio/output/reader.py

@@ -11,21 +11,24 @@ from fameprotobuf.data_storage_pb2 import DataStorage
 from google.protobuf.message import DecodeError
 
 import fameio
-from fameio.logs import log, log_critical_and_raise
+from fameio.logs import log, log_critical, log_error
+from fameio.output import OutputError
 
 
-class ProtobufReaderError(Exception):
-    """Indicates an error while reading a protobuf file"""
+class ProtobufReaderError(OutputError):
+    """Indicates an error while reading a protobuf file."""
 
 
 class Reader(ABC):
-    """Abstract base class for protobuf file readers"""
+    """Abstract base class for protobuf file readers."""
 
-    _WARN_NO_HEADER = "No header recognised in file. File might be deprecated or corrupted."  # noqa
+    _ERR_FILE_READ = "Could not read file content."
+    _ERR_HEADER_UNRECOGNISED = ""
     _ERR_FILE_CORRUPT_NEGATIVE_LENGTH = "Corrupt file, message length must be positive."
     _ERR_FILE_CORRUPT_MISSING_DATA = "Trying to read corrupt file caused by inconsistent message length."
     _ERR_UNSUPPORTED_MODE = "Ignoring memory saving mode: not supported for files created with `fame-core<1.4`."
     _ERR_PARSING_FAILED = "File Corrupt. Could not parse file content."
+    _WARN_NO_HEADER = "No header recognised in file. File might be deprecated or corrupted."
     _DEBUG_FILE_END_REACHED = "Reached expected end of file."
 
     _HEADER_LENGTH = 30
@@ -44,7 +47,7 @@ class Reader(ABC):
     @staticmethod
     @final
     def _raise_error(error_message: str) -> NoReturn:
-        log_critical_and_raise(ProtobufReaderError(error_message))
+        raise log_critical(ProtobufReaderError(error_message))
 
     def __init__(self, file: IO, read_single) -> None:
         self._file = file
@@ -52,12 +55,18 @@ class Reader(ABC):
 
     @abstractmethod
     def read(self) -> list[DataStorage]:
-        """Reads associated filestream and returns one or multiple DataStorage(s) or empty list"""
+        """Reads associated filestream and returns one or multiple DataStorage(s) or empty list.
+
+        Returns:
+            one or multiple DataStorage protobuf object(s) read from file
+
+        Raises:
+            ProtobufReaderError: if file is corrupted in any way, logged with level "ERROR"
+        """
 
     @staticmethod
     def get_reader(file: IO, read_single: bool = False) -> Reader:
-        """
-        Returns reader matching the given file header
+        """Returns reader matching the given file header.
 
         Args:
             file: to be read by the returned Reader
@@ -65,17 +74,30 @@ class Reader(ABC):
 
         Returns:
             Reader that can read the specified file
+
+        Raises:
+            ProtobufReaderError: if file has an unsupported header,logged with level "CRITICAL"
         """
         log().debug("Reading file headers...")
         try:
-            header = file.read(Reader._HEADER_LENGTH).decode(Reader.HEADER_ENCODING)
-            return Reader._READER_HEADERS[header](file, read_single)
-        except (KeyError, UnicodeDecodeError):
-            return Reader._READER_HEADERS[None](file, read_single)
+            header_content = file.read(Reader._HEADER_LENGTH)
+        except ValueError as e:
+            raise log_critical(ProtobufReaderError(Reader._ERR_FILE_READ)) from e
+
+        try:
+            header = header_content.decode(Reader.HEADER_ENCODING)
+        except UnicodeDecodeError:
+            header = None
+            log().warning(Reader._WARN_NO_HEADER)
+
+        if header not in Reader._READER_HEADERS:
+            header = None
+
+        return Reader._READER_HEADERS[header](file, read_single)
 
     @final
     def _read_message_length(self) -> int:
-        """Returns length of next DataStorage message in file"""
+        """Returns length of next DataStorage message in file."""
         message_length_byte = self._file.read(self.BYTES_DEFINING_MESSAGE_LENGTH)
         if not message_length_byte:
             log().debug(self._DEBUG_FILE_END_REACHED)
@@ -85,34 +107,55 @@ class Reader(ABC):
         return message_length_int
 
     @final
-    def _read_data_storage_message(self, message_length: int = None) -> DataStorage:
-        """
-        Returns given `data_storage` read from current file position and following `message_length` bytes.
+    def _read_data_storage_message(self, message_length: int | None = None) -> DataStorage:
+        """Returns data storage read from current file position and following `message_length` bytes.
+
         If `message_length` is omitted, the rest of the file is read. If no message is found, None is returned.
+
+        Args:
+            message_length: amounts of bytes to read - must correspond to the next DataStorage message in file
+
+        Returns:
+            Read and de-serialised DataStorage
+
+        Raises:
+            ProtobufReaderError: if message_length is corrupt or file is corrupt, logged with level "ERROR"
         """
         if message_length is None:
             message = self._file.read()
         elif message_length > 0:
             message = self._file.read(message_length)
         else:
-            raise IOError(self._ERR_FILE_CORRUPT_NEGATIVE_LENGTH)
+            raise log_error(ProtobufReaderError(self._ERR_FILE_CORRUPT_NEGATIVE_LENGTH))
         if message_length and len(message) != message_length:
-            log().error(self._ERR_FILE_CORRUPT_MISSING_DATA)
+            raise log_error(ProtobufReaderError(self._ERR_FILE_CORRUPT_MISSING_DATA))
         return self._parse_to_data_storage(message) if message else None
 
     @staticmethod
     @final
     def _parse_to_data_storage(message: bytes) -> DataStorage:
+        """
+        De-serialises a binary message into a DataStorage protobuf object
+
+        Args:
+            message: to be convert
+
+        Returns:
+            DataStorage initialised from the given message
+
+        Raises:
+            ProtobufReaderError: if message could not be converted, logged with level "ERROR"
+        """
         data_storage = DataStorage()
         try:
             data_storage.ParseFromString(message)
         except DecodeError as e:
-            raise IOError(Reader._ERR_PARSING_FAILED) from e
+            raise log_error(ProtobufReaderError(Reader._ERR_PARSING_FAILED)) from e
         return data_storage
 
 
 class ReaderV2(Reader):
-    """Reader class for `fame-core>=2.0` output with header of version v002"""
+    """Reader class for `fame-core>=2.0` output with header of version v002."""
 
     def read(self) -> list[DataStorage]:
         messages = []

fameio/output/yaml_writer.py

@@ -5,23 +5,31 @@ from pathlib import Path
 
 import yaml
 
-from fameio.logs import log
+from fameio.logs import log, log_error
+from fameio.output import OutputError
 
-ERR_WRITE_EXCEPTION = "Failed to save dictionary to YAML file `{}`"
-INFO_DESTINATION = "Saving scenario to file at {}"
+_ERR_OPEN_FILE = "Could not open file for reading: '{}'"
+
+_INFO_DESTINATION = "Saving scenario to file at {}"
+
+
+class YamlWriterError(OutputError):
+    """An error occurred during writing a YAML file."""
 
 
 def data_to_yaml_file(data: dict, file_path: Path) -> None:
-    """
-    Save the given data to a YAML file at given path
+    """Save the given data to a YAML file at given path.
 
     Args:
         data: to be saved to yaml file
         file_path: at which the file will be created
+
+    Raises:
+        YamlWriterError: if file could not be opened or written, logged with level "ERROR"
     """
-    log().info(INFO_DESTINATION.format(file_path))
+    log().info(_INFO_DESTINATION.format(file_path))
     try:
         with open(file_path, "w", encoding="utf-8") as f:
             yaml.dump(data, f, sort_keys=False, encoding="utf-8")
-    except Exception as e:
-        raise RuntimeError(ERR_WRITE_EXCEPTION.format(file_path)) from e
+    except OSError as e:
+        raise log_error(YamlWriterError(_ERR_OPEN_FILE.format(file_path))) from e

fameio/scripts/__init__.py

@@ -3,18 +3,36 @@ import sys
 
 from fameio.scripts.convert_results import DEFAULT_CONFIG as DEFAULT_CONVERT_CONFIG
 from fameio.scripts.convert_results import run as convert_results
+from fameio.scripts.exception import ScriptError
 from fameio.scripts.make_config import run as make_config
+from fameio.scripts.reformat import run as reformat
 from fameio.cli.convert_results import handle_args as handle_convert_results_args
 from fameio.cli.make_config import handle_args as handle_make_config_args
+from fameio.cli.reformat import handle_args as handle_reformat_args
 
 
 # noinspection PyPep8Naming
 def makeFameRunConfig():
-    run_config = handle_make_config_args(sys.argv[1:])
-    make_config(run_config)
+    cli_config = handle_make_config_args(sys.argv[1:])
+    try:
+        make_config(cli_config)
+    except ScriptError as e:
+        raise SystemExit(1) from e
 
 
 # noinspection PyPep8Naming
 def convertFameResults():
-    run_config = handle_convert_results_args(sys.argv[1:], DEFAULT_CONVERT_CONFIG)
-    convert_results(run_config)
+    cli_config = handle_convert_results_args(sys.argv[1:], DEFAULT_CONVERT_CONFIG)
+    try:
+        convert_results(cli_config)
+    except ScriptError as e:
+        raise SystemExit(1) from e
+
+
+# noinspection PyPep8Naming
+def reformatTimeSeries():
+    cli_config = handle_reformat_args(sys.argv[1:])
+    try:
+        reformat(cli_config)
+    except ScriptError as e:
+        raise SystemExit(1) from e