fameio 3.1.0__py3-none-any.whl → 3.2.0__py3-none-any.whl

fameio/output/agent_type.py

@@ -1,10 +1,13 @@
-# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
-from typing import Union
+from __future__ import annotations
 
 from fameprotobuf.services_pb2 import Output
 
+from fameio.logs import log_error
+from fameio.output import OutputError
+
 
 class AgentType:
     """Provides information derived from an underlying protobuf AgentType"""
@@ -28,18 +31,17 @@ class AgentType:
 
     def get_simple_column_mask(self) -> list[bool]:
         """Returns list of bool - where an entry is True if the output column with the same index is not complex"""
-        return [True if len(field.index_names) == 0 else False for field in self._agent_type.fields]
+        return [len(field.index_names) == 0 for field in self._agent_type.fields]
 
     def get_complex_column_ids(self) -> set[int]:
         """Returns set of IDs for complex columns, ignoring simple columns"""
-        return set([field.field_id for field in self._agent_type.fields if len(field.index_names) > 0])
+        return {field.field_id for field in self._agent_type.fields if len(field.index_names) > 0}
 
-    def get_column_name_for_id(self, column_index: int) -> Union[str, None]:
+    def get_column_name_for_id(self, column_index: int) -> str | None:
         """Returns name of column by given `column_index` or None, if column is not present"""
         if 0 <= column_index < len(self._agent_type.fields):
             return self._agent_type.fields[column_index].field_name
-        else:
-            return None
+        return None
 
     def get_inner_columns(self, column_index: int) -> tuple[str, ...]:
         """Returns tuple of inner column names for complex column with given `column_index`"""
@@ -50,40 +52,104 @@ class AgentType:
         return self._agent_type.class_name
 
 
+class AgentTypeError(OutputError):
+    """Indicates an error with the agent types definitions"""
+
+
 class AgentTypeLog:
     """Stores data about collected agent types"""
 
     _ERR_AGENT_TYPE_MISSING = "Requested AgentType `{}` not found."
     _ERR_DOUBLE_DEFINITION = "Just one definition allowed per AgentType. Found multiple for {}. File might be corrupt."
 
-    def __init__(self, requested_agents: list[str]) -> None:
-        self._requested_agents = [agent.upper() for agent in requested_agents] if requested_agents else None
-        self._requested_agent_types = {}
+    def __init__(self, _agent_name_filter_list: list[str]) -> None:
+        """
+        Initialises new AgentTypeLog
+
+        Args:
+            _agent_name_filter_list: list of agent type names that are requested for output data extraction
+        """
+        self._agent_name_filter_list: list[str] | None = (
+            [agent.upper() for agent in _agent_name_filter_list] if _agent_name_filter_list else None
+        )
+        self._requested_agent_types: dict[str, AgentType] = {}
+        self._agents_with_output: list[str] = []
 
     def update_agents(self, new_types: dict[str, Output.AgentType]) -> None:
-        """Saves new `agent_types` (if any) contained in given `output` if requested for extraction"""
-        if new_types:
-            if self._requested_agents:
-                new_types = {
-                    agent_name: agent_type
-                    for agent_name, agent_type in new_types.items()
-                    if agent_name.upper() in self._requested_agents
-                }
-            for agent_name in self._requested_agent_types.keys():
-                if agent_name in new_types:
-                    raise Exception(self._ERR_DOUBLE_DEFINITION.format(agent_name))
-            self._requested_agent_types.update(new_types)
+        """
+        If any new `agent_types` are provided, checks if they are requested for extraction, and, if so, saves them
+
+        Args:
+            new_types: to be saved (if requested for extraction)
+
+        Raises:
+            AgentTypeError: if agent type was already registered, logged with level "ERROR"
+        """
+        if not new_types:
+            return
+
+        self._agents_with_output.extend(list(new_types.keys()))
+        filtered_types = self._filter_agents_by_name(new_types)
+        self._ensure_no_duplication(filtered_types)
+        self._requested_agent_types.update(filtered_types)
+
+    def _filter_agents_by_name(self, new_types: dict[str, Output.AgentType]) -> dict[str, Output.AgentType]:
+        """
+        Removes and entries from `new_types` not on `agent_name_filter_list`
+
+        Args:
+            new_types: to be filtered
+
+        Returns:
+            filtered list, or original list if no filter is active
+        """
+        if self._agent_name_filter_list:
+            return {
+                agent_name: agent_type
+                for agent_name, agent_type in new_types.items()
+                if agent_name.upper() in self._agent_name_filter_list
+            }
+        return new_types
+
+    def _ensure_no_duplication(self, filtered_types: dict[str, Output.AgentType]) -> None:
+        """
+        Ensures no duplicate agent type definitions occur
+
+        Args:
+            filtered_types: to be checked for duplications with already registered types
+
+        Raises:
+            AgentTypeError: if duplicate agent type is found, logged with level "ERROR"
+        """
+        for agent_name in self._requested_agent_types:
+            if agent_name in filtered_types:
+                raise log_error(AgentTypeError(self._ERR_DOUBLE_DEFINITION.format(agent_name)))
 
     def has_any_agent_type(self) -> bool:
         """Returns True if any agent type was registered so far present"""
         return len(self._requested_agent_types) > 0
 
-    def get_agent_type(self, agent_name: str) -> AgentType:
-        """Returns `AgentType` of given agent `name`"""
-        if agent_name not in self._requested_agent_types:
-            raise Exception(self._ERR_AGENT_TYPE_MISSING.format(agent_name))
-        return AgentType(self._requested_agent_types[agent_name])
+    def get_agent_type(self, agent_type_name: str) -> AgentType:
+        """
+        Returns the requested type of agent
+
+        Args:
+            agent_type_name: requested name of agent type
+
+        Returns:
+            stored agent type
+
+        Raises:
+            AgentTypeError: if no agent type could be found with that name, logged with level "ERROR"
+        """
+        if agent_type_name not in self._requested_agent_types:
+            raise log_error(AgentTypeError(self._ERR_AGENT_TYPE_MISSING.format(agent_type_name)))
+        return AgentType(self._requested_agent_types[agent_type_name])
 
     def is_requested(self, agent_name: str) -> bool:
         """Returns True if given agent_name is known and requested"""
         return agent_name in self._requested_agent_types
+
+    def get_agents_with_output(self) -> list[str]:
+        """Returns all names of agents that had output"""
+        return self._agents_with_output

fameio/output/conversion.py

@@ -1,53 +1,62 @@
-# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
 
 import math
-from typing import Optional
 
 import pandas as pd
 
 from fameio.cli.options import TimeOptions
-from fameio.logs import log_error_and_raise, log
-from fameio.time import ConversionError, FameTime
+from fameio.logs import log_error, log
+from fameio.output import OutputError
+from fameio.time import FameTime, ConversionError as TimeConversionError
 
 _ERR_UNIMPLEMENTED = "Time conversion mode '{}' not implemented."
+_ERR_TIME_CONVERSION = "Conversion of timestamps failed."
 _ERR_NEGATIVE = "StepsBefore and StepsAfter must be Zero or positive integers"
 
 
-def _apply_time_merging(
-    dataframes: dict[Optional[str], pd.DataFrame], offset: int, period: int, first_positive_focal_point: int
-) -> None:
-    """Applies time merging to `data` based on given `offset`, `period`, and `first_positive_focal_point`"""
-    log().debug(f"Grouping TimeSteps...")
-    for key in dataframes.keys():
-        df = dataframes[key]
-        index_columns = df.index.names
-        df.reset_index(inplace=True)
-        df["TimeStep"] = df["TimeStep"].apply(lambda t: merge_time(t, first_positive_focal_point, offset, period))
-        dataframes[key] = df.groupby(by=index_columns).sum()
+class ConversionError(OutputError):
+    """An error that occurred during conversion of output data"""
 
 
-def apply_time_merging(data: dict[Optional[str], pd.DataFrame], config: Optional[list[int]]) -> None:
+def apply_time_merging(data: dict[str | None, pd.DataFrame], config: list[int] | None) -> None:
     """
     Applies merging of TimeSteps inplace for given `data`
 
     Args:
         data: one or multiple DataFrames of time series; depending on the given config, contents might be modified
         config: three integer values defining how to merge data within a range of time steps
+
+    Raises:
+        ConversionError: if parameters are not valid, logged with level "ERROR"
     """
     if not config or all(v == 0 for v in config):
         return
     focal_point, steps_before, steps_after = config
     if steps_before < 0 or steps_after < 0:
-        raise ValueError(_ERR_NEGATIVE)
+        raise log_error(ConversionError(_ERR_NEGATIVE))
 
     period = steps_before + steps_after + 1
     first_positive_focal_point = focal_point % period
     _apply_time_merging(data, offset=steps_before, period=period, first_positive_focal_point=first_positive_focal_point)
 
 
-def merge_time(time_step: int, focal_time: int, offset: int, period: int) -> int:
+def _apply_time_merging(
+    dataframes: dict[str | None, pd.DataFrame], offset: int, period: int, first_positive_focal_point: int
+) -> None:
+    """Applies time merging to `data` based on given `offset`, `period`, and `first_positive_focal_point`"""
+    log().debug("Grouping TimeSteps...")
+    for key in dataframes.keys():
+        df = dataframes[key]
+        index_columns = df.index.names
+        df.reset_index(inplace=True)
+        df["TimeStep"] = df["TimeStep"].apply(lambda t: _merge_time(t, first_positive_focal_point, offset, period))
+        dataframes[key] = df.groupby(by=index_columns).sum()
+
+
+def _merge_time(time_step: int, focal_time: int, offset: int, period: int) -> int:
     """
     Returns `time_step` rounded to its corresponding focal point
 
@@ -63,25 +72,31 @@ def merge_time(time_step: int, focal_time: int, offset: int, period: int) -> int
     return math.floor((time_step + offset - focal_time) / period) * period + focal_time
 
 
-def apply_time_option(data: dict[Optional[str], pd.DataFrame], mode: TimeOptions) -> None:
+def apply_time_option(data: dict[str | None, pd.DataFrame], mode: TimeOptions) -> None:
     """
     Applies time option based on given `mode` inplace of given `data`
 
     Args:
         data: one or multiple DataFrames of time series; column `TimeStep` might be modified (depending on mode)
         mode: name of time conversion mode (derived from Enum)
-    """
-    if mode == TimeOptions.INT:
-        log().debug("No time conversion...")
-    elif mode == TimeOptions.UTC:
-        _convert_time_index(data, "%Y-%m-%d %H:%M:%S")
-    elif mode == TimeOptions.FAME:
-        _convert_time_index(data, "%Y-%m-%d_%H:%M:%S")
-    else:
-        log_error_and_raise(ConversionError(_ERR_UNIMPLEMENTED.format(mode)))
-
 
-def _convert_time_index(data: dict[Optional[str], pd.DataFrame], datetime_format: str) -> None:
+    Raises:
+        ConversionError: if provided mode is not implemented , logged with level "ERROR"
+    """
+    try:
+        if mode == TimeOptions.INT:
+            log().debug("No time conversion...")
+        elif mode == TimeOptions.UTC:
+            _convert_time_index(data, "%Y-%m-%d %H:%M:%S")
+        elif mode == TimeOptions.FAME:
+            _convert_time_index(data, "%Y-%m-%d_%H:%M:%S")
+        else:
+            raise log_error(ConversionError(_ERR_UNIMPLEMENTED.format(mode)))
+    except TimeConversionError as e:
+        raise log_error(ConversionError(_ERR_TIME_CONVERSION.format())) from e
+
+
+def _convert_time_index(data: dict[str | None, pd.DataFrame], datetime_format: str) -> None:
     """
     Inplace replacement of `TimeStep` column in MultiIndex of each item of `data` from FAME's time steps` to DateTime
     in given `date_format`
@@ -89,6 +104,9 @@ def _convert_time_index(data: dict[Optional[str], pd.DataFrame], datetime_format
     Args:
         data: one or multiple DataFrames of time series; column `TimeStep` will be modified
         datetime_format: used for the conversion
+
+    Raises:
+        TimeConversionError: if time cannot be converted, logged with level "ERROR"
     """
     log().debug(f"Converting TimeStep to format '{datetime_format}'...")
     for _, df in data.items():

fameio/output/csv_writer.py

@@ -1,36 +1,50 @@
-# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
 from pathlib import Path
-from typing import Union
 
 import pandas as pd
 
-from fameio.logs import log
+from fameio.logs import log, log_error
+from fameio.output import OutputError
 from fameio.output.data_transformer import INDEX
 from fameio.series import TimeSeriesManager
 from fameio.tools import ensure_path_exists
 
 
+class CsvWriterError(OutputError):
+    """An error occurred during writing a CSV file"""
+
+
 class CsvWriter:
     """Writes dataframes to different csv files"""
 
+    _ERR_DIR_CREATE = "Could not create directory for output files: '{}'"
+    _ERR_FILE_OPEN = "Could not open file for writing: '{}'"
+    _ERR_FILE_WRITE = "Could not write to file '{}' due to: {}"
+
     _INFO_USING_PATH = "Using specified output path: {}"
     _INFO_USING_DERIVED_PATH = "No output path specified - writing to new local folder: {}"
 
     CSV_FILE_SUFFIX = ".csv"
 
     def __init__(self, config_output: Path, input_file_path: Path, single_export: bool) -> None:
+        """
+        Raises:
+             CsvWriterError: if output folder could not be created, logged with level "ERROR"
+        """
         self._single_export = single_export
         self._output_folder = self._get_output_folder_name(config_output, input_file_path)
-        self._files = {}
+        self._files: dict[str, Path] = {}
         self._create_output_folder()
 
     @staticmethod
     def _get_output_folder_name(config_output: Path, input_file_path: Path) -> Path:
         """Returns name of the output folder derived either from the specified `config_output` or `input_file_path`"""
         if config_output:
-            output_folder_name = config_output
+            output_folder_name: str | Path = config_output
             log().info(CsvWriter._INFO_USING_PATH.format(config_output))
         else:
             output_folder_name = input_file_path.stem
@@ -38,13 +52,30 @@ class CsvWriter:
         return Path(output_folder_name)
 
     def _create_output_folder(self) -> None:
-        """Creates output folder if not yet present"""
+        """
+        Creates output folder if not yet present
+
+        Raises:
+            CsvWriterError: if output folder could not be created, logged with level "ERROR"
+        """
         log().debug("Creating output folder if required...")
         if not self._output_folder.is_dir():
-            self._output_folder.mkdir(parents=True)
+            try:
+                self._output_folder.mkdir(parents=True)
+            except OSError as e:
+                raise log_error(CsvWriterError(self._ERR_DIR_CREATE.format(self._output_folder))) from e
+
+    def write_to_files(self, agent_name: str, data: dict[None | str, pd.DataFrame]) -> None:
+        """
+        Writes `data` for given `agent_name` to .csv file(s)
+
+        Args:
+            agent_name: name of agent whose data are to be written to file(s)
+            data: previously extracted data for that agent that are to be written
 
-    def write_to_files(self, agent_name: str, data: dict[Union[None, str], pd.DataFrame]) -> None:
-        """Writes `data` for given `agent_name` to .csv file(s)"""
+        Raises:
+            CsvWriterError: when file could not be written, logged on level "ERROR"
+        """
         for column_name, column_data in data.items():
             column_data.sort_index(inplace=True)
             if self._single_export:
@@ -56,16 +87,45 @@ class CsvWriter:
                 self._write_data_frame(column_data, identifier)
 
     def write_time_series_to_disk(self, timeseries_manager: TimeSeriesManager) -> None:
-        """Writes time_series of given `timeseries_manager` to disk"""
+        """
+        Writes time_series of given `timeseries_manager` to disk
+
+        Args:
+            timeseries_manager:
+
+        Raises:
+            CsvWriterError: if data could not be written to disk, logged on level "ERROR"
+        """
         for _, name, data in timeseries_manager.get_all_series():
             if data is not None:
                 target_path = Path(self._output_folder, name)
                 ensure_path_exists(target_path.parent)
-                # noinspection PyTypeChecker
-                data.to_csv(path_or_buf=target_path, sep=";", header=None, index=None)
+                self._dataframe_to_csv(data, target_path, header=False, index=False, mode="w")
 
     @staticmethod
-    def _get_identifier(agent_name: str, column_name: str, agent_id: str = None) -> str:
+    def _dataframe_to_csv(data: pd.DataFrame, file: Path, header: bool, index: bool, mode: str) -> None:
+        """
+        Write given data to specified CSV file with specified parameters using semicolon separators
+
+        Args:
+            data: to be written
+            file: target path of csv file
+            header: write column headers
+            index: write index column(s)
+            mode: append to or overwrite file
+
+        Raises:
+            CsvWriterError: if data could not be written to disk, logged on level "ERROR"
+        """
+        try:
+            data.to_csv(path_or_buf=file, sep=";", header=header, index=index, mode=mode)
+        except OSError as e:
+            raise log_error(CsvWriterError(CsvWriter._ERR_FILE_OPEN.format(file))) from e
+        except UnicodeError as e:
+            raise log_error(CsvWriterError(CsvWriter._ERR_FILE_WRITE.format(file, str(e)))) from e
+
+    @staticmethod
+    def _get_identifier(agent_name: str, column_name: str | None = None, agent_id: str | None = None) -> str:
         """Returns unique identifier for given `agent_name` and (optional) `agent_id` and `column_name`"""
         identifier = str(agent_name)
         if column_name:
@@ -78,14 +138,24 @@ class CsvWriter:
         """
         Appends `data` to existing csv file derived from `identifier` without headers,
         or writes new file with headers instead
+
+        Args:
+            data: to be written to file
+            identifier: to derive the file name from
+
+        Raises:
+            CsvWriterError: when file could not be written, logged on level "ERROR"
         """
         if self._has_file(identifier):
             outfile_name = self._get_outfile_name(identifier)
-            data.to_csv(outfile_name, sep=";", index=True, header=False, mode="a")
+            mode = "a"
+            header = False
         else:
             outfile_name = self._create_outfile_name(identifier)
             self._save_outfile_name(outfile_name, identifier)
-            data.to_csv(outfile_name, sep=";", index=True, header=True)
+            mode = "w"
+            header = True
+        self._dataframe_to_csv(data, outfile_name, header=header, index=True, mode=mode)
 
     def _has_file(self, identifier: str) -> bool:
         """Returns True if a file for given `identifier` was already written"""
@@ -97,12 +167,12 @@ class CsvWriter:
         self._files = {}
         return current_files
 
-    def _get_outfile_name(self, identifier: str) -> str:
-        """Returns file name for given `agent_name` and (optional) `agent_id`"""
+    def _get_outfile_name(self, identifier: str) -> Path:
+        """Returns file path for given `agent_name` and (optional) `agent_id`"""
         return self._files[identifier]
 
     def _create_outfile_name(self, identifier: str) -> Path:
-        """Returns fully qualified file name based on given `agent_name` and (optional) `agent_id`"""
+        """Returns fully qualified file path based on given `agent_name` and (optional) `agent_id`"""
         return Path(self._output_folder, f"{identifier}{self.CSV_FILE_SUFFIX}")
 
     def _save_outfile_name(self, outfile_name: Path, identifier: str) -> None:

fameio/output/data_transformer.py

@@ -1,11 +1,9 @@
-# SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
 from abc import ABC
-from builtins import staticmethod
-from typing import Union, Optional
 
 import pandas as pd
 from fameprotobuf.services_pb2 import Output
@@ -21,8 +19,8 @@ class DataTransformer(ABC):
     """Extracts and provides series data from parsed and processed output files for requested agents"""
 
     MODES = {
-        ResolveOptions.IGNORE: lambda: DataTransformerIgnore(),
-        ResolveOptions.SPLIT: lambda: DataTransformerSplit(),
+        ResolveOptions.IGNORE: lambda: DataTransformerIgnore(),  # pylint: disable=unnecessary-lambda
+        ResolveOptions.SPLIT: lambda: DataTransformerSplit(),  # pylint: disable=unnecessary-lambda
     }
     SIMPLE_COLUMN_INDEX = -1
 
@@ -30,13 +28,11 @@ class DataTransformer(ABC):
     def build(complex_column_mode: ResolveOptions) -> DataTransformer:
         return DataTransformer.MODES[complex_column_mode]()
 
-    def extract_agent_data(
-        self, series: list[Output.Series], agent_type: AgentType
-    ) -> dict[Optional[str], pd.DataFrame]:
+    def extract_agent_data(self, series: list[Output.Series], agent_type: AgentType) -> dict[str | None, pd.DataFrame]:
         """
         Returns dict of DataFrame(s) containing all data from given `series` of given `agent_type`.
         When ResolveOption is `SPLIT`, the dict maps each complex column's name to the associated DataFrame.
-        In any case, the dict maps `None` to a DataFrame with the content of all simple column / merged columns.
+        In any case, the dict maps `None` to a DataFrame with the content of all simple columns.
         """
         container = self._extract_agent_data(series, agent_type)
         data_frames = {}
@@ -45,7 +41,7 @@ class DataTransformer(ABC):
             column_name = agent_type.get_column_name_for_id(column_id)
             if column_id == DataTransformer.SIMPLE_COLUMN_INDEX:
                 data_frame.rename(columns=self._get_column_map(agent_type), inplace=True)
-                index = INDEX
+                index: tuple[str, ...] = INDEX
                 data_frame = data_frame.loc[:, agent_type.get_simple_column_mask()]
             else:
                 data_frame.rename(columns={0: column_name}, inplace=True)
@@ -59,7 +55,7 @@ class DataTransformer(ABC):
 
     def _extract_agent_data(
         self, series: list[Output.Series], agent_type: AgentType
-    ) -> dict[int, dict[tuple, list[Union[float, None, str]]]]:
+    ) -> dict[int, dict[tuple, list[float | None | str]]]:
         """Returns mapping of (agentId, timeStep) to fixed-length list of all output columns for given `class_name`"""
         container = DataTransformer._create_container(agent_type)
         mask_simple = agent_type.get_simple_column_mask()
@@ -78,7 +74,7 @@ class DataTransformer(ABC):
         self,
         series: Output.Series,
         mask_simple: list[bool],
-        container: dict[int, dict[tuple, list[Union[float, None, str]]]],
+        container: dict[int, dict[tuple, list[float | None | str]]],
     ) -> None:
         """Adds data from given `series` to specified `container` dict as list"""
         empty_list: list = [None] * len(mask_simple)
@@ -89,19 +85,12 @@ class DataTransformer(ABC):
                 if mask_simple[column.field_id]:
                     simple_values[column.field_id] = column.value
                 else:
-                    self._merge_complex_column(column, simple_values)
                     self._store_complex_values(column, container, index)
             container[DataTransformer.SIMPLE_COLUMN_INDEX][index] = simple_values
 
     @staticmethod
-    def _merge_complex_column(column: Output.Series.Line.Column, values: list) -> None:
-        """Does not merge complex column data"""
-        pass
-
-    @staticmethod
-    def _store_complex_values(column: Output.Series.Line.Column, container: dict[int, dict], index: tuple) -> None:
-        """Does not store complex column data"""
-        pass
+    def _store_complex_values(column: Output.Series.Line.Column, container: dict[int, dict], base_index: tuple) -> None:
+        """Stores complex column data"""
 
     @staticmethod
     def _get_column_map(agent_type: AgentType) -> dict[int, str]:
@@ -114,6 +103,8 @@ class DataTransformerIgnore(DataTransformer):
 
 
 class DataTransformerSplit(DataTransformer):
+    """Stores complex data columns split by column type"""
+
     @staticmethod
     def _store_complex_values(column: Output.Series.Line.Column, container: dict[int, dict], base_index: tuple) -> None:
         """Adds inner data from `column` to given `container` - split by column type"""