fameio 3.1.1__py3-none-any.whl → 3.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. fameio/cli/convert_results.py +10 -10
  2. fameio/cli/make_config.py +9 -9
  3. fameio/cli/options.py +6 -4
  4. fameio/cli/parser.py +87 -51
  5. fameio/cli/reformat.py +58 -0
  6. fameio/input/__init__.py +4 -4
  7. fameio/input/loader/__init__.py +13 -13
  8. fameio/input/loader/controller.py +64 -18
  9. fameio/input/loader/loader.py +25 -16
  10. fameio/input/metadata.py +57 -38
  11. fameio/input/resolver.py +9 -10
  12. fameio/input/scenario/agent.py +62 -26
  13. fameio/input/scenario/attribute.py +93 -40
  14. fameio/input/scenario/contract.py +160 -56
  15. fameio/input/scenario/exception.py +41 -18
  16. fameio/input/scenario/fameiofactory.py +57 -6
  17. fameio/input/scenario/generalproperties.py +22 -12
  18. fameio/input/scenario/scenario.py +117 -38
  19. fameio/input/scenario/stringset.py +29 -11
  20. fameio/input/schema/agenttype.py +27 -10
  21. fameio/input/schema/attribute.py +108 -45
  22. fameio/input/schema/java_packages.py +14 -12
  23. fameio/input/schema/schema.py +39 -15
  24. fameio/input/validator.py +198 -54
  25. fameio/input/writer.py +137 -46
  26. fameio/logs.py +28 -47
  27. fameio/output/__init__.py +5 -1
  28. fameio/output/agent_type.py +89 -28
  29. fameio/output/conversion.py +52 -37
  30. fameio/output/csv_writer.py +107 -27
  31. fameio/output/data_transformer.py +17 -24
  32. fameio/output/execution_dao.py +170 -0
  33. fameio/output/input_dao.py +71 -33
  34. fameio/output/output_dao.py +33 -11
  35. fameio/output/reader.py +64 -21
  36. fameio/output/yaml_writer.py +16 -8
  37. fameio/scripts/__init__.py +22 -4
  38. fameio/scripts/convert_results.py +126 -52
  39. fameio/scripts/convert_results.py.license +1 -1
  40. fameio/scripts/exception.py +7 -0
  41. fameio/scripts/make_config.py +34 -13
  42. fameio/scripts/make_config.py.license +1 -1
  43. fameio/scripts/reformat.py +71 -0
  44. fameio/scripts/reformat.py.license +3 -0
  45. fameio/series.py +174 -59
  46. fameio/time.py +79 -25
  47. fameio/tools.py +48 -8
  48. {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/METADATA +50 -34
  49. fameio-3.3.0.dist-info/RECORD +60 -0
  50. {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/WHEEL +1 -1
  51. {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/entry_points.txt +1 -0
  52. CHANGELOG.md +0 -288
  53. fameio-3.1.1.dist-info/RECORD +0 -56
  54. {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/LICENSE.txt +0 -0
  55. {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/LICENSES/Apache-2.0.txt +0 -0
  56. {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/LICENSES/CC-BY-4.0.txt +0 -0
  57. {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/LICENSES/CC0-1.0.txt +0 -0
@@ -1,55 +1,62 @@
1
1
  # SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
2
2
  #
3
3
  # SPDX-License-Identifier: Apache-2.0
4
+ from __future__ import annotations
4
5
 
5
6
  import math
6
- from typing import Optional
7
7
 
8
8
  import pandas as pd
9
9
 
10
10
  from fameio.cli.options import TimeOptions
11
- from fameio.logs import log_error_and_raise, log
12
- from fameio.time import ConversionError, FameTime
11
+ from fameio.logs import log_error, log
12
+ from fameio.output import OutputError
13
+ from fameio.time import FameTime, ConversionError as TimeConversionError
13
14
 
14
15
  _ERR_UNIMPLEMENTED = "Time conversion mode '{}' not implemented."
16
+ _ERR_TIME_CONVERSION = "Conversion of timestamps failed."
15
17
  _ERR_NEGATIVE = "StepsBefore and StepsAfter must be Zero or positive integers"
16
18
 
17
19
 
18
- def _apply_time_merging(
19
- dataframes: dict[Optional[str], pd.DataFrame], offset: int, period: int, first_positive_focal_point: int
20
- ) -> None:
21
- """Applies time merging to `data` based on given `offset`, `period`, and `first_positive_focal_point`"""
22
- log().debug("Grouping TimeSteps...")
23
- for key in dataframes.keys():
24
- df = dataframes[key]
25
- index_columns = df.index.names
26
- df.reset_index(inplace=True)
27
- df["TimeStep"] = df["TimeStep"].apply(lambda t: merge_time(t, first_positive_focal_point, offset, period))
28
- dataframes[key] = df.groupby(by=index_columns).sum()
20
+ class ConversionError(OutputError):
21
+ """An error that occurred during conversion of output data."""
29
22
 
30
23
 
31
- def apply_time_merging(data: dict[Optional[str], pd.DataFrame], config: Optional[list[int]]) -> None:
32
- """
33
- Applies merging of TimeSteps inplace for given `data`
24
+ def apply_time_merging(data: dict[str | None, pd.DataFrame], config: list[int] | None) -> None:
25
+ """Applies merging of TimeSteps inplace for given `data`.
34
26
 
35
27
  Args:
36
28
  data: one or multiple DataFrames of time series; depending on the given config, contents might be modified
37
29
  config: three integer values defining how to merge data within a range of time steps
30
+
31
+ Raises:
32
+ ConversionError: if parameters are not valid, logged with level "ERROR"
38
33
  """
39
34
  if not config or all(v == 0 for v in config):
40
35
  return
41
36
  focal_point, steps_before, steps_after = config
42
37
  if steps_before < 0 or steps_after < 0:
43
- raise ValueError(_ERR_NEGATIVE)
38
+ raise log_error(ConversionError(_ERR_NEGATIVE))
44
39
 
45
40
  period = steps_before + steps_after + 1
46
41
  first_positive_focal_point = focal_point % period
47
42
  _apply_time_merging(data, offset=steps_before, period=period, first_positive_focal_point=first_positive_focal_point)
48
43
 
49
44
 
50
- def merge_time(time_step: int, focal_time: int, offset: int, period: int) -> int:
51
- """
52
- Returns `time_step` rounded to its corresponding focal point
45
+ def _apply_time_merging(
46
+ dataframes: dict[str | None, pd.DataFrame], offset: int, period: int, first_positive_focal_point: int
47
+ ) -> None:
48
+ """Applies time merging to `data` based on given `offset`, `period`, and `first_positive_focal_point`."""
49
+ log().debug("Grouping TimeSteps...")
50
+ for key in dataframes.keys():
51
+ df = dataframes[key]
52
+ index_columns = df.index.names
53
+ df.reset_index(inplace=True)
54
+ df["TimeStep"] = df["TimeStep"].apply(lambda t: _merge_time(t, first_positive_focal_point, offset, period))
55
+ dataframes[key] = df.groupby(by=index_columns).sum()
56
+
57
+
58
+ def _merge_time(time_step: int, focal_time: int, offset: int, period: int) -> int:
59
+ """Returns `time_step` rounded to its corresponding focal point.
53
60
 
54
61
  Args:
55
62
  time_step: TimeStep to round
@@ -63,32 +70,40 @@ def merge_time(time_step: int, focal_time: int, offset: int, period: int) -> int
63
70
  return math.floor((time_step + offset - focal_time) / period) * period + focal_time
64
71
 
65
72
 
66
- def apply_time_option(data: dict[Optional[str], pd.DataFrame], mode: TimeOptions) -> None:
67
- """
68
- Applies time option based on given `mode` inplace of given `data`
73
+ def apply_time_option(data: dict[str | None, pd.DataFrame], mode: TimeOptions) -> None:
74
+ """Applies time option based on given `mode` inplace of given `data`.
69
75
 
70
76
  Args:
71
77
  data: one or multiple DataFrames of time series; column `TimeStep` might be modified (depending on mode)
72
78
  mode: name of time conversion mode (derived from Enum)
79
+
80
+ Raises:
81
+ ConversionError: if provided mode is not implemented , logged with level "ERROR"
73
82
  """
74
- if mode == TimeOptions.INT:
75
- log().debug("No time conversion...")
76
- elif mode == TimeOptions.UTC:
77
- _convert_time_index(data, "%Y-%m-%d %H:%M:%S")
78
- elif mode == TimeOptions.FAME:
79
- _convert_time_index(data, "%Y-%m-%d_%H:%M:%S")
80
- else:
81
- log_error_and_raise(ConversionError(_ERR_UNIMPLEMENTED.format(mode)))
83
+ try:
84
+ if mode == TimeOptions.INT:
85
+ log().debug("No time conversion...")
86
+ elif mode == TimeOptions.UTC:
87
+ _convert_time_index(data, "%Y-%m-%d %H:%M:%S")
88
+ elif mode == TimeOptions.FAME:
89
+ _convert_time_index(data, "%Y-%m-%d_%H:%M:%S")
90
+ else:
91
+ raise log_error(ConversionError(_ERR_UNIMPLEMENTED.format(mode)))
92
+ except TimeConversionError as e:
93
+ raise log_error(ConversionError(_ERR_TIME_CONVERSION.format())) from e
82
94
 
83
95
 
84
- def _convert_time_index(data: dict[Optional[str], pd.DataFrame], datetime_format: str) -> None:
85
- """
86
- Inplace replacement of `TimeStep` column in MultiIndex of each item of `data` from FAME's time steps` to DateTime
87
- in given `date_format`
96
+ def _convert_time_index(data: dict[str | None, pd.DataFrame], datetime_format: str) -> None:
97
+ """Replaces (inplace) `TimeStep` column in MultiIndex of each item of `data` to DateTime.
98
+
99
+ Format of the resulting DateTime is determined by given `date_format`.
88
100
 
89
101
  Args:
90
102
  data: one or multiple DataFrames of time series; column `TimeStep` will be modified
91
- datetime_format: used for the conversion
103
+ datetime_format: determines result of the conversion
104
+
105
+ Raises:
106
+ TimeConversionError: if time cannot be converted, logged with level "ERROR"
92
107
  """
93
108
  log().debug(f"Converting TimeStep to format '{datetime_format}'...")
94
109
  for _, df in data.items():
@@ -1,19 +1,29 @@
1
- # SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
1
+ # SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
2
2
  #
3
3
  # SPDX-License-Identifier: Apache-2.0
4
+ from __future__ import annotations
5
+
4
6
  from pathlib import Path
5
- from typing import Union
6
7
 
7
8
  import pandas as pd
8
9
 
9
- from fameio.logs import log
10
+ from fameio.logs import log, log_error
11
+ from fameio.output import OutputError
10
12
  from fameio.output.data_transformer import INDEX
11
13
  from fameio.series import TimeSeriesManager
12
14
  from fameio.tools import ensure_path_exists
13
15
 
14
16
 
17
+ class CsvWriterError(OutputError):
18
+ """An error occurred during writing a CSV file."""
19
+
20
+
15
21
  class CsvWriter:
16
- """Writes dataframes to different csv files"""
22
+ """Writes dataframes to different csv files."""
23
+
24
+ _ERR_DIR_CREATE = "Could not create directory for output files: '{}'"
25
+ _ERR_FILE_OPEN = "Could not open file for writing: '{}'"
26
+ _ERR_FILE_WRITE = "Could not write to file '{}' due to: {}"
17
27
 
18
28
  _INFO_USING_PATH = "Using specified output path: {}"
19
29
  _INFO_USING_DERIVED_PATH = "No output path specified - writing to new local folder: {}"
@@ -21,16 +31,21 @@ class CsvWriter:
21
31
  CSV_FILE_SUFFIX = ".csv"
22
32
 
23
33
  def __init__(self, config_output: Path, input_file_path: Path, single_export: bool) -> None:
34
+ """Constructs a new CsvWriter.
35
+
36
+ Raises:
37
+ CsvWriterError: if output folder could not be created, logged with level "ERROR"
38
+ """
24
39
  self._single_export = single_export
25
40
  self._output_folder = self._get_output_folder_name(config_output, input_file_path)
26
- self._files = {}
41
+ self._files: dict[str, Path] = {}
27
42
  self._create_output_folder()
28
43
 
29
44
  @staticmethod
30
45
  def _get_output_folder_name(config_output: Path, input_file_path: Path) -> Path:
31
- """Returns name of the output folder derived either from the specified `config_output` or `input_file_path`"""
46
+ """Returns name of the output folder derived either from the specified `config_output` or `input_file_path`."""
32
47
  if config_output:
33
- output_folder_name = config_output
48
+ output_folder_name: str | Path = config_output
34
49
  log().info(CsvWriter._INFO_USING_PATH.format(config_output))
35
50
  else:
36
51
  output_folder_name = input_file_path.stem
@@ -38,13 +53,28 @@ class CsvWriter:
38
53
  return Path(output_folder_name)
39
54
 
40
55
  def _create_output_folder(self) -> None:
41
- """Creates output folder if not yet present"""
56
+ """Creates output folder if not yet present.
57
+
58
+ Raises:
59
+ CsvWriterError: if output folder could not be created, logged with level "ERROR"
60
+ """
42
61
  log().debug("Creating output folder if required...")
43
62
  if not self._output_folder.is_dir():
44
- self._output_folder.mkdir(parents=True)
63
+ try:
64
+ self._output_folder.mkdir(parents=True)
65
+ except OSError as e:
66
+ raise log_error(CsvWriterError(self._ERR_DIR_CREATE.format(self._output_folder))) from e
67
+
68
+ def write_to_files(self, agent_name: str, data: dict[None | str, pd.DataFrame]) -> None:
69
+ """Writes `data` for given `agent_name` to .csv file(s).
70
+
71
+ Args:
72
+ agent_name: name of agent whose data are to be written to file(s)
73
+ data: previously extracted data for that agent that are to be written
45
74
 
46
- def write_to_files(self, agent_name: str, data: dict[Union[None, str], pd.DataFrame]) -> None:
47
- """Writes `data` for given `agent_name` to .csv file(s)"""
75
+ Raises:
76
+ CsvWriterError: when file could not be written, logged on level "ERROR"
77
+ """
48
78
  for column_name, column_data in data.items():
49
79
  column_data.sort_index(inplace=True)
50
80
  if self._single_export:
@@ -55,17 +85,57 @@ class CsvWriter:
55
85
  identifier = self._get_identifier(agent_name, column_name)
56
86
  self._write_data_frame(column_data, identifier)
57
87
 
58
- def write_time_series_to_disk(self, timeseries_manager: TimeSeriesManager) -> None:
59
- """Writes time_series of given `timeseries_manager` to disk"""
88
+ def write_all_time_series_to_disk(self, timeseries_manager: TimeSeriesManager) -> None:
89
+ """Writes time_series of given `timeseries_manager` to disk.
90
+
91
+ Args:
92
+ timeseries_manager: to provide the time series that are to be written
93
+
94
+ Raises:
95
+ CsvWriterError: if data could not be written to disk, logged on level "ERROR"
96
+ """
60
97
  for _, name, data in timeseries_manager.get_all_series():
61
98
  if data is not None:
62
99
  target_path = Path(self._output_folder, name)
63
100
  ensure_path_exists(target_path.parent)
64
- # noinspection PyTypeChecker
65
- data.to_csv(path_or_buf=target_path, sep=";", header=None, index=None)
101
+ self.write_single_time_series_to_disk(data, target_path)
66
102
 
67
103
  @staticmethod
68
- def _get_identifier(agent_name: str, column_name: str, agent_id: str = None) -> str:
104
+ def write_single_time_series_to_disk(data: pd.DataFrame, file: Path) -> None:
105
+ """Writes given timeseries the provided file path.
106
+
107
+ Args:
108
+ data: to be written
109
+ file: target path of csv file
110
+
111
+ Raises:
112
+ CsvWriterError: if data could not be written to disk, logged on level "ERROR"
113
+ """
114
+ CsvWriter._dataframe_to_csv(data, file, header=False, index=False, mode="w")
115
+
116
+ @staticmethod
117
+ def _dataframe_to_csv(data: pd.DataFrame, file: Path, header: bool, index: bool, mode: str) -> None:
118
+ """Write given data to specified CSV file with specified parameters using semicolon separators.
119
+
120
+ Args:
121
+ data: to be written
122
+ file: target path of csv file
123
+ header: write column headers
124
+ index: write index column(s)
125
+ mode: append to or overwrite file
126
+
127
+ Raises:
128
+ CsvWriterError: if data could not be written to disk, logged on level "ERROR"
129
+ """
130
+ try:
131
+ data.to_csv(path_or_buf=file, sep=";", header=header, index=index, mode=mode)
132
+ except OSError as e:
133
+ raise log_error(CsvWriterError(CsvWriter._ERR_FILE_OPEN.format(file))) from e
134
+ except UnicodeError as e:
135
+ raise log_error(CsvWriterError(CsvWriter._ERR_FILE_WRITE.format(file, str(e)))) from e
136
+
137
+ @staticmethod
138
+ def _get_identifier(agent_name: str, column_name: str | None = None, agent_id: str | None = None) -> str:
69
139
  """Returns unique identifier for given `agent_name` and (optional) `agent_id` and `column_name`"""
70
140
  identifier = str(agent_name)
71
141
  if column_name:
@@ -75,36 +145,46 @@ class CsvWriter:
75
145
  return identifier
76
146
 
77
147
  def _write_data_frame(self, data: pd.DataFrame, identifier: str) -> None:
78
- """
79
- Appends `data` to existing csv file derived from `identifier` without headers,
80
- or writes new file with headers instead
148
+ """Writes `data` to csv file derived from `identifier`.
149
+
150
+ Appends data if csv file exists, else writes new file with headers instead.
151
+
152
+ Args:
153
+ data: to be written to file
154
+ identifier: to derive the file name from
155
+
156
+ Raises:
157
+ CsvWriterError: when file could not be written, logged on level "ERROR"
81
158
  """
82
159
  if self._has_file(identifier):
83
160
  outfile_name = self._get_outfile_name(identifier)
84
- data.to_csv(outfile_name, sep=";", index=True, header=False, mode="a")
161
+ mode = "a"
162
+ header = False
85
163
  else:
86
164
  outfile_name = self._create_outfile_name(identifier)
87
165
  self._save_outfile_name(outfile_name, identifier)
88
- data.to_csv(outfile_name, sep=";", index=True, header=True)
166
+ mode = "w"
167
+ header = True
168
+ self._dataframe_to_csv(data, outfile_name, header=header, index=True, mode=mode)
89
169
 
90
170
  def _has_file(self, identifier: str) -> bool:
91
- """Returns True if a file for given `identifier` was already written"""
171
+ """Returns True if a file for given `identifier` was already written."""
92
172
  return identifier in self._files
93
173
 
94
174
  def pop_all_file_paths(self) -> dict[str, Path]:
95
- """Clears all stored file paths and returns their previous identifiers and their paths"""
175
+ """Clears all stored file paths and returns their previous identifiers and their paths."""
96
176
  current_files = self._files
97
177
  self._files = {}
98
178
  return current_files
99
179
 
100
- def _get_outfile_name(self, identifier: str) -> str:
101
- """Returns file name for given `agent_name` and (optional) `agent_id`"""
180
+ def _get_outfile_name(self, identifier: str) -> Path:
181
+ """Returns file path for given `agent_name` and (optional) `agent_id`."""
102
182
  return self._files[identifier]
103
183
 
104
184
  def _create_outfile_name(self, identifier: str) -> Path:
105
- """Returns fully qualified file name based on given `agent_name` and (optional) `agent_id`"""
185
+ """Returns fully qualified file path based on given `agent_name` and (optional) `agent_id`."""
106
186
  return Path(self._output_folder, f"{identifier}{self.CSV_FILE_SUFFIX}")
107
187
 
108
188
  def _save_outfile_name(self, outfile_name: Path, identifier: str) -> None:
109
- """Stores given name for given `agent_name` and (optional) `agent_id`"""
189
+ """Stores given name for given `agent_name` and (optional) `agent_id`."""
110
190
  self._files[identifier] = outfile_name
@@ -4,8 +4,6 @@
4
4
  from __future__ import annotations
5
5
 
6
6
  from abc import ABC
7
- from builtins import staticmethod
8
- from typing import Union, Optional
9
7
 
10
8
  import pandas as pd
11
9
  from fameprotobuf.services_pb2 import Output
@@ -18,7 +16,7 @@ INDEX = ("AgentId", "TimeStep")
18
16
 
19
17
 
20
18
  class DataTransformer(ABC):
21
- """Extracts and provides series data from parsed and processed output files for requested agents"""
19
+ """Extracts and provides series data from parsed and processed output files for requested agents."""
22
20
 
23
21
  MODES = {
24
22
  ResolveOptions.IGNORE: lambda: DataTransformerIgnore(), # pylint: disable=unnecessary-lambda
@@ -30,13 +28,11 @@ class DataTransformer(ABC):
30
28
  def build(complex_column_mode: ResolveOptions) -> DataTransformer:
31
29
  return DataTransformer.MODES[complex_column_mode]()
32
30
 
33
- def extract_agent_data(
34
- self, series: list[Output.Series], agent_type: AgentType
35
- ) -> dict[Optional[str], pd.DataFrame]:
36
- """
37
- Returns dict of DataFrame(s) containing all data from given `series` of given `agent_type`.
31
+ def extract_agent_data(self, series: list[Output.Series], agent_type: AgentType) -> dict[str | None, pd.DataFrame]:
32
+ """Returns dict of DataFrame(s) containing all data from given `series` of given `agent_type`.
33
+
38
34
  When ResolveOption is `SPLIT`, the dict maps each complex column's name to the associated DataFrame.
39
- In any case, the dict maps `None` to a DataFrame with the content of all simple column / merged columns.
35
+ In any case, the dict maps `None` to a DataFrame with the content of all simple columns.
40
36
  """
41
37
  container = self._extract_agent_data(series, agent_type)
42
38
  data_frames = {}
@@ -45,7 +41,7 @@ class DataTransformer(ABC):
45
41
  column_name = agent_type.get_column_name_for_id(column_id)
46
42
  if column_id == DataTransformer.SIMPLE_COLUMN_INDEX:
47
43
  data_frame.rename(columns=self._get_column_map(agent_type), inplace=True)
48
- index = INDEX
44
+ index: tuple[str, ...] = INDEX
49
45
  data_frame = data_frame.loc[:, agent_type.get_simple_column_mask()]
50
46
  else:
51
47
  data_frame.rename(columns={0: column_name}, inplace=True)
@@ -59,8 +55,8 @@ class DataTransformer(ABC):
59
55
 
60
56
  def _extract_agent_data(
61
57
  self, series: list[Output.Series], agent_type: AgentType
62
- ) -> dict[int, dict[tuple, list[Union[float, None, str]]]]:
63
- """Returns mapping of (agentId, timeStep) to fixed-length list of all output columns for given `class_name`"""
58
+ ) -> dict[int, dict[tuple, list[float | None | str]]]:
59
+ """Returns mapping of (agentId, timeStep) to fixed-length list of all output columns for given `class_name`."""
64
60
  container = DataTransformer._create_container(agent_type)
65
61
  mask_simple = agent_type.get_simple_column_mask()
66
62
  while series:
@@ -70,7 +66,7 @@ class DataTransformer(ABC):
70
66
 
71
67
  @staticmethod
72
68
  def _create_container(agent_type: AgentType) -> dict[int, dict]:
73
- """Returns map of complex columns IDs to an empty dict, and one more for the remaining simple columns"""
69
+ """Returns map of complex columns IDs to an empty dict, and one more for the remaining simple columns."""
74
70
  field_ids = agent_type.get_complex_column_ids().union([DataTransformer.SIMPLE_COLUMN_INDEX])
75
71
  return {field_id: {} for field_id in field_ids}
76
72
 
@@ -78,9 +74,9 @@ class DataTransformer(ABC):
78
74
  self,
79
75
  series: Output.Series,
80
76
  mask_simple: list[bool],
81
- container: dict[int, dict[tuple, list[Union[float, None, str]]]],
77
+ container: dict[int, dict[tuple, list[float | None | str]]],
82
78
  ) -> None:
83
- """Adds data from given `series` to specified `container` dict as list"""
79
+ """Adds data from given `series` to specified `container` dict as list."""
84
80
  empty_list: list = [None] * len(mask_simple)
85
81
  for line in series.lines:
86
82
  index = (series.agent_id, line.time_step)
@@ -89,32 +85,29 @@ class DataTransformer(ABC):
89
85
  if mask_simple[column.field_id]:
90
86
  simple_values[column.field_id] = column.value
91
87
  else:
92
- self._merge_complex_column(column, simple_values)
93
88
  self._store_complex_values(column, container, index)
94
89
  container[DataTransformer.SIMPLE_COLUMN_INDEX][index] = simple_values
95
90
 
96
- @staticmethod
97
- def _merge_complex_column(column: Output.Series.Line.Column, values: list) -> None:
98
- """Merges complex column data"""
99
-
100
91
  @staticmethod
101
92
  def _store_complex_values(column: Output.Series.Line.Column, container: dict[int, dict], base_index: tuple) -> None:
102
- """Stores complex column data"""
93
+ """Stores complex column data."""
103
94
 
104
95
  @staticmethod
105
96
  def _get_column_map(agent_type: AgentType) -> dict[int, str]:
106
- """Returns mapping of simple column IDs to their name for given `agent_type`"""
97
+ """Returns mapping of simple column IDs to their name for given `agent_type`."""
107
98
  return agent_type.get_simple_column_map()
108
99
 
109
100
 
110
101
  class DataTransformerIgnore(DataTransformer):
111
- """Ignores complex columns on output"""
102
+ """Ignores complex columns on output."""
112
103
 
113
104
 
114
105
  class DataTransformerSplit(DataTransformer):
106
+ """Stores complex data columns split by column type."""
107
+
115
108
  @staticmethod
116
109
  def _store_complex_values(column: Output.Series.Line.Column, container: dict[int, dict], base_index: tuple) -> None:
117
- """Adds inner data from `column` to given `container` - split by column type"""
110
+ """Adds inner data from `column` to given `container` - split by column type."""
118
111
  for entry in column.entries:
119
112
  index = base_index + tuple(entry.index_values)
120
113
  container[column.field_id][index] = entry.value
@@ -0,0 +1,170 @@
1
+ # SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
2
+ #
3
+ # SPDX-License-Identifier: Apache-2.0
4
+ from __future__ import annotations
5
+
6
+ from typing import Any, Final
7
+
8
+ from fameprotobuf.data_storage_pb2 import DataStorage
9
+ from fameprotobuf.execution_data_pb2 import ExecutionData
10
+ from google.protobuf import message
11
+
12
+ from fameio.logs import log_error
13
+ from fameio.output import OutputError
14
+
15
+
16
+ class ExecutionDataError(OutputError):
17
+ """Indicates an error during reconstruction of execution metadata from its protobuf representation."""
18
+
19
+
20
+ VERSION_MAP: Final[dict[str, str]] = {
21
+ "fame_protobuf": "FameProtobuf",
22
+ "fame_io": "FameIo",
23
+ "fame_core": "FameCore",
24
+ "python": "Python",
25
+ "jvm": "JavaVirtualMachine",
26
+ "os": "OperatingSystem",
27
+ }
28
+ PROCESS_MAP: Final[dict[str, str]] = {
29
+ "core_count": "NumberOfCores",
30
+ "output_interval": "OutputIntervalInTicks",
31
+ "output_process": "ProcessControllingOutputs",
32
+ }
33
+ STATISTICS_MAP: Final[dict[str, str]] = {
34
+ "start": "SimulationBeginInRealTime",
35
+ "duration_in_ms": "SimulationWallTimeInMillis",
36
+ "tick_count": "SimulatedTicks",
37
+ }
38
+
39
+
40
+ class ExecutionDao:
41
+ """Data access object for execution metadata saved in protobuf."""
42
+
43
+ _ERR_MULTIPLE_VERSIONS = "More than two version metadata sections found: File is corrupt."
44
+ _ERR_MULTIPLE_CONFIGURATIONS = "More than one configuration metadata section found: File is corrupt."
45
+ _ERR_MULTIPLE_SIMULATIONS = "More than one simulation metadata section found: File is corrupt."
46
+ _ERR_NO_VERSION = "No version data found: File is either corrupt or was created with fameio version < 3.0."
47
+
48
+ KEY_COMPILATION: Final[str] = "InputCompilation"
49
+ KEY_RUN: Final[str] = "ModelRun"
50
+ KEY_VERSIONS: Final[str] = "SoftwareVersions"
51
+ KEY_PROCESSES: Final[str] = "ProcessConfiguration"
52
+ KEY_STATISTICS: Final[str] = "Statistics"
53
+
54
+ def __init__(self) -> None:
55
+ self._compile_versions: ExecutionData.VersionData | None = None
56
+ self._run_versions: ExecutionData.VersionData | None = None
57
+ self._run_configuration: ExecutionData.ProcessConfiguration | None = None
58
+ self._run_simulation: ExecutionData.Simulation | None = None
59
+
60
+ def store_execution_metadata(self, data_storages: list[DataStorage]) -> None:
61
+ """Scans given data storages for execution metadata.
62
+
63
+ If metadata are present, they are extracted for later inspection
64
+
65
+ Args:
66
+ data_storages: to be scanned for execution metadata
67
+
68
+ Raises:
69
+ ExecutionDataError: if more execution sections are found than expected, logged with level "ERROR"
70
+ """
71
+ for entry in [storage.execution for storage in data_storages if storage.HasField("execution")]:
72
+ if entry.HasField("version_data"):
73
+ self._add_version_data(entry.version_data)
74
+ if entry.HasField("configuration"):
75
+ self._add_configuration(entry.configuration)
76
+ if entry.HasField("simulation"):
77
+ self._add_simulation(entry.simulation)
78
+
79
+ def _add_version_data(self, data: ExecutionData.VersionData) -> None:
80
+ """Stores given version metadata.
81
+
82
+ Args:
83
+ data: version data saved during compilation (first call), or during model run (second call)
84
+
85
+ Raises:
86
+ ExecutionDataError: if both version data are already set, logged with level "ERROR"
87
+ """
88
+ if not self._compile_versions:
89
+ self._compile_versions = data
90
+ elif not self._run_versions:
91
+ self._run_versions = data
92
+ else:
93
+ raise log_error(ExecutionDataError(self._ERR_MULTIPLE_VERSIONS))
94
+
95
+ def _add_configuration(self, data: ExecutionData.ProcessConfiguration) -> None:
96
+ """Stores given process configuration metadata.
97
+
98
+ Args:
99
+ data: process configuration data to be saved
100
+
101
+ Raises:
102
+ ExecutionDataError: if process configuration data are already set, logged with level "ERROR"
103
+ """
104
+ if not self._run_configuration:
105
+ self._run_configuration = data
106
+ else:
107
+ raise log_error(ExecutionDataError(self._ERR_MULTIPLE_CONFIGURATIONS))
108
+
109
+ def _add_simulation(self, data: ExecutionData.Simulation) -> None:
110
+ """Stores given simulation metadata.
111
+
112
+ Args:
113
+ data: simulation metadata to be stored
114
+
115
+ Raises:
116
+ ExecutionDataError: if simulation metadata are already set, logged with level "ERROR"
117
+ """
118
+ if not self._run_simulation:
119
+ self._run_simulation = data
120
+ else:
121
+ raise log_error(ExecutionDataError(self._ERR_MULTIPLE_SIMULATIONS))
122
+
123
+ def get_fameio_version(self) -> str:
124
+ """Gets version of fameio used to create the input data.
125
+
126
+ Returns:
127
+ fameio version that was used to create the input data
128
+
129
+ Raises:
130
+ ExecutionDataError: if fameio version could not be read, logged with level "ERROR"
131
+ """
132
+ if self._compile_versions:
133
+ return self._compile_versions.fame_io
134
+ raise log_error(ExecutionDataError(self._ERR_NO_VERSION))
135
+
136
+ def get_metadata_dict(self) -> dict[str, Any]:
137
+ """Creates a dictionary from all provided execution metadata.
138
+
139
+ Returns:
140
+ dictionary with all execution metadata currently stored in this DAO
141
+ """
142
+ result = {
143
+ self.KEY_COMPILATION: {self.KEY_VERSIONS: self._get_dict(self._compile_versions, VERSION_MAP)},
144
+ self.KEY_RUN: {
145
+ self.KEY_VERSIONS: self._get_dict(self._run_versions, VERSION_MAP),
146
+ self.KEY_PROCESSES: self._get_dict(self._run_configuration, PROCESS_MAP),
147
+ self.KEY_STATISTICS: self._get_dict(self._run_simulation, STATISTICS_MAP),
148
+ },
149
+ }
150
+ return result
151
+
152
+ @staticmethod
153
+ def _get_dict(data: message, replacements: dict[str, str]) -> dict[str, str]:
154
+ """Searches for `replacements.keys()` in provided `data`.
155
+
156
+ If key is available, saves the corresponding data item to dict, associated to a name matching the value in `replacements`.
157
+
158
+ Args:
159
+ data: to extract data from
160
+ replacements: keys to be replaced by their values in the resulting dict
161
+
162
+ Returns:
163
+ a dictionary matching entries from `data` with their new keys as specified under "replacements"
164
+ """
165
+ versions = {}
166
+ if data is not None:
167
+ for key, replacement in replacements.items():
168
+ if data.HasField(key):
169
+ versions[replacement] = getattr(data, key)
170
+ return versions