fameio 3.1.1__py3-none-any.whl → 3.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fameio/cli/convert_results.py +10 -10
- fameio/cli/make_config.py +9 -9
- fameio/cli/options.py +6 -4
- fameio/cli/parser.py +87 -51
- fameio/cli/reformat.py +58 -0
- fameio/input/__init__.py +4 -4
- fameio/input/loader/__init__.py +13 -13
- fameio/input/loader/controller.py +64 -18
- fameio/input/loader/loader.py +25 -16
- fameio/input/metadata.py +57 -38
- fameio/input/resolver.py +9 -10
- fameio/input/scenario/agent.py +62 -26
- fameio/input/scenario/attribute.py +93 -40
- fameio/input/scenario/contract.py +160 -56
- fameio/input/scenario/exception.py +41 -18
- fameio/input/scenario/fameiofactory.py +57 -6
- fameio/input/scenario/generalproperties.py +22 -12
- fameio/input/scenario/scenario.py +117 -38
- fameio/input/scenario/stringset.py +29 -11
- fameio/input/schema/agenttype.py +27 -10
- fameio/input/schema/attribute.py +108 -45
- fameio/input/schema/java_packages.py +14 -12
- fameio/input/schema/schema.py +39 -15
- fameio/input/validator.py +198 -54
- fameio/input/writer.py +137 -46
- fameio/logs.py +28 -47
- fameio/output/__init__.py +5 -1
- fameio/output/agent_type.py +89 -28
- fameio/output/conversion.py +52 -37
- fameio/output/csv_writer.py +107 -27
- fameio/output/data_transformer.py +17 -24
- fameio/output/execution_dao.py +170 -0
- fameio/output/input_dao.py +71 -33
- fameio/output/output_dao.py +33 -11
- fameio/output/reader.py +64 -21
- fameio/output/yaml_writer.py +16 -8
- fameio/scripts/__init__.py +22 -4
- fameio/scripts/convert_results.py +126 -52
- fameio/scripts/convert_results.py.license +1 -1
- fameio/scripts/exception.py +7 -0
- fameio/scripts/make_config.py +34 -13
- fameio/scripts/make_config.py.license +1 -1
- fameio/scripts/reformat.py +71 -0
- fameio/scripts/reformat.py.license +3 -0
- fameio/series.py +174 -59
- fameio/time.py +79 -25
- fameio/tools.py +48 -8
- {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/METADATA +50 -34
- fameio-3.3.0.dist-info/RECORD +60 -0
- {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/WHEEL +1 -1
- {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/entry_points.txt +1 -0
- CHANGELOG.md +0 -288
- fameio-3.1.1.dist-info/RECORD +0 -56
- {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/LICENSE.txt +0 -0
- {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/LICENSES/Apache-2.0.txt +0 -0
- {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/LICENSES/CC-BY-4.0.txt +0 -0
- {fameio-3.1.1.dist-info → fameio-3.3.0.dist-info}/LICENSES/CC0-1.0.txt +0 -0
fameio/output/conversion.py
CHANGED
@@ -1,55 +1,62 @@
|
|
1
1
|
# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
|
+
from __future__ import annotations
|
4
5
|
|
5
6
|
import math
|
6
|
-
from typing import Optional
|
7
7
|
|
8
8
|
import pandas as pd
|
9
9
|
|
10
10
|
from fameio.cli.options import TimeOptions
|
11
|
-
from fameio.logs import
|
12
|
-
from fameio.
|
11
|
+
from fameio.logs import log_error, log
|
12
|
+
from fameio.output import OutputError
|
13
|
+
from fameio.time import FameTime, ConversionError as TimeConversionError
|
13
14
|
|
14
15
|
_ERR_UNIMPLEMENTED = "Time conversion mode '{}' not implemented."
|
16
|
+
_ERR_TIME_CONVERSION = "Conversion of timestamps failed."
|
15
17
|
_ERR_NEGATIVE = "StepsBefore and StepsAfter must be Zero or positive integers"
|
16
18
|
|
17
19
|
|
18
|
-
|
19
|
-
|
20
|
-
) -> None:
|
21
|
-
"""Applies time merging to `data` based on given `offset`, `period`, and `first_positive_focal_point`"""
|
22
|
-
log().debug("Grouping TimeSteps...")
|
23
|
-
for key in dataframes.keys():
|
24
|
-
df = dataframes[key]
|
25
|
-
index_columns = df.index.names
|
26
|
-
df.reset_index(inplace=True)
|
27
|
-
df["TimeStep"] = df["TimeStep"].apply(lambda t: merge_time(t, first_positive_focal_point, offset, period))
|
28
|
-
dataframes[key] = df.groupby(by=index_columns).sum()
|
20
|
+
class ConversionError(OutputError):
|
21
|
+
"""An error that occurred during conversion of output data."""
|
29
22
|
|
30
23
|
|
31
|
-
def apply_time_merging(data: dict[
|
32
|
-
"""
|
33
|
-
Applies merging of TimeSteps inplace for given `data`
|
24
|
+
def apply_time_merging(data: dict[str | None, pd.DataFrame], config: list[int] | None) -> None:
|
25
|
+
"""Applies merging of TimeSteps inplace for given `data`.
|
34
26
|
|
35
27
|
Args:
|
36
28
|
data: one or multiple DataFrames of time series; depending on the given config, contents might be modified
|
37
29
|
config: three integer values defining how to merge data within a range of time steps
|
30
|
+
|
31
|
+
Raises:
|
32
|
+
ConversionError: if parameters are not valid, logged with level "ERROR"
|
38
33
|
"""
|
39
34
|
if not config or all(v == 0 for v in config):
|
40
35
|
return
|
41
36
|
focal_point, steps_before, steps_after = config
|
42
37
|
if steps_before < 0 or steps_after < 0:
|
43
|
-
raise
|
38
|
+
raise log_error(ConversionError(_ERR_NEGATIVE))
|
44
39
|
|
45
40
|
period = steps_before + steps_after + 1
|
46
41
|
first_positive_focal_point = focal_point % period
|
47
42
|
_apply_time_merging(data, offset=steps_before, period=period, first_positive_focal_point=first_positive_focal_point)
|
48
43
|
|
49
44
|
|
50
|
-
def
|
51
|
-
|
52
|
-
|
45
|
+
def _apply_time_merging(
|
46
|
+
dataframes: dict[str | None, pd.DataFrame], offset: int, period: int, first_positive_focal_point: int
|
47
|
+
) -> None:
|
48
|
+
"""Applies time merging to `data` based on given `offset`, `period`, and `first_positive_focal_point`."""
|
49
|
+
log().debug("Grouping TimeSteps...")
|
50
|
+
for key in dataframes.keys():
|
51
|
+
df = dataframes[key]
|
52
|
+
index_columns = df.index.names
|
53
|
+
df.reset_index(inplace=True)
|
54
|
+
df["TimeStep"] = df["TimeStep"].apply(lambda t: _merge_time(t, first_positive_focal_point, offset, period))
|
55
|
+
dataframes[key] = df.groupby(by=index_columns).sum()
|
56
|
+
|
57
|
+
|
58
|
+
def _merge_time(time_step: int, focal_time: int, offset: int, period: int) -> int:
|
59
|
+
"""Returns `time_step` rounded to its corresponding focal point.
|
53
60
|
|
54
61
|
Args:
|
55
62
|
time_step: TimeStep to round
|
@@ -63,32 +70,40 @@ def merge_time(time_step: int, focal_time: int, offset: int, period: int) -> int
|
|
63
70
|
return math.floor((time_step + offset - focal_time) / period) * period + focal_time
|
64
71
|
|
65
72
|
|
66
|
-
def apply_time_option(data: dict[
|
67
|
-
"""
|
68
|
-
Applies time option based on given `mode` inplace of given `data`
|
73
|
+
def apply_time_option(data: dict[str | None, pd.DataFrame], mode: TimeOptions) -> None:
|
74
|
+
"""Applies time option based on given `mode` inplace of given `data`.
|
69
75
|
|
70
76
|
Args:
|
71
77
|
data: one or multiple DataFrames of time series; column `TimeStep` might be modified (depending on mode)
|
72
78
|
mode: name of time conversion mode (derived from Enum)
|
79
|
+
|
80
|
+
Raises:
|
81
|
+
ConversionError: if provided mode is not implemented , logged with level "ERROR"
|
73
82
|
"""
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
83
|
+
try:
|
84
|
+
if mode == TimeOptions.INT:
|
85
|
+
log().debug("No time conversion...")
|
86
|
+
elif mode == TimeOptions.UTC:
|
87
|
+
_convert_time_index(data, "%Y-%m-%d %H:%M:%S")
|
88
|
+
elif mode == TimeOptions.FAME:
|
89
|
+
_convert_time_index(data, "%Y-%m-%d_%H:%M:%S")
|
90
|
+
else:
|
91
|
+
raise log_error(ConversionError(_ERR_UNIMPLEMENTED.format(mode)))
|
92
|
+
except TimeConversionError as e:
|
93
|
+
raise log_error(ConversionError(_ERR_TIME_CONVERSION.format())) from e
|
82
94
|
|
83
95
|
|
84
|
-
def _convert_time_index(data: dict[
|
85
|
-
"""
|
86
|
-
|
87
|
-
|
96
|
+
def _convert_time_index(data: dict[str | None, pd.DataFrame], datetime_format: str) -> None:
|
97
|
+
"""Replaces (inplace) `TimeStep` column in MultiIndex of each item of `data` to DateTime.
|
98
|
+
|
99
|
+
Format of the resulting DateTime is determined by given `date_format`.
|
88
100
|
|
89
101
|
Args:
|
90
102
|
data: one or multiple DataFrames of time series; column `TimeStep` will be modified
|
91
|
-
datetime_format:
|
103
|
+
datetime_format: determines result of the conversion
|
104
|
+
|
105
|
+
Raises:
|
106
|
+
TimeConversionError: if time cannot be converted, logged with level "ERROR"
|
92
107
|
"""
|
93
108
|
log().debug(f"Converting TimeStep to format '{datetime_format}'...")
|
94
109
|
for _, df in data.items():
|
fameio/output/csv_writer.py
CHANGED
@@ -1,19 +1,29 @@
|
|
1
|
-
# SPDX-FileCopyrightText:
|
1
|
+
# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
|
+
from __future__ import annotations
|
5
|
+
|
4
6
|
from pathlib import Path
|
5
|
-
from typing import Union
|
6
7
|
|
7
8
|
import pandas as pd
|
8
9
|
|
9
|
-
from fameio.logs import log
|
10
|
+
from fameio.logs import log, log_error
|
11
|
+
from fameio.output import OutputError
|
10
12
|
from fameio.output.data_transformer import INDEX
|
11
13
|
from fameio.series import TimeSeriesManager
|
12
14
|
from fameio.tools import ensure_path_exists
|
13
15
|
|
14
16
|
|
17
|
+
class CsvWriterError(OutputError):
|
18
|
+
"""An error occurred during writing a CSV file."""
|
19
|
+
|
20
|
+
|
15
21
|
class CsvWriter:
|
16
|
-
"""Writes dataframes to different csv files"""
|
22
|
+
"""Writes dataframes to different csv files."""
|
23
|
+
|
24
|
+
_ERR_DIR_CREATE = "Could not create directory for output files: '{}'"
|
25
|
+
_ERR_FILE_OPEN = "Could not open file for writing: '{}'"
|
26
|
+
_ERR_FILE_WRITE = "Could not write to file '{}' due to: {}"
|
17
27
|
|
18
28
|
_INFO_USING_PATH = "Using specified output path: {}"
|
19
29
|
_INFO_USING_DERIVED_PATH = "No output path specified - writing to new local folder: {}"
|
@@ -21,16 +31,21 @@ class CsvWriter:
|
|
21
31
|
CSV_FILE_SUFFIX = ".csv"
|
22
32
|
|
23
33
|
def __init__(self, config_output: Path, input_file_path: Path, single_export: bool) -> None:
|
34
|
+
"""Constructs a new CsvWriter.
|
35
|
+
|
36
|
+
Raises:
|
37
|
+
CsvWriterError: if output folder could not be created, logged with level "ERROR"
|
38
|
+
"""
|
24
39
|
self._single_export = single_export
|
25
40
|
self._output_folder = self._get_output_folder_name(config_output, input_file_path)
|
26
|
-
self._files = {}
|
41
|
+
self._files: dict[str, Path] = {}
|
27
42
|
self._create_output_folder()
|
28
43
|
|
29
44
|
@staticmethod
|
30
45
|
def _get_output_folder_name(config_output: Path, input_file_path: Path) -> Path:
|
31
|
-
"""Returns name of the output folder derived either from the specified `config_output` or `input_file_path
|
46
|
+
"""Returns name of the output folder derived either from the specified `config_output` or `input_file_path`."""
|
32
47
|
if config_output:
|
33
|
-
output_folder_name = config_output
|
48
|
+
output_folder_name: str | Path = config_output
|
34
49
|
log().info(CsvWriter._INFO_USING_PATH.format(config_output))
|
35
50
|
else:
|
36
51
|
output_folder_name = input_file_path.stem
|
@@ -38,13 +53,28 @@ class CsvWriter:
|
|
38
53
|
return Path(output_folder_name)
|
39
54
|
|
40
55
|
def _create_output_folder(self) -> None:
|
41
|
-
"""Creates output folder if not yet present
|
56
|
+
"""Creates output folder if not yet present.
|
57
|
+
|
58
|
+
Raises:
|
59
|
+
CsvWriterError: if output folder could not be created, logged with level "ERROR"
|
60
|
+
"""
|
42
61
|
log().debug("Creating output folder if required...")
|
43
62
|
if not self._output_folder.is_dir():
|
44
|
-
|
63
|
+
try:
|
64
|
+
self._output_folder.mkdir(parents=True)
|
65
|
+
except OSError as e:
|
66
|
+
raise log_error(CsvWriterError(self._ERR_DIR_CREATE.format(self._output_folder))) from e
|
67
|
+
|
68
|
+
def write_to_files(self, agent_name: str, data: dict[None | str, pd.DataFrame]) -> None:
|
69
|
+
"""Writes `data` for given `agent_name` to .csv file(s).
|
70
|
+
|
71
|
+
Args:
|
72
|
+
agent_name: name of agent whose data are to be written to file(s)
|
73
|
+
data: previously extracted data for that agent that are to be written
|
45
74
|
|
46
|
-
|
47
|
-
|
75
|
+
Raises:
|
76
|
+
CsvWriterError: when file could not be written, logged on level "ERROR"
|
77
|
+
"""
|
48
78
|
for column_name, column_data in data.items():
|
49
79
|
column_data.sort_index(inplace=True)
|
50
80
|
if self._single_export:
|
@@ -55,17 +85,57 @@ class CsvWriter:
|
|
55
85
|
identifier = self._get_identifier(agent_name, column_name)
|
56
86
|
self._write_data_frame(column_data, identifier)
|
57
87
|
|
58
|
-
def
|
59
|
-
"""Writes time_series of given `timeseries_manager` to disk
|
88
|
+
def write_all_time_series_to_disk(self, timeseries_manager: TimeSeriesManager) -> None:
|
89
|
+
"""Writes time_series of given `timeseries_manager` to disk.
|
90
|
+
|
91
|
+
Args:
|
92
|
+
timeseries_manager: to provide the time series that are to be written
|
93
|
+
|
94
|
+
Raises:
|
95
|
+
CsvWriterError: if data could not be written to disk, logged on level "ERROR"
|
96
|
+
"""
|
60
97
|
for _, name, data in timeseries_manager.get_all_series():
|
61
98
|
if data is not None:
|
62
99
|
target_path = Path(self._output_folder, name)
|
63
100
|
ensure_path_exists(target_path.parent)
|
64
|
-
|
65
|
-
data.to_csv(path_or_buf=target_path, sep=";", header=None, index=None)
|
101
|
+
self.write_single_time_series_to_disk(data, target_path)
|
66
102
|
|
67
103
|
@staticmethod
|
68
|
-
def
|
104
|
+
def write_single_time_series_to_disk(data: pd.DataFrame, file: Path) -> None:
|
105
|
+
"""Writes given timeseries the provided file path.
|
106
|
+
|
107
|
+
Args:
|
108
|
+
data: to be written
|
109
|
+
file: target path of csv file
|
110
|
+
|
111
|
+
Raises:
|
112
|
+
CsvWriterError: if data could not be written to disk, logged on level "ERROR"
|
113
|
+
"""
|
114
|
+
CsvWriter._dataframe_to_csv(data, file, header=False, index=False, mode="w")
|
115
|
+
|
116
|
+
@staticmethod
|
117
|
+
def _dataframe_to_csv(data: pd.DataFrame, file: Path, header: bool, index: bool, mode: str) -> None:
|
118
|
+
"""Write given data to specified CSV file with specified parameters using semicolon separators.
|
119
|
+
|
120
|
+
Args:
|
121
|
+
data: to be written
|
122
|
+
file: target path of csv file
|
123
|
+
header: write column headers
|
124
|
+
index: write index column(s)
|
125
|
+
mode: append to or overwrite file
|
126
|
+
|
127
|
+
Raises:
|
128
|
+
CsvWriterError: if data could not be written to disk, logged on level "ERROR"
|
129
|
+
"""
|
130
|
+
try:
|
131
|
+
data.to_csv(path_or_buf=file, sep=";", header=header, index=index, mode=mode)
|
132
|
+
except OSError as e:
|
133
|
+
raise log_error(CsvWriterError(CsvWriter._ERR_FILE_OPEN.format(file))) from e
|
134
|
+
except UnicodeError as e:
|
135
|
+
raise log_error(CsvWriterError(CsvWriter._ERR_FILE_WRITE.format(file, str(e)))) from e
|
136
|
+
|
137
|
+
@staticmethod
|
138
|
+
def _get_identifier(agent_name: str, column_name: str | None = None, agent_id: str | None = None) -> str:
|
69
139
|
"""Returns unique identifier for given `agent_name` and (optional) `agent_id` and `column_name`"""
|
70
140
|
identifier = str(agent_name)
|
71
141
|
if column_name:
|
@@ -75,36 +145,46 @@ class CsvWriter:
|
|
75
145
|
return identifier
|
76
146
|
|
77
147
|
def _write_data_frame(self, data: pd.DataFrame, identifier: str) -> None:
|
78
|
-
"""
|
79
|
-
|
80
|
-
|
148
|
+
"""Writes `data` to csv file derived from `identifier`.
|
149
|
+
|
150
|
+
Appends data if csv file exists, else writes new file with headers instead.
|
151
|
+
|
152
|
+
Args:
|
153
|
+
data: to be written to file
|
154
|
+
identifier: to derive the file name from
|
155
|
+
|
156
|
+
Raises:
|
157
|
+
CsvWriterError: when file could not be written, logged on level "ERROR"
|
81
158
|
"""
|
82
159
|
if self._has_file(identifier):
|
83
160
|
outfile_name = self._get_outfile_name(identifier)
|
84
|
-
|
161
|
+
mode = "a"
|
162
|
+
header = False
|
85
163
|
else:
|
86
164
|
outfile_name = self._create_outfile_name(identifier)
|
87
165
|
self._save_outfile_name(outfile_name, identifier)
|
88
|
-
|
166
|
+
mode = "w"
|
167
|
+
header = True
|
168
|
+
self._dataframe_to_csv(data, outfile_name, header=header, index=True, mode=mode)
|
89
169
|
|
90
170
|
def _has_file(self, identifier: str) -> bool:
|
91
|
-
"""Returns True if a file for given `identifier` was already written"""
|
171
|
+
"""Returns True if a file for given `identifier` was already written."""
|
92
172
|
return identifier in self._files
|
93
173
|
|
94
174
|
def pop_all_file_paths(self) -> dict[str, Path]:
|
95
|
-
"""Clears all stored file paths and returns their previous identifiers and their paths"""
|
175
|
+
"""Clears all stored file paths and returns their previous identifiers and their paths."""
|
96
176
|
current_files = self._files
|
97
177
|
self._files = {}
|
98
178
|
return current_files
|
99
179
|
|
100
|
-
def _get_outfile_name(self, identifier: str) ->
|
101
|
-
"""Returns file
|
180
|
+
def _get_outfile_name(self, identifier: str) -> Path:
|
181
|
+
"""Returns file path for given `agent_name` and (optional) `agent_id`."""
|
102
182
|
return self._files[identifier]
|
103
183
|
|
104
184
|
def _create_outfile_name(self, identifier: str) -> Path:
|
105
|
-
"""Returns fully qualified file
|
185
|
+
"""Returns fully qualified file path based on given `agent_name` and (optional) `agent_id`."""
|
106
186
|
return Path(self._output_folder, f"{identifier}{self.CSV_FILE_SUFFIX}")
|
107
187
|
|
108
188
|
def _save_outfile_name(self, outfile_name: Path, identifier: str) -> None:
|
109
|
-
"""Stores given name for given `agent_name` and (optional) `agent_id
|
189
|
+
"""Stores given name for given `agent_name` and (optional) `agent_id`."""
|
110
190
|
self._files[identifier] = outfile_name
|
@@ -4,8 +4,6 @@
|
|
4
4
|
from __future__ import annotations
|
5
5
|
|
6
6
|
from abc import ABC
|
7
|
-
from builtins import staticmethod
|
8
|
-
from typing import Union, Optional
|
9
7
|
|
10
8
|
import pandas as pd
|
11
9
|
from fameprotobuf.services_pb2 import Output
|
@@ -18,7 +16,7 @@ INDEX = ("AgentId", "TimeStep")
|
|
18
16
|
|
19
17
|
|
20
18
|
class DataTransformer(ABC):
|
21
|
-
"""Extracts and provides series data from parsed and processed output files for requested agents"""
|
19
|
+
"""Extracts and provides series data from parsed and processed output files for requested agents."""
|
22
20
|
|
23
21
|
MODES = {
|
24
22
|
ResolveOptions.IGNORE: lambda: DataTransformerIgnore(), # pylint: disable=unnecessary-lambda
|
@@ -30,13 +28,11 @@ class DataTransformer(ABC):
|
|
30
28
|
def build(complex_column_mode: ResolveOptions) -> DataTransformer:
|
31
29
|
return DataTransformer.MODES[complex_column_mode]()
|
32
30
|
|
33
|
-
def extract_agent_data(
|
34
|
-
|
35
|
-
|
36
|
-
"""
|
37
|
-
Returns dict of DataFrame(s) containing all data from given `series` of given `agent_type`.
|
31
|
+
def extract_agent_data(self, series: list[Output.Series], agent_type: AgentType) -> dict[str | None, pd.DataFrame]:
|
32
|
+
"""Returns dict of DataFrame(s) containing all data from given `series` of given `agent_type`.
|
33
|
+
|
38
34
|
When ResolveOption is `SPLIT`, the dict maps each complex column's name to the associated DataFrame.
|
39
|
-
In any case, the dict maps `None` to a DataFrame with the content of all simple
|
35
|
+
In any case, the dict maps `None` to a DataFrame with the content of all simple columns.
|
40
36
|
"""
|
41
37
|
container = self._extract_agent_data(series, agent_type)
|
42
38
|
data_frames = {}
|
@@ -45,7 +41,7 @@ class DataTransformer(ABC):
|
|
45
41
|
column_name = agent_type.get_column_name_for_id(column_id)
|
46
42
|
if column_id == DataTransformer.SIMPLE_COLUMN_INDEX:
|
47
43
|
data_frame.rename(columns=self._get_column_map(agent_type), inplace=True)
|
48
|
-
index = INDEX
|
44
|
+
index: tuple[str, ...] = INDEX
|
49
45
|
data_frame = data_frame.loc[:, agent_type.get_simple_column_mask()]
|
50
46
|
else:
|
51
47
|
data_frame.rename(columns={0: column_name}, inplace=True)
|
@@ -59,8 +55,8 @@ class DataTransformer(ABC):
|
|
59
55
|
|
60
56
|
def _extract_agent_data(
|
61
57
|
self, series: list[Output.Series], agent_type: AgentType
|
62
|
-
) -> dict[int, dict[tuple, list[
|
63
|
-
"""Returns mapping of (agentId, timeStep) to fixed-length list of all output columns for given `class_name
|
58
|
+
) -> dict[int, dict[tuple, list[float | None | str]]]:
|
59
|
+
"""Returns mapping of (agentId, timeStep) to fixed-length list of all output columns for given `class_name`."""
|
64
60
|
container = DataTransformer._create_container(agent_type)
|
65
61
|
mask_simple = agent_type.get_simple_column_mask()
|
66
62
|
while series:
|
@@ -70,7 +66,7 @@ class DataTransformer(ABC):
|
|
70
66
|
|
71
67
|
@staticmethod
|
72
68
|
def _create_container(agent_type: AgentType) -> dict[int, dict]:
|
73
|
-
"""Returns map of complex columns IDs to an empty dict, and one more for the remaining simple columns"""
|
69
|
+
"""Returns map of complex columns IDs to an empty dict, and one more for the remaining simple columns."""
|
74
70
|
field_ids = agent_type.get_complex_column_ids().union([DataTransformer.SIMPLE_COLUMN_INDEX])
|
75
71
|
return {field_id: {} for field_id in field_ids}
|
76
72
|
|
@@ -78,9 +74,9 @@ class DataTransformer(ABC):
|
|
78
74
|
self,
|
79
75
|
series: Output.Series,
|
80
76
|
mask_simple: list[bool],
|
81
|
-
container: dict[int, dict[tuple, list[
|
77
|
+
container: dict[int, dict[tuple, list[float | None | str]]],
|
82
78
|
) -> None:
|
83
|
-
"""Adds data from given `series` to specified `container` dict as list"""
|
79
|
+
"""Adds data from given `series` to specified `container` dict as list."""
|
84
80
|
empty_list: list = [None] * len(mask_simple)
|
85
81
|
for line in series.lines:
|
86
82
|
index = (series.agent_id, line.time_step)
|
@@ -89,32 +85,29 @@ class DataTransformer(ABC):
|
|
89
85
|
if mask_simple[column.field_id]:
|
90
86
|
simple_values[column.field_id] = column.value
|
91
87
|
else:
|
92
|
-
self._merge_complex_column(column, simple_values)
|
93
88
|
self._store_complex_values(column, container, index)
|
94
89
|
container[DataTransformer.SIMPLE_COLUMN_INDEX][index] = simple_values
|
95
90
|
|
96
|
-
@staticmethod
|
97
|
-
def _merge_complex_column(column: Output.Series.Line.Column, values: list) -> None:
|
98
|
-
"""Merges complex column data"""
|
99
|
-
|
100
91
|
@staticmethod
|
101
92
|
def _store_complex_values(column: Output.Series.Line.Column, container: dict[int, dict], base_index: tuple) -> None:
|
102
|
-
"""Stores complex column data"""
|
93
|
+
"""Stores complex column data."""
|
103
94
|
|
104
95
|
@staticmethod
|
105
96
|
def _get_column_map(agent_type: AgentType) -> dict[int, str]:
|
106
|
-
"""Returns mapping of simple column IDs to their name for given `agent_type
|
97
|
+
"""Returns mapping of simple column IDs to their name for given `agent_type`."""
|
107
98
|
return agent_type.get_simple_column_map()
|
108
99
|
|
109
100
|
|
110
101
|
class DataTransformerIgnore(DataTransformer):
|
111
|
-
"""Ignores complex columns on output"""
|
102
|
+
"""Ignores complex columns on output."""
|
112
103
|
|
113
104
|
|
114
105
|
class DataTransformerSplit(DataTransformer):
|
106
|
+
"""Stores complex data columns split by column type."""
|
107
|
+
|
115
108
|
@staticmethod
|
116
109
|
def _store_complex_values(column: Output.Series.Line.Column, container: dict[int, dict], base_index: tuple) -> None:
|
117
|
-
"""Adds inner data from `column` to given `container` - split by column type"""
|
110
|
+
"""Adds inner data from `column` to given `container` - split by column type."""
|
118
111
|
for entry in column.entries:
|
119
112
|
index = base_index + tuple(entry.index_values)
|
120
113
|
container[column.field_id][index] = entry.value
|
@@ -0,0 +1,170 @@
|
|
1
|
+
# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
|
2
|
+
#
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
4
|
+
from __future__ import annotations
|
5
|
+
|
6
|
+
from typing import Any, Final
|
7
|
+
|
8
|
+
from fameprotobuf.data_storage_pb2 import DataStorage
|
9
|
+
from fameprotobuf.execution_data_pb2 import ExecutionData
|
10
|
+
from google.protobuf import message
|
11
|
+
|
12
|
+
from fameio.logs import log_error
|
13
|
+
from fameio.output import OutputError
|
14
|
+
|
15
|
+
|
16
|
+
class ExecutionDataError(OutputError):
|
17
|
+
"""Indicates an error during reconstruction of execution metadata from its protobuf representation."""
|
18
|
+
|
19
|
+
|
20
|
+
VERSION_MAP: Final[dict[str, str]] = {
|
21
|
+
"fame_protobuf": "FameProtobuf",
|
22
|
+
"fame_io": "FameIo",
|
23
|
+
"fame_core": "FameCore",
|
24
|
+
"python": "Python",
|
25
|
+
"jvm": "JavaVirtualMachine",
|
26
|
+
"os": "OperatingSystem",
|
27
|
+
}
|
28
|
+
PROCESS_MAP: Final[dict[str, str]] = {
|
29
|
+
"core_count": "NumberOfCores",
|
30
|
+
"output_interval": "OutputIntervalInTicks",
|
31
|
+
"output_process": "ProcessControllingOutputs",
|
32
|
+
}
|
33
|
+
STATISTICS_MAP: Final[dict[str, str]] = {
|
34
|
+
"start": "SimulationBeginInRealTime",
|
35
|
+
"duration_in_ms": "SimulationWallTimeInMillis",
|
36
|
+
"tick_count": "SimulatedTicks",
|
37
|
+
}
|
38
|
+
|
39
|
+
|
40
|
+
class ExecutionDao:
|
41
|
+
"""Data access object for execution metadata saved in protobuf."""
|
42
|
+
|
43
|
+
_ERR_MULTIPLE_VERSIONS = "More than two version metadata sections found: File is corrupt."
|
44
|
+
_ERR_MULTIPLE_CONFIGURATIONS = "More than one configuration metadata section found: File is corrupt."
|
45
|
+
_ERR_MULTIPLE_SIMULATIONS = "More than one simulation metadata section found: File is corrupt."
|
46
|
+
_ERR_NO_VERSION = "No version data found: File is either corrupt or was created with fameio version < 3.0."
|
47
|
+
|
48
|
+
KEY_COMPILATION: Final[str] = "InputCompilation"
|
49
|
+
KEY_RUN: Final[str] = "ModelRun"
|
50
|
+
KEY_VERSIONS: Final[str] = "SoftwareVersions"
|
51
|
+
KEY_PROCESSES: Final[str] = "ProcessConfiguration"
|
52
|
+
KEY_STATISTICS: Final[str] = "Statistics"
|
53
|
+
|
54
|
+
def __init__(self) -> None:
|
55
|
+
self._compile_versions: ExecutionData.VersionData | None = None
|
56
|
+
self._run_versions: ExecutionData.VersionData | None = None
|
57
|
+
self._run_configuration: ExecutionData.ProcessConfiguration | None = None
|
58
|
+
self._run_simulation: ExecutionData.Simulation | None = None
|
59
|
+
|
60
|
+
def store_execution_metadata(self, data_storages: list[DataStorage]) -> None:
|
61
|
+
"""Scans given data storages for execution metadata.
|
62
|
+
|
63
|
+
If metadata are present, they are extracted for later inspection
|
64
|
+
|
65
|
+
Args:
|
66
|
+
data_storages: to be scanned for execution metadata
|
67
|
+
|
68
|
+
Raises:
|
69
|
+
ExecutionDataError: if more execution sections are found than expected, logged with level "ERROR"
|
70
|
+
"""
|
71
|
+
for entry in [storage.execution for storage in data_storages if storage.HasField("execution")]:
|
72
|
+
if entry.HasField("version_data"):
|
73
|
+
self._add_version_data(entry.version_data)
|
74
|
+
if entry.HasField("configuration"):
|
75
|
+
self._add_configuration(entry.configuration)
|
76
|
+
if entry.HasField("simulation"):
|
77
|
+
self._add_simulation(entry.simulation)
|
78
|
+
|
79
|
+
def _add_version_data(self, data: ExecutionData.VersionData) -> None:
|
80
|
+
"""Stores given version metadata.
|
81
|
+
|
82
|
+
Args:
|
83
|
+
data: version data saved during compilation (first call), or during model run (second call)
|
84
|
+
|
85
|
+
Raises:
|
86
|
+
ExecutionDataError: if both version data are already set, logged with level "ERROR"
|
87
|
+
"""
|
88
|
+
if not self._compile_versions:
|
89
|
+
self._compile_versions = data
|
90
|
+
elif not self._run_versions:
|
91
|
+
self._run_versions = data
|
92
|
+
else:
|
93
|
+
raise log_error(ExecutionDataError(self._ERR_MULTIPLE_VERSIONS))
|
94
|
+
|
95
|
+
def _add_configuration(self, data: ExecutionData.ProcessConfiguration) -> None:
|
96
|
+
"""Stores given process configuration metadata.
|
97
|
+
|
98
|
+
Args:
|
99
|
+
data: process configuration data to be saved
|
100
|
+
|
101
|
+
Raises:
|
102
|
+
ExecutionDataError: if process configuration data are already set, logged with level "ERROR"
|
103
|
+
"""
|
104
|
+
if not self._run_configuration:
|
105
|
+
self._run_configuration = data
|
106
|
+
else:
|
107
|
+
raise log_error(ExecutionDataError(self._ERR_MULTIPLE_CONFIGURATIONS))
|
108
|
+
|
109
|
+
def _add_simulation(self, data: ExecutionData.Simulation) -> None:
|
110
|
+
"""Stores given simulation metadata.
|
111
|
+
|
112
|
+
Args:
|
113
|
+
data: simulation metadata to be stored
|
114
|
+
|
115
|
+
Raises:
|
116
|
+
ExecutionDataError: if simulation metadata are already set, logged with level "ERROR"
|
117
|
+
"""
|
118
|
+
if not self._run_simulation:
|
119
|
+
self._run_simulation = data
|
120
|
+
else:
|
121
|
+
raise log_error(ExecutionDataError(self._ERR_MULTIPLE_SIMULATIONS))
|
122
|
+
|
123
|
+
def get_fameio_version(self) -> str:
|
124
|
+
"""Gets version of fameio used to create the input data.
|
125
|
+
|
126
|
+
Returns:
|
127
|
+
fameio version that was used to create the input data
|
128
|
+
|
129
|
+
Raises:
|
130
|
+
ExecutionDataError: if fameio version could not be read, logged with level "ERROR"
|
131
|
+
"""
|
132
|
+
if self._compile_versions:
|
133
|
+
return self._compile_versions.fame_io
|
134
|
+
raise log_error(ExecutionDataError(self._ERR_NO_VERSION))
|
135
|
+
|
136
|
+
def get_metadata_dict(self) -> dict[str, Any]:
|
137
|
+
"""Creates a dictionary from all provided execution metadata.
|
138
|
+
|
139
|
+
Returns:
|
140
|
+
dictionary with all execution metadata currently stored in this DAO
|
141
|
+
"""
|
142
|
+
result = {
|
143
|
+
self.KEY_COMPILATION: {self.KEY_VERSIONS: self._get_dict(self._compile_versions, VERSION_MAP)},
|
144
|
+
self.KEY_RUN: {
|
145
|
+
self.KEY_VERSIONS: self._get_dict(self._run_versions, VERSION_MAP),
|
146
|
+
self.KEY_PROCESSES: self._get_dict(self._run_configuration, PROCESS_MAP),
|
147
|
+
self.KEY_STATISTICS: self._get_dict(self._run_simulation, STATISTICS_MAP),
|
148
|
+
},
|
149
|
+
}
|
150
|
+
return result
|
151
|
+
|
152
|
+
@staticmethod
|
153
|
+
def _get_dict(data: message, replacements: dict[str, str]) -> dict[str, str]:
|
154
|
+
"""Searches for `replacements.keys()` in provided `data`.
|
155
|
+
|
156
|
+
If key is available, saves the corresponding data item to dict, associated to a name matching the value in `replacements`.
|
157
|
+
|
158
|
+
Args:
|
159
|
+
data: to extract data from
|
160
|
+
replacements: keys to be replaced by their values in the resulting dict
|
161
|
+
|
162
|
+
Returns:
|
163
|
+
a dictionary matching entries from `data` with their new keys as specified under "replacements"
|
164
|
+
"""
|
165
|
+
versions = {}
|
166
|
+
if data is not None:
|
167
|
+
for key, replacement in replacements.items():
|
168
|
+
if data.HasField(key):
|
169
|
+
versions[replacement] = getattr(data, key)
|
170
|
+
return versions
|