fameio 2.0.0__py3-none-any.whl → 2.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- CHANGELOG.md +45 -22
- fameio/scripts/convert_results.py +9 -9
- fameio/scripts/make_config.py +3 -3
- fameio/source/cli/convert_results.py +11 -2
- fameio/source/cli/make_config.py +2 -2
- fameio/source/cli/options.py +0 -1
- fameio/source/cli/parser.py +3 -3
- fameio/source/loader.py +10 -10
- fameio/source/logs.py +69 -38
- fameio/source/results/conversion.py +4 -4
- fameio/source/results/csv_writer.py +4 -4
- fameio/source/results/data_transformer.py +3 -20
- fameio/source/results/input_dao.py +3 -3
- fameio/source/results/output_dao.py +4 -1
- fameio/source/results/reader.py +8 -8
- fameio/source/results/yaml_writer.py +2 -2
- fameio/source/scenario/contract.py +2 -2
- fameio/source/scenario/exception.py +2 -2
- fameio/source/scenario/generalproperties.py +2 -2
- fameio/source/schema/agenttype.py +5 -5
- fameio/source/schema/attribute.py +4 -4
- fameio/source/schema/java_packages.py +69 -0
- fameio/source/schema/schema.py +23 -7
- fameio/source/series.py +5 -3
- fameio/source/validator.py +5 -5
- fameio/source/writer.py +23 -12
- {fameio-2.0.0.dist-info → fameio-2.1.1.dist-info}/METADATA +43 -31
- fameio-2.1.1.dist-info/RECORD +53 -0
- fameio-2.0.0.dist-info/RECORD +0 -52
- {fameio-2.0.0.dist-info → fameio-2.1.1.dist-info}/LICENSE.txt +0 -0
- {fameio-2.0.0.dist-info → fameio-2.1.1.dist-info}/LICENSES/Apache-2.0.txt +0 -0
- {fameio-2.0.0.dist-info → fameio-2.1.1.dist-info}/LICENSES/CC-BY-4.0.txt +0 -0
- {fameio-2.0.0.dist-info → fameio-2.1.1.dist-info}/LICENSES/CC0-1.0.txt +0 -0
- {fameio-2.0.0.dist-info → fameio-2.1.1.dist-info}/WHEEL +0 -0
- {fameio-2.0.0.dist-info → fameio-2.1.1.dist-info}/entry_points.txt +0 -0
@@ -22,7 +22,6 @@ class DataTransformer(ABC):
|
|
22
22
|
|
23
23
|
MODES = {
|
24
24
|
ResolveOptions.IGNORE: lambda: DataTransformerIgnore(),
|
25
|
-
ResolveOptions.MERGE: lambda: DataTransformerMerge(),
|
26
25
|
ResolveOptions.SPLIT: lambda: DataTransformerSplit(),
|
27
26
|
}
|
28
27
|
SIMPLE_COLUMN_INDEX = -1
|
@@ -47,6 +46,7 @@ class DataTransformer(ABC):
|
|
47
46
|
if column_id == DataTransformer.SIMPLE_COLUMN_INDEX:
|
48
47
|
data_frame.rename(columns=self._get_column_map(agent_type), inplace=True)
|
49
48
|
index = INDEX
|
49
|
+
data_frame = data_frame.loc[:, agent_type.get_simple_column_mask()]
|
50
50
|
else:
|
51
51
|
data_frame.rename(columns={0: column_name}, inplace=True)
|
52
52
|
index = INDEX + agent_type.get_inner_columns(column_id)
|
@@ -54,7 +54,6 @@ class DataTransformer(ABC):
|
|
54
54
|
if not data_frame.empty:
|
55
55
|
data_frame.index = pd.MultiIndex.from_tuples(data_frame.index)
|
56
56
|
data_frame.rename_axis(index, inplace=True)
|
57
|
-
data_frame.dropna(how="all", axis=1, inplace=True)
|
58
57
|
data_frames[column_name] = data_frame
|
59
58
|
return data_frames
|
60
59
|
|
@@ -82,10 +81,10 @@ class DataTransformer(ABC):
|
|
82
81
|
container: Dict[int, Dict[Tuple, List[Union[float, None, str]]]],
|
83
82
|
) -> None:
|
84
83
|
"""Adds data from given `series` to specified `container` dict as list"""
|
85
|
-
|
84
|
+
empty_list = [None] * len(mask_simple)
|
86
85
|
for line in series.line:
|
87
86
|
index = (series.agentId, line.timeStep)
|
88
|
-
simple_values =
|
87
|
+
simple_values = empty_list.copy()
|
89
88
|
for column in line.column:
|
90
89
|
if mask_simple[column.fieldId]:
|
91
90
|
simple_values[column.fieldId] = column.value
|
@@ -114,22 +113,6 @@ class DataTransformerIgnore(DataTransformer):
|
|
114
113
|
"""Ignores complex columns on output"""
|
115
114
|
|
116
115
|
|
117
|
-
class DataTransformerMerge(DataTransformer):
|
118
|
-
"""Merges complex columns on output into a single column entry"""
|
119
|
-
|
120
|
-
def _get_column_map(self, agent_type: AgentType) -> Dict[int, str]:
|
121
|
-
"""Returns mapping of simple (and merged) column IDs to their name (or enhanced name) for given `agent_type`"""
|
122
|
-
return agent_type.get_merged_column_map()
|
123
|
-
|
124
|
-
@staticmethod
|
125
|
-
def _merge_complex_column(column: Output.Series.Line.Column, values: List) -> None:
|
126
|
-
"""Merges given complex `column` content and saves it to given `values` list"""
|
127
|
-
result = []
|
128
|
-
for entry in column.entry:
|
129
|
-
result.append((tuple(entry.indexValue), entry.value))
|
130
|
-
values[column.fieldId] = result
|
131
|
-
|
132
|
-
|
133
116
|
class DataTransformerSplit(DataTransformer):
|
134
117
|
@staticmethod
|
135
118
|
def _store_complex_values(column: Output.Series.Line.Column, container: Dict[int, Dict], base_index: Tuple) -> None:
|
@@ -8,7 +8,7 @@ from fameprotobuf.DataStorage_pb2 import DataStorage
|
|
8
8
|
from fameprotobuf.Field_pb2 import NestedField
|
9
9
|
from fameprotobuf.InputFile_pb2 import InputData
|
10
10
|
|
11
|
-
from fameio.source.logs import
|
11
|
+
from fameio.source.logs import log
|
12
12
|
from fameio.source.scenario import GeneralProperties, Agent, Contract, Scenario
|
13
13
|
from fameio.source.schema import Schema, AttributeSpecs, AttributeType
|
14
14
|
from fameio.source.series import TimeSeriesManager
|
@@ -80,10 +80,10 @@ class InputDao:
|
|
80
80
|
InputConversionException: if no or more than one input is present
|
81
81
|
"""
|
82
82
|
if not self._inputs:
|
83
|
-
|
83
|
+
log().error(self._ERR_NO_INPUTS)
|
84
84
|
raise InputConversionException(self._ERR_NO_INPUTS)
|
85
85
|
if len(self._inputs) > 1:
|
86
|
-
|
86
|
+
log().error(self._ERR_MULTIPLE_INPUTS)
|
87
87
|
raise InputConversionException(self._ERR_MULTIPLE_INPUTS)
|
88
88
|
return self._inputs[0]
|
89
89
|
|
@@ -68,4 +68,7 @@ class OutputDAO:
|
|
68
68
|
"""
|
69
69
|
agent_series = self._all_series.pop(agent_name) if agent_name in self._all_series else []
|
70
70
|
agent_type = self._agent_type_log.get_agent_type(agent_name)
|
71
|
-
|
71
|
+
extracted_data = data_transformer.extract_agent_data(agent_series, agent_type)
|
72
|
+
if None in extracted_data and extracted_data[None].empty:
|
73
|
+
extracted_data.pop(None)
|
74
|
+
return extracted_data
|
fameio/source/results/reader.py
CHANGED
@@ -11,7 +11,7 @@ from typing import IO, List
|
|
11
11
|
from fameprotobuf.DataStorage_pb2 import DataStorage
|
12
12
|
from google.protobuf.message import DecodeError
|
13
13
|
|
14
|
-
from fameio.source.logs import
|
14
|
+
from fameio.source.logs import log
|
15
15
|
|
16
16
|
|
17
17
|
class Reader(ABC):
|
@@ -51,15 +51,15 @@ class Reader(ABC):
|
|
51
51
|
Returns:
|
52
52
|
Reader that can read the specified file
|
53
53
|
"""
|
54
|
-
|
54
|
+
log().debug("Reading file headers...")
|
55
55
|
try:
|
56
56
|
header = file.read(Reader._HEADER_LENGTH).decode(Reader.HEADER_ENCODING)
|
57
57
|
return Reader._READER_HEADERS[header](file, read_single)
|
58
58
|
except (KeyError, UnicodeDecodeError):
|
59
|
-
|
59
|
+
log().warning(Reader._WARN_NO_HEADER)
|
60
60
|
file.seek(0)
|
61
61
|
if read_single:
|
62
|
-
|
62
|
+
log().error(Reader._ERR_UNSUPPORTED_MODE)
|
63
63
|
return ReaderV0(file, False)
|
64
64
|
|
65
65
|
@typing.final
|
@@ -67,7 +67,7 @@ class Reader(ABC):
|
|
67
67
|
"""Returns length of next DataStorage message in file"""
|
68
68
|
message_length_byte = self._file.read(self.BYTES_DEFINING_MESSAGE_LENGTH)
|
69
69
|
if not message_length_byte:
|
70
|
-
|
70
|
+
log().debug(self._DEBUG_FILE_END_REACHED)
|
71
71
|
message_length_int = 0
|
72
72
|
else:
|
73
73
|
message_length_int = struct.unpack(">i", message_length_byte)[0]
|
@@ -86,7 +86,7 @@ class Reader(ABC):
|
|
86
86
|
else:
|
87
87
|
raise IOError(self._ERR_FILE_CORRUPT_NEGATIVE_LENGTH)
|
88
88
|
if message_length and len(message) != message_length:
|
89
|
-
|
89
|
+
log().error(self._ERR_FILE_CORRUPT_MISSING_DATA)
|
90
90
|
return self._parse_to_data_storage(message) if message else None
|
91
91
|
|
92
92
|
@staticmethod
|
@@ -107,7 +107,7 @@ class ReaderV0(Reader):
|
|
107
107
|
|
108
108
|
def __init__(self, file: IO, read_single):
|
109
109
|
super().__init__(file, read_single)
|
110
|
-
|
110
|
+
log().warning(self._WARN_DEPRECATED)
|
111
111
|
|
112
112
|
def read(self) -> List[DataStorage]:
|
113
113
|
result = self._read_data_storage_message()
|
@@ -126,5 +126,5 @@ class ReaderV1(Reader):
|
|
126
126
|
messages.append(self._read_data_storage_message(message_length))
|
127
127
|
if self._read_single:
|
128
128
|
break
|
129
|
-
|
129
|
+
log().debug(f"Read {len(messages)} messages from file.")
|
130
130
|
return messages
|
@@ -6,7 +6,7 @@ from typing import Dict
|
|
6
6
|
|
7
7
|
import yaml
|
8
8
|
|
9
|
-
from fameio.source.logs import
|
9
|
+
from fameio.source.logs import log
|
10
10
|
|
11
11
|
ERR_WRITE_EXCEPTION = "Failed to save dictionary to YAML file `{}`"
|
12
12
|
INFO_DESTINATION = "Saving scenario to file at {}"
|
@@ -20,7 +20,7 @@ def data_to_yaml_file(data: Dict, file_path: Path) -> None:
|
|
20
20
|
data: to be saved to yaml file
|
21
21
|
file_path: at which the file will be created
|
22
22
|
"""
|
23
|
-
|
23
|
+
log().info(INFO_DESTINATION.format(file_path))
|
24
24
|
try:
|
25
25
|
with open(file_path, "w") as f:
|
26
26
|
yaml.dump(data, f, sort_keys=False)
|
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|
5
5
|
|
6
6
|
from typing import Any, Dict, List, Optional
|
7
7
|
|
8
|
-
from fameio.source.logs import
|
8
|
+
from fameio.source.logs import log
|
9
9
|
from fameio.source.scenario.attribute import Attribute
|
10
10
|
from fameio.source.scenario.exception import get_or_default, get_or_raise, log_and_raise
|
11
11
|
from fameio.source.time import FameTime
|
@@ -47,7 +47,7 @@ class Contract:
|
|
47
47
|
"""Constructs a new Contract"""
|
48
48
|
assert product_name != ""
|
49
49
|
if sender_id == receiver_id:
|
50
|
-
|
50
|
+
log().warning(self._ERR_SENDER_IS_RECEIVER.format(sender_id))
|
51
51
|
if delivery_interval <= 0:
|
52
52
|
raise ValueError(self._ERR_INTERVAL_NOT_POSITIVE.format(delivery_interval))
|
53
53
|
self._sender_id = sender_id
|
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
from typing import NoReturn, Any, Union
|
6
6
|
|
7
|
-
from fameio.source.logs import log_error_and_raise,
|
7
|
+
from fameio.source.logs import log_error_and_raise, log
|
8
8
|
|
9
9
|
_DEFAULT_USED = "Using default value '{}' for missing key '{}'"
|
10
10
|
|
@@ -33,7 +33,7 @@ def get_or_default(dictionary: dict, key: str, default_value) -> Any:
|
|
33
33
|
if key in dictionary and dictionary[key] is not None:
|
34
34
|
return dictionary[key]
|
35
35
|
else:
|
36
|
-
|
36
|
+
log().debug(_DEFAULT_USED.format(default_value, key))
|
37
37
|
return default_value
|
38
38
|
|
39
39
|
|
@@ -3,7 +3,7 @@
|
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
4
|
from __future__ import annotations
|
5
5
|
|
6
|
-
from fameio.source.logs import
|
6
|
+
from fameio.source.logs import log
|
7
7
|
from fameio.source.scenario.exception import get_or_default, get_or_raise
|
8
8
|
from fameio.source.time import FameTime
|
9
9
|
from fameio.source.tools import keys_to_lower
|
@@ -34,7 +34,7 @@ class GeneralProperties:
|
|
34
34
|
output_process: int,
|
35
35
|
) -> None:
|
36
36
|
if simulation_stop_time < simulation_start_time:
|
37
|
-
|
37
|
+
log().warning(GeneralProperties._SIMULATION_DURATION)
|
38
38
|
self._run_id = run_id
|
39
39
|
self._simulation_start_time = simulation_start_time
|
40
40
|
self._simulation_stop_time = simulation_stop_time
|
@@ -5,7 +5,7 @@ from __future__ import annotations
|
|
5
5
|
|
6
6
|
from typing import Dict, List, Any
|
7
7
|
|
8
|
-
from fameio.source.logs import log_error_and_raise,
|
8
|
+
from fameio.source.logs import log_error_and_raise, log
|
9
9
|
from fameio.source.schema.exception import SchemaException
|
10
10
|
from fameio.source.schema.attribute import AttributeSpecs
|
11
11
|
from fameio.source.tools import keys_to_lower
|
@@ -63,26 +63,26 @@ class AgentType:
|
|
63
63
|
full_name = name + "." + attribute_name
|
64
64
|
agent_type._attributes[attribute_name] = AttributeSpecs(full_name, attribute_details)
|
65
65
|
else:
|
66
|
-
|
66
|
+
log().info(AgentType._NO_ATTRIBUTES.format(name))
|
67
67
|
|
68
68
|
if AgentType._KEY_PRODUCTS in definition and definition[AgentType._KEY_PRODUCTS]:
|
69
69
|
agent_type._products.update(AgentType._read_products(definition[AgentType._KEY_PRODUCTS], name))
|
70
70
|
else:
|
71
|
-
|
71
|
+
log().info(AgentType._NO_PRODUCTS.format(name))
|
72
72
|
|
73
73
|
if AgentType._KEY_OUTPUTS in definition:
|
74
74
|
outputs_to_add = definition[AgentType._KEY_OUTPUTS]
|
75
75
|
if outputs_to_add:
|
76
76
|
agent_type._outputs.update(outputs_to_add)
|
77
77
|
else:
|
78
|
-
|
78
|
+
log().debug(AgentType._NO_OUTPUTS.format(name))
|
79
79
|
|
80
80
|
if AgentType._KEY_METADATA in definition:
|
81
81
|
metadata_to_add = definition[AgentType._KEY_METADATA]
|
82
82
|
if metadata_to_add:
|
83
83
|
agent_type._metadata.update(metadata_to_add)
|
84
84
|
else:
|
85
|
-
|
85
|
+
log().debug(AgentType._NO_METADATA.format(name))
|
86
86
|
|
87
87
|
return agent_type
|
88
88
|
|
@@ -7,7 +7,7 @@ import typing
|
|
7
7
|
from enum import Enum, auto
|
8
8
|
from typing import Any, Dict
|
9
9
|
|
10
|
-
from fameio.source.logs import
|
10
|
+
from fameio.source.logs import log
|
11
11
|
from fameio.source.schema.exception import SchemaException
|
12
12
|
from fameio.source.time import FameTime
|
13
13
|
from fameio.source.tools import keys_to_lower
|
@@ -70,18 +70,18 @@ class AttributeSpecs:
|
|
70
70
|
self._is_mandatory = definition[AttributeSpecs._KEY_MANDATORY]
|
71
71
|
else:
|
72
72
|
self._is_mandatory = True
|
73
|
-
|
73
|
+
log().warning(AttributeSpecs._MISSING_SPEC_DEFAULT.format(AttributeSpecs._KEY_MANDATORY, name, True))
|
74
74
|
|
75
75
|
if AttributeSpecs._KEY_LIST in definition:
|
76
76
|
self._is_list = definition[AttributeSpecs._KEY_LIST]
|
77
77
|
else:
|
78
78
|
self._is_list = False
|
79
|
-
|
79
|
+
log().warning(AttributeSpecs._MISSING_SPEC_DEFAULT.format(AttributeSpecs._KEY_LIST, name, False))
|
80
80
|
|
81
81
|
if AttributeSpecs._KEY_TYPE in definition:
|
82
82
|
self._attr_type = AttributeSpecs._get_type_for_name(definition[AttributeSpecs._KEY_TYPE])
|
83
83
|
else:
|
84
|
-
|
84
|
+
log().error(AttributeSpecs._MISSING_TYPE.format(name))
|
85
85
|
raise SchemaException(AttributeSpecs._MISSING_TYPE.format(name))
|
86
86
|
|
87
87
|
if self._attr_type == AttributeType.TIME_SERIES and self._is_list:
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
|
2
|
+
#
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
4
|
+
from __future__ import annotations
|
5
|
+
|
6
|
+
from typing import List, Dict
|
7
|
+
|
8
|
+
from fameio.source.logs import log_error_and_raise
|
9
|
+
from fameio.source.schema import SchemaException
|
10
|
+
from fameio.source.tools import keys_to_lower
|
11
|
+
|
12
|
+
|
13
|
+
class JavaPackages:
|
14
|
+
"""Schema definitions for Java package names in which model classes reside"""
|
15
|
+
|
16
|
+
_ERR_MISSING_AGENTS = "JavaPackages requires non-empty list for `Agents`. Key was missing or list was empty."
|
17
|
+
_ERR_MISSING_DATA_ITEMS = "JavaPackages requires non-empty list for `DataItems`. Key was missing or list was empty."
|
18
|
+
_ERR_MISSING_PORTABLES = "JavaPackages require non-empty list for `Portables`. Key was missing or list was empty."
|
19
|
+
|
20
|
+
_KEY_AGENT = "Agents".lower()
|
21
|
+
_KEY_DATA_ITEM = "DataItems".lower()
|
22
|
+
_KEY_PORTABLE = "Portables".lower()
|
23
|
+
|
24
|
+
def __init__(self):
|
25
|
+
self._agents: List[str] = []
|
26
|
+
self._data_items: List[str] = []
|
27
|
+
self._portables: List[str] = []
|
28
|
+
|
29
|
+
@classmethod
|
30
|
+
def from_dict(cls, definitions: Dict[str, List[str]]) -> JavaPackages:
|
31
|
+
"""
|
32
|
+
Creates JavaPackages from a dictionary representation
|
33
|
+
|
34
|
+
Args:
|
35
|
+
definitions: dictionary representation of JavaPackages
|
36
|
+
|
37
|
+
Returns:
|
38
|
+
new instance of JavaPackages
|
39
|
+
"""
|
40
|
+
java_packages = cls()
|
41
|
+
definitions = keys_to_lower(definitions)
|
42
|
+
|
43
|
+
java_packages._agents = definitions.get(JavaPackages._KEY_AGENT, [])
|
44
|
+
java_packages._data_items = definitions.get(JavaPackages._KEY_DATA_ITEM, [])
|
45
|
+
java_packages._portables = definitions.get(JavaPackages._KEY_PORTABLE, [])
|
46
|
+
|
47
|
+
if not java_packages._agents:
|
48
|
+
log_error_and_raise(SchemaException(JavaPackages._ERR_MISSING_AGENTS))
|
49
|
+
if not java_packages._data_items:
|
50
|
+
log_error_and_raise(SchemaException(JavaPackages._ERR_MISSING_DATA_ITEMS))
|
51
|
+
if not java_packages._portables:
|
52
|
+
log_error_and_raise(SchemaException(JavaPackages._ERR_MISSING_PORTABLES))
|
53
|
+
|
54
|
+
return java_packages
|
55
|
+
|
56
|
+
@property
|
57
|
+
def agents(self) -> List[str]:
|
58
|
+
"""Return list of java package names that contain the model's Agents"""
|
59
|
+
return self._agents
|
60
|
+
|
61
|
+
@property
|
62
|
+
def data_items(self) -> List[str]:
|
63
|
+
"""Return list of java package names that contain the model's DataItems"""
|
64
|
+
return self._data_items
|
65
|
+
|
66
|
+
@property
|
67
|
+
def portables(self) -> List[str]:
|
68
|
+
"""Return list of java package names that contain the model's Portables"""
|
69
|
+
return self._portables
|
fameio/source/schema/schema.py
CHANGED
@@ -1,42 +1,53 @@
|
|
1
|
-
# SPDX-FileCopyrightText:
|
1
|
+
# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
4
|
from __future__ import annotations
|
5
5
|
|
6
6
|
import ast
|
7
|
-
from typing import Dict
|
7
|
+
from typing import Dict, Optional
|
8
8
|
|
9
|
-
from fameio.source.logs import log_error_and_raise
|
9
|
+
from fameio.source.logs import log_error_and_raise, log
|
10
10
|
from fameio.source.schema.agenttype import AgentType
|
11
11
|
from fameio.source.schema.exception import SchemaException
|
12
|
+
from fameio.source.schema.java_packages import JavaPackages
|
12
13
|
from fameio.source.tools import keys_to_lower
|
13
14
|
|
14
15
|
|
15
16
|
class Schema:
|
16
17
|
"""Definition of a schema"""
|
17
18
|
|
18
|
-
|
19
|
-
|
19
|
+
_ERR_AGENT_TYPES_MISSING = "Required keyword `AgentTypes` missing in Schema."
|
20
|
+
_ERR_AGENT_TYPES_EMPTY = "`AgentTypes` must not be empty - at least one type of agent is required."
|
21
|
+
_WARN_MISSING_PACKAGES = "No `JavaPackages` defined Schema. Future versions of fameio will require it."
|
22
|
+
|
20
23
|
_KEY_AGENT_TYPE = "AgentTypes".lower()
|
24
|
+
_KEY_PACKAGES = "JavaPackages".lower()
|
21
25
|
|
22
26
|
def __init__(self, definitions: dict):
|
23
27
|
self._original_input_dict = definitions
|
24
28
|
self._agent_types = {}
|
29
|
+
self._packages: Optional[JavaPackages] = None
|
25
30
|
|
26
31
|
@classmethod
|
27
32
|
def from_dict(cls, definitions: dict) -> Schema:
|
28
33
|
"""Load given dictionary `definitions` into a new Schema"""
|
29
34
|
definitions = keys_to_lower(definitions)
|
30
35
|
if Schema._KEY_AGENT_TYPE not in definitions:
|
31
|
-
log_error_and_raise(SchemaException(Schema.
|
36
|
+
log_error_and_raise(SchemaException(Schema._ERR_AGENT_TYPES_MISSING))
|
32
37
|
schema = cls(definitions)
|
33
38
|
agent_types = definitions[Schema._KEY_AGENT_TYPE]
|
34
39
|
if len(agent_types) == 0:
|
35
|
-
log_error_and_raise(SchemaException(Schema.
|
40
|
+
log_error_and_raise(SchemaException(Schema._ERR_AGENT_TYPES_EMPTY))
|
36
41
|
|
37
42
|
for agent_type_name, agent_definition in agent_types.items():
|
38
43
|
agent_type = AgentType.from_dict(agent_type_name, agent_definition)
|
39
44
|
schema._agent_types[agent_type_name] = agent_type
|
45
|
+
|
46
|
+
if Schema._KEY_PACKAGES in definitions:
|
47
|
+
schema._packages = JavaPackages.from_dict(definitions[Schema._KEY_PACKAGES])
|
48
|
+
else:
|
49
|
+
log().warning(Schema._WARN_MISSING_PACKAGES)
|
50
|
+
schema._packages = JavaPackages()
|
40
51
|
return schema
|
41
52
|
|
42
53
|
@classmethod
|
@@ -56,3 +67,8 @@ class Schema:
|
|
56
67
|
def agent_types(self) -> Dict[str, AgentType]:
|
57
68
|
"""Returns all the agent types by their name"""
|
58
69
|
return self._agent_types
|
70
|
+
|
71
|
+
@property
|
72
|
+
def packages(self) -> JavaPackages:
|
73
|
+
"""Returns JavaPackages, i.e. names where model classes are defined in"""
|
74
|
+
return self._packages
|
fameio/source/series.py
CHANGED
@@ -9,9 +9,10 @@ from typing import Dict, Union, Tuple, Any, List
|
|
9
9
|
|
10
10
|
import pandas as pd
|
11
11
|
from fameprotobuf.InputFile_pb2 import InputData
|
12
|
+
from google.protobuf.internal.wire_format import INT64_MIN, INT64_MAX
|
12
13
|
|
13
14
|
from fameio.source import PathResolver
|
14
|
-
from fameio.source.logs import log_error_and_raise,
|
15
|
+
from fameio.source.logs import log_error_and_raise, log
|
15
16
|
from fameio.source.time import ConversionException, FameTime
|
16
17
|
from fameio.source.tools import clean_up_file_name
|
17
18
|
|
@@ -114,7 +115,8 @@ class TimeSeriesManager:
|
|
114
115
|
"""Returns name and dataframe for a new static timeseries created from the given `value`"""
|
115
116
|
if math.isnan(value):
|
116
117
|
log_error_and_raise(TimeSeriesException(TimeSeriesManager._ERR_NAN_VALUE))
|
117
|
-
|
118
|
+
data = pd.DataFrame({0: [INT64_MIN, INT64_MAX], 1: [value, value]})
|
119
|
+
return TimeSeriesManager._CONSTANT_IDENTIFIER.format(value), data
|
118
120
|
|
119
121
|
def get_series_id_by_identifier(self, identifier: Union[str, int, float]) -> int:
|
120
122
|
"""
|
@@ -136,7 +138,7 @@ class TimeSeriesManager:
|
|
136
138
|
def get_all_series(self) -> List[Tuple[int, str, pd.DataFrame]]:
|
137
139
|
"""Returns iterator over id, name and dataframe of all stored series"""
|
138
140
|
if len(self._series_by_id) == 0:
|
139
|
-
|
141
|
+
log().warning(self._WARN_NO_DATA)
|
140
142
|
return [(v[Entry.ID], v[Entry.NAME], v[Entry.DATA]) for v in self._series_by_id.values()]
|
141
143
|
|
142
144
|
def reconstruct_time_series(self, timeseries: List[InputData.TimeSeriesDao]) -> None:
|
fameio/source/validator.py
CHANGED
@@ -7,7 +7,7 @@ from collections import Counter
|
|
7
7
|
from typing import Any, Dict, List
|
8
8
|
|
9
9
|
from fameio.source import PathResolver
|
10
|
-
from fameio.source.logs import log_error_and_raise,
|
10
|
+
from fameio.source.logs import log_error_and_raise, log
|
11
11
|
from fameio.source.scenario import Agent, Attribute, Contract, Scenario
|
12
12
|
from fameio.source.schema.agenttype import AgentType
|
13
13
|
from fameio.source.schema.attribute import AttributeSpecs, AttributeType
|
@@ -125,9 +125,9 @@ class SchemaValidator:
|
|
125
125
|
)
|
126
126
|
else:
|
127
127
|
if specification.has_default_value:
|
128
|
-
|
128
|
+
log().warning(SchemaValidator._DEFAULT_IGNORED.format(specification.full_name))
|
129
129
|
else:
|
130
|
-
|
130
|
+
log().info(SchemaValidator._OPTIONAL_MISSING.format(specification.full_name))
|
131
131
|
if name in attributes and specification.has_nested_attributes:
|
132
132
|
attribute = attributes[name]
|
133
133
|
if specification.is_list:
|
@@ -179,7 +179,7 @@ class SchemaValidator:
|
|
179
179
|
attribute_type = specification.attr_type
|
180
180
|
if specification.is_list:
|
181
181
|
if not is_list:
|
182
|
-
|
182
|
+
log().warning(SchemaValidator._IS_NO_LIST.format(specification.full_name, value_or_values))
|
183
183
|
return SchemaValidator._is_compatible_value(attribute_type, value_or_values)
|
184
184
|
for value in value_or_values:
|
185
185
|
if not SchemaValidator._is_compatible_value(attribute_type, value):
|
@@ -280,4 +280,4 @@ class SchemaValidator:
|
|
280
280
|
|
281
281
|
if inactive_agents:
|
282
282
|
for agent_id, agent_name in inactive_agents.items():
|
283
|
-
|
283
|
+
log().warning(SchemaValidator._MISSING_CONTRACTS_FOR_AGENTS.format(agent_id, agent_name))
|
fameio/source/writer.py
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
# SPDX-FileCopyrightText:
|
1
|
+
# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
4
|
|
@@ -9,8 +9,9 @@ from fameprotobuf.DataStorage_pb2 import DataStorage
|
|
9
9
|
from fameprotobuf.InputFile_pb2 import InputData
|
10
10
|
from fameprotobuf.Field_pb2 import NestedField
|
11
11
|
from fameprotobuf.Contract_pb2 import ProtoContract
|
12
|
+
from fameprotobuf.Model_pb2 import ModelData
|
12
13
|
|
13
|
-
from fameio.source.logs import log_error_and_raise,
|
14
|
+
from fameio.source.logs import log_error_and_raise, log
|
14
15
|
from fameio.source.results.reader import Reader
|
15
16
|
from fameio.source.scenario import (
|
16
17
|
Agent,
|
@@ -21,6 +22,7 @@ from fameio.source.scenario import (
|
|
21
22
|
)
|
22
23
|
from fameio.source.schema import Schema
|
23
24
|
from fameio.source.schema.attribute import AttributeSpecs, AttributeType
|
25
|
+
from fameio.source.schema.java_packages import JavaPackages
|
24
26
|
from fameio.source.series import TimeSeriesManager
|
25
27
|
from fameio.source.time import FameTime
|
26
28
|
from fameio.source.tools import ensure_is_list
|
@@ -59,7 +61,7 @@ class ProtoWriter:
|
|
59
61
|
|
60
62
|
def _create_protobuf_from_scenario(self, scenario: Scenario) -> DataStorage:
|
61
63
|
"""Returns given `scenario` written to new DataStorage protobuf"""
|
62
|
-
|
64
|
+
log().info("Converting scenario to protobuf.")
|
63
65
|
pb_data_storage = DataStorage()
|
64
66
|
pb_input = pb_data_storage.input
|
65
67
|
|
@@ -69,12 +71,13 @@ class ProtoWriter:
|
|
69
71
|
self._set_time_series(pb_input)
|
70
72
|
self._set_schema(pb_input, scenario.schema)
|
71
73
|
|
74
|
+
self._set_java_package_names(pb_data_storage.model, scenario.schema.packages)
|
72
75
|
return pb_data_storage
|
73
76
|
|
74
77
|
@staticmethod
|
75
78
|
def _set_general_properties(pb_input: InputData, general_properties: GeneralProperties) -> None:
|
76
79
|
"""Saves a scenario's general properties to specified protobuf `pb_input` container"""
|
77
|
-
|
80
|
+
log().info("Adding General Properties")
|
78
81
|
pb_input.runId = general_properties.run_id
|
79
82
|
pb_input.simulation.startTime = general_properties.simulation_start_time
|
80
83
|
pb_input.simulation.stopTime = general_properties.simulation_stop_time
|
@@ -84,7 +87,7 @@ class ProtoWriter:
|
|
84
87
|
|
85
88
|
def _add_agents(self, pb_input: InputData, agents: List[Agent], schema: Schema) -> None:
|
86
89
|
"""Triggers setting of `agents` to `pb_input`"""
|
87
|
-
|
90
|
+
log().info("Adding Agents")
|
88
91
|
for agent in agents:
|
89
92
|
pb_agent = self._set_agent(pb_input.agent.add(), agent)
|
90
93
|
attribute_specs = schema.agent_types[agent.type_name].attributes
|
@@ -125,7 +128,7 @@ class ProtoWriter:
|
|
125
128
|
if attribute_specs.is_mandatory:
|
126
129
|
pb_field = self._add_field(pb_parent, name)
|
127
130
|
self._set_attribute(pb_field, attribute_specs.default_value, attribute_specs.attr_type)
|
128
|
-
|
131
|
+
log().info(self._USING_DEFAULT.format(name))
|
129
132
|
|
130
133
|
@staticmethod
|
131
134
|
def _add_field(pb_parent: Union[InputData.AgentDao, NestedField], name: str) -> NestedField:
|
@@ -154,7 +157,7 @@ class ProtoWriter:
|
|
154
157
|
@staticmethod
|
155
158
|
def _add_contracts(pb_input: InputData, contracts: List[Contract]) -> None:
|
156
159
|
"""Triggers setting of `contracts` to `pb_input`"""
|
157
|
-
|
160
|
+
log().info("Adding Contracts")
|
158
161
|
for contract in contracts:
|
159
162
|
pb_contract = ProtoWriter._set_contract(pb_input.contract.add(), contract)
|
160
163
|
ProtoWriter._set_contract_attributes(pb_contract, contract.attributes)
|
@@ -178,7 +181,7 @@ class ProtoWriter:
|
|
178
181
|
) -> None:
|
179
182
|
"""Assign (nested) Attributes to given protobuf container `pb_parent`"""
|
180
183
|
for name, attribute in attributes.items():
|
181
|
-
|
184
|
+
log().debug("Assigning contract attribute `{}`.".format(name))
|
182
185
|
pb_field = ProtoWriter._add_field(pb_parent, name)
|
183
186
|
|
184
187
|
if attribute.has_value:
|
@@ -196,7 +199,7 @@ class ProtoWriter:
|
|
196
199
|
|
197
200
|
def _set_time_series(self, pb_input: InputData) -> None:
|
198
201
|
"""Adds all time series from TimeSeriesManager to given `pb_input`"""
|
199
|
-
|
202
|
+
log().info("Adding TimeSeries")
|
200
203
|
for unique_id, identifier, data in self._time_series_manager.get_all_series():
|
201
204
|
pb_series = pb_input.timeSeries.add()
|
202
205
|
pb_series.seriesId = unique_id
|
@@ -213,12 +216,20 @@ class ProtoWriter:
|
|
213
216
|
@staticmethod
|
214
217
|
def _set_schema(pb_input: InputData, schema: Schema) -> None:
|
215
218
|
"""Sets the given `schema` `pb_input`"""
|
216
|
-
|
219
|
+
log().info("Adding Schema")
|
217
220
|
pb_input.schema = schema.to_string()
|
218
221
|
|
222
|
+
@staticmethod
|
223
|
+
def _set_java_package_names(pb_model: ModelData, java_packages: JavaPackages) -> None:
|
224
|
+
"""Adds given JavaPackages names to given ModelData section"""
|
225
|
+
pb_packages = pb_model.packages
|
226
|
+
pb_packages.agent.extend(java_packages.agents)
|
227
|
+
pb_packages.dataItem.extend(java_packages.data_items)
|
228
|
+
pb_packages.portable.extend(java_packages.portables)
|
229
|
+
|
219
230
|
def _write_protobuf_to_disk(self, pb_data_storage: DataStorage) -> None:
|
220
231
|
"""Writes given `protobuf_input_data` to disk"""
|
221
|
-
|
232
|
+
log().info(self._INFO_WRITING.format(self.file_path))
|
222
233
|
try:
|
223
234
|
with open(self.file_path, "wb") as file:
|
224
235
|
serialised_data_storage = pb_data_storage.SerializeToString()
|
@@ -227,4 +238,4 @@ class ProtoWriter:
|
|
227
238
|
file.write(serialised_data_storage)
|
228
239
|
except OSError as e:
|
229
240
|
log_error_and_raise(ProtoWriterException(ProtoWriter._NO_FILE_SPECIFIED.format(self.file_path), e))
|
230
|
-
|
241
|
+
log().info(self._INFO_WRITING_COMPLETED.format(self.file_path))
|