fameio 1.8.2__py3-none-any.whl → 2.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- CHANGELOG.md +224 -0
- fameio/scripts/__init__.py +8 -6
- fameio/scripts/__init__.py.license +3 -0
- fameio/scripts/convert_results.py +31 -35
- fameio/scripts/convert_results.py.license +3 -0
- fameio/scripts/make_config.py +14 -17
- fameio/scripts/make_config.py.license +3 -0
- fameio/source/cli/__init__.py +3 -0
- fameio/source/cli/convert_results.py +84 -0
- fameio/source/cli/make_config.py +62 -0
- fameio/source/cli/options.py +58 -0
- fameio/source/cli/parser.py +238 -0
- fameio/source/loader.py +10 -11
- fameio/source/logs.py +90 -35
- fameio/source/results/conversion.py +11 -13
- fameio/source/results/csv_writer.py +16 -5
- fameio/source/results/data_transformer.py +6 -22
- fameio/source/results/input_dao.py +163 -0
- fameio/source/results/reader.py +25 -14
- fameio/source/results/yaml_writer.py +28 -0
- fameio/source/scenario/agent.py +56 -39
- fameio/source/scenario/attribute.py +9 -12
- fameio/source/scenario/contract.py +55 -40
- fameio/source/scenario/exception.py +11 -9
- fameio/source/scenario/generalproperties.py +11 -17
- fameio/source/scenario/scenario.py +19 -14
- fameio/source/schema/agenttype.py +75 -27
- fameio/source/schema/attribute.py +8 -7
- fameio/source/schema/java_packages.py +69 -0
- fameio/source/schema/schema.py +44 -15
- fameio/source/series.py +148 -25
- fameio/source/time.py +8 -8
- fameio/source/tools.py +13 -2
- fameio/source/validator.py +138 -58
- fameio/source/writer.py +120 -113
- fameio-2.1.0.dist-info/LICENSES/Apache-2.0.txt +178 -0
- fameio-2.1.0.dist-info/LICENSES/CC-BY-4.0.txt +395 -0
- fameio-2.1.0.dist-info/LICENSES/CC0-1.0.txt +121 -0
- {fameio-1.8.2.dist-info → fameio-2.1.0.dist-info}/METADATA +706 -660
- fameio-2.1.0.dist-info/RECORD +53 -0
- {fameio-1.8.2.dist-info → fameio-2.1.0.dist-info}/WHEEL +1 -2
- fameio-2.1.0.dist-info/entry_points.txt +4 -0
- fameio/source/cli.py +0 -253
- fameio-1.8.2.dist-info/RECORD +0 -40
- fameio-1.8.2.dist-info/entry_points.txt +0 -3
- fameio-1.8.2.dist-info/top_level.txt +0 -1
- {fameio-1.8.2.dist-info → fameio-2.1.0.dist-info}/LICENSE.txt +0 -0
fameio/source/schema/schema.py
CHANGED
@@ -1,45 +1,74 @@
|
|
1
|
-
# SPDX-FileCopyrightText:
|
1
|
+
# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
|
+
from __future__ import annotations
|
4
5
|
|
5
|
-
|
6
|
+
import ast
|
7
|
+
from typing import Dict, Optional
|
6
8
|
|
7
|
-
from fameio.source.logs import log_error_and_raise
|
9
|
+
from fameio.source.logs import log_error_and_raise, log
|
8
10
|
from fameio.source.schema.agenttype import AgentType
|
9
11
|
from fameio.source.schema.exception import SchemaException
|
12
|
+
from fameio.source.schema.java_packages import JavaPackages
|
10
13
|
from fameio.source.tools import keys_to_lower
|
11
14
|
|
12
15
|
|
13
16
|
class Schema:
|
14
17
|
"""Definition of a schema"""
|
15
18
|
|
16
|
-
|
19
|
+
_ERR_AGENT_TYPES_MISSING = "Required keyword `AgentTypes` missing in Schema."
|
20
|
+
_ERR_AGENT_TYPES_EMPTY = "`AgentTypes` must not be empty - at least one type of agent is required."
|
21
|
+
_WARN_MISSING_PACKAGES = "No `JavaPackages` defined Schema. Future versions of fameio will require it."
|
22
|
+
|
17
23
|
_KEY_AGENT_TYPE = "AgentTypes".lower()
|
24
|
+
_KEY_PACKAGES = "JavaPackages".lower()
|
18
25
|
|
19
26
|
def __init__(self, definitions: dict):
|
20
|
-
# the current Schema class design is read-only, so it's much simpler to remember the original schema dict
|
21
|
-
# in order to implement to_dict()
|
22
27
|
self._original_input_dict = definitions
|
23
|
-
|
24
|
-
# fill the agent types
|
25
28
|
self._agent_types = {}
|
26
|
-
|
27
|
-
agent_type = AgentType.from_dict(agent_type_name, agent_definition)
|
28
|
-
self._agent_types[agent_type_name] = agent_type
|
29
|
+
self._packages: Optional[JavaPackages] = None
|
29
30
|
|
30
31
|
@classmethod
|
31
|
-
def from_dict(cls, definitions: dict) ->
|
32
|
-
"""Load definitions
|
32
|
+
def from_dict(cls, definitions: dict) -> Schema:
|
33
|
+
"""Load given dictionary `definitions` into a new Schema"""
|
33
34
|
definitions = keys_to_lower(definitions)
|
34
35
|
if Schema._KEY_AGENT_TYPE not in definitions:
|
35
|
-
log_error_and_raise(SchemaException(Schema.
|
36
|
-
|
36
|
+
log_error_and_raise(SchemaException(Schema._ERR_AGENT_TYPES_MISSING))
|
37
|
+
schema = cls(definitions)
|
38
|
+
agent_types = definitions[Schema._KEY_AGENT_TYPE]
|
39
|
+
if len(agent_types) == 0:
|
40
|
+
log_error_and_raise(SchemaException(Schema._ERR_AGENT_TYPES_EMPTY))
|
41
|
+
|
42
|
+
for agent_type_name, agent_definition in agent_types.items():
|
43
|
+
agent_type = AgentType.from_dict(agent_type_name, agent_definition)
|
44
|
+
schema._agent_types[agent_type_name] = agent_type
|
45
|
+
|
46
|
+
if Schema._KEY_PACKAGES in definitions:
|
47
|
+
schema._packages = JavaPackages.from_dict(definitions[Schema._KEY_PACKAGES])
|
48
|
+
else:
|
49
|
+
log().warning(Schema._WARN_MISSING_PACKAGES)
|
50
|
+
schema._packages = JavaPackages()
|
51
|
+
return schema
|
52
|
+
|
53
|
+
@classmethod
|
54
|
+
def from_string(cls, definitions: str) -> Schema:
|
55
|
+
"""Load given string `definitions` into a new Schema"""
|
56
|
+
return cls.from_dict(ast.literal_eval(definitions))
|
37
57
|
|
38
58
|
def to_dict(self) -> dict:
|
39
59
|
"""Serializes the schema content to a dict"""
|
40
60
|
return self._original_input_dict
|
41
61
|
|
62
|
+
def to_string(self) -> str:
|
63
|
+
"""Returns a string representation of the Schema of which the class can be rebuilt"""
|
64
|
+
return repr(self.to_dict())
|
65
|
+
|
42
66
|
@property
|
43
67
|
def agent_types(self) -> Dict[str, AgentType]:
|
44
68
|
"""Returns all the agent types by their name"""
|
45
69
|
return self._agent_types
|
70
|
+
|
71
|
+
@property
|
72
|
+
def packages(self) -> JavaPackages:
|
73
|
+
"""Returns JavaPackages, i.e. names where model classes are defined in"""
|
74
|
+
return self._packages
|
fameio/source/series.py
CHANGED
@@ -1,48 +1,171 @@
|
|
1
1
|
# SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
|
+
import math
|
5
|
+
import os
|
6
|
+
from enum import Enum, auto
|
7
|
+
from pathlib import Path
|
8
|
+
from typing import Dict, Union, Tuple, Any, List
|
4
9
|
|
5
|
-
|
10
|
+
import pandas as pd
|
11
|
+
from fameprotobuf.InputFile_pb2 import InputData
|
12
|
+
from google.protobuf.internal.wire_format import INT64_MIN, INT64_MAX
|
6
13
|
|
7
|
-
from fameio.source
|
14
|
+
from fameio.source import PathResolver
|
15
|
+
from fameio.source.logs import log_error_and_raise, log
|
16
|
+
from fameio.source.time import ConversionException, FameTime
|
17
|
+
from fameio.source.tools import clean_up_file_name
|
8
18
|
|
9
19
|
|
10
|
-
class
|
20
|
+
class TimeSeriesException(Exception):
|
11
21
|
"""Indicates that an error occurred during management of time series"""
|
12
22
|
|
13
23
|
pass
|
14
24
|
|
15
25
|
|
26
|
+
class Entry(Enum):
|
27
|
+
ID = auto()
|
28
|
+
NAME = auto()
|
29
|
+
DATA = auto()
|
30
|
+
|
31
|
+
|
16
32
|
class TimeSeriesManager:
|
17
33
|
"""Manages matching of files to time series ids and their protobuf representation"""
|
18
34
|
|
19
|
-
|
35
|
+
_TIMESERIES_RECONSTRUCTION_PATH = "./timeseries/"
|
36
|
+
_CONSTANT_IDENTIFIER = "Constant value: {}"
|
37
|
+
_KEY_ROW_TIME = "timeStep"
|
38
|
+
_KEY_ROW_VALUE = "value"
|
39
|
+
|
40
|
+
_ERR_FILE_NOT_FOUND = "Cannot find Timeseries file '{}'"
|
41
|
+
_ERR_CORRUPT_TIME_SERIES_KEY = "TimeSeries file '{}' corrupt: At least one entry in first column isn't a timestamp."
|
42
|
+
_ERR_CORRUPT_TIME_SERIES_VALUE = "TimeSeries file '{}' corrupt: At least one entry in value column isn't numeric."
|
43
|
+
_ERR_NON_NUMERIC = "Values in TimeSeries must be numeric but was: '{}'"
|
44
|
+
_ERR_NAN_VALUE = "Values in TimeSeries must not be missing or NaN."
|
45
|
+
_ERR_UNREGISTERED_SERIES = "No timeseries registered with identifier '{}' - was the Scenario validated?"
|
46
|
+
_WARN_NO_DATA = "No timeseries stored in timeseries manager. Double check if you expected timeseries."
|
20
47
|
|
21
|
-
def __init__(self):
|
48
|
+
def __init__(self, path_resolver: PathResolver = PathResolver()) -> None:
|
49
|
+
self._path_resolver = path_resolver
|
22
50
|
self._id_count = -1
|
23
|
-
self.
|
51
|
+
self._series_by_id: Dict[Union[str, int, float], Dict[Entry, Any]] = {}
|
24
52
|
|
25
|
-
def
|
26
|
-
"""
|
27
|
-
|
53
|
+
def register_and_validate(self, identifier: Union[str, int, float]) -> None:
|
54
|
+
"""
|
55
|
+
Registers given timeseries `identifier` and validates associated timeseries
|
28
56
|
|
29
|
-
|
30
|
-
|
31
|
-
return name in self._ids_of_time_series.keys()
|
57
|
+
Args:
|
58
|
+
identifier: to be registered - either a single numeric value or a string pointing to a timeseries file
|
32
59
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
self.
|
60
|
+
Raises:
|
61
|
+
TimeSeriesException: if file was not found, ill-formatted, or value was invalid
|
62
|
+
"""
|
63
|
+
if not self._time_series_is_registered(identifier):
|
64
|
+
self._register_time_series(identifier)
|
65
|
+
|
66
|
+
def _time_series_is_registered(self, identifier: Union[str, int, float]) -> bool:
|
67
|
+
"""Returns True if the value was already registered"""
|
68
|
+
return identifier in self._series_by_id.keys()
|
69
|
+
|
70
|
+
def _register_time_series(self, identifier: Union[str, int, float]) -> None:
|
71
|
+
"""Assigns an id to the given `identifier` and loads the time series into a dataframe"""
|
72
|
+
self._id_count += 1
|
73
|
+
name, series = self._get_name_and_dataframe(identifier)
|
74
|
+
self._series_by_id[identifier] = {Entry.ID: self._id_count, Entry.NAME: name, Entry.DATA: series}
|
75
|
+
|
76
|
+
def _get_name_and_dataframe(self, identifier: Union[str, int, float]) -> Tuple[str, pd.DataFrame]:
|
77
|
+
"""Returns name and DataFrame containing the series obtained from the given `identifier`"""
|
78
|
+
if isinstance(identifier, str):
|
79
|
+
series_path = self._path_resolver.resolve_series_file_path(Path(identifier).as_posix())
|
80
|
+
if series_path and os.path.exists(series_path):
|
81
|
+
data = pd.read_csv(series_path, sep=";", header=None, comment="#")
|
82
|
+
try:
|
83
|
+
return identifier, self._check_and_convert_series(data)
|
84
|
+
except TypeError as e:
|
85
|
+
log_error_and_raise(TimeSeriesException(self._ERR_CORRUPT_TIME_SERIES_VALUE.format(identifier), e))
|
86
|
+
except ConversionException:
|
87
|
+
log_error_and_raise(TimeSeriesException(self._ERR_CORRUPT_TIME_SERIES_KEY.format(identifier)))
|
88
|
+
else:
|
89
|
+
log_error_and_raise(TimeSeriesException(self._ERR_FILE_NOT_FOUND.format(identifier)))
|
38
90
|
else:
|
39
|
-
|
91
|
+
return self._create_timeseries_from_value(identifier)
|
92
|
+
|
93
|
+
def _check_and_convert_series(self, data: pd.DataFrame) -> pd.DataFrame:
|
94
|
+
"""Ensures validity of time series and convert to required format for writing to disk"""
|
95
|
+
data = data.apply(
|
96
|
+
lambda r: [FameTime.convert_string_if_is_datetime(r[0]), self._assert_valid(r[1])],
|
97
|
+
axis=1,
|
98
|
+
result_type="expand",
|
99
|
+
)
|
100
|
+
return data
|
101
|
+
|
102
|
+
@staticmethod
|
103
|
+
def _assert_valid(value: Any) -> float:
|
104
|
+
"""Returns the given `value` if it is a numeric value other than NaN"""
|
105
|
+
try:
|
106
|
+
value = float(value)
|
107
|
+
except ValueError:
|
108
|
+
log_error_and_raise(TypeError(TimeSeriesManager._ERR_NON_NUMERIC.format(value)))
|
109
|
+
if math.isnan(value):
|
110
|
+
log_error_and_raise(TypeError(TimeSeriesManager._ERR_NAN_VALUE))
|
111
|
+
return value
|
40
112
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
113
|
+
@staticmethod
|
114
|
+
def _create_timeseries_from_value(value: Union[int, float]) -> Tuple[str, pd.DataFrame]:
|
115
|
+
"""Returns name and dataframe for a new static timeseries created from the given `value`"""
|
116
|
+
if math.isnan(value):
|
117
|
+
log_error_and_raise(TimeSeriesException(TimeSeriesManager._ERR_NAN_VALUE))
|
118
|
+
data = pd.DataFrame({0: [INT64_MIN, INT64_MAX], 1: [value, value]})
|
119
|
+
return TimeSeriesManager._CONSTANT_IDENTIFIER.format(value), data
|
120
|
+
|
121
|
+
def get_series_id_by_identifier(self, identifier: Union[str, int, float]) -> int:
|
122
|
+
"""
|
123
|
+
Returns id for a previously stored time series by given `identifier`
|
124
|
+
|
125
|
+
Args:
|
126
|
+
identifier: to get the unique ID for
|
127
|
+
|
128
|
+
Returns:
|
129
|
+
unique ID for the given identifier
|
130
|
+
|
131
|
+
Raises:
|
132
|
+
TimeSeriesException: if identifier was not yet registered
|
133
|
+
"""
|
134
|
+
if not self._time_series_is_registered(identifier):
|
135
|
+
log_error_and_raise(TimeSeriesException(self._ERR_UNREGISTERED_SERIES.format(identifier)))
|
136
|
+
return self._series_by_id.get(identifier)[Entry.ID]
|
137
|
+
|
138
|
+
def get_all_series(self) -> List[Tuple[int, str, pd.DataFrame]]:
|
139
|
+
"""Returns iterator over id, name and dataframe of all stored series"""
|
140
|
+
if len(self._series_by_id) == 0:
|
141
|
+
log().warning(self._WARN_NO_DATA)
|
142
|
+
return [(v[Entry.ID], v[Entry.NAME], v[Entry.DATA]) for v in self._series_by_id.values()]
|
143
|
+
|
144
|
+
def reconstruct_time_series(self, timeseries: List[InputData.TimeSeriesDao]) -> None:
|
145
|
+
"""Reconstructs and stores time series from given list of `timeseries_dao`"""
|
146
|
+
for one_series in timeseries:
|
147
|
+
self._id_count += 1
|
148
|
+
reconstructed = {Entry.ID: one_series.seriesId}
|
149
|
+
if len(one_series.row) == 1:
|
150
|
+
reconstructed[Entry.NAME] = one_series.row[0].value
|
151
|
+
reconstructed[Entry.DATA] = None
|
152
|
+
else:
|
153
|
+
reconstructed[Entry.NAME] = self._get_cleaned_file_name(one_series.seriesName)
|
154
|
+
reconstructed[Entry.DATA] = pd.DataFrame(
|
155
|
+
[{self._KEY_ROW_TIME: item.timeStep, self._KEY_ROW_VALUE: item.value} for item in one_series.row]
|
156
|
+
)
|
157
|
+
self._series_by_id[one_series.seriesId] = reconstructed
|
158
|
+
|
159
|
+
def _get_cleaned_file_name(self, timeseries_name: str):
|
160
|
+
if timeseries_name.lower().endswith(".csv"):
|
161
|
+
filename = Path(timeseries_name).name
|
162
|
+
else:
|
163
|
+
filename = clean_up_file_name(timeseries_name) + ".csv"
|
164
|
+
return str(Path(self._TIMESERIES_RECONSTRUCTION_PATH, filename))
|
46
165
|
|
47
|
-
def
|
48
|
-
|
166
|
+
def get_reconstructed_series_by_id(self, series_id: int) -> str:
|
167
|
+
"""Return name or path for given `series_id` if series these are identified by their number.
|
168
|
+
Use this only if series were added via `reconstruct_time_series`"""
|
169
|
+
if series_id < 0 or series_id > self._id_count:
|
170
|
+
log_error_and_raise(TimeSeriesException(self._ERR_UNREGISTERED_SERIES.format(series_id)))
|
171
|
+
return self._series_by_id[series_id][Entry.NAME]
|
fameio/source/time.py
CHANGED
@@ -5,7 +5,7 @@
|
|
5
5
|
import datetime as dt
|
6
6
|
import math
|
7
7
|
import re
|
8
|
-
from enum import Enum
|
8
|
+
from enum import Enum, auto
|
9
9
|
from typing import Union
|
10
10
|
|
11
11
|
from fameio.source.logs import log_error_and_raise
|
@@ -25,13 +25,13 @@ class ConversionException(Exception):
|
|
25
25
|
class TimeUnit(Enum):
|
26
26
|
"""Time units defined in FAME"""
|
27
27
|
|
28
|
-
SECONDS =
|
29
|
-
MINUTES =
|
30
|
-
HOURS =
|
31
|
-
DAYS =
|
32
|
-
WEEKS =
|
33
|
-
MONTHS =
|
34
|
-
YEARS =
|
28
|
+
SECONDS = auto()
|
29
|
+
MINUTES = auto()
|
30
|
+
HOURS = auto()
|
31
|
+
DAYS = auto()
|
32
|
+
WEEKS = auto()
|
33
|
+
MONTHS = auto()
|
34
|
+
YEARS = auto()
|
35
35
|
|
36
36
|
|
37
37
|
class Constants:
|
fameio/source/tools.py
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
# SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
|
-
|
5
|
-
from typing import Any, Dict
|
4
|
+
from pathlib import Path
|
5
|
+
from typing import Any, Dict, Union
|
6
6
|
|
7
7
|
|
8
8
|
def keys_to_lower(dictionary: Dict[str, Any]) -> Dict[str, Any]:
|
@@ -16,3 +16,14 @@ def ensure_is_list(value: Any) -> list:
|
|
16
16
|
return value
|
17
17
|
else:
|
18
18
|
return [value]
|
19
|
+
|
20
|
+
|
21
|
+
def ensure_path_exists(path: Union[Path, str]):
|
22
|
+
"""Creates a specified path if not already existent"""
|
23
|
+
Path(path).mkdir(parents=True, exist_ok=True)
|
24
|
+
|
25
|
+
|
26
|
+
def clean_up_file_name(name: str) -> str:
|
27
|
+
"""Returns given `name` with replacements defined in `replace_map`"""
|
28
|
+
replace_map = {" ": "_", ":": "_", "/": "-"}
|
29
|
+
return name.translate(str.maketrans(replace_map))
|
fameio/source/validator.py
CHANGED
@@ -2,15 +2,17 @@
|
|
2
2
|
#
|
3
3
|
# SPDX-License-Identifier: Apache-2.0
|
4
4
|
|
5
|
-
import
|
5
|
+
import math
|
6
6
|
from collections import Counter
|
7
7
|
from typing import Any, Dict, List
|
8
8
|
|
9
|
-
from fameio.source
|
9
|
+
from fameio.source import PathResolver
|
10
|
+
from fameio.source.logs import log_error_and_raise, log
|
10
11
|
from fameio.source.scenario import Agent, Attribute, Contract, Scenario
|
11
12
|
from fameio.source.schema.agenttype import AgentType
|
12
13
|
from fameio.source.schema.attribute import AttributeSpecs, AttributeType
|
13
14
|
from fameio.source.schema.schema import Schema
|
15
|
+
from fameio.source.series import TimeSeriesManager, TimeSeriesException
|
14
16
|
from fameio.source.time import FameTime
|
15
17
|
|
16
18
|
|
@@ -36,6 +38,77 @@ class SchemaValidator:
|
|
36
38
|
_DEFAULT_IGNORED = "Optional Attribute: '{}': not specified - provided Default ignored for optional Attributes."
|
37
39
|
_OPTIONAL_MISSING = "Optional Attribute: '{}': not specified."
|
38
40
|
_IS_NO_LIST = "Attribute '{}' is list but assigned value '{}' is not a list."
|
41
|
+
_TIME_SERIES_INVALID = "Timeseries at '{}' is invalid."
|
42
|
+
_MISSING_CONTRACTS_FOR_AGENTS = "No contracts defined for Agent '{}' of type '{}'"
|
43
|
+
|
44
|
+
@staticmethod
|
45
|
+
def validate_scenario_and_timeseries(
|
46
|
+
scenario: Scenario, path_resolver: PathResolver = PathResolver()
|
47
|
+
) -> TimeSeriesManager:
|
48
|
+
"""
|
49
|
+
Validates the given `scenario` and its timeseries using given `path_resolver`
|
50
|
+
Raises an exception if schema requirements are not met or timeseries data are erroneous.
|
51
|
+
|
52
|
+
Args:
|
53
|
+
scenario: to be validated against the encompassed schema
|
54
|
+
path_resolver: to resolve paths of timeseries
|
55
|
+
|
56
|
+
Returns:
|
57
|
+
a new TimeSeriesManager initialised with validated time series from scenario
|
58
|
+
Raises:
|
59
|
+
ValidationException: if an error in the scenario or in timeseries is spotted
|
60
|
+
"""
|
61
|
+
schema = scenario.schema
|
62
|
+
agents = scenario.agents
|
63
|
+
timeseries_manager = TimeSeriesManager(path_resolver)
|
64
|
+
|
65
|
+
SchemaValidator.ensure_unique_agent_ids(agents)
|
66
|
+
for agent in agents:
|
67
|
+
SchemaValidator.ensure_agent_and_timeseries_are_valid(agent, schema, timeseries_manager)
|
68
|
+
|
69
|
+
agent_types_by_id = {agent.id: agent.type_name for agent in agents}
|
70
|
+
for contract in scenario.contracts:
|
71
|
+
SchemaValidator.ensure_is_valid_contract(contract, schema, agent_types_by_id)
|
72
|
+
|
73
|
+
return timeseries_manager
|
74
|
+
|
75
|
+
@staticmethod
|
76
|
+
def ensure_unique_agent_ids(agents: List[Agent]) -> None:
|
77
|
+
"""Raises exception if any id for given `agents` is not unique"""
|
78
|
+
list_of_ids = [agent.id for agent in agents]
|
79
|
+
non_unique_ids = [agent_id for agent_id, count in Counter(list_of_ids).items() if count > 1]
|
80
|
+
if non_unique_ids:
|
81
|
+
log_error_and_raise(ValidationException(SchemaValidator._AGENT_ID_NOT_UNIQUE.format(non_unique_ids)))
|
82
|
+
|
83
|
+
@staticmethod
|
84
|
+
def ensure_agent_and_timeseries_are_valid(agent: Agent, schema: Schema, timeseries_manager: TimeSeriesManager):
|
85
|
+
"""Validates given `agent` against `schema` plus loads and validates its timeseries"""
|
86
|
+
SchemaValidator.ensure_agent_type_in_schema(agent, schema)
|
87
|
+
SchemaValidator.ensure_is_valid_agent(agent, schema)
|
88
|
+
SchemaValidator.load_and_validate_timeseries(agent, schema, timeseries_manager)
|
89
|
+
|
90
|
+
@staticmethod
|
91
|
+
def ensure_agent_type_in_schema(agent: Agent, schema: Schema) -> None:
|
92
|
+
"""Raises exception if type for given `agent` is not specified in given `schema`"""
|
93
|
+
if agent.type_name not in schema.agent_types:
|
94
|
+
log_error_and_raise(ValidationException(SchemaValidator._AGENT_TYPE_UNKNOWN.format(agent.type_name)))
|
95
|
+
|
96
|
+
@staticmethod
|
97
|
+
def ensure_is_valid_agent(agent: Agent, schema: Schema) -> None:
|
98
|
+
"""Raises an exception if given `agent` does not meet the specified `schema` requirements"""
|
99
|
+
scenario_attributes = agent.attributes
|
100
|
+
schema_attributes = SchemaValidator._get_agent(schema, agent.type_name).attributes
|
101
|
+
SchemaValidator._ensure_mandatory_present(scenario_attributes, schema_attributes)
|
102
|
+
SchemaValidator._ensure_attributes_exist(scenario_attributes, schema_attributes)
|
103
|
+
SchemaValidator._ensure_value_and_type_match(scenario_attributes, schema_attributes)
|
104
|
+
|
105
|
+
@staticmethod
|
106
|
+
def _get_agent(schema: Schema, name: str) -> AgentType:
|
107
|
+
"""Returns agent specified by `name` or raises Exception if this agent is not present in given `schema`"""
|
108
|
+
if name in schema.agent_types:
|
109
|
+
return schema.agent_types[name]
|
110
|
+
else:
|
111
|
+
log_error_and_raise(ValidationException(SchemaValidator._AGENT_TYPE_UNKNOWN.format(name)))
|
39
112
|
|
40
113
|
@staticmethod
|
41
114
|
def _ensure_mandatory_present(attributes: Dict[str, Attribute], specifications: Dict[str, AttributeSpecs]) -> None:
|
@@ -47,12 +120,14 @@ class SchemaValidator:
|
|
47
120
|
if name not in attributes:
|
48
121
|
if specification.is_mandatory:
|
49
122
|
if not specification.has_default_value:
|
50
|
-
log_error_and_raise(
|
123
|
+
log_error_and_raise(
|
124
|
+
ValidationException(SchemaValidator._ATTRIBUTE_MISSING.format(specification.full_name))
|
125
|
+
)
|
51
126
|
else:
|
52
127
|
if specification.has_default_value:
|
53
|
-
log.warning(SchemaValidator._DEFAULT_IGNORED.format(
|
128
|
+
log().warning(SchemaValidator._DEFAULT_IGNORED.format(specification.full_name))
|
54
129
|
else:
|
55
|
-
log.info(SchemaValidator._OPTIONAL_MISSING.format(
|
130
|
+
log().info(SchemaValidator._OPTIONAL_MISSING.format(specification.full_name))
|
56
131
|
if name in attributes and specification.has_nested_attributes:
|
57
132
|
attribute = attributes[name]
|
58
133
|
if specification.is_list:
|
@@ -61,23 +136,6 @@ class SchemaValidator:
|
|
61
136
|
else:
|
62
137
|
SchemaValidator._ensure_mandatory_present(attribute.nested, specification.nested_attributes)
|
63
138
|
|
64
|
-
@staticmethod
|
65
|
-
def _get_agent(schema: Schema, name: str) -> AgentType:
|
66
|
-
"""Returns agent specified by `name` or raises Exception if this agent is not present in given `schema`"""
|
67
|
-
if name in schema.agent_types:
|
68
|
-
return schema.agent_types[name]
|
69
|
-
else:
|
70
|
-
log_error_and_raise(ValidationException(SchemaValidator._AGENT_TYPE_UNKNOWN.format(name)))
|
71
|
-
|
72
|
-
@staticmethod
|
73
|
-
def ensure_is_valid_agent(agent: Agent, schema: Schema) -> None:
|
74
|
-
"""Raises an exception if given `agent` does not meet the specified `schema` requirements"""
|
75
|
-
scenario_attributes = agent.attributes
|
76
|
-
schema_attributes = SchemaValidator._get_agent(schema, agent.type_name).attributes
|
77
|
-
SchemaValidator._ensure_mandatory_present(scenario_attributes, schema_attributes)
|
78
|
-
SchemaValidator._ensure_attributes_exist(scenario_attributes, schema_attributes)
|
79
|
-
SchemaValidator._ensure_value_matches_type(scenario_attributes, schema_attributes)
|
80
|
-
|
81
139
|
@staticmethod
|
82
140
|
def _ensure_attributes_exist(attributes: Dict[str, Attribute], specifications: Dict[str, AttributeSpecs]) -> None:
|
83
141
|
"""Raises exception any entry of given `attributes` has no corresponding type `specification`"""
|
@@ -93,7 +151,9 @@ class SchemaValidator:
|
|
93
151
|
SchemaValidator._ensure_attributes_exist(entry, specification.nested_attributes)
|
94
152
|
|
95
153
|
@staticmethod
|
96
|
-
def
|
154
|
+
def _ensure_value_and_type_match(
|
155
|
+
attributes: Dict[str, Attribute], specifications: Dict[str, AttributeSpecs]
|
156
|
+
) -> None:
|
97
157
|
"""Raises exception if in given list of `attributes` its value does not match associated type `specification`"""
|
98
158
|
for name, attribute in attributes.items():
|
99
159
|
specification = specifications[name]
|
@@ -104,14 +164,13 @@ class SchemaValidator:
|
|
104
164
|
message = SchemaValidator._INCOMPATIBLE.format(value, type_spec, specification.full_name)
|
105
165
|
log_error_and_raise(ValidationException(message))
|
106
166
|
if not SchemaValidator._is_allowed_value(specification, value):
|
107
|
-
|
108
|
-
|
109
|
-
)
|
167
|
+
message = SchemaValidator._DISALLOWED.format(value, specification.full_name)
|
168
|
+
log_error_and_raise(ValidationException(message))
|
110
169
|
if attribute.has_nested:
|
111
|
-
SchemaValidator.
|
170
|
+
SchemaValidator._ensure_value_and_type_match(attribute.nested, specification.nested_attributes)
|
112
171
|
if attribute.has_nested_list:
|
113
172
|
for entry in attribute.nested_list:
|
114
|
-
SchemaValidator.
|
173
|
+
SchemaValidator._ensure_value_and_type_match(entry, specification.nested_attributes)
|
115
174
|
|
116
175
|
@staticmethod
|
117
176
|
def _is_compatible(specification: AttributeSpecs, value_or_values: Any) -> bool:
|
@@ -120,7 +179,7 @@ class SchemaValidator:
|
|
120
179
|
attribute_type = specification.attr_type
|
121
180
|
if specification.is_list:
|
122
181
|
if not is_list:
|
123
|
-
log.warning(SchemaValidator._IS_NO_LIST.format(specification.full_name, value_or_values))
|
182
|
+
log().warning(SchemaValidator._IS_NO_LIST.format(specification.full_name, value_or_values))
|
124
183
|
return SchemaValidator._is_compatible_value(attribute_type, value_or_values)
|
125
184
|
for value in value_or_values:
|
126
185
|
if not SchemaValidator._is_compatible_value(attribute_type, value):
|
@@ -131,7 +190,7 @@ class SchemaValidator:
|
|
131
190
|
|
132
191
|
@staticmethod
|
133
192
|
def _is_compatible_value(attribute_type: AttributeType, value) -> bool:
|
134
|
-
"""Returns True if given single value is compatible to specified `attribute_type`"""
|
193
|
+
"""Returns True if given single value is compatible to specified `attribute_type` and is not a NaN float"""
|
135
194
|
if attribute_type is AttributeType.INTEGER:
|
136
195
|
if isinstance(value, int):
|
137
196
|
return -2147483648 < value < 2147483647
|
@@ -139,13 +198,13 @@ class SchemaValidator:
|
|
139
198
|
if attribute_type is AttributeType.LONG:
|
140
199
|
return isinstance(value, int)
|
141
200
|
elif attribute_type is AttributeType.DOUBLE:
|
142
|
-
return isinstance(value, (int, float))
|
201
|
+
return isinstance(value, (int, float)) and not math.isnan(value)
|
143
202
|
elif attribute_type in (AttributeType.ENUM, AttributeType.STRING):
|
144
203
|
return isinstance(value, str)
|
145
204
|
elif attribute_type is AttributeType.TIME_STAMP:
|
146
205
|
return FameTime.is_fame_time_compatible(value)
|
147
206
|
elif attribute_type is AttributeType.TIME_SERIES:
|
148
|
-
return isinstance(value, (str, int, float))
|
207
|
+
return isinstance(value, (str, int)) or (isinstance(value, float) and not math.isnan(value))
|
149
208
|
else:
|
150
209
|
log_error_and_raise(ValidationException(SchemaValidator._TYPE_NOT_IMPLEMENTED.format(attribute_type)))
|
151
210
|
|
@@ -157,6 +216,44 @@ class SchemaValidator:
|
|
157
216
|
else:
|
158
217
|
return value in attribute.values
|
159
218
|
|
219
|
+
@staticmethod
|
220
|
+
def load_and_validate_timeseries(agent: Agent, schema: Schema, timeseries_manager: TimeSeriesManager) -> None:
|
221
|
+
"""
|
222
|
+
Loads all timeseries specified in given `schema` of given `agent` into given `timeseries_manager`
|
223
|
+
|
224
|
+
Args:
|
225
|
+
agent: definition in scenario
|
226
|
+
schema: schema encompassed in scenario
|
227
|
+
timeseries_manager: to be filled with timeseries
|
228
|
+
|
229
|
+
Raises:
|
230
|
+
ValidationException: if timeseries is not found, ill-formatted or invalid
|
231
|
+
"""
|
232
|
+
scenario_attributes = agent.attributes
|
233
|
+
schema_attributes = SchemaValidator._get_agent(schema, agent.type_name).attributes
|
234
|
+
SchemaValidator._ensure_valid_timeseries(scenario_attributes, schema_attributes, timeseries_manager)
|
235
|
+
|
236
|
+
@staticmethod
|
237
|
+
def _ensure_valid_timeseries(
|
238
|
+
attributes: Dict[str, Attribute], specifications: Dict[str, AttributeSpecs], manager: TimeSeriesManager
|
239
|
+
) -> None:
|
240
|
+
"""Recursively searches for time_series in agent attributes and registers them at given `manager`"""
|
241
|
+
for name, attribute in attributes.items():
|
242
|
+
specification = specifications[name]
|
243
|
+
if attribute.has_value:
|
244
|
+
attribute_type = specification.attr_type
|
245
|
+
if attribute_type is AttributeType.TIME_SERIES:
|
246
|
+
try:
|
247
|
+
manager.register_and_validate(attribute.value)
|
248
|
+
except TimeSeriesException as e:
|
249
|
+
message = SchemaValidator._TIME_SERIES_INVALID.format(specification.full_name)
|
250
|
+
log_error_and_raise(ValidationException(message, e))
|
251
|
+
if attribute.has_nested:
|
252
|
+
SchemaValidator._ensure_valid_timeseries(attribute.nested, specification.nested_attributes, manager)
|
253
|
+
if attribute.has_nested_list:
|
254
|
+
for entry in attribute.nested_list:
|
255
|
+
SchemaValidator._ensure_valid_timeseries(entry, specification.nested_attributes, manager)
|
256
|
+
|
160
257
|
@staticmethod
|
161
258
|
def ensure_is_valid_contract(contract: Contract, schema: Schema, agent_types_by_id: Dict[int, str]) -> None:
|
162
259
|
"""Raises exception if given `contract` does not meet the `schema`'s requirements, using `agent_types_by_id`"""
|
@@ -174,30 +271,13 @@ class SchemaValidator:
|
|
174
271
|
log_error_and_raise(ValidationException(SchemaValidator._PRODUCT_MISSING.format(product, sender_type_name)))
|
175
272
|
|
176
273
|
@staticmethod
|
177
|
-
def
|
178
|
-
"""Raises
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
274
|
+
def check_agents_have_contracts(scenario: Scenario) -> None:
|
275
|
+
"""Raises warning for each agent without any assigned contract"""
|
276
|
+
senders = [contract.sender_id for contract in scenario.contracts]
|
277
|
+
receivers = [contract.receiver_id for contract in scenario.contracts]
|
278
|
+
active_agents = set(senders + receivers)
|
279
|
+
inactive_agents = {agent.id: agent.type_name for agent in scenario.agents if agent.id not in active_agents}
|
183
280
|
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
if agent.type_name not in schema.agent_types:
|
188
|
-
log_error_and_raise(ValidationException(SchemaValidator._AGENT_TYPE_UNKNOWN.format(agent.type_name)))
|
189
|
-
|
190
|
-
@staticmethod
|
191
|
-
def ensure_is_valid_scenario(scenario: Scenario) -> None:
|
192
|
-
"""Raises exception if given `scenario` does not meet its own schema requirements"""
|
193
|
-
schema = scenario.schema
|
194
|
-
agents = scenario.agents
|
195
|
-
|
196
|
-
SchemaValidator.ensure_unique_agent_ids(agents)
|
197
|
-
for agent in agents:
|
198
|
-
SchemaValidator.ensure_agent_type_in_schema(agent, schema)
|
199
|
-
SchemaValidator.ensure_is_valid_agent(agent, schema)
|
200
|
-
|
201
|
-
agent_types_by_id = {agent.id: agent.type_name for agent in agents}
|
202
|
-
for contract in scenario.contracts:
|
203
|
-
SchemaValidator.ensure_is_valid_contract(contract, schema, agent_types_by_id)
|
281
|
+
if inactive_agents:
|
282
|
+
for agent_id, agent_name in inactive_agents.items():
|
283
|
+
log().warning(SchemaValidator._MISSING_CONTRACTS_FOR_AGENTS.format(agent_id, agent_name))
|