fameio 1.8.2__py3-none-any.whl → 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- CHANGELOG.md +204 -0
- fameio/scripts/__init__.py +8 -6
- fameio/scripts/__init__.py.license +3 -0
- fameio/scripts/convert_results.py +30 -34
- fameio/scripts/convert_results.py.license +3 -0
- fameio/scripts/make_config.py +13 -16
- fameio/scripts/make_config.py.license +3 -0
- fameio/source/cli/__init__.py +3 -0
- fameio/source/cli/convert_results.py +75 -0
- fameio/source/cli/make_config.py +62 -0
- fameio/source/cli/options.py +59 -0
- fameio/source/cli/parser.py +238 -0
- fameio/source/loader.py +10 -11
- fameio/source/logs.py +49 -25
- fameio/source/results/conversion.py +11 -13
- fameio/source/results/csv_writer.py +16 -5
- fameio/source/results/data_transformer.py +3 -2
- fameio/source/results/input_dao.py +163 -0
- fameio/source/results/reader.py +25 -14
- fameio/source/results/yaml_writer.py +28 -0
- fameio/source/scenario/agent.py +56 -39
- fameio/source/scenario/attribute.py +9 -12
- fameio/source/scenario/contract.py +55 -40
- fameio/source/scenario/exception.py +11 -9
- fameio/source/scenario/generalproperties.py +11 -17
- fameio/source/scenario/scenario.py +19 -14
- fameio/source/schema/agenttype.py +75 -27
- fameio/source/schema/attribute.py +8 -7
- fameio/source/schema/schema.py +24 -11
- fameio/source/series.py +146 -25
- fameio/source/time.py +8 -8
- fameio/source/tools.py +13 -2
- fameio/source/validator.py +138 -58
- fameio/source/writer.py +108 -112
- fameio-2.0.0.dist-info/LICENSES/Apache-2.0.txt +178 -0
- fameio-2.0.0.dist-info/LICENSES/CC-BY-4.0.txt +395 -0
- fameio-2.0.0.dist-info/LICENSES/CC0-1.0.txt +121 -0
- {fameio-1.8.2.dist-info → fameio-2.0.0.dist-info}/METADATA +694 -660
- fameio-2.0.0.dist-info/RECORD +52 -0
- {fameio-1.8.2.dist-info → fameio-2.0.0.dist-info}/WHEEL +1 -2
- fameio-2.0.0.dist-info/entry_points.txt +4 -0
- fameio/source/cli.py +0 -253
- fameio-1.8.2.dist-info/RECORD +0 -40
- fameio-1.8.2.dist-info/entry_points.txt +0 -3
- fameio-1.8.2.dist-info/top_level.txt +0 -1
- {fameio-1.8.2.dist-info → fameio-2.0.0.dist-info}/LICENSE.txt +0 -0
CHANGELOG.md
ADDED
@@ -0,0 +1,204 @@
|
|
1
|
+
<!-- SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
|
2
|
+
|
3
|
+
SPDX-License-Identifier: CC0-1.0 -->
|
4
|
+
|
5
|
+
# Changelog
|
6
|
+
|
7
|
+
## [2.0.0 - 2024-04-03](https://gitlab.com/fame-framework/fame-io/-/tags/v2.0.0)
|
8
|
+
### Changed
|
9
|
+
- **Breaking**: Removed support for `python==3.8` #163 (@dlr-cjs @dlr_fn)
|
10
|
+
- **Breaking**: Signature of `run` functions in `make_config.py` and `convert_results.py` changed: the input file is now read from the configuration dictionary #163 (@dlr-cjs @dlr_fn)
|
11
|
+
- **Breaking**: Created protobuf files now have a header section -> minimum required FAME-Core version is now 1.6.0 #183 (@dlr-cjs @dlr_fn)
|
12
|
+
- Raise error for NaN float values in scenario and time series #165 (@dlr-cjs @dlr_fn)
|
13
|
+
- Enhance Schema to include metadata and output fields #156 (@dlr-cjs @litotes18 @dlr_fn)
|
14
|
+
- Enhance Contracts to include metadata #158 (@dlr-cjs @litotes18 @dlr_fn)
|
15
|
+
- Enhance Agents to include metadata #159 (@dlr-cjs @litotes18 @dlr_fn)
|
16
|
+
- Improve general handling of CLI arguments #163 (@dlr_fn @dlr-cjs)
|
17
|
+
- Ensure `fameio` logger is used consistently !126 (@dlr-cjs @dlr_fn)
|
18
|
+
- Enhanced error message if mandatory attribute is not defined by logging `full_name` #177 (@dlr_fn)
|
19
|
+
- Switch to pyproject.toml #173 (@dlr-cjs)
|
20
|
+
- Restrict supported pandas versions #171 (@dlr-cjs)
|
21
|
+
- Enable to specify defaults for MERGE_TIME parameters #179 (@dlr-cjs)
|
22
|
+
- Conserve order of keys in YAML files #186 (@dlr-cjs @dlr_fn)
|
23
|
+
- Update to `fameprotobuf==1.4.0` #189 (@dlr-cjs @dlr_fn)
|
24
|
+
- Update `CHANGELOG.md` to conform with Common Changelog format #172 (@dlr-cjs @dlr_fn)
|
25
|
+
|
26
|
+
### Added
|
27
|
+
- Write Schema and Metadata of Contracts and Agents to protobuf file #160 (@dlr-cjs @litotes18 @dlr_fn)
|
28
|
+
- Add option to recover input data `--input-recovery`/`--no-input-recovery` #163 (@litotes18 @dlr_fn @dlr-cjs)
|
29
|
+
- Add pipeline tests for all major Python versions >= 3.8 #173 (@dlr-cjs)
|
30
|
+
- Read all input from protobuf file #162 (@dlr-cjs @litotes18 @dlr_fn)
|
31
|
+
- Write all input from protobuf to disk #163 (@litotes18 @dlr_fn @dlr-cjs)
|
32
|
+
- Add header section to input protobuf enabling recovering of inputs from protobuf #183 (@dlr-cjs @dlr_fn)
|
33
|
+
- Add pipeline tests for all major Python versions >= 3.8, < 3.12 #173 (@dlr-cjs)
|
34
|
+
- Enable comments in timeseries using '#' #184 (@dlr-cjs)
|
35
|
+
- Raise Warning if Agent has no Contracts attributed #187 (@dlr_fn)
|
36
|
+
- Add JOSS Paper in folder paper/ #139 (@dlr-cjs @litotes18 @dlr_fn)
|
37
|
+
- Add `CONTRIBUTING.md` #102 (@dlr-cjs @dlr_fn)
|
38
|
+
- Add `Citation.cff` #166 (@dlr-cjs)
|
39
|
+
|
40
|
+
### Removed
|
41
|
+
- Remove deprecated protobuf test !127 (@dlr-cjs @dlr_fn)
|
42
|
+
|
43
|
+
### Fixed
|
44
|
+
- Fix deprecated arguments in pandas groupby !129 (@maurerle)
|
45
|
+
- Fix breaking tests in Pytest 8.0 #176 (@dlr-cjs)
|
46
|
+
- Fix PyTests for Python 3.12 #182 (@dlr_fn)
|
47
|
+
|
48
|
+
## [1.8.1 - 2023-05-04](https://gitlab.com/fame-framework/fame-io/-/tags/v1.8.1)
|
49
|
+
### Fixed
|
50
|
+
- Fix fail of `ConvertFameResults` when `merge-times` was not specified
|
51
|
+
|
52
|
+
## [1.8.0 - 2023-04-14](https://gitlab.com/fame-framework/fame-io/-/tags/v1.8)
|
53
|
+
### Changed
|
54
|
+
- Update repository to be compliant to `REUSE` standard
|
55
|
+
- Accept custom `date_format` (default: `"%Y-%m-%d_%H:%M:%S"`) for `FameTime.convert_fame_time_step_to_datetime()`
|
56
|
+
- Parse command-line arguments case-insensitive for arguments with predefined options
|
57
|
+
- Handle potentially missing cli arguments in `cli.update_default_config` for `makeFameRunConfig` and `convertFameResults` in a robust way.
|
58
|
+
|
59
|
+
### Added
|
60
|
+
- **Breaking**: Add option to define conversion of time steps to given format (default=`UTC`) by `-t/--time {UTC, INT, FAME}` for `convertFameResults`
|
61
|
+
- Add option to merge time steps in results with `convertFameResults`
|
62
|
+
- Add pre-commit hooks enforcing high coding standards and reducing CI runner minutes during development
|
63
|
+
|
64
|
+
## [1.7.0 - 2023-02-20](https://gitlab.com/fame-framework/fame-io/-/tags/v1.7)
|
65
|
+
### Added
|
66
|
+
- Support dictionaries in Schema for field `Products` in elements of `AgentTypes`
|
67
|
+
- Support dictionaries in Schema for field `Values` in elements of `Attributes`
|
68
|
+
|
69
|
+
### Changed
|
70
|
+
- Use `Pathlib` for path handling
|
71
|
+
- Improve error message when no valid `YAML` file is specified for `makeFameRunConfig`
|
72
|
+
|
73
|
+
### Remove
|
74
|
+
- **Breaking**: `Products` in Schema no longer support single non-list values
|
75
|
+
|
76
|
+
## [1.6.3 - 2022-11-04](https://gitlab.com/fame-framework/fame-io/-/tags/v1.6.3)
|
77
|
+
### Added
|
78
|
+
- Allow parsing `Help` for `Attributes` in `schema`
|
79
|
+
|
80
|
+
## [1.6.1 - 2022-11-02](https://gitlab.com/fame-framework/fame-io/-/tags/v1.6.1)
|
81
|
+
### Changed
|
82
|
+
- Use existing logger if already set up to avoid duplicates when `fameio` is used as dependency in third party workflows
|
83
|
+
|
84
|
+
## [1.6.0 - 2022-07-08](https://gitlab.com/fame-framework/fame-io/-/tags/v1.6)
|
85
|
+
### Added
|
86
|
+
- Add option to enable memory saving mode using the flag `-m` or `--memory-saving`
|
87
|
+
- Add options to deal with complex indexed output columns using the flag `-cc` or `--complex-column` with
|
88
|
+
options `IGNORE`, `MERGE` or `SPLIT`
|
89
|
+
|
90
|
+
### Changed
|
91
|
+
- **Breaking**: Update requirement to `python>=3.8`
|
92
|
+
- **Breaking**: Update requirement to `fameprotobuf==v1.2`
|
93
|
+
- Enable parsing of protobuf output files > 2 GB
|
94
|
+
- Reduce memory profile for `convertFameResults`
|
95
|
+
- Extract `source` scripts relevant for `convertFameResults` to be hosted in subpackage `results`
|
96
|
+
|
97
|
+
## [1.5.4 - 2022-06-01](https://gitlab.com/fame-framework/fame-io/-/tags/v1.5.4)
|
98
|
+
### Changed
|
99
|
+
- Limit `protobuf` dependency to `>=3.19,<4.0`
|
100
|
+
|
101
|
+
## [1.5.3 - 2022-03-18](https://gitlab.com/fame-framework/fame-io/-/tags/v1.5.3)
|
102
|
+
### Changed
|
103
|
+
- Harmonize interface with `famegui`
|
104
|
+
- Return `None` on failure of `resolve_series_file_path` instead of raising a `FileNotFoundError`
|
105
|
+
|
106
|
+
## [1.5.2 - 2022-03-10](https://gitlab.com/fame-framework/fame-io/-/tags/v1.5.2)
|
107
|
+
### Changed
|
108
|
+
- Allow interfacing of `famegui` with `scenario` (e.g. serialization, error handling)
|
109
|
+
- Move `scenario` validation to `validator.py`
|
110
|
+
- Extract `path_resolver.py`
|
111
|
+
- Increase test coverage by incorporating [AMIRIS examples](https://gitlab.com/dlr-ve/esy/amiris/examples)
|
112
|
+
|
113
|
+
## [1.5.1 - 2022-01-10](https://gitlab.com/fame-framework/fame-io/-/tags/v1.5.1)
|
114
|
+
### Added
|
115
|
+
- Provide documentation on installation using `pipx`
|
116
|
+
- Add optional argument `-se`/`--singleexport` for exporting individual files for each agent
|
117
|
+
- Add compatibility hook for `famegui` integration
|
118
|
+
|
119
|
+
### Changed
|
120
|
+
- Refactor `scenario.py`
|
121
|
+
- Ensure code formatting using `black`
|
122
|
+
|
123
|
+
## [1.5.0 - 2021-06-30](https://gitlab.com/fame-framework/fame-io/-/tags/v1.5)
|
124
|
+
### Added
|
125
|
+
- Support specifying an output folder in command line interface of `convert_results.py`
|
126
|
+
|
127
|
+
### Changed
|
128
|
+
- Update to latest protobuf package
|
129
|
+
- Refactor code
|
130
|
+
|
131
|
+
## [1.4.0 - 2021-06-10](https://gitlab.com/fame-framework/fame-io/-/tags/v1.4)
|
132
|
+
### Added
|
133
|
+
- Enable "Default" values for Attributes - these are used in case a mandatory attribute is not specified in the Scenario
|
134
|
+
- Allow "List" Attributes with multiple values
|
135
|
+
- Add new AttributeTypes "Long", "String" and "TimeStamp"
|
136
|
+
- Add compact definition of multiple contracts: enable lists for senders and receivers
|
137
|
+
|
138
|
+
### Changed
|
139
|
+
- **Breaking**: Update requirement to `fameprotobuf==1.1.4`
|
140
|
+
- Refactor `make_config.py`: split into several classes and packages, improved exception handling
|
141
|
+
- Switch to pytest and improved test coverage
|
142
|
+
- Make keywords in Schema and Scenario case-insensitive
|
143
|
+
- Improve validations for Schema and Scenario
|
144
|
+
|
145
|
+
### Fixed
|
146
|
+
- Fixed minor bugs
|
147
|
+
|
148
|
+
## [1.3.0 - 2021-04-13](https://gitlab.com/fame-framework/fame-io/-/tags/v1.3)
|
149
|
+
### Added
|
150
|
+
- Enable `Attributes` in agents (formerly known as `Fields`) to be structured in complex tree-like data dictionaries
|
151
|
+
- Allow contracts to support `Attributes` of type `int`, `float`, `enum` or `dict`
|
152
|
+
- Add coverage report badge
|
153
|
+
- Add `CHANGELOG.md`
|
154
|
+
|
155
|
+
### Changed
|
156
|
+
- **Breaking**: Use new format `DataStorage` for input and output protobuf files allowing `FAME-Core` input and output to be written to the same file (requires `FAME-Core > 1.0`)
|
157
|
+
- **Breaking**: Update requirement to `fameprotobuf==1.1.2`
|
158
|
+
- Enable automatic detection of `TimeStamps` by string format and conversion to int64
|
159
|
+
- Raise proper error when file can not be loaded triggered by `!include` command
|
160
|
+
- Raise critical error when trying to convert empty protobuf output file
|
161
|
+
- Check if `product` in `contract` is valid according to `schema.yaml`
|
162
|
+
|
163
|
+
## [1.2.4 - 2021-02-26](https://gitlab.com/fame-framework/fame-io/-/tags/v1.2.4)
|
164
|
+
### Changed
|
165
|
+
- Move `is_compatible` function to class `AttributeType`
|
166
|
+
|
167
|
+
## [1.2.3 - 2021-02-24](https://gitlab.com/fame-framework/fame-io/-/tags/v1.2.3)
|
168
|
+
### Fixed
|
169
|
+
- Fix file prefix `IGNORE_` (used when loading a set of contract files with the !include argument) is now working consistently
|
170
|
+
|
171
|
+
## [1.2.2 - 2021-02-18](https://gitlab.com/fame-framework/fame-io/-/tags/v1.2.2)
|
172
|
+
### Changed
|
173
|
+
- **Breaking**: Rename `fieldtype` to `attributetype` in `schema.yaml`
|
174
|
+
- Derive protobuf imports from `fameprotobuf` package
|
175
|
+
- Improve handling of cases for keys in `scenario.yaml`
|
176
|
+
- Improve handling of time stamp strings
|
177
|
+
|
178
|
+
## [1.2.1 - 2021-02-10](https://gitlab.com/fame-framework/fame-io/-/tags/v1.2.1)
|
179
|
+
### Changed
|
180
|
+
- Improve key handling for contracts which are now case-insensitive
|
181
|
+
|
182
|
+
## [1.2.0 - 2021-02-04](https://gitlab.com/fame-framework/fame-io/-/tags/v1.2)
|
183
|
+
### Added
|
184
|
+
- Add `!include` command to yaml loading to allow integrating additional yaml files
|
185
|
+
|
186
|
+
### Changed
|
187
|
+
- **Breaking**: Rename package to `fameio`
|
188
|
+
- Improve executables
|
189
|
+
- Restructure logging
|
190
|
+
- Improve documentation
|
191
|
+
|
192
|
+
### Fixed
|
193
|
+
- Fix bugs
|
194
|
+
|
195
|
+
## [1.1.0 - 2020-12-09](https://gitlab.com/fame-framework/fame-io/-/tags/v1.1)
|
196
|
+
### Added
|
197
|
+
- Package to PyPI
|
198
|
+
- Provide executables for calling `makeFameRunConfig` and `convertFameResults`
|
199
|
+
|
200
|
+
### Changed
|
201
|
+
- Improve documentation
|
202
|
+
|
203
|
+
## [1.0.0 - 2020-11-17](https://gitlab.com/fame-framework/fame-io/-/tags/v1.0)
|
204
|
+
_Initial release of `famepy`_
|
fameio/scripts/__init__.py
CHANGED
@@ -3,16 +3,18 @@ import sys
|
|
3
3
|
|
4
4
|
from fameio.scripts.convert_results import DEFAULT_CONFIG as DEFAULT_CONVERT_CONFIG
|
5
5
|
from fameio.scripts.convert_results import run as convert_results
|
6
|
-
from fameio.scripts.make_config import DEFAULT_CONFIG as DEFAULT_MAKE_CONFIG
|
7
6
|
from fameio.scripts.make_config import run as make_config
|
8
|
-
from fameio.source.cli import
|
7
|
+
from fameio.source.cli.convert_results import handle_args as handle_convert_results_args
|
8
|
+
from fameio.source.cli.make_config import handle_args as handle_make_config_args
|
9
9
|
|
10
10
|
|
11
|
+
# noinspection PyPep8Naming
|
11
12
|
def makeFameRunConfig():
|
12
|
-
|
13
|
-
make_config(
|
13
|
+
run_config = handle_make_config_args(sys.argv[1:])
|
14
|
+
make_config(run_config)
|
14
15
|
|
15
16
|
|
17
|
+
# noinspection PyPep8Naming
|
16
18
|
def convertFameResults():
|
17
|
-
|
18
|
-
convert_results(
|
19
|
+
run_config = handle_convert_results_args(sys.argv[1:], DEFAULT_CONVERT_CONFIG)
|
20
|
+
convert_results(run_config)
|
@@ -1,75 +1,71 @@
|
|
1
1
|
#!/usr/bin/env python
|
2
2
|
|
3
|
-
import logging as log
|
4
3
|
import sys
|
5
4
|
from pathlib import Path
|
6
|
-
from typing import Union
|
7
5
|
|
8
|
-
from fameio.source.cli import
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
update_default_config,
|
13
|
-
TimeOptions,
|
14
|
-
)
|
15
|
-
from fameio.source.logs import log_and_raise_critical, set_up_logger
|
6
|
+
from fameio.source.cli.convert_results import handle_args, CLI_DEFAULTS as DEFAULT_CONFIG
|
7
|
+
from fameio.source.cli.options import Options
|
8
|
+
from fameio.source.cli.parser import update_default_config
|
9
|
+
from fameio.source.logs import log_and_raise_critical, set_up_logger, logger
|
16
10
|
from fameio.source.results.agent_type import AgentTypeLog
|
17
11
|
from fameio.source.results.conversion import apply_time_option, apply_time_merging
|
18
12
|
from fameio.source.results.csv_writer import CsvWriter
|
19
13
|
from fameio.source.results.data_transformer import DataTransformer
|
14
|
+
from fameio.source.results.input_dao import InputDao
|
20
15
|
from fameio.source.results.output_dao import OutputDAO
|
21
16
|
from fameio.source.results.reader import Reader
|
22
|
-
|
23
|
-
DEFAULT_CONFIG = {
|
24
|
-
Options.LOG_LEVEL: "info",
|
25
|
-
Options.LOG_FILE: None,
|
26
|
-
Options.AGENT_LIST: None,
|
27
|
-
Options.OUTPUT: None,
|
28
|
-
Options.SINGLE_AGENT_EXPORT: False,
|
29
|
-
Options.MEMORY_SAVING: False,
|
30
|
-
Options.RESOLVE_COMPLEX_FIELD: ResolveOptions.SPLIT.name,
|
31
|
-
Options.TIME: TimeOptions.UTC.name,
|
32
|
-
Options.TIME_MERGING: {},
|
33
|
-
}
|
17
|
+
from fameio.source.results.yaml_writer import data_to_yaml_file
|
34
18
|
|
35
19
|
ERR_MEMORY_ERROR = "Out of memory. Try using `-m` or `--memory-saving` option."
|
36
20
|
ERR_MEMORY_SEVERE = "Out of memory despite memory-saving mode. Reduce output interval in `FAME-Core` and rerun model"
|
37
21
|
|
38
22
|
|
39
|
-
def run(
|
40
|
-
"""Reads file in protobuf format
|
23
|
+
def run(config: dict = None) -> None:
|
24
|
+
"""Reads file in protobuf format for configures FILE and extracts its content to .csv file(s)"""
|
41
25
|
config = update_default_config(config, DEFAULT_CONFIG)
|
42
26
|
set_up_logger(level_name=config[Options.LOG_LEVEL], file_name=config[Options.LOG_FILE])
|
43
27
|
|
44
|
-
|
28
|
+
file_path = config[Options.FILE]
|
29
|
+
output_writer = CsvWriter(config[Options.OUTPUT], Path(file_path), config[Options.SINGLE_AGENT_EXPORT])
|
45
30
|
file_stream = open(Path(file_path), "rb")
|
46
31
|
|
47
32
|
if config[Options.MEMORY_SAVING]:
|
48
|
-
|
33
|
+
logger().info("Memory saving mode enabled: Disable on conversion of small files for performance improvements.")
|
49
34
|
|
50
|
-
|
35
|
+
logger().info("Reading and extracting data...")
|
51
36
|
reader = Reader.get_reader(file=file_stream, read_single=config[Options.MEMORY_SAVING])
|
52
37
|
agent_type_log = AgentTypeLog(requested_agents=config[Options.AGENT_LIST])
|
53
38
|
data_transformer = DataTransformer.build(config[Options.RESOLVE_COMPLEX_FIELD])
|
54
39
|
try:
|
40
|
+
input_dao = InputDao()
|
55
41
|
while data_storages := reader.read():
|
42
|
+
if config[Options.INPUT_RECOVERY]:
|
43
|
+
input_dao.store_inputs(data_storages)
|
56
44
|
output = OutputDAO(data_storages, agent_type_log)
|
57
45
|
for agent_name in output.get_sorted_agents_to_extract():
|
58
|
-
|
46
|
+
logger().debug(f"Extracting data for {agent_name}...")
|
59
47
|
data_frames = output.get_agent_data(agent_name, data_transformer)
|
60
48
|
apply_time_merging(data_frames, config[Options.TIME_MERGING])
|
61
49
|
apply_time_option(data_frames, config[Options.TIME])
|
62
|
-
|
63
|
-
|
64
|
-
|
50
|
+
logger().debug(f"Writing data for {agent_name}...")
|
51
|
+
output_writer.write_to_files(agent_name, data_frames)
|
52
|
+
|
53
|
+
if config[Options.INPUT_RECOVERY]:
|
54
|
+
logger().info("Recovering inputs...")
|
55
|
+
timeseries, scenario = input_dao.recover_inputs()
|
56
|
+
series_writer = CsvWriter(Path(config[Options.OUTPUT], "./recovered"), Path("./"), False)
|
57
|
+
series_writer.write_time_series_to_disk(timeseries)
|
58
|
+
data_to_yaml_file(scenario.to_dict(), Path(config[Options.OUTPUT], "./recovered/scenario.yaml"))
|
59
|
+
|
60
|
+
logger().info("Data conversion completed.")
|
65
61
|
except MemoryError:
|
66
62
|
log_and_raise_critical(ERR_MEMORY_SEVERE if Options.MEMORY_SAVING else ERR_MEMORY_ERROR)
|
67
63
|
|
68
64
|
file_stream.close()
|
69
65
|
if not agent_type_log.has_any_agent_type():
|
70
|
-
|
66
|
+
logger().error("Provided file did not contain any output data.")
|
71
67
|
|
72
68
|
|
73
69
|
if __name__ == "__main__":
|
74
|
-
|
75
|
-
run(
|
70
|
+
run_config = handle_args(sys.argv[1:])
|
71
|
+
run(run_config)
|
fameio/scripts/make_config.py
CHANGED
@@ -1,37 +1,34 @@
|
|
1
1
|
#!/usr/bin/env python
|
2
|
-
import logging as log
|
3
2
|
import sys
|
4
3
|
from pathlib import Path
|
5
|
-
from typing import Union
|
6
4
|
|
7
|
-
from fameio.source.cli import
|
5
|
+
from fameio.source.cli.make_config import handle_args, CLI_DEFAULTS as DEFAULT_CONFIG
|
6
|
+
from fameio.source.cli.options import Options
|
7
|
+
from fameio.source.cli.parser import update_default_config
|
8
8
|
from fameio.source.loader import load_yaml, check_for_yaml_file_type
|
9
|
-
from fameio.source.logs import set_up_logger
|
9
|
+
from fameio.source.logs import set_up_logger, logger
|
10
10
|
from fameio.source.scenario import Scenario
|
11
11
|
from fameio.source.validator import SchemaValidator
|
12
12
|
from fameio.source.writer import ProtoWriter
|
13
13
|
|
14
|
-
DEFAULT_CONFIG = {
|
15
|
-
Options.LOG_LEVEL: "info",
|
16
|
-
Options.LOG_FILE: None,
|
17
|
-
Options.OUTPUT: Path("config.pb"),
|
18
|
-
}
|
19
14
|
|
20
|
-
|
21
|
-
def run(file: Union[str, Path], config: dict = None) -> None:
|
15
|
+
def run(config: dict = None) -> None:
|
22
16
|
"""Executes the main workflow for the building of a FAME configuration file"""
|
23
17
|
config = update_default_config(config, DEFAULT_CONFIG)
|
24
18
|
set_up_logger(level_name=config[Options.LOG_LEVEL], file_name=config[Options.LOG_FILE])
|
25
19
|
|
20
|
+
file = config[Options.FILE]
|
26
21
|
check_for_yaml_file_type(Path(file))
|
27
22
|
scenario = Scenario.from_dict(load_yaml(Path(file)))
|
28
|
-
SchemaValidator.
|
29
|
-
|
23
|
+
SchemaValidator.check_agents_have_contracts(scenario)
|
24
|
+
|
25
|
+
timeseries_manager = SchemaValidator.validate_scenario_and_timeseries(scenario)
|
26
|
+
writer = ProtoWriter(config[Options.OUTPUT], timeseries_manager)
|
30
27
|
writer.write_validated_scenario(scenario)
|
31
28
|
|
32
|
-
|
29
|
+
logger().info("Configuration completed.")
|
33
30
|
|
34
31
|
|
35
32
|
if __name__ == "__main__":
|
36
|
-
|
37
|
-
run(
|
33
|
+
run_config = handle_args(sys.argv[1:])
|
34
|
+
run(run_config)
|
@@ -0,0 +1,75 @@
|
|
1
|
+
# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
|
2
|
+
#
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
4
|
+
import argparse
|
5
|
+
from typing import List, Dict, Any, Optional
|
6
|
+
|
7
|
+
from fameio.source.cli.options import Options, ResolveOptions, TimeOptions
|
8
|
+
from fameio.source.cli.parser import (
|
9
|
+
add_file_argument,
|
10
|
+
add_log_level_argument,
|
11
|
+
add_logfile_argument,
|
12
|
+
add_output_argument,
|
13
|
+
add_select_agents_argument,
|
14
|
+
add_single_export_argument,
|
15
|
+
add_memory_saving_argument,
|
16
|
+
add_resolve_complex_argument,
|
17
|
+
add_time_argument,
|
18
|
+
add_merge_time_parser,
|
19
|
+
add_inputs_recovery_argument,
|
20
|
+
map_namespace_to_options_dict,
|
21
|
+
)
|
22
|
+
|
23
|
+
CLI_DEFAULTS = {
|
24
|
+
Options.FILE: None,
|
25
|
+
Options.LOG_LEVEL: "info",
|
26
|
+
Options.LOG_FILE: None,
|
27
|
+
Options.AGENT_LIST: None,
|
28
|
+
Options.OUTPUT: None,
|
29
|
+
Options.SINGLE_AGENT_EXPORT: False,
|
30
|
+
Options.MEMORY_SAVING: False,
|
31
|
+
Options.RESOLVE_COMPLEX_FIELD: ResolveOptions.SPLIT,
|
32
|
+
Options.TIME: TimeOptions.UTC,
|
33
|
+
Options.TIME_MERGING: {},
|
34
|
+
Options.INPUT_RECOVERY: False,
|
35
|
+
}
|
36
|
+
|
37
|
+
_INFILE_PATH_HELP = "Provide path to protobuf file"
|
38
|
+
_OUTFILE_PATH_HELP = "Provide path to folder to store output .csv files"
|
39
|
+
|
40
|
+
|
41
|
+
def handle_args(args: List[str], defaults: Optional[Dict[Options, Any]] = None) -> Dict[Options, Any]:
|
42
|
+
"""Handles command line arguments and returns `input_file` and `run_config` for convert_results script"""
|
43
|
+
parser = _prepare_parser(defaults)
|
44
|
+
parsed = parser.parse_args(args)
|
45
|
+
return map_namespace_to_options_dict(parsed)
|
46
|
+
|
47
|
+
|
48
|
+
def _prepare_parser(defaults: Optional[Dict[Options, Any]]) -> argparse.ArgumentParser:
|
49
|
+
"""
|
50
|
+
Creates a parser with given defaults to handle `make_config` configuration arguments
|
51
|
+
|
52
|
+
Returns:
|
53
|
+
new parser using given defaults for its arguments; if a default is not specified, hard-coded defaults are used
|
54
|
+
"""
|
55
|
+
defaults = defaults if (defaults is not None) else {}
|
56
|
+
parser = argparse.ArgumentParser()
|
57
|
+
|
58
|
+
add_file_argument(parser, _get_default(defaults, Options.FILE), _INFILE_PATH_HELP)
|
59
|
+
add_log_level_argument(parser, _get_default(defaults, Options.LOG_LEVEL))
|
60
|
+
add_logfile_argument(parser, _get_default(defaults, Options.LOG_FILE))
|
61
|
+
add_output_argument(parser, _get_default(defaults, Options.OUTPUT), _OUTFILE_PATH_HELP)
|
62
|
+
add_select_agents_argument(parser, _get_default(defaults, Options.AGENT_LIST))
|
63
|
+
add_single_export_argument(parser, _get_default(defaults, Options.SINGLE_AGENT_EXPORT))
|
64
|
+
add_memory_saving_argument(parser, _get_default(defaults, Options.MEMORY_SAVING))
|
65
|
+
add_resolve_complex_argument(parser, _get_default(defaults, Options.RESOLVE_COMPLEX_FIELD))
|
66
|
+
add_time_argument(parser, _get_default(defaults, Options.TIME))
|
67
|
+
add_merge_time_parser(parser)
|
68
|
+
add_inputs_recovery_argument(parser, _get_default(defaults, Options.INPUT_RECOVERY))
|
69
|
+
|
70
|
+
return parser
|
71
|
+
|
72
|
+
|
73
|
+
def _get_default(defaults: dict, option: Options) -> Any:
|
74
|
+
"""Returns default for given `option` or its cli default"""
|
75
|
+
return defaults.get(option, CLI_DEFAULTS[option])
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
|
2
|
+
#
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
4
|
+
import argparse
|
5
|
+
from pathlib import Path
|
6
|
+
from typing import List, Dict, Any, Optional
|
7
|
+
|
8
|
+
from fameio.source.cli.options import Options
|
9
|
+
from fameio.source.cli.parser import (
|
10
|
+
add_file_argument,
|
11
|
+
add_log_level_argument,
|
12
|
+
add_logfile_argument,
|
13
|
+
add_output_argument,
|
14
|
+
map_namespace_to_options_dict,
|
15
|
+
)
|
16
|
+
|
17
|
+
CLI_DEFAULTS = {
|
18
|
+
Options.FILE: None,
|
19
|
+
Options.LOG_LEVEL: "info",
|
20
|
+
Options.LOG_FILE: None,
|
21
|
+
Options.OUTPUT: Path("config.pb"),
|
22
|
+
}
|
23
|
+
|
24
|
+
_INFILE_PATH_HELP = "provide path to configuration file"
|
25
|
+
_OUTFILE_PATH_HELP = "provide file-path for the file to generate"
|
26
|
+
|
27
|
+
|
28
|
+
def handle_args(args: List[str], defaults: Optional[Dict[Options, Any]] = None) -> Dict[Options, Any]:
|
29
|
+
"""
|
30
|
+
Converts given `arguments` and returns a configuration for the make_config script
|
31
|
+
|
32
|
+
Args:
|
33
|
+
args: list of (command line) arguments, e.g., ['-f', 'my_file']; arg values take precedence over defaults
|
34
|
+
defaults: optional default values used for unspecified parameters; missing defaults are replaced by CLI defaults
|
35
|
+
|
36
|
+
Returns:
|
37
|
+
final configuration compiled from (given) defaults and
|
38
|
+
"""
|
39
|
+
parser = _prepare_parser(defaults)
|
40
|
+
parsed = parser.parse_args(args)
|
41
|
+
return map_namespace_to_options_dict(parsed)
|
42
|
+
|
43
|
+
|
44
|
+
def _prepare_parser(defaults: Optional[Dict[Options, Any]]) -> argparse.ArgumentParser:
|
45
|
+
"""
|
46
|
+
Creates a parser with given defaults to handle `make_config` configuration arguments
|
47
|
+
|
48
|
+
Returns:
|
49
|
+
new parser using given defaults for its arguments; if a default is not specified, hard-coded defaults are used
|
50
|
+
"""
|
51
|
+
defaults = defaults if (defaults is not None) else {}
|
52
|
+
parser = argparse.ArgumentParser()
|
53
|
+
add_file_argument(parser, _get_default(defaults, Options.FILE), _INFILE_PATH_HELP)
|
54
|
+
add_log_level_argument(parser, _get_default(defaults, Options.LOG_LEVEL))
|
55
|
+
add_logfile_argument(parser, _get_default(defaults, Options.LOG_FILE))
|
56
|
+
add_output_argument(parser, _get_default(defaults, Options.OUTPUT), _OUTFILE_PATH_HELP)
|
57
|
+
return parser
|
58
|
+
|
59
|
+
|
60
|
+
def _get_default(defaults: dict, option: Options) -> Any:
|
61
|
+
"""Returns default for given `option` or, if missing, its cli default"""
|
62
|
+
return defaults.get(option, CLI_DEFAULTS[option])
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
|
2
|
+
#
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
4
|
+
import argparse
|
5
|
+
from enum import Enum, auto
|
6
|
+
|
7
|
+
|
8
|
+
class ParsableEnum(Enum):
|
9
|
+
"""Extend this to create an enum that can be parsed with argparse"""
|
10
|
+
|
11
|
+
@classmethod
|
12
|
+
def instantiate(cls, name: str) -> Enum:
|
13
|
+
try:
|
14
|
+
return cls[name]
|
15
|
+
except KeyError:
|
16
|
+
raise argparse.ArgumentTypeError(f"'{name}' is not a valid option")
|
17
|
+
|
18
|
+
def __str__(self):
|
19
|
+
return self.name
|
20
|
+
|
21
|
+
|
22
|
+
class Options(Enum):
|
23
|
+
"""Specifies command line configuration options"""
|
24
|
+
|
25
|
+
FILE = auto()
|
26
|
+
LOG_LEVEL = auto()
|
27
|
+
LOG_FILE = auto()
|
28
|
+
OUTPUT = auto()
|
29
|
+
AGENT_LIST = auto()
|
30
|
+
SINGLE_AGENT_EXPORT = auto()
|
31
|
+
MEMORY_SAVING = auto()
|
32
|
+
RESOLVE_COMPLEX_FIELD = auto()
|
33
|
+
TIME = auto()
|
34
|
+
TIME_MERGING = auto()
|
35
|
+
INPUT_RECOVERY = auto()
|
36
|
+
|
37
|
+
|
38
|
+
class TimeOptions(ParsableEnum, Enum):
|
39
|
+
"""Specifies options for conversion of time in output"""
|
40
|
+
|
41
|
+
INT = auto()
|
42
|
+
UTC = auto()
|
43
|
+
FAME = auto()
|
44
|
+
|
45
|
+
|
46
|
+
class ResolveOptions(ParsableEnum, Enum):
|
47
|
+
"""Specifies options for resolving complex fields in output files"""
|
48
|
+
|
49
|
+
IGNORE = auto()
|
50
|
+
SPLIT = auto()
|
51
|
+
MERGE = auto()
|
52
|
+
|
53
|
+
|
54
|
+
class MergingOptions(Enum):
|
55
|
+
"""Specifies options for merging TimeSteps"""
|
56
|
+
|
57
|
+
FOCAL_POINT = auto()
|
58
|
+
STEPS_BEFORE = auto()
|
59
|
+
STEPS_AFTER = auto()
|