fameio 2.2.0__py3-none-any.whl → 2.3.1__py3-none-any.whl

CHANGELOG.md

@@ -3,9 +3,22 @@
 SPDX-License-Identifier: CC0-1.0 -->
 
 # Changelog
-## [2.2.0](https://gitlab.com/fame-framework/fame-io/-/tags/v2.2.0) - TBA
+## [2.3.1](https://gitlab.com/fame-framework/fame-io/-/tags/v2.3.1) - 2024-08-26
+### Fixed
+- Fix ignored default values of `convert_results` for `merge-times` arguments #211 (@dlr-cjs, dlr_fn)
+
+## [2.3.0](https://gitlab.com/fame-framework/fame-io/-/tags/v2.3.0) - 2024-08-12
+### Added
+- New attribute type `string_set` #175 (@dlr_fn @dlr-cjs)
+- Add warning if a timeseries file has additional, non-empty columns #155 (@LeonardWilleke)
+- Ensure `CHANGELOG.md` is updated in automated testing pipeline #207 (@dlr_fn)
+
+### Fixed
+- ConvertFameResults: Fix bug on `merge-times` when `--memory-saving` is active #201 (@dlr_fn @dlr-cjs)
+
+## [2.2.0](https://gitlab.com/fame-framework/fame-io/-/tags/v2.2.0) - 2024-05-28
 ### Changed
-- New command line option `-enc --encoding` to change encoding when reading yaml-files
+- New command line option `-enc --encoding` to change encoding when reading yaml-files #170 (@dlr-cjs)
 - Improve error message when timeseries is not found and is number string #178 (@dlr-cjs)
 
 ### Added
@@ -14,7 +27,7 @@ SPDX-License-Identifier: CC0-1.0 -->
 
 ## [2.1.1](https://gitlab.com/fame-framework/fame-io/-/tags/v2.1.1) - 2024-05-28
 ### Fixed
-- ConvertFameResults: Fix crash on complex column conversion if Agent has no simple columns #204 (@dlr_fn @dlr-cjs) 
+- ConvertFameResults: Fix crash on complex column conversion if Agent has no simple columns #204 (@dlr_fn @dlr-cjs)
 
 ## [2.1.0](https://gitlab.com/fame-framework/fame-io/-/tags/v2.1.0) - 2024-05-11
 ### Changed
@@ -30,8 +43,8 @@ SPDX-License-Identifier: CC0-1.0 -->
 - Fix potential duplicates in logging #191 (@dlr_fn @dlr-cjs)
 
 ## [2.0.1](https://gitlab.com/fame-framework/fame-io/-/tags/v2.0.1) - 2024-04-05
-### Fix
-- Fix potential missing columns when memory-saving-mode `-m` is enabled #194 (@dlr_fn @dlr-cjs) 
+### Fixed
+- Fix potential missing columns when memory-saving-mode `-m` is enabled #194 (@dlr_fn @dlr-cjs)
 
 ### Remove
 - Remove convert results option `-cc MERGE` #194 (@dlr_fn @dlr-cjs)

fameio/scripts/convert_results.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
-
 import sys
 from pathlib import Path
 
+import pandas as pd
+
 from fameio.source.cli.convert_results import handle_args, CLI_DEFAULTS as DEFAULT_CONFIG
 from fameio.source.cli.options import Options
 from fameio.source.cli.parser import update_default_config
@@ -10,7 +11,7 @@ from fameio.source.logs import log_and_raise_critical, fameio_logger, log
 from fameio.source.results.agent_type import AgentTypeLog
 from fameio.source.results.conversion import apply_time_option, apply_time_merging
 from fameio.source.results.csv_writer import CsvWriter
-from fameio.source.results.data_transformer import DataTransformer
+from fameio.source.results.data_transformer import DataTransformer, INDEX
 from fameio.source.results.input_dao import InputDao
 from fameio.source.results.output_dao import OutputDAO
 from fameio.source.results.reader import Reader
@@ -45,8 +46,9 @@ def run(config: dict = None) -> None:
             for agent_name in output.get_sorted_agents_to_extract():
                 log().debug(f"Extracting data for {agent_name}...")
                 data_frames = output.get_agent_data(agent_name, data_transformer)
-                apply_time_merging(data_frames, config[Options.TIME_MERGING])
-                apply_time_option(data_frames, config[Options.TIME])
+                if not config[Options.MEMORY_SAVING]:
+                    apply_time_merging(data_frames, config[Options.TIME_MERGING])
+                    apply_time_option(data_frames, config[Options.TIME])
                 log().debug(f"Writing data for {agent_name}...")
                 output_writer.write_to_files(agent_name, data_frames)
 
@@ -57,6 +59,14 @@ def run(config: dict = None) -> None:
             series_writer.write_time_series_to_disk(timeseries)
             data_to_yaml_file(scenario.to_dict(), Path(config[Options.OUTPUT], "./recovered/scenario.yaml"))
 
+        if config[Options.MEMORY_SAVING]:
+            written_files = output_writer.pop_all_file_paths()
+            for agent_name, file_path in written_files.items():
+                parsed_data = {None: pd.read_csv(file_path, sep=";", index_col=INDEX)}
+                apply_time_merging(parsed_data, config[Options.TIME_MERGING])
+                apply_time_option(parsed_data, config[Options.TIME])
+                output_writer.write_to_files(agent_name, parsed_data)
+
         log().info("Data conversion completed.")
     except MemoryError:
         log_and_raise_critical(ERR_MEMORY_SEVERE if Options.MEMORY_SAVING else ERR_MEMORY_ERROR)

fameio/source/cli/convert_results.py

@@ -73,7 +73,7 @@ def _prepare_parser(defaults: Optional[Dict[Options, Any]]) -> argparse.Argument
     add_memory_saving_argument(parser, _get_default(defaults, Options.MEMORY_SAVING))
     add_resolve_complex_argument(parser, _get_default(defaults, Options.RESOLVE_COMPLEX_FIELD))
     add_time_argument(parser, _get_default(defaults, Options.TIME))
-    add_merge_time_parser(parser)
+    add_merge_time_parser(parser, _get_default(defaults, Options.TIME_MERGING))
     add_inputs_recovery_argument(parser, _get_default(defaults, Options.INPUT_RECOVERY))
 
     return parser

fameio/source/cli/parser.py

@@ -138,25 +138,40 @@ def add_time_argument(parser: ArgumentParser, default_value: Union[TimeOptions,
     )
 
 
-def add_merge_time_parser(parser: ArgumentParser) -> None:
-    """Adds subparser for merging of TimeSteps to given `parser`"""
-    subparser = parser.add_subparsers(dest="time_merging", required=False, help="Optional merging of TimeSteps")
+def add_merge_time_parser(parser: ArgumentParser, defaults: Optional[Dict[MergingOptions, int]]) -> None:
+    """
+    Adds subparser for merging of TimeSteps to given `parser`
+    If at least one valid time merging option is specified in given defaults, calling the subparser becomes mandatory
+    """
+    defaults = defaults if (defaults is not None) and (isinstance(defaults, dict)) else {}
+    if any([option in defaults.keys() for option in MergingOptions]):
+        subparser = parser.add_subparsers(dest="time_merging", required=True, help="Optional merging of TimeSteps")
+    else:
+        subparser = parser.add_subparsers(dest="time_merging", required=False, help="Optional merging of TimeSteps")
     group_parser = subparser.add_parser("merge-times")
-    add_focal_point_argument(group_parser)
-    add_steps_before_argument(group_parser)
-    add_steps_after_argument(group_parser)
+    add_focal_point_argument(group_parser, defaults.get(MergingOptions.FOCAL_POINT, None))
+    add_steps_before_argument(group_parser, defaults.get(MergingOptions.STEPS_BEFORE, None))
+    add_steps_after_argument(group_parser, defaults.get(MergingOptions.STEPS_AFTER, None))
 
 
-def add_focal_point_argument(parser: ArgumentParser) -> None:
+def add_focal_point_argument(parser: ArgumentParser, default_value: Optional[int]) -> None:
     """Adds `focal-point` argument to given `parser`"""
     help_text = "TimeStep on which `steps_before` earlier and `steps_after` later TimeSteps are merged on"
-    parser.add_argument("-fp", "--focal-point", required=True, type=int, help=help_text)
+    if default_value is not None:
+        parser.add_argument("-fp", "--focal-point", required=False, type=int, help=help_text, default=default_value)
+    else:
+        parser.add_argument("-fp", "--focal-point", required=True, type=int, help=help_text)
 
 
-def add_steps_before_argument(parser: ArgumentParser) -> None:
+def add_steps_before_argument(parser: ArgumentParser, default_value: Optional[int]) -> None:
     """Adds `steps-before` argument to given `parser`"""
     help_text = "Range of TimeSteps before the `focal-point` they get merged to"
-    parser.add_argument("-sb", "--steps-before", required=True, type=_non_negative_int, help=help_text)
+    if default_value is not None:
+        parser.add_argument(
+            "-sb", "--steps-before", required=False, type=_non_negative_int, help=help_text, default=default_value
+        )
+    else:
+        parser.add_argument("-sb", "--steps-before", required=True, type=_non_negative_int, help=help_text)
 
 
 def _non_negative_int(value: Any) -> int:
@@ -181,10 +196,15 @@ def _non_negative_int(value: Any) -> int:
     return value
 
 
-def add_steps_after_argument(parser: ArgumentParser) -> None:
+def add_steps_after_argument(parser: ArgumentParser, default_value: Optional[int]) -> None:
     """Adds `steps-after` argument to given `parser`"""
     help_text = "Range of TimeSteps after the `focal-point` they get merged to"
-    parser.add_argument("-sa", "--steps-after", required=True, type=_non_negative_int, help=help_text)
+    if default_value is not None:
+        parser.add_argument(
+            "-sa", "--steps-after", required=False, type=_non_negative_int, help=help_text, default=default_value
+        )
+    else:
+        parser.add_argument("-sa", "--steps-after", required=True, type=_non_negative_int, help=help_text)
 
 
 def add_inputs_recovery_argument(parser: ArgumentParser, default: bool) -> None:

fameio/source/metadata.py

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+from abc import ABC
+from typing import Dict, Any
+
+from fameio.source.tools import keys_to_lower
+
+
+class Metadata(ABC):
+    """Hosts Metadata"""
+
+    _KEY_METADATA = "MetaData".lower()
+
+    def __init__(self):
+        self._metadata = {}
+
+    @property
+    def metadata(self) -> dict:
+        """Returns list of metadata or an empty list if no metadata are defined"""
+        return self._metadata
+
+    def _extract_metadata(self, definitions: Dict[str, Any]) -> None:
+        """If metadata is found in definitions, it is extracted and set"""
+        definitions = keys_to_lower(definitions)
+        if self._KEY_METADATA in definitions:
+            self._metadata = definitions[self._KEY_METADATA]
+
+    def _enrich_with_metadata(self, data: Dict) -> Dict:
+        """Returns data enriched with metadata field"""
+        data[self._KEY_METADATA] = self._metadata
+        return data

fameio/source/results/conversion.py

@@ -7,10 +7,9 @@ from typing import Dict, Optional
 
 import pandas as pd
 
-from fameio.source import FameTime
 from fameio.source.cli.options import TimeOptions, MergingOptions
 from fameio.source.logs import log_error_and_raise, log
-from fameio.source.time import ConversionException
+from fameio.source.time import ConversionException, FameTime
 
 _ERR_UNIMPLEMENTED = "Time conversion mode '{}' not implemented."
 

fameio/source/results/csv_writer.py

@@ -1,7 +1,6 @@
 # SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
-
 from pathlib import Path
 from typing import Dict, Union
 
@@ -19,6 +18,8 @@ class CsvWriter:
     _INFO_USING_PATH = "Using specified output path: {}"
     _INFO_USING_DERIVED_PATH = "No output path specified - writing to new local folder: {}"
 
+    CSV_FILE_SUFFIX = ".csv"
+
     def __init__(self, config_output: Path, input_file_path: Path, single_export: bool) -> None:
         self._single_export = single_export
         self._output_folder = self._get_output_folder_name(config_output, input_file_path)
@@ -90,13 +91,19 @@ class CsvWriter:
         """Returns True if a file for given `identifier` was already written"""
         return identifier in self._files
 
+    def pop_all_file_paths(self) -> Dict[str, Path]:
+        """Clears all stored file paths and returns their previous identifiers and their paths"""
+        current_files = self._files
+        self._files = {}
+        return current_files
+
     def _get_outfile_name(self, identifier: str) -> str:
         """Returns file name for given `agent_name` and (optional) `agent_id`"""
         return self._files[identifier]
 
     def _create_outfile_name(self, identifier: str) -> Path:
         """Returns fully qualified file name based on given `agent_name` and (optional) `agent_id`"""
-        return Path(self._output_folder, f"{identifier}.csv")
+        return Path(self._output_folder, f"{identifier}{self.CSV_FILE_SUFFIX}")
 
     def _save_outfile_name(self, outfile_name: Path, identifier: str) -> None:
         """Stores given name for given `agent_name` and (optional) `agent_id`"""

fameio/source/scenario/attribute.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
@@ -16,6 +16,8 @@ class Attribute:
     _DICT_EMPTY = "Attribute '{}' was assigned an empty dictionary - please remove or fill empty assignments."
     _MIXED_DATA = "Attribute '{}' was assigned a list with mixed complex and simple entries - please fix."
 
+    _NAME_STRING_SEPARATOR = "."
+
     def __init__(self, name: str, definitions) -> None:
         """Parses an Attribute's definition"""
         self._full_name = name
@@ -46,7 +48,7 @@ class Attribute:
 
         inner_elements = {}
         for nested_name, value in definitions.items():
-            full_name = name + "." + nested_name
+            full_name = name + Attribute._NAME_STRING_SEPARATOR + nested_name
             inner_elements[nested_name] = Attribute(full_name, value)
         return inner_elements
 

fameio/source/scenario/fameiofactory.py

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
-
+from fameio.source.scenario.stringset import StringSet
 from fameio.source.schema.schema import Schema
 from fameio.source.scenario.generalproperties import GeneralProperties
 from fameio.source.scenario.agent import Agent
@@ -32,3 +32,8 @@ class FameIOFactory:
     def new_contract_from_dict(definitions: dict) -> Contract:
         """Constructs a new Contract from provided `definitions`"""
         return Contract.from_dict(definitions)
+
+    @staticmethod
+    def new_string_set_from_dict(definition: StringSet.StringSetType) -> StringSet:
+        """Constructs a new StringSet from provided `definitions`"""
+        return StringSet.from_dict(definition)

fameio/source/scenario/scenario.py

@@ -3,13 +3,14 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-from typing import List
+from typing import List, Dict
 
 from fameio.source.scenario.agent import Agent
 from fameio.source.scenario.contract import Contract
 from fameio.source.scenario.exception import get_or_default, get_or_raise
 from fameio.source.scenario.fameiofactory import FameIOFactory
 from fameio.source.scenario.generalproperties import GeneralProperties
+from fameio.source.scenario.stringset import StringSet
 from fameio.source.schema.schema import Schema
 from fameio.source.tools import keys_to_lower
 
@@ -21,12 +22,14 @@ class Scenario:
     _KEY_GENERAL = "GeneralProperties".lower()
     _KEY_AGENTS = "Agents".lower()
     _KEY_CONTRACTS = "Contracts".lower()
+    _KEY_STRING_SETS = "StringSets".lower()
 
     _MISSING_KEY = "Scenario definition misses required key '{}'."
 
     def __init__(self, schema: Schema, general_props: GeneralProperties) -> None:
         self._schema = schema
         self._general_props = general_props
+        self._string_sets = {}
         self._agents = []
         self._contracts = []
 
@@ -41,6 +44,9 @@ class Scenario:
         )
         scenario = cls(schema, general_props)
 
+        for name, string_set_definition in get_or_default(definitions, Scenario._KEY_STRING_SETS, {}).items():
+            scenario.add_string_set(name, factory.new_string_set_from_dict(string_set_definition))
+
         for agent_definition in get_or_default(definitions, Scenario._KEY_AGENTS, []):
             scenario.add_agent(factory.new_agent_from_dict(agent_definition))
 
@@ -56,12 +62,14 @@ class Scenario:
             Scenario._KEY_GENERAL: self.general_properties.to_dict(),
             Scenario._KEY_SCHEMA: self.schema.to_dict(),
         }
-
+        if self.string_sets:
+            result[Scenario._KEY_STRING_SETS] = {
+                name: string_set.to_dict() for name, string_set in self.string_sets.items()
+            }
         if self.agents:
             result[Scenario._KEY_AGENTS] = []
             for agent in self.agents:
                 result[Scenario._KEY_AGENTS].append(agent.to_dict())
-
         if self.contracts:
             result[Scenario._KEY_CONTRACTS] = []
             for contract in self.contracts:
@@ -95,3 +103,12 @@ class Scenario:
     def general_properties(self) -> GeneralProperties:
         """Returns General properties of this scenario"""
         return self._general_props
+
+    @property
+    def string_sets(self) -> Dict[str, StringSet]:
+        """Returns StringSets of this scenario"""
+        return self._string_sets
+
+    def add_string_set(self, name: str, string_set: StringSet) -> None:
+        """Adds `string_set` with `name`"""
+        self._string_sets[name] = string_set

fameio/source/scenario/stringset.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+from typing import Dict, List, Union
+
+from fameio.source.metadata import Metadata
+from fameio.source.scenario.exception import log_and_raise
+from fameio.source.tools import keys_to_lower
+
+
+class StringSet(Metadata):
+    """Hosts a StringSet in the given format"""
+
+    ValueType = Union[List[str], Dict[str, Dict]]
+    StringSetType = Dict[str, Union[Dict, ValueType]]
+
+    _ERR_NO_STRING_SET_VALUES = "Missing mandatory key '{}' in StringSet definition {}."
+
+    _KEY_VALUES = "Values".lower()
+
+    def __init__(self):
+        super().__init__()
+        self._values = {}
+
+    @classmethod
+    def from_dict(cls, definition: StringSetType) -> "StringSet":
+        """Returns StringSet initialised from `definition`"""
+        string_set = cls()
+        string_set._extract_metadata(definition)
+        definition = keys_to_lower(definition)
+        if cls._KEY_VALUES in definition:
+            string_set._values = string_set._read_values(definition)
+        else:
+            log_and_raise(cls._ERR_NO_STRING_SET_VALUES.format(cls._KEY_VALUES, definition))
+        return string_set
+
+    def _read_values(self, definition: ValueType) -> Dict[str, Dict]:
+        """Ensures values are returned as dictionary representation by converting `definitions` of type 'List[str]'"""
+        values = definition[self._KEY_VALUES]
+        if isinstance(values, list):
+            return {name: {} for name in values}
+        return values
+
+    def to_dict(self) -> Dict:
+        """Serializes the StringSet to a dict"""
+        result = {self._KEY_VALUES: self._values}
+        return self._enrich_with_metadata(result)
+
+    def is_in_set(self, key: str) -> bool:
+        """Returns True if `key` is a valid name in this StringSet"""
+        return key in self._values

fameio/source/schema/attribute.py

@@ -21,6 +21,7 @@ class AttributeType(Enum):
     LONG = auto()
     TIME_STAMP = auto()
     STRING = auto()
+    STRING_SET = auto()
     ENUM = auto()
     TIME_SERIES = auto()
     BLOCK = auto()
@@ -33,7 +34,7 @@ class AttributeType(Enum):
             return float(value)
         elif self is AttributeType.TIME_STAMP:
             return FameTime.convert_string_if_is_datetime(value)
-        elif self is AttributeType.ENUM or self is AttributeType.STRING:
+        elif self is AttributeType.ENUM or self is AttributeType.STRING or self is AttributeType.STRING_SET:
             return str(value)
         elif self is AttributeType.TIME_SERIES:
             return float(value)

fameio/source/series.py

@@ -45,6 +45,7 @@ class TimeSeriesManager:
     _ERR_NAN_VALUE = "Values in TimeSeries must not be missing or NaN."
     _ERR_UNREGISTERED_SERIES = "No timeseries registered with identifier '{}' - was the Scenario validated?"
     _WARN_NO_DATA = "No timeseries stored in timeseries manager. Double check if you expected timeseries."
+    _WARN_DATA_IGNORED = "Timeseries contains additional columns with data which will be ignored."
 
     def __init__(self, path_resolver: PathResolver = PathResolver()) -> None:
         self._path_resolver = path_resolver
@@ -96,6 +97,10 @@ class TimeSeriesManager:
 
     def _check_and_convert_series(self, data: pd.DataFrame) -> pd.DataFrame:
         """Ensures validity of time series and convert to required format for writing to disk"""
+        additional_columns = data.loc[:, 2:]
+        is_empty = additional_columns.dropna(how="all").empty
+        if not is_empty:
+            log().warning(self._WARN_DATA_IGNORED)
         data = data.apply(
             lambda r: [FameTime.convert_string_if_is_datetime(r[0]), self._assert_valid(r[1])],
             axis=1,

fameio/source/tools.py

@@ -6,7 +6,7 @@ from typing import Any, Dict, Union
 
 
 def keys_to_lower(dictionary: Dict[str, Any]) -> Dict[str, Any]:
-    """Returns new dictionary content of given `dictionary` but its `keys` in lower case"""
+    """Returns new dictionary content of given `dictionary` but its top-level `keys` in lower case"""
     return {keys.lower(): value for keys, value in dictionary.items()}
 
 

fameio/source/validator.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2023 German Aerospace Center <fame@dlr.de>
+# SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
 #
 # SPDX-License-Identifier: Apache-2.0
 
@@ -9,6 +9,7 @@ from typing import Any, Dict, List
 from fameio.source import PathResolver
 from fameio.source.logs import log_error_and_raise, log
 from fameio.source.scenario import Agent, Attribute, Contract, Scenario
+from fameio.source.scenario.stringset import StringSet
 from fameio.source.schema.agenttype import AgentType
 from fameio.source.schema.attribute import AttributeSpecs, AttributeType
 from fameio.source.schema.schema import Schema
@@ -40,6 +41,8 @@ class SchemaValidator:
     _IS_NO_LIST = "Attribute '{}' is list but assigned value '{}' is not a list."
     _TIME_SERIES_INVALID = "Timeseries at '{}' is invalid."
     _MISSING_CONTRACTS_FOR_AGENTS = "No contracts defined for Agent '{}' of type '{}'"
+    _MISSING_STRING_SET = "StringSet '{}' not defined in scenario."
+    _MISSING_STRING_SET_ENTRY = "Value '{}' of Attribute '{}' not defined in StringSet '{}'"
 
     @staticmethod
     def validate_scenario_and_timeseries(
@@ -65,6 +68,7 @@ class SchemaValidator:
         SchemaValidator.ensure_unique_agent_ids(agents)
         for agent in agents:
             SchemaValidator.ensure_agent_and_timeseries_are_valid(agent, schema, timeseries_manager)
+            SchemaValidator.ensure_string_set_consistency(agent, schema, scenario.string_sets)
 
         agent_types_by_id = {agent.id: agent.type_name for agent in agents}
         for contract in scenario.contracts:
@@ -199,7 +203,7 @@ class SchemaValidator:
             return isinstance(value, int)
         elif attribute_type is AttributeType.DOUBLE:
             return isinstance(value, (int, float)) and not math.isnan(value)
-        elif attribute_type in (AttributeType.ENUM, AttributeType.STRING):
+        elif attribute_type in (AttributeType.ENUM, AttributeType.STRING, AttributeType.STRING_SET):
             return isinstance(value, str)
         elif attribute_type is AttributeType.TIME_STAMP:
             return FameTime.is_fame_time_compatible(value)
@@ -254,6 +258,50 @@ class SchemaValidator:
                 for entry in attribute.nested_list:
                     SchemaValidator._ensure_valid_timeseries(entry, specification.nested_attributes, manager)
 
+    @staticmethod
+    def ensure_string_set_consistency(agent: Agent, schema: Schema, string_sets: Dict[str, StringSet]) -> None:
+        """
+        Raises exception if
+            a) an agent's attribute is of type StringSet but the corresponding StringSet is not defined in the scenario
+            b) the value assigned to an attribute of type StringSet is not contained in the corresponding StringSet
+        """
+        scenario_attributes = agent.attributes
+        schema_attributes = SchemaValidator._get_agent(schema, agent.type_name).attributes
+        SchemaValidator._ensure_string_set_consistency(scenario_attributes, schema_attributes, string_sets)
+
+    @staticmethod
+    def _ensure_string_set_consistency(
+        attributes: Dict[str, Attribute], specifications: Dict[str, AttributeSpecs], string_sets: Dict[str, StringSet]
+    ) -> None:
+        """
+        Recursively iterates through all attributes of an agent, applying tests if attribute type is `StringSet`
+        Raises:
+            ValidationException: if
+                a) StringSet mentioned in schema is not defined in the scenario
+                b) the value assigned to an attribute of type StringSet is not contained in the corresponding StringSet
+        """
+        for name, attribute in attributes.items():
+            specification = specifications[name]
+            if attribute.has_value:
+                attribute_type = specification.attr_type
+                if attribute_type is AttributeType.STRING_SET:
+                    if name in string_sets:
+                        if not string_sets[name].is_in_set(attribute.value):
+                            msg = SchemaValidator._MISSING_STRING_SET_ENTRY.format(
+                                attribute.value, str(attribute), name
+                            )
+                            log_error_and_raise(ValidationException(msg))
+                    else:
+                        msg = SchemaValidator._MISSING_STRING_SET.format(specification.full_name)
+                        log_error_and_raise(ValidationException(msg))
+            if attribute.has_nested:
+                SchemaValidator._ensure_string_set_consistency(
+                    attribute.nested, specification.nested_attributes, string_sets
+                )
+            if attribute.has_nested_list:
+                for entry in attribute.nested_list:
+                    SchemaValidator._ensure_string_set_consistency(entry, specification.nested_attributes, string_sets)
+
     @staticmethod
     def ensure_is_valid_contract(contract: Contract, schema: Schema, agent_types_by_id: Dict[int, str]) -> None:
         """Raises exception if given `contract` does not meet the `schema`'s requirements, using `agent_types_by_id`"""

fameio/source/writer.py

@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 import sys
-from importlib.metadata import version
+import importlib.metadata as metadata
 from pathlib import Path
 from typing import Any, Dict, List, Union
 
@@ -150,7 +150,7 @@ class ProtoWriter:
             pb_field.longValue.extend(ensure_is_list(value))
         elif attribute_type is AttributeType.TIME_STAMP:
             pb_field.longValue.extend(ensure_is_list(FameTime.convert_string_if_is_datetime(value)))
-        elif attribute_type in (AttributeType.ENUM, AttributeType.STRING):
+        elif attribute_type in (AttributeType.ENUM, AttributeType.STRING, AttributeType.STRING_SET):
             pb_field.stringValue.extend(ensure_is_list(value))
         elif attribute_type is AttributeType.TIME_SERIES:
             pb_field.seriesId = self._time_series_manager.get_series_id_by_identifier(value)
@@ -246,6 +246,6 @@ class ProtoWriter:
     @staticmethod
     def _set_execution_versions(pb_versions: ExecutionData.Versions) -> None:
         """Adds version strings for fameio, fameprotobuf, and python to the given Versions message"""
-        pb_versions.fameProtobuf = version("fameprotobuf")
-        pb_versions.fameIo = version("fameio")
+        pb_versions.fameProtobuf = metadata.version("fameprotobuf")
+        pb_versions.fameIo = metadata.version("fameio")
         pb_versions.python = sys.version

{fameio-2.2.0.dist-info → fameio-2.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: fameio
-Version: 2.2.0
+Version: 2.3.1
 Summary: Python scripts for operation of FAME models
 Home-page: https://gitlab.com/fame-framework/wiki/-/wikis/home
 License: Apache-2.0
@@ -127,18 +127,19 @@ make_config(run_config)
 
 ### Scenario YAML
 The "scenario.yaml" file contains all configuration options for a FAME-based simulation.
-It consists of the sections `Schema`, `GeneralProperties`, `Agents` and `Contracts`, all of them described below.
+It consists of the sections `Schema`, `GeneralProperties`, `Agents` and `Contracts`, and the optional section `StringSets`.
+All of them are described below.
 
 #### Schema
-The Schema describes a model's components such as its types of agents, their inputs, what data they exchange, etc. 
-It is also used to validate the model inputs provided in the `scenario.yaml`. 
+The Schema describes a model's components such as its types of agents, their inputs, what data they exchange, etc.
+It is also used to validate the model inputs provided in the `scenario.yaml`.
 Since the Schema is valid until the model itself is changed, it is recommended to defined it in a separate file and include the file here.
 
 Currently, the schema specifies:
 * which type of Agents can be created
 * what type of input attributes an Agent uses
 * what type of Products an Agent can send in Contracts, and
-* the names of the Java packages for the classes corresponding to Agents, DataItems and Portables. 
+* the names of the Java packages for the classes corresponding to Agents, DataItems and Portables.
 
 The Schema consists of the sections `JavaPackages` and `AgentTypes`.
 
@@ -154,7 +155,7 @@ Also, package names occur on multiple lists for Agent, DataItem or Portable.
 For example, for a project with all its
 * Agent-derived java classes located in packages below the package named "agents",
 * DataItem implementation classes in a subpackage named "msg",
-* Portable implementation classes in a subpackages named "portableItems" and "otherPortables", 
+* Portable implementation classes in a subpackages named "portableItems" and "otherPortables",
 
 the corresponding section in the schema would look like this:
 
@@ -235,7 +236,8 @@ MyComplexAttribute:
 | `long`        | a 64-bit integer value                                                                                                  |
 | `time_stamp`  | either a FAME time stamp string or 64-bit integer value                                                                 |
 | `string`      | any string                                                                                                              |
-| `enum`        | any string, however, usually tied to a set of allowed `Values`                                                          |
+| `string_set`  | a string from a set of allowed `Values` defined in `StringSet` section in `scenario`                                    |
+| `enum`        | a string from a set of allowed `Values` defined in `schema`                                                             |
 | `time_series` | either a path to a .csv-file or a single 64-bit floating-point value; does not support `List: true`                     |
 | `block`       | this attribute has no value of its own but hosts a group of nested Attributes; implies `NestedAttributes` to be defined |
 
@@ -352,7 +354,7 @@ Contract Parameters:
 * `ProductName` name of the product to be sent
 * `FirstDeliveryTime` first time of delivery in the format "seconds after the January 1st 2000, 00:00:00"
 * `DeliveryIntervalInSteps` delay time in between deliveries in seconds
-* `Attributes` can be set to include additional information as `int`, `float`, `enum` or `dict` data types
+* `Attributes` can be set to include additional information as `int`, `float`, `enum`, or `dict` data types
 
 ##### Definition of Multiple Similar Contracts
 Often, scenarios contain multiple agents of similar type that also have similar chains of contracts.
@@ -414,11 +416,47 @@ Contracts:
     DeliveryIntervalInSteps: 3600
 ```
 
+#### StringSets
+This optional section defines values of type `string_set`.
+In contrast to `enum` values, which are **statically** defined in the `Schema`, `string_set` values can be **dynamically** defined in this section.
+If an agent attribute is of type `string_set` and the attribute is set in the `scenario`, then
+1. the section `StringSets` in the `scenario` must contain an entry named exactly like the attribute, and
+2. the attribute value must be contained in the string set's `Values` declaration.
+
+For instance:
+
+In `schema`:
+
+``` yaml
+AgentTypes:
+  FuelsMarket:
+    Attributes:
+      FuelType:
+        AttributeType: string_set
+```
+
+In `scenario`:
+
+``` yaml
+StringSets:
+  FuelType:
+    Values: ['OIL', 'HARD_COAL', 'LIGNITE']
+
+Agents:
+ - Type: FuelsMarket
+   Id: 1
+   Attributes:
+     FuelType: OIL
+```
+
+Important: If different types of Agents shall refer to the same StringSet, their attributes in schema must have the **exact** same name.
+
 ### CSV files
 TIME_SERIES inputs are not directly fed into the Scenario YAML file.
 Instead, TIME_SERIES reference a CSV file that can be stored some place else.
 These CSV files follow a specific structure:
-* They must contain exactly two columns.
+* They should contain exactly two columns - any other columns are ignored.
+  A warning is raised if more than two non-empty columns are detected.
 * The first column must be a time stamp in form `YYYY-MM-DD_hh:mm:ss`
 * The second column must be a numerical value (either integer or floating-point)
 * The separator of the two columns is a semicolon
@@ -612,7 +650,7 @@ You may also specify any of the following arguments:
 | `-t` or `--time` <option>                   | Option to define conversion of time steps to given format (default=`UTC`) by `-t/--time {UTC, INT, FAME}`                                                                                 |
 | `--input-recovery` or `--no-input-recovery` | If True, all input data are recovered as well as the outputs (default=False).                                                                                                             |
 
-Additionally, you may merge TimeSteps of a certain range of steps in the output files to
+Additionally, you may merge `TimeSteps` of a certain range of steps in the output files to
 i) associate multiple time steps with a common logical time in your simulation
 ii) reduce number of lines in output files
 

{fameio-2.2.0.dist-info → fameio-2.3.1.dist-info}/RECORD

@@ -1,24 +1,25 @@
-CHANGELOG.md,sha256=H8YzcsZiM8WHQSzP9TOrah5y7nH4fuP4M6SHN4VszYY,11358
+CHANGELOG.md,sha256=0T5FhyDSLmIo_tr4aXo_tOgJ5IsQH4WTY2JG3cnIjiI,12025
 fameio/__init__.py,sha256=IQm0MNOXkhBexiMXBoNZDK5xHUYgazH7oXm-lc0Vm04,109
 fameio/scripts/__init__.py,sha256=Bdu79kajJvvmPWdSP82Y6G8MCpP4n9ftTR-snWSbMJY,741
 fameio/scripts/__init__.py.license,sha256=2-OqCNxP4504xY2XQqseYypJi1_Qx4xJSzO3t7c3ACM,107
-fameio/scripts/convert_results.py,sha256=UHTRnE8LkhO6lHEkfgettdbhRcRHeZqQw5MkxYpD8T8,3616
+fameio/scripts/convert_results.py,sha256=ljrfPTwhauW98w-wJs4Fu4vtTreGzsMEd-io33UC1z0,4189
 fameio/scripts/convert_results.py.license,sha256=2-OqCNxP4504xY2XQqseYypJi1_Qx4xJSzO3t7c3ACM,107
 fameio/scripts/make_config.py,sha256=KQsj-aPXXtch8ITITCOcQUj6nYhvDs9LO7tdSikUe6w,1388
 fameio/scripts/make_config.py.license,sha256=2-OqCNxP4504xY2XQqseYypJi1_Qx4xJSzO3t7c3ACM,107
 fameio/source/__init__.py,sha256=14CnWOIkdSeKQg6FioQSgO7UtEFF6pO4MUtDAUfwNmA,278
 fameio/source/cli/__init__.py,sha256=UqrdWnYMoy-o2--m0a4c-MtAorsc4eILXaPq27EXM1Q,112
-fameio/source/cli/convert_results.py,sha256=LY9ySXs3dAyyos0yoc-bY8M1Z9Oti-AgGYEEK5K-Ctg,3412
+fameio/source/cli/convert_results.py,sha256=oijIHua5gCjJ7ZAHNQJXNAoNRK1-pt11LlCDiwudhMQ,3458
 fameio/source/cli/make_config.py,sha256=phZKimvItl538xnwv9IhBoDkh__YqNlRk4YNDyTOFaA,2738
 fameio/source/cli/options.py,sha256=5DOFq4LMcObrSOlbt74HGE1vghByn-I30YY46tm5Snc,1422
-fameio/source/cli/parser.py,sha256=cVC6j_oy3lBQOC7nrk5oF2m5CpcohOh5y689lhvu4gw,9726
+fameio/source/cli/parser.py,sha256=WK3vV7lrCpY2GCwS0N4ppNhJfVUcJ8pRZrM6NmGE8Ks,11015
 fameio/source/loader.py,sha256=-Y0j3OrV2MwAlfxFP2YA3ZwZhP-eFVlT6hdHO4Qze2U,7536
 fameio/source/logs.py,sha256=9Iq9jcglGyALYir-Luxfa-m4uUJX074JkUM0uMwzPNc,3684
+fameio/source/metadata.py,sha256=zdqKEIEQ0NKqEmBTBdPEq0fw1MdsXwCpzs1O6rFTq_Y,1024
 fameio/source/path_resolver.py,sha256=cIEVvz7Eh4e3Rh87XkwyGiyj9iKxUI7TzWtq6ClhLd8,1321
 fameio/source/results/__init__.py,sha256=IQm0MNOXkhBexiMXBoNZDK5xHUYgazH7oXm-lc0Vm04,109
 fameio/source/results/agent_type.py,sha256=pW5cLduLlNOcBBvS6sCLpTpZt3D6B8UMsizEZhe5lJ0,4321
-fameio/source/results/conversion.py,sha256=F7rDJ_iJGN1DFt_9jkweGap_TGMJV3UnUyssfIJehDM,3709
-fameio/source/results/csv_writer.py,sha256=A4hWnuBmU0cOfFLkW57L0NlH2tHc4H-ME1fmJitZLI8,4856
+fameio/source/results/conversion.py,sha256=5s6gPWNIOOLKn0O0-9gVvmJ8wq7m0Dh5NgtXIe1PIJs,3683
+fameio/source/results/csv_writer.py,sha256=lp5fhnhD99ksYLJwUguC9o8mujM8tkyCfq2vQ5LT1zw,5152
 fameio/source/results/data_transformer.py,sha256=tz58m7_TZPE3YjBoDyiMWiP_sz9mWaj1JfW2FjKLVhY,5490
 fameio/source/results/input_dao.py,sha256=5V_7dUhO1XZGNuwH5041HnMGCcQjC9d9_TBR4FZ9QJE,6919
 fameio/source/results/output_dao.py,sha256=-tMewCUS4m3RebTAf8Z33gtlj-SxqRdxnzvGIkg6Ah0,4106
@@ -26,28 +27,29 @@ fameio/source/results/reader.py,sha256=g3cfbNq7eHO98dxtpqjjPE8_YJVM56Wp0MCnNi3ds
 fameio/source/results/yaml_writer.py,sha256=rOAbeZQgRojYvofUcXreIdIJtdSbOrHAohiNLkmPgCI,837
 fameio/source/scenario/__init__.py,sha256=YfJvz275GWX8kVWMSMRcF3KsUQNqSAPrwBgNV7tqBTY,372
 fameio/source/scenario/agent.py,sha256=Z8jfMfUjrTJ4R1rEpwNdKCblhEx6-9xioewxGuajuqw,4713
-fameio/source/scenario/attribute.py,sha256=-Ed6jHlztwf9NhvpfRNt6ohGKKqkPBx_hLvGFE6O6K0,4834
+fameio/source/scenario/attribute.py,sha256=X63ZFpAdObTjMHpf0FD_H3v9sfOOqn7KDdjiGmyMZj4,4899
 fameio/source/scenario/contract.py,sha256=lM2NN_CUDh9iqDghq_F6JE9rEW3ZGUhBJOD0qkjuKBU,9442
 fameio/source/scenario/exception.py,sha256=7MYN4h40ptMZFD1lLtnK5PWXbAGlsmUSDmUQ5FAKeN8,1554
-fameio/source/scenario/fameiofactory.py,sha256=7P-x8i9sSU9sIkOlgF6gVRFuNacFMUBrBh4KdHoAUtU,1376
+fameio/source/scenario/fameiofactory.py,sha256=kV2GXGjNxYrt4lv5KoK8wYN6lfN3Z3hWaRNOl89gYv8,1654
 fameio/source/scenario/generalproperties.py,sha256=TkQV9dEo5zMfNiFITolJU1rQUBi-yDJLpoello39e_0,4796
-fameio/source/scenario/scenario.py,sha256=j2IVVZlpBJjYECXzBouFtrDXPPES0uSf_Q2l0MzcQkQ,3727
+fameio/source/scenario/scenario.py,sha256=w64SHQXpRFIMhDUNOIRkuK81jY_cXvT_M4CNhtULbMQ,4585
+fameio/source/scenario/stringset.py,sha256=ZfObpTbAsYZcTti-1xoq3KaZf3D59fMToRkkZaTXIcc,1920
 fameio/source/schema/__init__.py,sha256=ZGTyliDbjlYGPAjB9bbggzACOYWdhm4I1VSmm7YGmTk,270
 fameio/source/schema/agenttype.py,sha256=fe6GWJwl2S-J7KmH_eR5Y8Ieez1difez1w7v-r26yGU,5183
-fameio/source/schema/attribute.py,sha256=o2eZjurrYfvBiaZm1WE1XW0u8joHSbbXEKu-385cuW8,8296
+fameio/source/schema/attribute.py,sha256=wzu4563_zUU863dhjZQOuoPBBW4a7TjED5orKWuEujM,8357
 fameio/source/schema/exception.py,sha256=NMftGnrFOaS3MwrjZl8qbx3bi6KYUxhzHJIX1v--B5M,224
 fameio/source/schema/java_packages.py,sha256=-2ZbAG_htEowB-fBSMizsLWrphyiKfESDDHspzdEQP4,2656
 fameio/source/schema/schema.py,sha256=-J8VtO4W9Z-OnTV7MFO1umZWIC_epbLCYd6-q4lNQ8Y,2975
-fameio/source/series.py,sha256=o5tNcZfzfWEmhtzqXeR6hLaX3Wb1lvHLokSHnbvCP3Y,8754
+fameio/source/series.py,sha256=wwq7Nm0i0mOvMOjmqPW5x1gSDnhpWEy14VLceUhWcYw,9041
 fameio/source/time.py,sha256=Vsd95nTubL8x5cwiznJYeti6clZCCHHJv7xjSpYqkF0,6984
-fameio/source/tools.py,sha256=7YxX-CmsakJJB2RbBhf2u8y1-RyhDhsASHXFztkDIKE,1052
-fameio/source/validator.py,sha256=cXTDl75zt6NVVyHUUFDL9Y_lg3QuJJfTdW83lv4Krvs,15719
-fameio/source/writer.py,sha256=n9ZGRqzkPsuY-hrjGlSeQDuc6OHtgeK9hzM_d_Bvs_U,12456
-fameio-2.2.0.dist-info/entry_points.txt,sha256=jvQVfwJjZXPWQjJlhj1Dt6PTeblryTc1GxjKeK90twI,123
-fameio-2.2.0.dist-info/LICENSE.txt,sha256=eGHBZnhr9CWjE95SWjRfmhtK1lvVn5X4Fpf3KrrAZDg,10391
-fameio-2.2.0.dist-info/LICENSES/Apache-2.0.txt,sha256=eGHBZnhr9CWjE95SWjRfmhtK1lvVn5X4Fpf3KrrAZDg,10391
-fameio-2.2.0.dist-info/LICENSES/CC-BY-4.0.txt,sha256=y9WvMYKGt0ZW8UXf9QkZB8wj1tjJrQngKR7CSXeSukE,19051
-fameio-2.2.0.dist-info/LICENSES/CC0-1.0.txt,sha256=9Ofzc7m5lpUDN-jUGkopOcLZC3cl6brz1QhKInF60yg,7169
-fameio-2.2.0.dist-info/METADATA,sha256=8T_pXeAjdSsE1A24QKLpyWNWnKdUimDiZgMwi0wMa0o,31231
-fameio-2.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-fameio-2.2.0.dist-info/RECORD,,
+fameio/source/tools.py,sha256=B94U7Idq9ZuvsQmc9HOti016z4jn2oTTMIElWhs-5_w,1062
+fameio/source/validator.py,sha256=44wAlSxzSlaP9cm3YpNXOcs_kXIUNagctHLguY6H3T4,18596
+fameio/source/writer.py,sha256=d35rZGPLil3yku8_N6l1B45J4agKemBhAHJJPynQiJE,12499
+fameio-2.3.1.dist-info/entry_points.txt,sha256=jvQVfwJjZXPWQjJlhj1Dt6PTeblryTc1GxjKeK90twI,123
+fameio-2.3.1.dist-info/LICENSE.txt,sha256=eGHBZnhr9CWjE95SWjRfmhtK1lvVn5X4Fpf3KrrAZDg,10391
+fameio-2.3.1.dist-info/LICENSES/Apache-2.0.txt,sha256=eGHBZnhr9CWjE95SWjRfmhtK1lvVn5X4Fpf3KrrAZDg,10391
+fameio-2.3.1.dist-info/LICENSES/CC-BY-4.0.txt,sha256=y9WvMYKGt0ZW8UXf9QkZB8wj1tjJrQngKR7CSXeSukE,19051
+fameio-2.3.1.dist-info/LICENSES/CC0-1.0.txt,sha256=9Ofzc7m5lpUDN-jUGkopOcLZC3cl6brz1QhKInF60yg,7169
+fameio-2.3.1.dist-info/METADATA,sha256=kYlnlq-NqQnaJpgxUgfehySdfLTA0wIUF8OU66a8Wqw,32471
+fameio-2.3.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+fameio-2.3.1.dist-info/RECORD,,