fameio 3.1.1__py3-none-any.whl → 3.3.0__py3-none-any.whl

fameio/scripts/convert_results.py

@@ -1,101 +1,175 @@
 #!/usr/bin/env python
+from __future__ import annotations
+
 import sys
 from pathlib import Path
+from typing import Any, BinaryIO
 
 import pandas as pd
 
+from fameio.cli import update_default_config
 from fameio.cli.convert_results import handle_args, CLI_DEFAULTS as DEFAULT_CONFIG
 from fameio.cli.options import Options
-from fameio.cli import update_default_config
-from fameio.logs import log_critical_and_raise, fameio_logger, log
+from fameio.input import InputError
+from fameio.logs import fameio_logger, log, log_error, log_critical
+from fameio.output import OutputError
 from fameio.output.agent_type import AgentTypeLog
 from fameio.output.conversion import apply_time_option, apply_time_merging
 from fameio.output.csv_writer import CsvWriter
 from fameio.output.data_transformer import DataTransformer, INDEX
+from fameio.output.execution_dao import ExecutionDao
 from fameio.output.input_dao import InputDao
 from fameio.output.output_dao import OutputDAO
 from fameio.output.reader import Reader
 from fameio.output.yaml_writer import data_to_yaml_file
+from fameio.scripts.exception import ScriptError
 
-ERR_OUT_OF_MEMORY = "Out of memory. Retry result conversion using `-m` or `--memory-saving` option."
-ERR_MEMORY_SEVERE = "Out of memory despite memory-saving mode. Reduce output interval in `FAME-Core` and rerun model."
+_ERR_OUT_OF_MEMORY = "Out of memory. Retry result conversion using `-m` or `--memory-saving` option."
+_ERR_MEMORY_SEVERE = "Out of memory despite memory-saving mode. Reduce output interval in `FAME-Core` and rerun model."
+_ERR_FILE_OPEN_FAIL = "Could not open file: '{}'"
+_ERR_RECOVER_INPUT = "Input recovery failed: File was created with `fameio=={}`. Use that version to recover inputs."
+_ERR_FAIL = "Results conversion script failed."
 
-WARN_OUTPUT_MISSING = "Provided file did not contain any output data, only input recovery available."
+_WARN_OUTPUT_SUPPRESSED = "All output data suppressed by agent filter, but there is data available for agent types: {}"
+_WARN_OUTPUT_MISSING = "Provided file did not contain any output data, only input recovery available."
+_INFO_MEMORY_SAVING = "Memory saving mode enabled: Disable on conversion of small files for performance improvements."
 
-INFO_MEMORY_SAVING = "Memory saving mode enabled: Disable on conversion of small files for performance improvements."
-INFO_RECOVERY = "Recovering inputs..."
-INFO_CONVERSION = "Applying time conversion and merging options to extracted files..."
 
+def _read_and_extract_data(config: dict[Options, Any]) -> None:
+    """Read protobuf file, extracts, converts, and saves the converted data.
 
-def _extract_data(config: dict) -> bool:
-    """Extracts, converts, and saves the converted data; Returns false if no result data was found"""
+    Args:
+        config: script configuration options
+
+    Raises:
+        OutputError: if file could not be opened or converted, logged with level "ERROR"
+    """
     file_path = Path(config[Options.FILE])
+    log().info("Opening file for reading...")
+    try:
+        with open(file_path, "rb") as file_stream:
+            _extract_and_convert_data(config, file_stream, file_path)
+    except OSError as ex:
+        raise log_error(OutputError(_ERR_FILE_OPEN_FAIL.format(file_path))) from ex
+
+
+def _extract_and_convert_data(config: dict[Options, Any], file_stream: BinaryIO, file_path: Path) -> None:
+    """Extracts data from provided input file stream, converts it, and writes the result to output files.
+
+    Args:
+        config: script configuration options
+        file_stream: opened input file
+        file_path: path to input file
+
+    Raises:
+        OutputError: if file could not be opened or converted, logged with level "ERROR"
+    """
+    log().info("Reading and extracting data...")
     output_writer = CsvWriter(config[Options.OUTPUT], file_path, config[Options.SINGLE_AGENT_EXPORT])
-    agent_type_log = AgentTypeLog(requested_agents=config[Options.AGENT_LIST])
+    agent_type_log = AgentTypeLog(_agent_name_filter_list=config[Options.AGENT_LIST])
     data_transformer = DataTransformer.build(config[Options.RESOLVE_COMPLEX_FIELD])
+    reader = Reader.get_reader(file=file_stream, read_single=config[Options.MEMORY_SAVING])
+    input_dao = InputDao()
+    execution_dao = ExecutionDao()
+    while data_storages := reader.read():
+        execution_dao.store_execution_metadata(data_storages)
+        if config[Options.INPUT_RECOVERY]:
+            input_dao.store_inputs(data_storages)
+        output = OutputDAO(data_storages, agent_type_log)
+        for agent_name in output.get_sorted_agents_to_extract():
+            log().debug(f"Extracting data for {agent_name}...")
+            data_frames = output.get_agent_data(agent_name, data_transformer)
+            if not config[Options.MEMORY_SAVING]:
+                apply_time_merging(data_frames, config[Options.TIME_MERGING])
+                apply_time_option(data_frames, config[Options.TIME])
+            log().debug(f"Writing data for {agent_name}...")
+            output_writer.write_to_files(agent_name, data_frames)
+
+    if config[Options.INPUT_RECOVERY]:
+        _recover_inputs(config, input_dao, execution_dao.get_fameio_version())
+    if config[Options.MEMORY_SAVING]:
+        _memory_saving_apply_conversions(config, output_writer)
 
-    log().info("Reading and extracting data...")
-    with open(file_path, "rb") as file_stream:
-        reader = Reader.get_reader(file=file_stream, read_single=config[Options.MEMORY_SAVING])
-        input_dao = InputDao()
-        while data_storages := reader.read():
-            if config[Options.INPUT_RECOVERY]:
-                input_dao.store_inputs(data_storages)
-            output = OutputDAO(data_storages, agent_type_log)
-            for agent_name in output.get_sorted_agents_to_extract():
-                log().debug(f"Extracting data for {agent_name}...")
-                data_frames = output.get_agent_data(agent_name, data_transformer)
-                if not config[Options.MEMORY_SAVING]:
-                    apply_time_merging(data_frames, config[Options.TIME_MERGING])
-                    apply_time_option(data_frames, config[Options.TIME])
-                log().debug(f"Writing data for {agent_name}...")
-                output_writer.write_to_files(agent_name, data_frames)
+    if not agent_type_log.has_any_agent_type():
+        if len(agent_type_log.get_agents_with_output()) > 0:
+            log().warning(_WARN_OUTPUT_SUPPRESSED.format(agent_type_log.get_agents_with_output()))
+        else:
+            log().warning(_WARN_OUTPUT_MISSING)
+    log().info("Data conversion completed.")
 
-        if config[Options.INPUT_RECOVERY]:
-            _recover_inputs(config, input_dao)
-        if config[Options.MEMORY_SAVING]:
-            _memory_saving_apply_conversions(config, output_writer)
-        log().info("Data conversion completed.")
-    return agent_type_log.has_any_agent_type()
 
+def _recover_inputs(config: dict[Options, Any], input_dao: InputDao, fameio_version: str) -> None:
+    """Reads scenario configuration from provided `input_dao`.
+
+    Args:
+        config: script configuration options
+        input_dao: to recover the input data from
+        fameio_version: version of fameio that was used to create the input data
 
-def _recover_inputs(config: dict, input_dao: InputDao) -> None:
-    """Reads scenario configuration from provided input_dao"""
-    log().info(INFO_RECOVERY)
-    timeseries, scenario = input_dao.recover_inputs()
+    Raises:
+        OutputError: if inputs could not be recovered or saved to files, logged with level "ERROR"
+    """
+    log().info("Recovering inputs...")
+    try:
+        timeseries, scenario = input_dao.recover_inputs()
+    except InputError as ex:
+        raise log_error(OutputError(_ERR_RECOVER_INPUT.format(fameio_version))) from ex
     base_path = config[Options.OUTPUT] if config[Options.OUTPUT] is not None else "./"
     series_writer = CsvWriter(
         config_output=Path(base_path, "./recovered"), input_file_path=Path("./"), single_export=False
     )
-    series_writer.write_time_series_to_disk(timeseries)
+    series_writer.write_all_time_series_to_disk(timeseries)
     data_to_yaml_file(scenario.to_dict(), Path(base_path, "./recovered/scenario.yaml"))
 
 
-def _memory_saving_apply_conversions(config: dict, output_writer: CsvWriter) -> None:
-    """Rewrite result files in memory saving mode: apply time-merging and time conversion options on a per-file basis"""
-    log().info(INFO_CONVERSION)
+def _memory_saving_apply_conversions(config: dict[Options, Any], output_writer: CsvWriter) -> None:
+    """Rewrite result files: applies time-merging and time conversion options on a per-file basis.
+
+    This is only required in memory saving mode.
+
+    Args:
+        config: script configuration options
+        output_writer: to rewrite the previously written files
+
+    Raises:
+        OutputError: in case files could not be read, converted, or re-written, logged with level "ERROR"
+    """
+    log().info("Applying time conversion and merging options to extracted files...")
     written_files = output_writer.pop_all_file_paths()
     for agent_name, file_path in written_files.items():
-        parsed_data = {None: pd.read_csv(file_path, sep=";", index_col=INDEX)}
+        parsed_data: dict[str | None, pd.DataFrame] = {None: pd.read_csv(file_path, sep=";", index_col=INDEX)}
         apply_time_merging(parsed_data, config[Options.TIME_MERGING])
         apply_time_option(parsed_data, config[Options.TIME])
         output_writer.write_to_files(agent_name, parsed_data)
 
 
-def run(config: dict = None) -> None:
-    """Reads configured file in protobuf format and extracts its content to .CSV and .YAML file(s)"""
+def run(config: dict[Options, Any] | None = None) -> None:
+    """Reads configured file in protobuf format and extracts its content to .CSV and .YAML file(s).
+
+    Args:
+        config: script configuration options
+
+    Raises:
+        ScriptError: if any kind of expected error or a memory error occurred, logged with level "CRITICAL"
+    """
     config = update_default_config(config, DEFAULT_CONFIG)
     fameio_logger(log_level_name=config[Options.LOG_LEVEL], file_name=config[Options.LOG_FILE])
     if config[Options.MEMORY_SAVING]:
-        log().info(INFO_MEMORY_SAVING)
+        log().info(_INFO_MEMORY_SAVING)
+
     try:
-        found_result_data = _extract_data(config)
-        if not found_result_data:
-            log().warning(WARN_OUTPUT_MISSING)
-    except MemoryError:
-        log_critical_and_raise(MemoryError(ERR_MEMORY_SEVERE if config[Options.MEMORY_SAVING] else ERR_OUT_OF_MEMORY))
+        try:
+            _read_and_extract_data(config)
+        except MemoryError as ex:
+            error = OutputError(_ERR_MEMORY_SEVERE if config[Options.MEMORY_SAVING] else _ERR_OUT_OF_MEMORY)
+            raise log_critical(error) from ex
+    except OutputError as ex:
+        raise log_critical(ScriptError(_ERR_FAIL)) from ex
 
 
 if __name__ == "__main__":
-    run_config = handle_args(sys.argv[1:])
-    run(run_config)
+    cli_config = handle_args(sys.argv[1:])
+    try:
+        run(cli_config)
+    except ScriptError as e:
+        raise SystemExit(1) from e

fameio/scripts/convert_results.py.license

@@ -1,3 +1,3 @@
-SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 
 SPDX-License-Identifier: Apache-2.0

fameio/scripts/exception.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+class ScriptError(Exception):
+    """Any kind of expected error that occurred during execution of FAME-Io scripts."""

fameio/scripts/make_config.py

@@ -1,34 +1,55 @@
 #!/usr/bin/env python
+from __future__ import annotations
+
 import sys
 from pathlib import Path
+from typing import Any
 
+from fameio.cli import update_default_config
 from fameio.cli.make_config import handle_args, CLI_DEFAULTS as DEFAULT_CONFIG
 from fameio.cli.options import Options
-from fameio.cli import update_default_config
+from fameio.input import InputError
 from fameio.input.loader import load_yaml, validate_yaml_file_suffix
-from fameio.logs import fameio_logger, log
 from fameio.input.scenario import Scenario
 from fameio.input.validator import SchemaValidator
 from fameio.input.writer import ProtoWriter
+from fameio.logs import fameio_logger, log, log_critical
+from fameio.scripts.exception import ScriptError
+
+_ERR_FAIL: str = "Creation of run configuration file failed."
+
+
+def run(config: dict[Options, Any] | None = None) -> None:
+    """Executes the main workflow of building a FAME configuration file.
 
+    Args:
+        config: configuration options
 
-def run(config: dict = None) -> None:
-    """Executes the main workflow for the building of a FAME configuration file"""
+    Raises:
+        ScriptError: if any kind of expected error occurred, logged with level "CRITICAL"
+    """
     config = update_default_config(config, DEFAULT_CONFIG)
     fameio_logger(log_level_name=config[Options.LOG_LEVEL], file_name=config[Options.LOG_FILE])
 
-    file = config[Options.FILE]
-    validate_yaml_file_suffix(Path(file))
-    scenario = Scenario.from_dict(load_yaml(Path(file), encoding=config[Options.INPUT_ENCODING]))
-    SchemaValidator.check_agents_have_contracts(scenario)
+    try:
+        file = config[Options.FILE]
+        validate_yaml_file_suffix(Path(file))
+        scenario_definition = load_yaml(Path(file), encoding=config[Options.INPUT_ENCODING])
+        scenario = Scenario.from_dict(scenario_definition)
+        SchemaValidator.check_agents_have_contracts(scenario)
 
-    timeseries_manager = SchemaValidator.validate_scenario_and_timeseries(scenario)
-    writer = ProtoWriter(config[Options.OUTPUT], timeseries_manager)
-    writer.write_validated_scenario(scenario)
+        timeseries_manager = SchemaValidator.validate_scenario_and_timeseries(scenario)
+        writer = ProtoWriter(config[Options.OUTPUT], timeseries_manager)
+        writer.write_validated_scenario(scenario)
+    except InputError as ex:
+        raise log_critical(ScriptError(_ERR_FAIL)) from ex
 
     log().info("Configuration completed.")
 
 
 if __name__ == "__main__":
-    run_config = handle_args(sys.argv[1:])
-    run(run_config)
+    cli_config = handle_args(sys.argv[1:])
+    try:
+        run(cli_config)
+    except ScriptError as e:
+        raise SystemExit(1) from e

fameio/scripts/make_config.py.license

@@ -1,3 +1,3 @@
-SPDX-FileCopyrightText: 2024 German Aerospace Center <fame@dlr.de>
+SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
 
 SPDX-License-Identifier: Apache-2.0

fameio/scripts/reformat.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Any
+
+from fameio.cli import update_default_config
+from fameio.cli.options import Options
+from fameio.cli.reformat import handle_args, CLI_DEFAULTS as DEFAULT_CONFIG
+from fameio.logs import fameio_logger, log_error, log_and_print
+from fameio.output.csv_writer import CsvWriter, CsvWriterError
+from fameio.scripts.exception import ScriptError
+from fameio.series import TimeSeriesManager, TimeSeriesError
+from fameio.tools import get_csv_files_with_pattern, extend_file_name
+
+FILE_NAME_APPENDIX = "_reformatted"
+
+_ERR_FAIL = "Timeseries reformatting script failed."
+_ERR_NO_FILES = "No file found matching this pattern: '{}'"
+_ERR_FILE_CONVERSION = "Could not reformat file: '{}'"
+
+
+def reformat_file(file: Path, replace: bool) -> None:
+    """Transforms content of specified CSV file to FAME format.
+
+    Args:
+        file: whose content is to be reformatted
+        replace: if true, original file will be replaced; otherwise, a new file will be created instead
+
+    Raises:
+        ScriptError: if file could not be read, file reformatting failed, or result file could not be written;
+            logged with level "ERROR"
+    """
+    try:
+        data = TimeSeriesManager.read_timeseries_file(file)
+        data = TimeSeriesManager.check_and_convert_series(data, str(file), warn=False)
+        target_path = file if replace else extend_file_name(file, FILE_NAME_APPENDIX)
+        CsvWriter.write_single_time_series_to_disk(data, target_path)
+    except (TimeSeriesError, CsvWriterError) as ex:
+        raise log_error(ScriptError(_ERR_FILE_CONVERSION.format(file))) from ex
+
+
+def run(config: dict[Options, Any] | None = None) -> None:
+    """Executes the workflow of transforming time series file(s).
+
+    Args:
+        config: configuration options
+
+    Raises:
+        ScriptError: if no file could be found, or if any file could not be transformed, logged with level "ERROR"
+    """
+    config = update_default_config(config, DEFAULT_CONFIG)
+    fameio_logger(log_level_name=config[Options.LOG_LEVEL], file_name=config[Options.LOG_FILE])
+    try:
+        files = get_csv_files_with_pattern(Path("."), config[Options.FILE_PATTERN])
+    except ValueError as ex:
+        raise log_error(ScriptError(_ERR_NO_FILES.format(config[Options.FILE_PATTERN]))) from ex
+    if not files:
+        raise log_error(ScriptError(_ERR_NO_FILES.format(config[Options.FILE_PATTERN])))
+    for file in files:
+        log_and_print(f"Reformatting file: {file}")
+        reformat_file(file, config[Options.REPLACE])
+
+
+if __name__ == "__main__":
+    cli_config = handle_args(sys.argv[1:])
+    try:
+        run(cli_config)
+    except ScriptError as e:
+        raise SystemExit(1) from e

fameio/scripts/reformat.py.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2025 German Aerospace Center <fame@dlr.de>
+
+SPDX-License-Identifier: Apache-2.0