gammasimtools 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/testing/helpers.py

@@ -1,25 +1,25 @@
 """Helper functions for integration testing."""
 
 import os
-
-import pytest
+from pathlib import Path
 
 
 def skip_camera_efficiency(config):
     """Skip camera efficiency tests if the old version of testeff is used."""
     if "camera-efficiency" in config["APPLICATION"]:
         if not _new_testeff_version():
-            pytest.skip(
+            return (
                 "Any applications calling the old version of testeff are skipped "
                 "due to a limitation of the old testeff not allowing to specify "
                 "the include directory. Please update your sim_telarray tarball."
             )
         full_test_name = f"{config['APPLICATION']}_{config['TEST_NAME']}"
         if "simtools-validate-camera-efficiency_SSTS" == full_test_name:
-            pytest.skip(
+            return (
                 "The test simtools-validate-camera-efficiency_SSTS is skipped "
                 "since the fake SST mirrors are not yet implemented (#1155)"
             )
+    return None
 
 
 def _new_testeff_version():
@@ -28,7 +28,7 @@ def _new_testeff_version():
 
     This test checks if the new version is used.
     """
-    testeff_path = os.path.join(os.getenv("SIMTOOLS_SIMTEL_PATH"), "sim_telarray/testeff.c")
+    testeff_path = Path(os.getenv("SIMTOOLS_SIMTEL_PATH")) / "sim_telarray/testeff.c"
     try:
         with open(testeff_path, encoding="utf-8") as file:
             file_content = file.read()

simtools/testing/validate_output.py

@@ -55,8 +55,13 @@ def validate_application_output(config):
         if "REFERENCE_OUTPUT_FILE" in integration_test:
             _validate_reference_output_file(config, integration_test)
 
+        if "TEST_OUTPUT_FILES" in integration_test:
+            _validate_output_path_and_file(config, integration_test["TEST_OUTPUT_FILES"])
         if "OUTPUT_FILE" in integration_test:
-            _validate_output_path_and_file(config, integration_test)
+            _validate_output_path_and_file(
+                config,
+                [{"PATH_DESCRIPTOR": "OUTPUT_PATH", "FILE": integration_test["OUTPUT_FILE"]}],
+            )
 
         if "FILE_TYPE" in integration_test:
             assert assertions.assert_file_type(
@@ -79,31 +84,25 @@ def _validate_reference_output_file(config, integration_test):
     )
 
 
-def _validate_output_path_and_file(config, integration_test):
-    """Check if output path and file exist."""
-    _logger.info(f"PATH {config['CONFIGURATION']['OUTPUT_PATH']}")
-    _logger.info(f"File {integration_test['OUTPUT_FILE']}")
-
-    data_path = config["CONFIGURATION"].get(
-        "DATA_DIRECTORY", config["CONFIGURATION"]["OUTPUT_PATH"]
-    )
-    output_file_path = Path(data_path) / integration_test["OUTPUT_FILE"]
-
-    _logger.info(f"Checking path: {output_file_path}")
-    assert output_file_path.exists()
-
-    expected_output = [
-        d["EXPECTED_OUTPUT"] for d in config["INTEGRATION_TESTS"] if "EXPECTED_OUTPUT" in d
-    ]
-    if expected_output and "log_hist" not in integration_test["OUTPUT_FILE"]:
-        # Get the expected output from the configuration file
-        expected_output = expected_output[0]
-        _logger.info(
-            f"Checking the output of {integration_test['OUTPUT_FILE']} "
-            "complies with the expected output: "
-            f"{expected_output}"
-        )
-        assert assertions.check_output_from_sim_telarray(output_file_path, expected_output)
+def _validate_output_path_and_file(config, integration_file_tests):
+    """Check if output paths and files exist."""
+    for file_test in integration_file_tests:
+        try:
+            output_path = config["CONFIGURATION"][file_test["PATH_DESCRIPTOR"]]
+        except KeyError as exc:
+            raise KeyError(
+                f"Path {file_test['PATH_DESCRIPTOR']} not found in integration test configuration."
+            ) from exc
+
+        output_file_path = Path(output_path) / file_test["FILE"]
+        _logger.info(f"Checking path: {output_file_path}")
+        assert output_file_path.exists()
+
+        if "EXPECTED_OUTPUT" in file_test:
+            assert assertions.check_output_from_sim_telarray(
+                output_file_path,
+                file_test["EXPECTED_OUTPUT"],
+            )
 
 
 def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None):
@@ -124,7 +123,7 @@ def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None):
     Returns
     -------
     bool
-        True if the files are equal, False otherwise.
+        True if the files are equal.
 
     """
     _file1_suffix = Path(file1).suffix
@@ -158,7 +157,7 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2):
     Returns
     -------
     bool
-        True if the files are equal, False otherwise.
+        True if the files are equal.
 
     """
     data1 = gen.collect_data_from_file(file1)
@@ -235,6 +234,7 @@ def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
 
         if np.issubdtype(table1_masked[col_name].dtype, np.floating):
             if not np.allclose(table1_masked[col_name], table2_masked[col_name], rtol=tolerance):
+                _logger.warning(f"Column {col_name} outside of relative tolerance {tolerance}")
                 return False
 
     return True

simtools/utils/general.py

@@ -164,6 +164,9 @@ def collect_data_from_file(file_name):
             return yaml.safe_load(file)
         except yaml.constructor.ConstructorError:
             return _load_yaml_using_astropy(file)
+        except yaml.composer.ComposerError:
+            file.seek(0)
+            return list(yaml.safe_load_all(file))
 
 
 def collect_kwargs(label, in_kwargs):
@@ -333,9 +336,10 @@ def program_is_executable(program):
     Follows https://stackoverflow.com/questions/377017/
 
     """
+    program = Path(program)
 
     def is_exe(fpath):
-        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+        return fpath.is_file() and os.access(fpath, os.X_OK)
 
     fpath, _ = os.path.split(program)
     if fpath:
@@ -344,7 +348,7 @@ def program_is_executable(program):
     else:
         try:
             for path in os.environ["PATH"].split(os.pathsep):
-                exe_file = os.path.join(path, program)
+                exe_file = Path(path) / program
                 if is_exe(exe_file):
                     return exe_file
         except KeyError:
@@ -442,7 +446,7 @@ def get_file_age(file_path):
     if not Path(file_path).is_file():
         raise FileNotFoundError(f"'{file_path}' does not exist or is not a file.")
 
-    file_stats = os.stat(file_path)
+    file_stats = Path(file_path).stat()
     modification_time = file_stats.st_mtime
     current_time = time.time()
 
@@ -785,3 +789,46 @@ def read_file_encoded_in_utf_or_latin(file_name):
             raise UnicodeDecodeError("Unable to decode file using UTF-8 or Latin-1.") from exc
 
     return lines
+
+
+def get_structure_array_from_table(table, column_names):
+    """
+    Get a structured array from an astropy table for a selected list of columns.
+
+    Parameters
+    ----------
+    table: astropy.table.Table
+        Table to be converted.
+    column_names: list
+        List of column names to be included in the structured array.
+
+    Returns
+    -------
+    numpy.ndarray
+        Structured array containing the table data.
+    """
+    return np.array(
+        list(zip(*[np.array(table[col]) for col in column_names])),
+        dtype=[(col, np.array(table[col]).dtype) for col in column_names],
+    )
+
+
+def convert_keys_in_dict_to_lowercase(data):
+    """
+    Recursively convert all dictionary keys to lowercase.
+
+    Parameters
+    ----------
+    data: dict
+        Dictionary to be converted.
+
+    Returns
+    -------
+    dict
+        Dictionary with all keys converted to lowercase.
+    """
+    if isinstance(data, dict):
+        return {k.lower(): convert_keys_in_dict_to_lowercase(v) for k, v in data.items()}
+    if isinstance(data, list):
+        return [convert_keys_in_dict_to_lowercase(i) for i in data]
+    return data

simtools/utils/names.py

@@ -1,9 +1,9 @@
 """Validation of names."""
 
-import glob
 import logging
 import re
 from functools import cache
+from importlib.resources import files
 from pathlib import Path
 
 import yaml
@@ -34,8 +34,7 @@ def array_elements():
     dict
         Array elements.
     """
-    base_path = Path(__file__).parent
-    with open(base_path / "../schemas/array_elements.yml", encoding="utf-8") as file:
+    with open(files("simtools") / "schemas/array_elements.yml", encoding="utf-8") as file:
         return yaml.safe_load(file)["data"]
 
 
@@ -60,7 +59,7 @@ def site_names():
 @cache
 def load_model_parameters(class_key_list):
     model_parameters = {}
-    schema_files = glob.glob(str(Path(__file__).parent / "../schemas/model_parameters") + "/*.yml")
+    schema_files = list(Path(files("simtools") / "schemas/model_parameters").rglob("*.yml"))
     for schema_file in schema_files:
         with open(schema_file, encoding="utf-8") as f:
             data = yaml.safe_load(f)

simtools/utils/value_conversion.py

@@ -84,7 +84,7 @@ def get_value_unit_type(value, unit_str=None):
             pass
         base_unit = unit_str
 
-    return base_value, base_unit, base_type
+    return base_value, _unit_as_string(base_unit), base_type
 
 
 def split_value_and_unit(value):
@@ -174,3 +174,11 @@ def get_value_as_quantity(value, unit):
             _logger.error(f"Cannot convert {value.unit} to {unit}.")
             raise
     return value * unit
+
+
+def _unit_as_string(unit):
+    """Return the string representation of a unit. Collapse if it is a list of identical items."""
+    if not isinstance(unit, list):
+        unit = [unit]
+    unit = [str(element) if element is not None else None for element in unit]
+    return unit[0] if len(set(unit)) == 1 else unit

simtools/version.py

@@ -6,7 +6,7 @@
 
 try:
     try:
-        from ._dev.scm_version import version
+        from ._dev_version import version
     except ImportError:
         from ._version import version
 except Exception:

simtools/visualization/plot_tables.py

@@ -0,0 +1,106 @@
+#!/usr/bin/python3
+"""Plot tabular data."""
+
+import numpy as np
+
+import simtools.utils.general as gen
+from simtools.io_operations import legacy_data_handler
+from simtools.model.site_model import SiteModel
+from simtools.model.telescope_model import TelescopeModel
+from simtools.visualization import visualize
+
+
+def plot(config, output_file, db_config=None):
+    """
+    Plot tabular data from data or from model parameter files.
+
+    Parameters
+    ----------
+    config: dict
+        Configuration dictionary for plotting.
+    output_file: str
+        Output file.
+    """
+    data = read_table_data(config, db_config)
+
+    fig = visualize.plot_1d(
+        data,
+        **config,
+    )
+    visualize.save_figure(fig, output_file)
+
+
+def read_table_data(config, db_config):
+    """
+    Read table data from file or parameter database.
+
+    Parameters
+    ----------
+    config: dict
+        Configuration dictionary for plotting.
+
+    Returns
+    -------
+    Dict
+        Dict with table data (astropy tables).
+    """
+    data = {}
+
+    for _config in config["tables"]:
+        if "parameter" in _config:
+            table = _read_table_from_model_database(_config, db_config)
+        elif "file_name" in _config:
+            table = legacy_data_handler.read_legacy_data_as_table(
+                _config["file_name"], _config["type"]
+            )
+        else:
+            raise ValueError("No table data defined in configuration.")
+        if _config.get("normalize_y"):
+            table[_config["column_y"]] = (
+                table[_config["column_y"]] / table[_config["column_y"]].max()
+            )
+        if _config.get("select_values"):
+            table = _select_values_from_table(
+                table,
+                _config["select_values"]["column_name"],
+                _config["select_values"]["value"],
+            )
+        data[_config["label"]] = gen.get_structure_array_from_table(
+            table, [_config["column_x"], _config["column_y"]]
+        )
+    return data
+
+
+def _read_table_from_model_database(table_config, db_config):
+    """
+    Read table data from model parameter database.
+
+    Parameters
+    ----------
+    table_config: dict
+        Configuration dictionary for table data.
+
+    Returns
+    -------
+    Table
+        Astropy table.
+    """
+    if "telescope" in table_config:
+        model = TelescopeModel(
+            site=table_config["site"],
+            telescope_name=table_config["telescope"],
+            model_version=table_config["model_version"],
+            mongo_db_config=db_config,
+        )
+    else:
+        model = SiteModel(
+            site=table_config["site"],
+            model_version=table_config["model_version"],
+            mongo_db_config=db_config,
+        )
+    return model.get_model_file_as_table(table_config["parameter"])
+
+
+def _select_values_from_table(table, column_name, value):
+    """Return a table with only the rows where column_name == value."""
+    return table[np.isclose(table[column_name], value)]

simtools/visualization/visualize.py

@@ -5,6 +5,7 @@ import copy
 import logging
 import re
 from collections import OrderedDict
+from pathlib import Path
 
 import astropy.units as u
 import matplotlib.patches as mpatches
@@ -27,6 +28,7 @@ __all__ = [
     "plot_array",
     "plot_hist_2d",
     "plot_table",
+    "save_figure",
     "set_style",
 ]
 
@@ -366,6 +368,12 @@ def handle_kwargs(kwargs):
         "empty_markers": False,
         "plot_ratio": False,
         "plot_difference": False,
+        "xscale": "linear",
+        "yscale": "linear",
+        "xlim": (None, None),
+        "ylim": (None, None),
+        "xtitle": None,
+        "ytitle": None,
     }
     for key, default in kwargs_defaults.items():
         kwargs[key] = kwargs.pop(key, default)
@@ -414,11 +422,17 @@ def plot_main_data(data_dict, kwargs, plot_args):
     """Plot the main data."""
     for label, data_now in data_dict.items():
         assert len(data_now.dtype.names) == 2, "Input array must have two columns with titles."
-        x_title, y_title = data_now.dtype.names[0], data_now.dtype.names[1]
-        x_title_unit = _add_unit(x_title, data_now[x_title])
-        y_title_unit = _add_unit(y_title, data_now[y_title])
-        plt.plot(data_now[x_title], data_now[y_title], label=label, **plot_args)
-
+        x_column_name, y_column_name = data_now.dtype.names[0], data_now.dtype.names[1]
+        x_title = kwargs["xtitle"] if kwargs.get("xtitle") else x_column_name
+        y_title = kwargs["ytitle"] if kwargs.get("ytitle") else y_column_name
+        x_title_unit = _add_unit(x_title, data_now[x_column_name])
+        y_title_unit = _add_unit(y_title, data_now[y_column_name])
+        plt.plot(data_now[x_column_name], data_now[y_column_name], label=label, **plot_args)
+
+    plt.xscale(kwargs["xscale"])
+    plt.yscale(kwargs["yscale"])
+    plt.xlim(kwargs["xlim"])
+    plt.ylim(kwargs["ylim"])
     plt.ylabel(y_title_unit)
     if not (kwargs["plot_ratio"] or kwargs["plot_difference"]):
         plt.xlabel(x_title_unit)
@@ -878,3 +892,27 @@ def plot_simtel_ctapipe(filename, cleaning_args, distance, return_cleaned=False)
     ax.set_axis_off()
     fig.tight_layout()
     return fig
+
+
+def save_figure(fig, output_file, figure_format=None, log_title=""):
+    """
+    Save figure to output file(s).
+
+    Parameters
+    ----------
+    fig: plt.figure
+        Figure to save.
+    output_file: Path, str
+        Path to save the figure (without suffix).
+    figure_format: list
+        List of formats to save the figure.
+    title: str
+        Title of the figure to be added to the log message.
+    """
+    figure_format = figure_format or ["pdf", "png"]
+    for fmt in figure_format:
+        _file = Path(output_file).with_suffix(f".{fmt}")
+        fig.savefig(_file, format=fmt, bbox_inches="tight")
+        logging.info(f"Saved plot {log_title} to {_file}")
+
+    fig.clf()