gammasimtools 0.15.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/testing/sim_telarray_metadata.py

@@ -0,0 +1,212 @@
+"""Test consistency of sim_telarray metadata."""
+
+import logging
+
+import numpy as np
+
+from simtools.simtel.simtel_config_reader import SimtelConfigReader
+from simtools.simtel.simtel_config_writer import sim_telarray_random_seeds
+from simtools.simtel.simtel_io_file_info import get_corsika_run_number
+from simtools.simtel.simtel_io_metadata import (
+    get_sim_telarray_telescope_id,
+    read_sim_telarray_metadata,
+)
+
+_logger = logging.getLogger(__name__)
+
+
+def assert_sim_telarray_metadata(file, array_model):
+    """
+    Assert consistency of sim_telarray metadata with given array model.
+
+    Parameters
+    ----------
+    file: Path
+        Path to the sim_telarray file.
+    array_model: ArrayModel
+        Array model to compare with.
+    """
+    global_meta, telescope_meta = read_sim_telarray_metadata(file)
+    _logger.info(f"Found metadata in sim_telarray file for {len(telescope_meta)} telescopes")
+    site_parameter_mismatch = _assert_model_parameters(global_meta, array_model.site_model)
+    sim_telarray_seed_mismatch = _assert_sim_telarray_seed(
+        global_meta, array_model.sim_telarray_seeds, file
+    )
+    if sim_telarray_seed_mismatch:
+        site_parameter_mismatch.append(sim_telarray_seed_mismatch)
+
+    if len(telescope_meta) != len(array_model.telescope_model):
+        raise ValueError(
+            f"Number of telescopes in sim_telarray file ({len(telescope_meta)}) does not match "
+            f"number of telescopes in array model ({len(array_model.telescope_model)})"
+        )
+
+    for telescope_name in array_model.telescope_model.keys():
+        if not get_sim_telarray_telescope_id(telescope_name, file):
+            raise ValueError(f"Telescope {telescope_name} not found in sim_telarray file metadata")
+
+    telescope_parameter_mismatch = [
+        _assert_model_parameters(telescope_meta[i], model)
+        for i, model in enumerate(array_model.telescope_model.values(), start=1)
+    ]
+
+    # ensure printout of all mismatches, not only those found first
+    telescope_parameter_mismatch.append(site_parameter_mismatch)
+    if any(len(m) > 0 for m in telescope_parameter_mismatch):
+        mismatches = [m for m in telescope_parameter_mismatch if len(m) > 0]
+        raise ValueError(
+            f"Telescope or site model parameters do not match sim_telarray metadata: {mismatches}"
+        )
+
+
+def _assert_model_parameters(metadata, model):
+    """
+    Assert that model parameter values matches the values in the sim_telarray metadata.
+
+    Parameters
+    ----------
+    metadata: dict
+        Metadata dictionary.
+    model: SiteModel, TelescopeModel
+        Model to compare with.
+
+    Returns
+    -------
+    invalid_parameter_list: list
+        List of parameters that do not match.
+
+    """
+    config_reader = SimtelConfigReader()
+
+    invalid_parameter_list = []
+
+    for param in model.parameters:
+        sim_telarray_name = _sim_telarray_name_from_parameter_name(param)
+        if sim_telarray_name in metadata.keys():
+            parameter_type = model.parameters[param]["type"]
+            if parameter_type not in ("string", "dict", "boolean"):
+                value, _ = config_reader.extract_value_from_sim_telarray_column(
+                    [metadata[sim_telarray_name]], parameter_type
+                )
+            else:
+                value = metadata[sim_telarray_name]
+                value = (int)(value) if value.isnumeric() else value
+
+            if not is_equal(value, model.parameters[param]["value"], parameter_type):
+                invalid_parameter_list.append(
+                    f"Parameter {param} mismatch between sim_telarray file: {value}, "
+                    f"and model: {model.parameters[param]['value']}"
+                )
+
+    return invalid_parameter_list
+
+
+def _assert_sim_telarray_seed(metadata, sim_telarray_seeds, file=None):
+    """
+    Assert that sim_telarray seed matches the values in the sim_telarray metadata.
+
+    Regenerate seeds using the sim_telarray_random_seeds function and compare with the metadata.
+
+    Parameters
+    ----------
+    metadata: dict
+        Metadata dictionary.
+    sim_telarray_seeds: dict
+        Dictionary of sim_telarray seeds.
+    file : Path
+        Path to the sim_telarray file.
+
+    Returns
+    -------
+    invalid_parameter_list: list
+        Error message if sim_telarray seeds do not match.
+
+    """
+    if not sim_telarray_seeds or not metadata:
+        return None
+
+    if "instrument_seed" in metadata.keys() and "instrument_instances" in metadata.keys():
+        if str(metadata.get("instrument_seed")) != str(sim_telarray_seeds.get("seed")):
+            return (
+                "Parameter instrument_seed mismatch between sim_telarray file: "
+                f"{metadata['instrument_seed']}, and model: {sim_telarray_seeds.get('seed')}"
+            )
+        _logger.info(
+            f"sim_telarray_seed in sim_telarray file: {metadata['instrument_seed']}, "
+            f"and model: {sim_telarray_seeds.get('seed')}"
+        )
+        if file:
+            run_number_modified = get_corsika_run_number(file) - 1
+            test_seeds = sim_telarray_random_seeds(
+                int(metadata["instrument_seed"]), int(metadata["instrument_instances"])
+            )
+            # no +1 as in sim_telarray (as we count from 0)
+            seed_used = run_number_modified % int(metadata["instrument_instances"])
+            if str(metadata.get("rng_select_seed")) != str(test_seeds[seed_used]):
+                return (
+                    "Parameter rng_select_seed mismatch between sim_telarray file: "
+                    f"{metadata['rng_select_seed']}, and model: {test_seeds[seed_used]}"
+                )
+
+    return None
+
+
+def _sim_telarray_name_from_parameter_name(parameter_name):
+    """Return sim_telarray parameter name. Some specific fine tuning."""
+    # parameters like "reference_point_latitude"
+    sim_telarray_name = parameter_name.replace("reference_point_", "")
+
+    if sim_telarray_name == "altitude":
+        return "corsika_observation_level"
+    if sim_telarray_name == "array_triggers":
+        return None
+
+    return sim_telarray_name
+
+
+def is_equal(value1, value2, value_type):
+    """
+    Check if two values are equal based on their type.
+
+    The complexity of this function reflects the complexity of the sim_telarray
+    metadata output.
+
+    Parameters
+    ----------
+    value1: any
+        First value to compare.
+    value2: any
+        Second value to compare.
+    value_type: str
+        Type of the values ('string', 'dict', etc.).
+
+    Returns
+    -------
+    bool
+        True if the values are equal, False otherwise.
+    """
+    value1 = value1[0] if isinstance(value1, tuple) else value1
+    value2 = value2[0] if isinstance(value2, tuple) else value2
+    if value1 is None or value2 is None:
+        if value1 in ("none", None) and value2 in ("none", None):
+            return True
+    if value_type == "string":
+        return str(value1).strip() == str(value2).strip()
+    if value_type == "dict":
+        return value1 == value2
+    if value_type == "boolean":
+        return bool(value1) == bool(value2)
+    return _is_equal_floats_or_ints(value1, value2)
+
+
+def _is_equal_floats_or_ints(value1, value2):
+    """Check if floats and ints are equal."""
+    if isinstance(value1, np.ndarray | list) and isinstance(value2, np.ndarray | list):
+        return bool(np.allclose(np.array(value1), np.array(value2), rtol=1e-10))
+    if isinstance(value1, list) and isinstance(value2, float | int | np.integer | np.floating):
+        if all(x == value1[0] for x in value1):
+            return bool(np.isclose(float(value1[0]), float(value2), rtol=1e-10))
+    if isinstance(value2, list) and isinstance(value1, float | int | np.integer | np.floating):
+        if all(x == value2[0] for x in value2):
+            return bool(np.isclose(float(value1), float(value2[0]), rtol=1e-10))
+    return bool(np.isclose(float(value1), float(value2), rtol=1e-10))

simtools/testing/validate_output.py

@@ -64,11 +64,18 @@ def _validate_output_files(config, integration_test):
 
 def _test_simtel_cfg_files(config, integration_test, from_command_line, from_config_file):
     """Test simtel cfg files."""
-    test_simtel_cfg_file = integration_test.get("TEST_SIMTEL_CFG_FILES", {}).get(
-        from_command_line or from_config_file
-    )
-    if test_simtel_cfg_file:
-        _validate_simtel_cfg_files(config, test_simtel_cfg_file)
+    cfg_files = integration_test.get("TEST_SIMTEL_CFG_FILES", {})
+    if isinstance(from_command_line, list):
+        sources = from_command_line
+    elif isinstance(from_config_file, list):
+        sources = from_config_file
+    else:
+        sources = [from_command_line or from_config_file]
+    for version in sources:
+        cfg = cfg_files.get(version)
+        if cfg:
+            _validate_simtel_cfg_files(config, cfg)
+            break
 
 
 def _validate_reference_output_file(config, integration_test):
@@ -95,7 +102,10 @@ def _validate_output_path_and_file(config, integration_file_tests):
 
         output_file_path = Path(output_path) / file_test["FILE"]
         _logger.info(f"Checking path: {output_file_path}")
-        assert output_file_path.exists()
+        try:
+            assert output_file_path.exists()
+        except AssertionError as exc:
+            raise AssertionError(f"Output file {output_file_path} does not exist. ") from exc
 
         if "EXPECTED_OUTPUT" in file_test:
             assert assertions.check_output_from_sim_telarray(

simtools/utils/general.py

@@ -133,28 +133,13 @@ def collect_data_from_http(url):
     try:
         with tempfile.NamedTemporaryFile(mode="w+t") as tmp_file:
             urllib.request.urlretrieve(url, tmp_file.name)
-            if url.endswith("yml") or url.endswith("yaml"):
-                try:
-                    data = yaml.safe_load(tmp_file)
-                except yaml.constructor.ConstructorError:
-                    data = _load_yaml_using_astropy(tmp_file)
-            elif url.endswith("json"):
-                data = json.load(tmp_file)
-            elif url.endswith("list"):
-                lines = tmp_file.readlines()
-                data = [line.strip() for line in lines]
-            else:
-                msg = f"File extension of {url} not supported (should be json or yaml)"
-                _logger.error(msg)
-                raise TypeError(msg)
+            data = _collect_data_from_different_file_types(
+                tmp_file, url, Path(url).suffix.lower(), None
+            )
     except TypeError as exc:
-        msg = "Invalid url {url}"
-        _logger.error(msg)
-        raise TypeError(msg) from exc
+        raise TypeError(f"Invalid url {url}") from exc
     except urllib.error.HTTPError as exc:
-        msg = f"Failed to download file from {url}"
-        _logger.error(msg)
-        raise FileNotFoundError(msg) from exc
+        raise FileNotFoundError(f"Failed to download file from {url}") from exc
 
     _logger.debug(f"Downloaded file from {url}")
     return data
@@ -182,20 +167,26 @@ def collect_data_from_file(file_name, yaml_document=None):
     suffix = Path(file_name).suffix.lower()
     try:
         with open(file_name, encoding="utf-8") as file:
-            if suffix == ".json":
-                return json.load(file)
-            if suffix == ".list":
-                return [line.strip() for line in file.readlines()]
-            if suffix in [".yml", ".yaml"]:
-                return _collect_data_from_yaml_file(file, file_name, yaml_document)
+            return _collect_data_from_different_file_types(file, file_name, suffix, yaml_document)
     # broad exception to catch all possible errors in reading the file
     except Exception as exc:  # pylint: disable=broad-except
         raise type(exc)(f"Failed to read file {file_name}: {exc}") from exc
     return None
 
 
+def _collect_data_from_different_file_types(file, file_name, suffix, yaml_document):
+    """Collect data from different file types."""
+    if suffix == ".json":
+        return json.load(file)
+    if suffix in (".list", ".txt"):
+        return [line.strip() for line in file.readlines()]
+    if suffix in [".yml", ".yaml"]:
+        return _collect_data_from_yaml_file(file, file_name, yaml_document)
+    raise TypeError(f"File type {suffix} not supported.")
+
+
 def _collect_data_from_yaml_file(file, file_name, yaml_document):
-    """Collect data from a yaml file."""
+    """Collect data from a yaml file (allow for multi-document yaml files)."""
     try:
         return yaml.safe_load(file)
     except yaml.constructor.ConstructorError:

simtools/utils/names.py

@@ -19,7 +19,11 @@ from pathlib import Path
 
 import yaml
 
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH, SCHEMA_PATH
+from simtools.constants import (
+    MODEL_PARAMETER_DESCRIPTION_METASCHEMA,
+    MODEL_PARAMETER_SCHEMA_PATH,
+    SCHEMA_PATH,
+)
 
 _logger = logging.getLogger(__name__)
 
@@ -59,6 +63,21 @@ def array_elements():
         return yaml.safe_load(file)["data"]
 
 
+@cache
+def simulation_software():
+    """
+    Get simulation software names from the meta schema definition.
+
+    Returns
+    -------
+    list
+        List of simulation software names.
+    """
+    with open(Path(MODEL_PARAMETER_DESCRIPTION_METASCHEMA), encoding="utf-8") as file:
+        schema = yaml.safe_load(file)
+        return schema["definitions"]["SimulationSoftwareName"]["enum"]
+
+
 @cache
 def site_names():
     """
@@ -129,15 +148,18 @@ def _load_model_parameters():
     """
     Get model parameters properties from schema files.
 
+    For schema files including multiple schemas, only the first one is returned
+    (as this is the most recent definition).
+
     Returns
     -------
     dict
         Model parameters definitions for all model parameters.
     """
     _parameters = {}
-    for schema_file in list(Path(MODEL_PARAMETER_SCHEMA_PATH).rglob("*.yml")):
+    for schema_file in Path(MODEL_PARAMETER_SCHEMA_PATH).rglob("*.yml"):
         with open(schema_file, encoding="utf-8") as f:
-            data = yaml.safe_load(f)
+            data = next(yaml.safe_load_all(f))
             _parameters[data["name"]] = data
     return _parameters
 
@@ -512,7 +534,7 @@ def get_collection_name_from_parameter_name(parameter_name):
 
 def get_simulation_software_name_from_parameter_name(
     parameter_name,
-    simulation_software="sim_telarray",
+    software_name="sim_telarray",
     set_meta_parameter=False,
 ):
     """
@@ -541,7 +563,7 @@ def get_simulation_software_name_from_parameter_name(
 
     for software in _parameter.get("simulation_software", []):
         if (
-            software.get("name") == simulation_software
+            software.get("name") == software_name
             and software.get("set_meta_parameter", False) is set_meta_parameter
         ):
             return software.get("internal_parameter_name", parameter_name)
@@ -636,7 +658,7 @@ def generate_file_name(
     """
     Generate a file name for output, config, or plotting.
 
-    Used e.g., to generate camera-efficiency and ray-tracing output files.
+    Used e.g., to generate camera_efficiency and ray_tracing output files.
 
     Parameters
     ----------
@@ -668,10 +690,10 @@ def generate_file_name(
     str
         File name.
     """
-    name = f"{file_type}-{site}-{telescope_model_name}"
-    name += f"-d{source_distance:.1f}km" if source_distance is not None else ""
-    name += f"-za{float(zenith_angle):.1f}deg"
-    name += f"-off{off_axis_angle:.3f}deg" if off_axis_angle is not None else ""
+    name = f"{file_type}_{site}_{telescope_model_name}"
+    name += f"_d{source_distance:.1f}km" if source_distance is not None else ""
+    name += f"_za{float(zenith_angle):.1f}deg"
+    name += f"_off{off_axis_angle:.3f}deg" if off_axis_angle is not None else ""
     name += f"_azm{round(azimuth_angle):03}deg" if azimuth_angle is not None else ""
     name += f"_mirror{mirror_number}" if mirror_number is not None else ""
     name += f"_{label}" if label is not None else ""

simtools/visualization/plot_array_layout.py

@@ -0,0 +1,242 @@
+#!/usr/bin/python3
+"""Plot array elements for a layout."""
+
+from collections import Counter
+
+import astropy.units as u
+import matplotlib.patches as mpatches
+import matplotlib.pyplot as plt
+from astropy.table import Column
+from matplotlib.collections import PatchCollection
+
+from simtools.utils import geometry as transf
+from simtools.utils import names
+from simtools.visualization import legend_handlers as leg_h
+
+__all__ = ["get_telescope_patch", "plot_array_layout"]
+
+
+def plot_array_layout(
+    telescopes,
+    show_tel_label=False,
+    axes_range=None,
+    marker_scaling=1.0,
+    background_telescopes=None,
+):
+    """
+    Plot telescope array layout.
+
+    Parameters
+    ----------
+    telescopes : Table
+        Telescope data table.
+    show_tel_label : bool
+        Show telescope labels (default False).
+    axes_range : float or None
+        Axis range, auto if None.
+    marker_scaling : float
+        Marker size scale factor.
+    background_telescopes : Table or None
+        Optional background telescope table.
+
+    Returns
+    -------
+    fig : Figure
+        Matplotlib figure object.
+    """
+    fig, ax = plt.subplots(1)
+
+    patches, plot_range = get_patches(ax, telescopes, show_tel_label, axes_range, marker_scaling)
+
+    if background_telescopes is not None:
+        bg_patches, bg_range = get_patches(
+            ax, background_telescopes, False, axes_range, marker_scaling
+        )
+        ax.add_collection(PatchCollection(bg_patches, match_original=True, alpha=0.1))
+        if axes_range is None:
+            plot_range = max(plot_range, bg_range)
+
+    update_legend(ax, telescopes)
+    finalize_plot(ax, patches, "Easting [m]", "Northing [m]", plot_range)
+
+    return fig
+
+
+def get_patches(ax, telescopes, show_tel_label, axes_range, marker_scaling):
+    """
+    Get plot patches and axis range.
+
+    Returns
+    -------
+    patches : list
+        List of telescope patches.
+    axes_range : float
+        Calculated or input axis range.
+    """
+    pos_x, pos_y = get_positions(telescopes)
+    telescopes["pos_x_rotated"] = Column(pos_x)
+    telescopes["pos_y_rotated"] = Column(pos_y)
+
+    patches, radii = create_patches(telescopes, marker_scaling, show_tel_label, ax)
+
+    if axes_range:
+        return patches, axes_range
+
+    r = max(radii).value
+    max_x = max(abs(pos_x.min().value), abs(pos_x.max().value)) + r
+    max_y = max(abs(pos_y.min().value), abs(pos_y.max().value)) + r
+    updated_axes_range = max(max_x, max_y) * 1.1
+
+    return patches, updated_axes_range
+
+
+@u.quantity_input(x=u.m, y=u.m, radius=u.m)
+def get_telescope_patch(name, x, y, radius):
+    """
+    Create patch for a telescope.
+
+    Returns
+    -------
+    patch : Patch
+        Circle or rectangle patch.
+    """
+    tel_obj = leg_h.TelescopeHandler()
+    tel_type = names.get_array_element_type_from_name(name)
+    x, y, r = x.to(u.m), y.to(u.m), radius.to(u.m)
+
+    if tel_type == "SCTS":
+        return mpatches.Rectangle(
+            ((x - r / 2).value, (y - r / 2).value),
+            width=r.value,
+            height=r.value,
+            fill=False,
+            color=tel_obj.colors_dict[tel_type],
+        )
+
+    return mpatches.Circle(
+        (x.value, y.value),
+        radius=r.value,
+        fill=tel_type.startswith("MST"),
+        color=tel_obj.colors_dict[tel_type],
+    )
+
+
+def get_positions(telescopes):
+    """
+    Get X/Y positions depending on coordinate system.
+
+    For ground coordinates, rotates the positions by 90 degrees.
+
+    Returns
+    -------
+    x_rot, y_rot : Quantity
+        Position coordinates.
+    """
+    if "position_x" in telescopes.colnames:
+        x, y = telescopes["position_x"], telescopes["position_y"]
+        locale_rotate_angle = 90 * u.deg
+    elif "utm_east" in telescopes.colnames:
+        x, y = telescopes["utm_east"], telescopes["utm_north"]
+        locale_rotate_angle = 0 * u.deg
+    else:
+        raise ValueError("Missing required position columns.")
+
+    return transf.rotate(x, y, locale_rotate_angle) if locale_rotate_angle != 0 else (x, y)
+
+
+def create_patches(telescopes, scale, show_label, ax):
+    """
+    Create telescope patches and labels.
+
+    Returns
+    -------
+    patches : list
+        Shape patches.
+    radii : list
+        Telescope radii.
+    """
+    patches, radii = [], []
+    fontsize, scale_factor = (4, 2) if len(telescopes) > 30 else (8, 1)
+
+    for tel in telescopes:
+        name = get_telescope_name(tel)
+        radius = get_sphere_radius(tel)
+        radii.append(radius)
+        tel_type = names.get_array_element_type_from_name(name)
+
+        patches.append(
+            get_telescope_patch(
+                tel_type,
+                tel["pos_x_rotated"],
+                tel["pos_y_rotated"],
+                scale_factor * radius * scale,
+            )
+        )
+
+        if show_label:
+            ax.text(
+                tel["pos_x_rotated"].value,
+                tel["pos_y_rotated"].value + scale_factor * radius.value,
+                name,
+                ha="center",
+                va="bottom",
+                fontsize=fontsize,
+            )
+
+    return patches, radii
+
+
+def get_telescope_name(tel):
+    """
+    Get telescope name.
+
+    Returns
+    -------
+    name : str
+        Telescope name or fallback identifier.
+    """
+    if "telescope_name" in tel.colnames:
+        return tel["telescope_name"]
+    if "asset_code" in tel.colnames and "sequence_number" in tel.colnames:
+        return f"{tel['asset_code']}-{tel['sequence_number']}"
+    return f"tel_{tel.index}"
+
+
+def get_sphere_radius(tel):
+    """
+    Get telescope sphere radius.
+
+    Returns
+    -------
+    radius : Quantity
+        Radius with units.
+    """
+    return tel["sphere_radius"] if "sphere_radius" in tel.colnames else 10.0 * u.m
+
+
+def update_legend(ax, telescopes):
+    """Add legend for telescope types and counts."""
+    names_list = [get_telescope_name(tel) for tel in telescopes]
+    types = [names.get_array_element_type_from_name(n) for n in names_list]
+    counts = Counter(types)
+
+    objs, labels = [], []
+    for t in names.get_list_of_array_element_types():
+        if counts[t]:
+            objs.append(leg_h.all_telescope_objects[t]())
+            labels.append(f"{t} ({counts[t]})")
+
+    handler_map = {k: v() for k, v in leg_h.legend_handler_map.items()}
+    ax.legend(objs, labels, handler_map=handler_map, prop={"size": 11}, loc="best")
+
+
+def finalize_plot(ax, patches, x_title, y_title, axes_range):
+    """Finalize plot appearance and limits."""
+    ax.add_collection(PatchCollection(patches, match_original=True))
+    ax.set(xlabel=x_title, ylabel=y_title)
+    ax.tick_params(labelsize=8)
+    ax.axis("square")
+    if axes_range:
+        ax.set_xlim(-axes_range, axes_range)
+        ax.set_ylim(-axes_range, axes_range)
+    plt.tight_layout()