gammasimtools 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

simtools/data_model/model_data_writer.py

@@ -6,7 +6,6 @@ from pathlib import Path
 
 import astropy.units as u
 import numpy as np
-import yaml
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
@@ -95,7 +94,7 @@ class ModelDataWriter:
             Dictionary with configuration parameters (including output file name and path).
         output_file: string or Path
             Name of output file (args["output_file"] is used if this parameter is not set).
-        metadata: dict
+        metadata: MetadataCollector object
             Metadata to be written.
         product_data: astropy Table
             Model data to be written
@@ -174,11 +173,11 @@ class ModelDataWriter:
         if metadata_input_dict is not None:
             metadata_input_dict["output_file"] = output_file
             metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
-            metadata = MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata()
-            writer.write_metadata_to_yml(
-                metadata=metadata, yml_file=output_path / f"{Path(output_file).stem}"
+            metadata = MetadataCollector(args_dict=metadata_input_dict)
+            metadata.write(output_path / Path(output_file))
+            unique_id = (
+                metadata.get_top_level_metadata().get("cta", {}).get("product", {}).get("id")
             )
-            unique_id = metadata.get("cta", {}).get("product", {}).get("id")
 
         _json_dict = writer.get_validated_parameter_dict(
             parameter_name, value, instrument, parameter_version, unique_id
@@ -364,7 +363,7 @@ class ModelDataWriter:
         ----------
         product_data: astropy Table
             Model data to be written
-        metadata: dict
+        metadata: MetadataCollector object
             Metadata to be written.
 
         Raises
@@ -377,7 +376,9 @@ class ModelDataWriter:
             return
 
         if metadata is not None:
-            product_data.meta.update(gen.change_dict_keys_case(metadata, False))
+            product_data.meta.update(
+                gen.change_dict_keys_case(metadata.get_top_level_metadata(), False)
+            )
 
         self._logger.info(f"Writing data to {self.product_data_file}")
         if isinstance(product_data, dict) and Path(self.product_data_file).suffix == ".json":
@@ -390,6 +391,8 @@ class ModelDataWriter:
         except IORegistryError:
             self._logger.error(f"Error writing model data to {self.product_data_file}.")
             raise
+        if metadata is not None:
+            metadata.write(self.product_data_file, add_activity_name=True)
 
     def write_dict_to_model_parameter_json(self, file_name, data_dict):
         """
@@ -449,53 +452,6 @@ class ModelDataWriter:
             pass
         return data_dict
 
-    def write_metadata_to_yml(self, metadata, yml_file=None, keys_lower_case=False):
-        """
-        Write model metadata file (yaml file format).
-
-        Parameters
-        ----------
-        metadata: dict
-            Metadata to be stored
-        yml_file: str
-            Name of output file.
-        keys_lower_case: bool
-            Write yaml keys in lower case.
-
-        Returns
-        -------
-        str
-            Name of output file
-
-        Raises
-        ------
-        FileNotFoundError
-            If yml_file not found.
-        TypeError
-            If yml_file is not defined.
-        """
-        try:
-            yml_file = names.file_name_with_version(
-                yml_file or self.product_data_file, ".metadata.yml"
-            )
-            with open(yml_file, "w", encoding="UTF-8") as file:
-                yaml.safe_dump(
-                    gen.change_dict_keys_case(metadata, keys_lower_case),
-                    file,
-                    sort_keys=False,
-                )
-            self._logger.info(f"Writing metadata to {yml_file}")
-            return yml_file
-        except FileNotFoundError:
-            self._logger.error(f"Error writing model data to {yml_file}")
-            raise
-        except AttributeError:
-            self._logger.error("No metadata defined for writing")
-            raise
-        except TypeError:
-            self._logger.error("No output file for metadata defined")
-            raise
-
     @staticmethod
     def _astropy_data_format(product_data_format):
         """

simtools/data_model/schema.py

@@ -108,7 +108,7 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     """
     if json_schema is None and schema_file is None:
         _logger.warning(f"No schema provided for validation of {data}")
-        return
+        return None
     if json_schema is None:
         json_schema = load_schema(
             schema_file,
@@ -131,6 +131,7 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
         raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
 
     _logger.debug(f"Successful validation of data using schema ({json_schema.get('name')})")
+    return data
 
 
 def load_schema(schema_file=None, schema_version=None):

simtools/data_model/validate_data.py

@@ -220,7 +220,7 @@ class DataValidator:
                 json_schema=self._get_data_description(index).get("json_schema"),
             )
         else:
-            self._check_data_type(np.array(value).dtype, index)
+            self._check_data_type(np.array(value).dtype, index, value)
 
         if self.data_dict.get("type") not in ("string", "dict", "file"):
             self._check_for_not_a_number(value, index)
@@ -436,7 +436,7 @@ class DataValidator:
 
         return u.Unit(reference_unit)
 
-    def _check_data_type(self, dtype, column_name):
+    def _check_data_type(self, dtype, column_name, value=None):
         """
         Check column data type.
 
@@ -446,6 +446,8 @@ class DataValidator:
             data type
         column_name: str
             column name
+        value: value
+            value to be tested (optional)
 
         Raises
         ------
@@ -456,7 +458,7 @@ class DataValidator:
         reference_dtype = self._get_data_description(column_name).get("type", None)
         if not gen.validate_data_type(
             reference_dtype=reference_dtype,
-            value=None,
+            value=value,
             dtype=dtype,
             allow_subtypes=(not self.check_exact_data_type),
         ):

simtools/db/db_handler.py

@@ -2,6 +2,7 @@
 
 import logging
 import re
+from collections import defaultdict
 from pathlib import Path
 from threading import Lock
 
@@ -13,6 +14,7 @@ from pymongo import MongoClient
 
 from simtools.data_model import validate_data
 from simtools.io_operations import io_handler
+from simtools.simtel import simtel_table_reader
 from simtools.utils import names, value_conversion
 
 __all__ = ["DatabaseHandler"]
@@ -113,6 +115,7 @@ class DatabaseHandler:
         direct_connection = self.mongo_db_config["db_server"] in (
             "localhost",
             "simtools-mongodb",
+            "mongodb",
         )
         return MongoClient(
             self.mongo_db_config["db_server"],
@@ -177,48 +180,62 @@ class DatabaseHandler:
     def get_model_parameter(
         self,
         parameter,
-        parameter_version,
         site,
         array_element_name,
+        parameter_version=None,
+        model_version=None,
     ):
         """
-        Get a model parameter using the parameter version.
+        Get a single model parameter using model or parameter version.
+
+        Note that this function should not be called in a loop for many parameters,
+        as it each call queries the database.
 
         Parameters
         ----------
         parameter: str
             Name of the parameter.
-        parameter_version: str
-            Version of the parameter.
         site: str
             Site name.
         array_element_name: str
-            Name of the array element model (e.g. MSTN, SSTS).
+            Name of the array element model.
+        parameter_version: str
+            Version of the parameter.
+        model_version: str
+            Version of the model.
 
         Returns
         -------
         dict containing the parameter
 
         """
+        collection_name = names.get_collection_name_from_parameter_name(parameter)
+        if model_version:
+            production_table = self._read_production_table_from_mongo_db(
+                collection_name, model_version
+            )
+            array_element_list = self._get_array_element_list(
+                array_element_name, site, production_table, collection_name
+            )
+            for array_element in reversed(array_element_list):
+                parameter_version = (
+                    production_table["parameters"].get(array_element, {}).get(parameter)
+                )
+                if parameter_version:
+                    array_element_name = array_element
+                    break
+
         query = {
             "parameter_version": parameter_version,
             "parameter": parameter,
         }
-        if array_element_name is not None:
+        if array_element_name:
             query["instrument"] = array_element_name
-        if site is not None:
+        if site:
             query["site"] = site
-        return self._read_mongo_db(
-            query=query, collection_name=names.get_collection_name_from_parameter_name(parameter)
-        )
+        return self._read_mongo_db(query=query, collection_name=collection_name)
 
-    def get_model_parameters(
-        self,
-        site,
-        array_element_name,
-        collection,
-        model_version=None,
-    ):
+    def get_model_parameters(self, site, array_element_name, collection, model_version):
         """
         Get model parameters using the model version.
 
@@ -239,22 +256,44 @@ class DatabaseHandler:
         -------
         dict containing the parameters
         """
-        model_versions = (
-            self.get_model_versions(collection) if model_version is None else [model_version]
+        pars = {}
+        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        array_element_list = self._get_array_element_list(
+            array_element_name, site, production_table, collection
         )
+        for array_element in array_element_list:
+            pars.update(
+                self._get_parameter_for_model_version(
+                    array_element, model_version, site, collection, production_table
+                )
+            )
+        return pars
 
-        pars = {}
-        for _model_version in model_versions:
-            production_table = self._read_production_table_from_mongo_db(collection, _model_version)
-            array_element_list = self._get_array_element_list(
-                array_element_name, site, production_table, collection
+    def get_model_parameters_for_all_model_versions(self, site, array_element_name, collection):
+        """
+        Get model parameters for all model versions.
+
+        Queries parameters for design and for the specified array element (if necessary).
+
+        Parameters
+        ----------
+        site: str
+            Site name.
+        array_element_name: str
+            Name of the array element model (e.g. LSTN-01, MSTx-FlashCam, ILLN-01).
+        collection: str
+            Collection of array element (e.g. telescopes, calibration_devices).
+
+        Returns
+        -------
+        dict containing the parameters with model version as first key
+        """
+        pars = defaultdict(dict)
+        for _model_version in self.get_model_versions(collection):
+            parameter_data = self.get_model_parameters(
+                site, array_element_name, collection, _model_version
             )
-            for array_element in array_element_list:
-                pars.update(
-                    self._get_parameter_for_model_version(
-                        array_element, _model_version, site, collection, production_table
-                    )
-                )
+            pars[_model_version].update(parameter_data)
         return pars
 
     def _get_parameter_for_model_version(
@@ -331,9 +370,54 @@ class DatabaseHandler:
             return [collection for collection in collections if not collection.startswith("fs.")]
         return collections
 
+    def export_model_file(
+        self,
+        parameter,
+        site,
+        array_element_name,
+        model_version=None,
+        parameter_version=None,
+        export_file_as_table=False,
+    ):
+        """
+        Export single model file from the DB identified by the parameter name.
+
+        The parameter can be identified by model or parameter version.
+        Files can be exported as astropy tables (ecsv format).
+
+        Parameters
+        ----------
+        parameter: str
+            Name of the parameter.
+        site: str
+            Site name.
+        array_element_name: str
+            Name of the array element model (e.g. MSTN, SSTS).
+        parameter_version: str
+            Version of the parameter.
+        model_version: str
+            Version of the model.
+        export_file_as_table: bool
+            If True, export the file as an astropy table (ecsv format).
+        """
+        parameters = self.get_model_parameter(
+            parameter,
+            site,
+            array_element_name,
+            parameter_version=parameter_version,
+            model_version=model_version,
+        )
+        self.export_model_files(parameters=parameters, dest=self.io_handler.get_output_directory())
+        if export_file_as_table:
+            return simtel_table_reader.read_simtel_table(
+                parameter,
+                self.io_handler.get_output_directory().joinpath(parameters[parameter]["value"]),
+            )
+        return None
+
     def export_model_files(self, parameters=None, file_names=None, dest=None, db_name=None):
         """
-        Export files from the DB to the model directory.
+        Export models files from the DB to given directory.
 
         The files to be exported can be specified by file_name or retrieved from the database
         using the parameters dictionary.

simtools/model/model_parameter.py

@@ -6,12 +6,10 @@ import shutil
 from copy import copy
 
 import astropy.units as u
-from astropy.table import Table
 
 import simtools.utils.general as gen
 from simtools.db import db_handler
 from simtools.io_operations import io_handler
-from simtools.simtel import simtel_table_reader
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
 from simtools.utils import names
 
@@ -517,35 +515,6 @@ class ModelParameter:
         self.db.export_model_files(parameters=pars_from_db, dest=self.config_file_directory)
         self._is_exported_model_files_up_to_date = True
 
-    def get_model_file_as_table(self, par_name):
-        """
-        Return tabular data from file as astropy table.
-
-        Parameters
-        ----------
-        par_name: str
-            Name of the parameter.
-
-        Returns
-        -------
-        Table
-            Astropy table.
-        """
-        _par_entry = {}
-        try:
-            _par_entry[par_name] = self._parameters[par_name]
-        except KeyError as exc:
-            raise ValueError(f"Parameter {par_name} not found in the model.") from exc
-        self.db.export_model_files(parameters=_par_entry, dest=self.config_file_directory)
-        if _par_entry[par_name]["value"].endswith("ecsv"):
-            return Table.read(
-                self.config_file_directory.joinpath(_par_entry[par_name]["value"]),
-                format="ascii.ecsv",
-            )
-        return simtel_table_reader.read_simtel_table(
-            par_name, self.config_file_directory.joinpath(_par_entry[par_name]["value"])
-        )
-
     def export_config_file(self):
         """Export the config file used by sim_telarray."""
         # Exporting model file

simtools/production_configuration/derive_corsika_limits.py

@@ -0,0 +1,260 @@
+"""Calculate the thresholds for energy, radial distance, and viewcone."""
+
+import astropy.units as u
+import matplotlib.pyplot as plt
+import numpy as np
+import tables
+from astropy.coordinates import AltAz
+from ctapipe.coordinates import GroundFrame, TiltedGroundFrame
+
+
+class LimitCalculator:
+    """
+    Compute thresholds/limits for energy, radial distance, and viewcone.
+
+    Event data is read from the reduced MC event data file.
+
+    Parameters
+    ----------
+    event_data_file : str
+        Path to the HDF5 file containing the event data.
+    telescope_list : list, optional
+        List of telescope IDs to filter the events (default is None).
+    """
+
+    def __init__(self, event_data_file, telescope_list=None):
+        """
+        Initialize the LimitCalculator with the given event data file.
+
+        Parameters
+        ----------
+        event_data_file : str
+            Path to the reduced MC event data file.
+        telescope_list : list, optional
+            List of telescope IDs to filter the events (default is None).
+        """
+        self.event_data_file = event_data_file
+        self.telescope_list = telescope_list
+        self.event_x_core = None
+        self.event_y_core = None
+        self.simulated = None
+        self.shower_id_triggered = None
+        self.list_of_files = None
+        self.shower_sim_azimuth = None
+        self.shower_sim_altitude = None
+        self.array_azimuth = None
+        self.array_altitude = None
+        self.trigger_telescope_list_list = None
+        self.units = {}
+        self._read_event_data()
+
+    def _read_event_data(self):
+        """Read the event data from the reduced MC event data file."""
+        with tables.open_file(self.event_data_file, mode="r") as f:
+            reduced_data = f.root.data.reduced_data
+            triggered_data = f.root.data.triggered_data
+            file_names = f.root.data.file_names
+            trigger_telescope_list_list = f.root.data.trigger_telescope_list_list
+
+            self.event_x_core = reduced_data.col("core_x")
+            self.event_y_core = reduced_data.col("core_y")
+            self.simulated = reduced_data.col("simulated")
+            self.shower_id_triggered = triggered_data.col("shower_id_triggered")
+            self.list_of_files = file_names.col("file_names")
+            self.shower_sim_azimuth = reduced_data.col("shower_sim_azimuth")
+            self.shower_sim_altitude = reduced_data.col("shower_sim_altitude")
+            self.array_altitude = reduced_data.col("array_altitudes")
+            self.array_azimuth = reduced_data.col("array_azimuths")
+
+            self.trigger_telescope_list_list = [
+                [np.int16(tel) for tel in event] for event in trigger_telescope_list_list
+            ]
+
+    def _compute_limits(self, hist, bin_edges, loss_fraction, limit_type="lower"):
+        """
+        Compute the limits based on the loss fraction.
+
+        Parameters
+        ----------
+        hist : np.ndarray
+            1D histogram array.
+        bin_edges : np.ndarray
+            Array of bin edges.
+        loss_fraction : float
+            Fraction of events to be lost.
+        limit_type : str, optional
+            Type of limit ('lower' or 'upper'). Default is 'lower'.
+
+        Returns
+        -------
+        float
+            Bin edge value corresponding to the threshold.
+        """
+        cumulative_sum = np.cumsum(hist) if limit_type == "upper" else np.cumsum(hist[::-1])
+        total_events = np.sum(hist)
+        threshold = (1 - loss_fraction) * total_events
+        bin_index = np.searchsorted(cumulative_sum, threshold)
+        return bin_edges[bin_index] if limit_type == "upper" else bin_edges[-bin_index]
+
+    def _prepare_data_for_limits(self):
+        """
+        Prepare the data required for computing limits.
+
+        Returns
+        -------
+        tuple
+            Tuple containing core distances, triggered energies, core bins, and energy bins.
+        """
+        shower_id_triggered_masked = self.shower_id_triggered
+        if self.telescope_list is not None:
+            mask = np.array(
+                [
+                    all(tel in event for tel in self.telescope_list)
+                    for event in self.trigger_telescope_list_list
+                ]
+            )
+            shower_id_triggered_masked = self.shower_id_triggered[mask]
+
+        triggered_energies = self.simulated[shower_id_triggered_masked]
+        energy_bins = np.logspace(
+            np.log10(triggered_energies.min()), np.log10(triggered_energies.max()), 1000
+        )
+        event_x_core_shower, event_y_core_shower = self._transform_to_shower_coordinates()
+        core_distances_all = np.sqrt(event_x_core_shower**2 + event_y_core_shower**2)
+        core_distances_triggered = core_distances_all[shower_id_triggered_masked]
+        core_bins = np.linspace(
+            core_distances_triggered.min(), core_distances_triggered.max(), 1000
+        )
+
+        return core_distances_triggered, triggered_energies, core_bins, energy_bins
+
+    def compute_lower_energy_limit(self, loss_fraction):
+        """
+        Compute the lower energy limit in TeV based on the event loss fraction.
+
+        Parameters
+        ----------
+        loss_fraction : float
+            Fraction of events to be lost.
+
+        Returns
+        -------
+        astropy.units.Quantity
+            Lower energy limit.
+        """
+        _, triggered_energies, _, energy_bins = self._prepare_data_for_limits()
+
+        hist, _ = np.histogram(triggered_energies, bins=energy_bins)
+        lower_bin_edge_value = self._compute_limits(
+            hist, energy_bins, loss_fraction, limit_type="lower"
+        )
+        return lower_bin_edge_value * u.TeV
+
+    def compute_upper_radial_distance(self, loss_fraction):
+        """
+        Compute the upper radial distance based on the event loss fraction.
+
+        Parameters
+        ----------
+        loss_fraction : float
+            Fraction of events to be lost.
+
+        Returns
+        -------
+        astropy.units.Quantity
+            Upper radial distance in m.
+        """
+        core_distances_triggered, _, core_bins, _ = self._prepare_data_for_limits()
+
+        hist, _ = np.histogram(core_distances_triggered, bins=core_bins)
+        upper_bin_edge_value = self._compute_limits(
+            hist, core_bins, loss_fraction, limit_type="upper"
+        )
+        return upper_bin_edge_value * u.m
+
+    def compute_viewcone(self, loss_fraction):
+        """
+        Compute the viewcone based on the event loss fraction.
+
+        Parameters
+        ----------
+        loss_fraction : float
+            Fraction of events to be lost.
+
+        Returns
+        -------
+        astropy.units.Quantity
+            Viewcone radius in degrees.
+        """
+        # already in radians
+        azimuth_diff = self.array_azimuth - self.shower_sim_azimuth  # * (np.pi / 180.0)
+        sim_altitude_rad = self.shower_sim_altitude  # * (np.pi / 180.0)
+        array_altitude_rad = self.array_altitude  # * (np.pi / 180.0)
+        x_1 = np.cos(azimuth_diff) * np.cos(sim_altitude_rad)
+        y_1 = np.sin(azimuth_diff) * np.cos(sim_altitude_rad)
+        z_1 = np.sin(sim_altitude_rad)
+        x_2 = x_1 * np.sin(array_altitude_rad) - z_1 * np.cos(array_altitude_rad)
+        y_2 = y_1
+        z_2 = x_1 * np.cos(array_altitude_rad) + z_1 * np.sin(array_altitude_rad)
+        off_angles = np.arctan2(np.sqrt(x_2**2 + y_2**2), z_2) * (180.0 / np.pi)
+        angle_bins = np.linspace(off_angles.min(), off_angles.max(), 400)
+        hist, _ = np.histogram(off_angles, bins=angle_bins)
+
+        upper_bin_edge_value = self._compute_limits(
+            hist, angle_bins, loss_fraction, limit_type="upper"
+        )
+        return upper_bin_edge_value * u.deg
+
+    def _transform_to_shower_coordinates(self):
+        """
+        Transform core positions from ground coordinates to shower coordinates.
+
+        Returns
+        -------
+        tuple
+            Core positions in shower coordinates (x, y).
+        """
+        pointing_az = self.shower_sim_azimuth * u.rad
+        pointing_alt = self.shower_sim_altitude * u.rad
+
+        pointing = AltAz(az=pointing_az, alt=pointing_alt)
+        ground = GroundFrame(x=self.event_x_core * u.m, y=self.event_y_core * u.m, z=0 * u.m)
+        shower_frame = ground.transform_to(TiltedGroundFrame(pointing_direction=pointing))
+
+        return shower_frame.x.value, shower_frame.y.value
+
+    def plot_data(self):
+        """Plot the core distances and energies of triggered events."""
+        shower_id_triggered_masked = self.shower_id_triggered
+        if self.telescope_list is not None:
+            mask = np.array(
+                [
+                    all(tel in event for tel in self.telescope_list)
+                    for event in self.trigger_telescope_list_list
+                ]
+            )
+            shower_id_triggered_masked = self.shower_id_triggered[mask]
+
+        core_distances_all = np.sqrt(self.event_x_core**2 + self.event_y_core**2)
+        core_distances_triggered = core_distances_all[shower_id_triggered_masked]
+        triggered_energies = self.simulated[shower_id_triggered_masked]
+
+        core_bins = np.linspace(core_distances_triggered.min(), core_distances_triggered.max(), 400)
+        energy_bins = np.logspace(
+            np.log10(triggered_energies.min()), np.log10(triggered_energies.max()), 400
+        )
+        plt.figure(figsize=(8, 6))
+        plt.hist2d(
+            core_distances_triggered,
+            triggered_energies,
+            bins=[core_bins, energy_bins],
+            norm="log",
+            cmap="viridis",
+        )
+
+        plt.colorbar(label="Event Count")
+        plt.xlabel("Core Distance [m]")
+        plt.ylabel("Energy [TeV]")
+        plt.yscale("log")
+        plt.title("2D Histogram of Triggered Core Distance vs Energy")
+        plt.show()