gammasimtools 0.12.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

simtools/simtel/simtel_config_reader.py

@@ -4,13 +4,38 @@
 import logging
 import re
 
+import astropy.units as u
 import numpy as np
 
 import simtools.utils.general as gen
+from simtools.utils import names
 
 __all__ = ["SimtelConfigReader"]
 
 
+def get_list_of_simtel_parameters(simtel_config_file):
+    """
+    Return list of simtel parameters found in simtel configuration file.
+
+    Parameters
+    ----------
+    simtel_config_file: str
+        File name for sim_telarray configuration
+
+    Returns
+    -------
+    list
+        List of parameters found in simtel configuration file.
+
+    """
+    simtel_parameter_set = set()
+    with open(simtel_config_file, encoding="utf-8") as file:
+        for line in file:
+            parts_of_lines = re.split(r",\s*|\s+", line.strip())
+            simtel_parameter_set.add(parts_of_lines[1].lower())
+    return sorted(simtel_parameter_set)
+
+
 class SimtelConfigReader:
     """
     Reads model parameters from configuration files and converts to the simtools representation.
@@ -26,7 +51,7 @@ class SimtelConfigReader:
             -C limits=no-internal -C initlist=no-internal -C list=no-internal\
             -C typelist=no-internal -C maximum_telescopes=30\
             -DNSB_AUTOSCALE -DNECTARCAM -DHYPER_LAYOUT\
-            -DNUM_TELESCOPES=30 /dev/null 2>|/dev/null | grep '(@cfg)'
+            -DNUM_TELESCOPES=30 /dev/null 2>|/dev/null | grep '(@cfg)' | sed 's/^(@cfg)
 
     Parameters
     ----------
@@ -61,7 +86,12 @@ class SimtelConfigReader:
             else None
         )
         self.parameter_name = self.schema_dict.get("name") if self.schema_dict else parameter_name
-        self.simtel_parameter_name = self._get_simtel_parameter_name(self.parameter_name)
+        try:
+            self.simtel_parameter_name = names.get_simulation_software_name_from_parameter_name(
+                self.parameter_name
+            ).upper()
+        except (KeyError, AttributeError):
+            self.simtel_parameter_name = self.parameter_name.upper()
         self.simtel_telescope_name = simtel_telescope_name
         self.camera_pixels = camera_pixels
         self.parameter_dict = self.read_simtel_config_file(
@@ -73,7 +103,7 @@ class SimtelConfigReader:
         return data_type == "limits" and self.parameter_dict.get("type") == "bool"
 
     def _get_schema_values(self, data_type):
-        """Check schema values for limits and defaults."""
+        """Check schema values for limits, unit, and default."""
         try:
             if data_type == "limits":
                 _from_schema = [
@@ -82,14 +112,25 @@ class SimtelConfigReader:
                 ]
                 return _from_schema[0] if _from_schema[1] is None else _from_schema
             if len(self.schema_dict["data"]) == 1:
-                return self.schema_dict["data"][0]["default"]
-            return [data.get("default") for data in self.schema_dict["data"]]
+                return self.schema_dict["data"][0].get(data_type)
+            return [data.get(data_type) for data in self.schema_dict["data"]]
         except (KeyError, IndexError):
             return None
 
     @staticmethod
     def _values_match(_from_simtel, _from_schema):
-        """Check if values match (are close for floats)."""
+        """
+        Check if values match (are close for floats).
+
+        Convert where necessary astropy.Quantity to float.
+
+        """
+        if isinstance(_from_simtel, u.Quantity):
+            _from_simtel = _from_simtel.value
+        if isinstance(_from_simtel, np.ndarray) and len(_from_simtel) > 0:
+            _from_simtel = np.array(
+                [v.value if isinstance(v, u.Quantity) else v for v in _from_simtel]
+            )
         try:
             if not isinstance(_from_schema, list | np.ndarray) and _from_simtel == _from_schema:
                 return True
@@ -184,6 +225,7 @@ class SimtelConfigReader:
                     dtype=_para_dict.get("type"),
                     n_dim=_para_dict.get("dimension"),
                     default=_para_dict.get("default"),
+                    is_limit=(key == "limits"),
                 )
             except KeyError:
                 pass
@@ -232,7 +274,7 @@ class SimtelConfigReader:
 
         return column, except_from_all
 
-    def _add_value_from_simtel_cfg(self, column, dtype=None, n_dim=1, default=None):
+    def _add_value_from_simtel_cfg(self, column, dtype=None, n_dim=1, default=None, is_limit=False):
         """
         Extract value(s) from simtel configuration file columns.
 
@@ -276,7 +318,42 @@ class SimtelConfigReader:
         if dtype == "bool":
             column = np.array([bool(int(item)) for item in column])
 
-        return self._process_column(column, dtype)
+        column, ndim = self._process_column(column, dtype)
+        if not is_limit:
+            column = self._add_units(column)
+        return column, ndim
+
+    def _add_units(self, column):
+        """
+        Add units as given in schema file to column.
+
+        Take into account array types and dimensionless units.
+        Ensure that integer values are returned as integers (astropy converts
+        values to floats when multiplying them with units).
+
+        """
+        try:
+            unit = self._get_schema_values("unit")
+        except TypeError:  # no schema defined
+            return column
+        if unit is None or unit == "dimensionless":
+            return column
+
+        if isinstance(column, np.ndarray) and len(column) == len(unit):
+            return np.array(
+                [
+                    col * u.Unit(un) if un != "dimensionless" else col
+                    for col, un in zip(column, unit)
+                ],
+                dtype=object,
+            )
+        if isinstance(unit, str):
+            column_with_unit = column * u.Unit(unit)
+            if isinstance(column, int | np.integer):
+                return u.Quantity(int(column_with_unit.value), unit, dtype=type(column))
+            return column_with_unit
+
+        return None
 
     def _process_column(self, column, dtype):
         """
@@ -324,29 +401,3 @@ class SimtelConfigReader:
         if self.camera_pixels is not None and self.simtel_parameter_name in ["NIGHTSKY_BACKGROUND"]:
             return str(np.dtype(column[0].lower())), self.camera_pixels
         return str(np.dtype(column[0].lower())), int(column[1])
-
-    def _get_simtel_parameter_name(self, parameter_name):
-        """
-        Return parameter name as used in sim_telarray.
-
-        This is documented in the schema file.
-
-        Parameters
-        ----------
-        parameter_name: str
-            Model parameter name (as used in simtools)
-
-        Returns
-        -------
-        str
-            Parameter name as used in sim_telarray.
-
-        """
-        try:
-            for sim_soft in self.schema_dict["simulation_software"]:
-                if sim_soft["name"] == "sim_telarray":
-                    return sim_soft["internal_parameter_name"].upper()
-        except (KeyError, TypeError):
-            pass
-
-        return parameter_name.upper()

simtools/simtel/simtel_config_writer.py

@@ -76,16 +76,24 @@ class SimtelConfigWriter:
             )
             file.write("#endif\n\n")
 
-            for _simtel_name, value in parameters.items():
+            for par, value in parameters.items():
+                _simtel_name = names.get_simulation_software_name_from_parameter_name(
+                    par, simulation_software="sim_telarray"
+                )
                 # array trigger is a site parameter, not a telescope parameter
                 # fake_mirror_list is not a sim_telarray parameter (used for testeff only)
-                if _simtel_name.startswith("array_trigger") or _simtel_name == "fake_mirror_list":
+                if (
+                    not _simtel_name
+                    or _simtel_name.startswith("array_trigger")
+                    or _simtel_name == "fake_mirror_list"
+                ):
                     continue
-                if _simtel_name:
-                    file.write(f"{_simtel_name} = {self._get_value_string_for_simtel(value)}\n")
-            _config_meta = self._get_simtel_metadata("telescope")
-            for _simtel_name, value in _config_meta.items():
-                file.write(f"{_simtel_name} = {value}\n")
+                file.write(
+                    f"{_simtel_name} = {self._get_value_string_for_simtel(value['value'])}\n"
+                )
+            _config_meta = self._get_simtel_metadata("telescope", parameters)
+            for value in _config_meta:
+                file.write(f"{value}\n")
 
     def _get_value_string_for_simtel(self, value):
         """
@@ -108,7 +116,7 @@ class SimtelConfigWriter:
             value = gen.convert_list_to_string(value, shorten_list=True)
         return value
 
-    def _get_simtel_metadata(self, config_type):
+    def _get_simtel_metadata(self, config_type, model_parameters):
         """
         Return simtel metadata.
 
@@ -116,35 +124,60 @@ class SimtelConfigWriter:
         ----------
         type: str
             Type of the configuration file (telescope, site)
+        model_parameters: dict
+            Model parameters dictionary.
 
         Returns
         -------
-        dict
-            Dictionary with simtel metadata.
+        list
+            List with simtel metadata.
 
         """
-        parameters = {}
-        parameters["config_release"] = (
-            f"{self._model_version} written by simtools v{simtools.version.__version__}"
-        )
-        parameters["config_version"] = self._model_version
+        meta_parameters = [
+            f"config_release = {self._model_version} with simtools v{simtools.version.__version__}",
+            f"config_version = {self._model_version}",
+        ]
         if config_type == "telescope":
-            parameters["camera_config_name"] = self._telescope_model_name
-            parameters["camera_config_variant"] = ""
-            parameters["camera_config_version"] = self._model_version
-            parameters["optics_config_name"] = self._telescope_model_name
-            parameters["optics_config_variant"] = ""
-            parameters["optics_config_version"] = self._model_version
+            meta_parameters.extend(
+                [
+                    f"camera_config_name = {self._telescope_model_name}",
+                    "camera_config_variant = ",
+                    f"camera_config_version = {self._model_version}",
+                    f"optics_config_name = {self._telescope_model_name}",
+                    "optics_config_variant = ",
+                    f"optics_config_version = {self._model_version}",
+                ]
+            )
+            prefix = "metaparam telescope"
         elif config_type == "site":
-            parameters["site_config_name"] = self._site
-            parameters["site_config_variant"] = ""
-            parameters["site_config_version"] = self._model_version
-            parameters["array_config_name"] = self._layout_name
-            parameters["array_config_variant"] = ""
-            parameters["array_config_version"] = self._model_version
+            meta_parameters.extend(
+                [
+                    f"site_config_name = {self._site}",
+                    "site_config_variant = ",
+                    f"site_config_version = {self._model_version}",
+                    f"array_config_name = {self._layout_name}",
+                    "array_config_variant = ",
+                    f"array_config_version = {self._model_version}",
+                ]
+            )
+            prefix = "metaparam global"
         else:
             raise ValueError(f"Unknown metadata type {config_type}")
-        return parameters
+
+        if model_parameters:
+            for key, value in model_parameters.items():
+                simtel_name = names.get_simulation_software_name_from_parameter_name(
+                    key, simulation_software="sim_telarray", set_meta_parameter=False
+                )
+                if simtel_name and value.get("meta_parameter"):
+                    meta_parameters.append(f"{prefix} add {simtel_name}")
+                simtel_name = names.get_simulation_software_name_from_parameter_name(
+                    key, simulation_software="sim_telarray", set_meta_parameter=True
+                )
+                if simtel_name and value.get("meta_parameter"):
+                    meta_parameters.append(f"{prefix} set {simtel_name}={value['value']}")
+
+        return meta_parameters
 
     def write_array_config_file(self, config_file_path, telescope_model, site_model):
         """
@@ -177,7 +210,7 @@ class SimtelConfigWriter:
 
             # Writing site parameters
             self._write_site_parameters(
-                file, site_model, Path(config_file_path).parent, telescope_model
+                file, site_model.parameters, Path(config_file_path).parent, telescope_model
             )
 
             # Maximum telescopes
@@ -276,7 +309,7 @@ class SimtelConfigWriter:
         header += f"{comment_char}\n"
         file.write(header)
 
-    def _write_site_parameters(self, file, site_model, model_path, telescope_model):
+    def _write_site_parameters(self, file, site_parameters, model_path, telescope_model):
         """
         Write site parameters.
 
@@ -284,27 +317,26 @@ class SimtelConfigWriter:
         ----------
         file: file
             File to write on.
-        site_model: SiteModel
-            Site model.
+        site_parameters: site parameters
+            Site parameters.
         model_path: Path
             Path to the model for writing of additional files.
         telescope_model: dict of TelescopeModel
             Telescope models.
         """
         file.write(self.TAB + "% Site parameters\n")
-        _site_parameters = site_model.get_simtel_parameters()
-        for par, value in _site_parameters.items():
+        for par, value in site_parameters.items():
             _simtel_name = names.get_simulation_software_name_from_parameter_name(
                 par, simulation_software="sim_telarray"
             )
             _simtel_name, value = self._convert_model_parameters_to_simtel_format(
-                _simtel_name, value, model_path, telescope_model
+                _simtel_name, value["value"], model_path, telescope_model
             )
             if _simtel_name is not None:
                 file.write(f"{self.TAB}{_simtel_name} = {value}\n")
-        _simtel_meta = self._get_simtel_metadata("site")
-        for _simtel_name, value in _simtel_meta.items():
-            file.write(f"{self.TAB}{_simtel_name} = {value}\n")
+        _simtel_meta = self._get_simtel_metadata("site", site_parameters)
+        for value in _simtel_meta:
+            file.write(f"{self.TAB}{value}\n")
         file.write("\n")
 
     def _convert_model_parameters_to_simtel_format(

simtools/simtel/simtel_io_event_reader.py

@@ -0,0 +1,278 @@
+"""
+Read a reduced dataset from file.
+
+Allow to filter the events based on the triggered telescopes.
+Provide functionality to list events, e.g. through
+
+.. code-block:: console
+
+    from simtools.simtel.simtel_io_event_reader import SimtelIOEventDataReader
+    reader = SimtelIOEventDataReader("gamma_diffuse_60deg.hdf5", [1,2,3,4])
+    reader.print_event_table()
+
+"""
+
+import logging
+
+import astropy.units as u
+import numpy as np
+import tables
+from astropy.coordinates import AltAz, angular_separation
+from ctapipe.coordinates import GroundFrame, TiltedGroundFrame
+
+from simtools.simtel.simtel_io_event_writer import ShowerEventData, TriggeredEventData
+
+
+class SimtelIOEventDataReader:
+    """
+    Read reduced MC data set from file.
+
+    Calculate some standard derivation like core position in shower coordinates.
+
+    Parameters
+    ----------
+    event_data_file : str
+        Path to the HDF5 file containing the event data.
+    telescope_list : list, optional
+        List of telescope IDs to filter the events (default is None).
+    """
+
+    def __init__(self, event_data_file, telescope_list=None):
+        """Initialize SimtelIOEventDataReader with the given event data file."""
+        self._logger = logging.getLogger(__name__)
+        self.telescope_list = telescope_list
+
+        self.shower_data, self.triggered_shower_data, self.triggered_data = self.read_event_data(
+            event_data_file
+        )
+        self._derived_event_data()
+
+    def read_event_data(self, event_data_file):
+        """
+        Read event data from the reduced MC event data file.
+
+        Parameters
+        ----------
+        event_data_file : str, Path
+            Path to the HDF5 file containing the event data.
+
+        Returns
+        -------
+        ShowerEventData, TriggeredEventData
+            Event data and triggered event data.
+        """
+        event_data = ShowerEventData()
+        triggered_event_data = TriggeredEventData()
+
+        with tables.open_file(event_data_file, mode="r") as f:
+            reduced_data = f.root.data.reduced_data
+            event_data.simulated_energy = reduced_data.col("simulated_energy")
+            event_data.x_core = reduced_data.col("x_core")
+            event_data.y_core = reduced_data.col("y_core")
+            event_data.shower_azimuth = reduced_data.col("shower_azimuth")
+            event_data.shower_altitude = reduced_data.col("shower_altitude")
+            event_data.shower_id = reduced_data.col("shower_id")
+            event_data.area_weight = reduced_data.col("area_weight")
+
+            triggered_data = f.root.data.triggered_data
+            triggered_event_data.triggered_id = triggered_data.col("triggered_id")
+            triggered_event_data.array_altitudes = triggered_data.col("array_altitudes")
+            triggered_event_data.array_azimuths = triggered_data.col("array_azimuths")
+
+            telescope_indices = triggered_data.col("telescope_list_index")
+            telescope_list_array = f.root.data.trigger_telescope_list_list
+
+            triggered_event_data.trigger_telescope_list_list = []
+            for index in telescope_indices:
+                if index < telescope_list_array.nrows:
+                    triggered_event_data.trigger_telescope_list_list.append(
+                        telescope_list_array[index]
+                    )
+                else:
+                    self._logger.warning(f"Invalid telescope list index: {index}")
+                    triggered_event_data.trigger_telescope_list_list.append(
+                        np.array([], dtype=np.int16)
+                    )
+
+        triggered_shower, triggered_data = self._reduce_to_triggered_events(
+            event_data, triggered_event_data
+        )
+        return event_data, triggered_shower, triggered_data
+
+    def _reduce_to_triggered_events(self, event_data, triggered_data):
+        """
+        Reduce event data to triggered events only. Apply filter on telescope list if specified.
+
+        Parameters
+        ----------
+        event_data : ShowerEventData
+            Event data.
+        triggered_data : TriggeredEventData
+            Triggered event data.
+
+        Returns
+        -------
+        ShowerEventData, TriggeredEventData
+            Filtered event data and triggered event data
+        """
+        filtered_shower_ids, triggered_indices = self._get_mask_triggered_telescopes(
+            self.telescope_list,
+            triggered_data.triggered_id,
+            triggered_data.trigger_telescope_list_list,
+        )
+        filtered_event_data = ShowerEventData(
+            x_core=event_data.x_core[filtered_shower_ids],
+            y_core=event_data.y_core[filtered_shower_ids],
+            simulated_energy=event_data.simulated_energy[filtered_shower_ids],
+            shower_azimuth=event_data.shower_azimuth[filtered_shower_ids],
+            shower_altitude=event_data.shower_altitude[filtered_shower_ids],
+        )
+
+        filtered_telescope_list = [
+            triggered_data.trigger_telescope_list_list[i] for i in triggered_indices
+        ]
+
+        filtered_triggered_data = TriggeredEventData(
+            array_azimuths=triggered_data.array_azimuths[triggered_indices],
+            array_altitudes=triggered_data.array_altitudes[triggered_indices],
+            trigger_telescope_list_list=filtered_telescope_list,
+        )
+        self._logger.info(
+            f"Events reduced to triggered events: {len(filtered_event_data.simulated_energy)}"
+        )
+        return filtered_event_data, filtered_triggered_data
+
+    def _get_mask_triggered_telescopes(
+        self, telescope_list, triggered_id, trigger_telescope_list_list
+    ):
+        """
+        Return indices of events that triggered the specified telescopes.
+
+        Parameters
+        ----------
+        telescope_list : list
+            List of telescope IDs to filter the events
+        triggered_id : np.ndarray
+            Array of event IDs that triggered.
+        trigger_telescope_list_list : list
+            List of triggered telescopes for each event.
+
+        Returns
+        -------
+        tuple
+            Filtered triggered IDs and indices.
+        """
+        triggered_indices = np.arange(len(triggered_id))
+        if telescope_list:
+            mask = np.array(
+                [
+                    any(tel in event for tel in telescope_list)
+                    for event in trigger_telescope_list_list
+                ]
+            )
+            triggered_indices = triggered_indices[mask]
+            return triggered_id[mask], triggered_indices
+        return triggered_id, triggered_indices
+
+    def _derived_event_data(self):
+        """Calculate core positions in shower coordinates and angular distances."""
+        for event_data in (self.shower_data, self.triggered_shower_data):
+            event_data.x_core_shower, event_data.y_core_shower = (
+                self._transform_to_shower_coordinates(
+                    event_data.x_core,
+                    event_data.y_core,
+                    event_data.shower_azimuth,
+                    event_data.shower_altitude,
+                )
+            )
+            event_data.core_distance_shower = np.sqrt(
+                event_data.x_core_shower**2 + event_data.y_core_shower**2
+            )
+
+        self.triggered_data.angular_distance = (
+            angular_separation(
+                self.triggered_shower_data.shower_azimuth,
+                self.triggered_shower_data.shower_altitude,
+                self.triggered_data.array_azimuths,
+                self.triggered_data.array_altitudes,
+            )
+            * 180
+            / np.pi
+        )
+
+    def _transform_to_shower_coordinates(self, x_core, y_core, shower_azimuth, shower_altitude):
+        """
+        Transform core positions from ground coordinates to shower coordinates.
+
+        Parameters
+        ----------
+        x_core : np.ndarray
+            Core x positions in ground coordinates.
+        y_core : np.ndarray
+            Core y positions in ground coordinates.
+        shower_azimuth : np.ndarray
+            Shower azimuth angles.
+        shower_altitude : np.ndarray
+            Shower altitude angles.
+
+        Returns
+        -------
+        tuple
+            Core positions in shower coordinates (x, y).
+        """
+        pointing = AltAz(az=shower_azimuth * u.rad, alt=shower_altitude * u.rad)
+        ground = GroundFrame(x=x_core * u.m, y=y_core * u.m, z=0 * u.m)
+        shower_frame = ground.transform_to(TiltedGroundFrame(pointing_direction=pointing))
+
+        return shower_frame.x.value, shower_frame.y.value
+
+    def print_dataset_information(self, n_events=1):
+        """Print information about the datasets."""
+
+        def print_event_data(data, name):
+            """Print event data."""
+            print(f"{name}: {data[:n_events]}")
+
+        print_event_data(self.triggered_shower_data.simulated_energy, "Simulated energy (TeV)")
+        print_event_data(self.triggered_shower_data.x_core, "Core x (m)")
+        print_event_data(self.triggered_shower_data.y_core, "Core y (m)")
+        print_event_data(self.triggered_shower_data.shower_azimuth, "Shower azimuth (rad)")
+        print_event_data(self.triggered_shower_data.shower_altitude, "Shower altitude (rad)")
+        print_event_data(self.triggered_shower_data.x_core_shower, "Core x shower (m)")
+        print_event_data(self.triggered_shower_data.y_core_shower, "Core y shower (m)")
+        print_event_data(
+            self.triggered_shower_data.core_distance_shower, "Core distance shower (m)"
+        )
+        print_event_data(self.triggered_data.array_azimuths, "Array azimuth (rad)")
+        print_event_data(self.triggered_data.array_altitudes, "Array altitude (rad)")
+        print_event_data(self.triggered_data.trigger_telescope_list_list, "Triggered telescopes")
+        print_event_data(
+            self.triggered_data.angular_distance, "Angular distance to pointing direction (deg)"
+        )
+        print("")
+
+    def print_event_table(self, lines_per_page=20):
+        """Print event table with pagination.
+
+        Parameters
+        ----------
+        lines_per_page : int
+            Number of lines to display before pausing (default: 20)
+        """
+        print(
+            f"{'Counter':<10} {'Simulated Energy (TeV)':<20} {'Triggered Telescopes':<20} "
+            f"{'Core distance shower (m)':<20}"
+        )
+
+        for i, telescope_list in enumerate(self.triggered_data.trigger_telescope_list_list):
+            if i > 0 and i % lines_per_page == 0:
+                response = input("Press Enter to continue or 'q' to quit...")
+                if response.lower() == "q":
+                    break
+
+            print(
+                f"{i:<10} {self.triggered_shower_data.simulated_energy[i]:<20.3f}"
+                f"{telescope_list} "
+                f"{self.triggered_shower_data.core_distance_shower[i]:<20.3f}"
+            )
+        print("")