gammasimtools 0.13.0__py3-none-any.whl → 0.15.0__py3-none-any.whl

simtools/simtel/simtel_io_event_writer.py

@@ -0,0 +1,317 @@
+"""Generate a reduced dataset from given simulation event list and save the output to file."""
+
+import logging
+from dataclasses import dataclass, field
+
+import numpy as np
+import tables
+from eventio import EventIOFile
+from eventio.simtel import (
+    ArrayEvent,
+    MCEvent,
+    MCRunHeader,
+    MCShower,
+    TrackingPosition,
+    TriggerInformation,
+)
+
+from simtools.utils.geometry import calculate_circular_mean
+
+DEFAULT_FILTERS = tables.Filters(complevel=5, complib="zlib", shuffle=True, bitshuffle=False)
+
+
+@dataclass
+class ShowerEventData:
+    """Shower event data."""
+
+    simulated_energy: list = field(default_factory=list)
+    x_core: list = field(default_factory=list)
+    y_core: list = field(default_factory=list)
+    shower_azimuth: list = field(default_factory=list)
+    shower_altitude: list = field(default_factory=list)
+    shower_id: list = field(default_factory=list)
+    area_weight: list = field(default_factory=list)
+
+    x_core_shower: list = field(default_factory=list)
+    y_core_shower: list = field(default_factory=list)
+    core_distance_shower: list = field(default_factory=list)
+
+
+@dataclass
+class TriggeredEventData:
+    """Triggered event data."""
+
+    triggered_id: list = field(default_factory=list)
+    array_altitudes: list = field(default_factory=list)
+    array_azimuths: list = field(default_factory=list)
+    trigger_telescope_list_list: list = field(default_factory=list)
+    angular_distance: list = field(default_factory=list)
+
+
+class SimtelIOEventDataWriter:
+    """
+    Generate a reduced dataset from given simulation event list and save the output to file.
+
+    Attributes
+    ----------
+    input_files : list
+        List of input file paths to process.
+    output_file : str
+        Path to the output file.
+    max_files : int, optional
+        Maximum number of files to process.
+    """
+
+    def __init__(self, input_files, output_file, max_files=100):
+        """Initialize class."""
+        self._logger = logging.getLogger(__name__)
+        self.input_files = input_files
+        self.output_file = output_file
+        try:
+            self.max_files = max_files if max_files < len(input_files) else len(input_files)
+        except TypeError as exc:
+            raise TypeError("No input files provided.") from exc
+        self.shower = None
+        self.n_use = None
+        self.shower_id_offset = 0
+        self.event_data = ShowerEventData()
+        self.triggered_data = TriggeredEventData()
+        self.file_names = []
+
+    def process_files(self):
+        """Process the input files and store them in an file."""
+        self.shower_id_offset = 0
+
+        for i, file in enumerate(self.input_files[: self.max_files], start=1):
+            self._logger.info(f"Processing file {i}/{self.max_files}: {file}")
+            self._process_file(file)
+            if i == 1 or len(self.event_data.simulated_energy) >= 1e7:
+                self._write_data(mode="w" if i == 1 else "a")
+                self.shower_id_offset += len(self.event_data.simulated_energy)
+                self._reset_data()
+
+        self._write_data(mode="a")
+
+    def get_event_data(self):
+        """
+        Return shower and triggered event data.
+
+        Returns
+        -------
+        ShowerEventData, TriggeredEventData
+            Shower and triggered event data.
+        """
+        return self.event_data, self.triggered_data
+
+    def _process_file(self, file):
+        """Process a single file and update data lists."""
+        with EventIOFile(file) as f:
+            for eventio_object in f:
+                if isinstance(eventio_object, MCRunHeader):
+                    self._process_mc_run_header(eventio_object)
+                elif isinstance(eventio_object, MCShower):
+                    self._process_mc_shower(eventio_object)
+                elif isinstance(eventio_object, MCEvent):
+                    self._process_mc_event(eventio_object)
+                elif isinstance(eventio_object, ArrayEvent):
+                    self._process_array_event(eventio_object)
+            self.file_names.append(str(file))
+
+    def _process_mc_run_header(self, eventio_object):
+        """Process MC run header and update data lists."""
+        mc_head = eventio_object.parse()
+        self.n_use = mc_head["n_use"]  # reuse factor n_use needed to extend the values below
+        self._logger.info(f"Shower reuse factor: {self.n_use} (viewcone: {mc_head['viewcone']})")
+
+    def _process_mc_shower(self, eventio_object):
+        """
+        Process MC shower and update shower event list.
+
+        Duplicated entries 'self.n_use' times to match the number simulated events with
+        different core positions.
+        """
+        self.shower = eventio_object.parse()
+
+        self.event_data.simulated_energy.extend([self.shower["energy"]] * self.n_use)
+        self.event_data.shower_azimuth.extend([self.shower["azimuth"]] * self.n_use)
+        self.event_data.shower_altitude.extend([self.shower["altitude"]] * self.n_use)
+
+    def _process_mc_event(self, eventio_object):
+        """Process MC event and update shower event list."""
+        event = eventio_object.parse()
+
+        self.event_data.shower_id.append(event["shower_num"])
+        self.event_data.x_core.append(event["xcore"])
+        self.event_data.y_core.append(event["ycore"])
+        self.event_data.area_weight.append(event["aweight"])
+
+    def _process_array_event(self, eventio_object):
+        """Process array event and update triggered event list."""
+        tracking_positions = []
+
+        for _, obj in enumerate(eventio_object):
+            if isinstance(obj, TriggerInformation):
+                self._process_trigger_information(obj)
+
+            if isinstance(obj, TrackingPosition):
+                tracking_position = obj.parse()
+                tracking_positions.append(
+                    {
+                        "altitude": tracking_position["altitude_raw"],
+                        "azimuth": tracking_position["azimuth_raw"],
+                    }
+                )
+
+        if tracking_positions:
+            self._process_tracking_positions(tracking_positions)
+
+    def _process_tracking_positions(self, tracking_positions):
+        """
+        Process collected tracking positions and update triggered event list.
+
+        Use mean telescope tracking positions, averaged over all triggered telescopes.
+        """
+        altitudes = [pos["altitude"] for pos in tracking_positions]
+        azimuths = [pos["azimuth"] for pos in tracking_positions]
+
+        self.triggered_data.array_altitudes.append(np.mean(altitudes))
+        self.triggered_data.array_azimuths.append(calculate_circular_mean(azimuths))
+
+    def _process_trigger_information(self, trigger_info):
+        """Process trigger information and update triggered event list."""
+        trigger_info = trigger_info.parse()
+        telescopes = trigger_info["triggered_telescopes"]
+        if len(telescopes) > 0:
+            # add offset to obtain unique shower IDs among all files
+            self.triggered_data.triggered_id.append(self.shower["shower"] + self.shower_id_offset)
+            self.triggered_data.trigger_telescope_list_list.append(
+                np.array(telescopes, dtype=np.int16)
+            )
+
+    def _table_descriptions(self):
+        """HDF5 table descriptions for shower data, triggered data, and file names."""
+        shower_data_desc = {
+            "shower_id": tables.Int32Col(),
+            "simulated_energy": tables.Float32Col(),
+            "x_core": tables.Float32Col(),
+            "y_core": tables.Float32Col(),
+            "area_weight": tables.Float32Col(),
+            "shower_azimuth": tables.Float32Col(),
+            "shower_altitude": tables.Float32Col(),
+        }
+        triggered_data_desc = {
+            "triggered_id": tables.Int32Col(),
+            "array_altitudes": tables.Float32Col(),
+            "array_azimuths": tables.Float32Col(),
+            "telescope_list_index": tables.Int32Col(),  # Index into VLArray
+        }
+        file_names_desc = {
+            "file_names": tables.StringCol(256),
+        }
+        return shower_data_desc, triggered_data_desc, file_names_desc
+
+    def _tables(self, output_file, data_group, mode="a"):
+        """Create or get HDF5 tables."""
+        descriptions = self._table_descriptions()
+        table_names = ["reduced_data", "triggered_data", "file_names"]
+
+        table_dict = {}
+        for name, desc in zip(table_names, descriptions):
+            path = f"/data/{name}"
+            table_dict[name] = (
+                output_file.create_table(
+                    data_group, name, desc, name.replace("_", " ").title(), filters=DEFAULT_FILTERS
+                )
+                if mode == "w" or path not in output_file
+                else output_file.get_node(path)
+            )
+
+        return table_dict["reduced_data"], table_dict["triggered_data"], table_dict["file_names"]
+
+    def _write_event_data(self, reduced_table):
+        """Fill event data tables."""
+        if len(self.event_data.simulated_energy) == 0:
+            return
+        row = reduced_table.row
+        for i, energy in enumerate(self.event_data.simulated_energy):
+            row["shower_id"] = (
+                self.event_data.shower_id[i] if i < len(self.event_data.shower_id) else 0
+            )
+            row["simulated_energy"] = energy
+            row["x_core"] = self.event_data.x_core[i] if i < len(self.event_data.x_core) else 0
+            row["y_core"] = self.event_data.y_core[i] if i < len(self.event_data.y_core) else 0
+            row["area_weight"] = (
+                self.event_data.area_weight[i] if i < len(self.event_data.area_weight) else 0
+            )
+            row["shower_azimuth"] = (
+                self.event_data.shower_azimuth[i] if i < len(self.event_data.shower_azimuth) else 0
+            )
+            row["shower_altitude"] = (
+                self.event_data.shower_altitude[i]
+                if i < len(self.event_data.shower_altitude)
+                else 0
+            )
+            row.append()
+        reduced_table.flush()
+
+    def _writer_triggered_data(self, triggered_table, vlarray):
+        """Fill triggered event data tables."""
+        # Get or create VLArray for telescope lists
+        if len(self.triggered_data.triggered_id) == 0:
+            return
+        row = triggered_table.row
+        start_idx = vlarray.nrows
+        for i, triggered_id in enumerate(self.triggered_data.triggered_id):
+            row["triggered_id"] = triggered_id
+            row["array_altitudes"] = (
+                self.triggered_data.array_altitudes[i]
+                if i < len(self.triggered_data.array_altitudes)
+                else 0
+            )
+            row["array_azimuths"] = (
+                self.triggered_data.array_azimuths[i]
+                if i < len(self.triggered_data.array_azimuths)
+                else 0
+            )
+            row["telescope_list_index"] = start_idx + i  # Index into the VLArray
+            row.append()
+            vlarray.append(
+                self.triggered_data.trigger_telescope_list_list[i]
+                if i < len(self.triggered_data.trigger_telescope_list_list)
+                else []
+            )
+        triggered_table.flush()
+
+    def _write_data(self, mode="a"):
+        """Write data to HDF5 file."""
+        with tables.open_file(self.output_file, mode=mode) as f:
+            data_group = (
+                f.create_group("/", "data", "Data group")
+                if mode == "w" or "/data" not in f
+                else f.get_node("/data")
+            )
+
+            reduced_table, triggered_table, file_names_table = self._tables(f, data_group, mode)
+            self._write_event_data(reduced_table)
+
+            vlarray = (
+                f.create_vlarray(
+                    data_group,
+                    "trigger_telescope_list_list",
+                    tables.Int16Atom(),
+                    "List of triggered telescope IDs",
+                )
+                if mode == "w" or "/data/trigger_telescope_list_list" not in f
+                else f.get_node("/data/trigger_telescope_list_list")
+            )
+            self._writer_triggered_data(triggered_table, vlarray)
+
+            if self.file_names:
+                file_names_table.append([[name] for name in self.file_names])
+                file_names_table.flush()
+
+    def _reset_data(self):
+        """Reset data structures for batch processing."""
+        self.event_data = ShowerEventData()
+        self.triggered_data = TriggeredEventData()
+        self.file_names = []

simtools/simtel/simulator_light_emission.py

@@ -2,14 +2,17 @@
 
 import logging
 import stat
+import subprocess
 from pathlib import Path
 
 import astropy.units as u
 import numpy as np
 
+from simtools.corsika.corsika_histograms_visualize import save_figs_to_pdf
 from simtools.io_operations import io_handler
 from simtools.runners.simtel_runner import SimtelRunner
 from simtools.utils.general import clear_default_sim_telarray_cfg_directories
+from simtools.visualization.visualize import plot_simtel_ctapipe
 
 __all__ = ["SimulatorLightEmission"]
 
@@ -18,36 +21,7 @@ class SimulatorLightEmission(SimtelRunner):
     """
     Interface with sim_telarray to perform light emission package simulations.
 
-    The light emission package is used to simulate a artificial light source, used for calibration.
-
-    The angle and pointing vector calculations use the convention north (x) towards
-    east (-y).
-
-    Parameters
-    ----------
-    telescope_model:
-        Instance of the TelescopeModel class.
-    calibration_model:
-        CalibrationModel instance to define calibration device.
-    site_model:
-        SiteModel instance to define the site specific parameters.
-    default_le_config: dict
-        defines parameters for running the sim_telarray light emission application.
-    le_application: str
-        Name of the application. Default sim_telarray application running
-        the sim_telarray LightEmission package is xyzls.
-    simtel_path: str or Path
-        Location of sim_telarray installation.
-    light_source_type: str
-        The light source type.
-    label: str
-        Label for output directory.
-    config_data: dict
-        Dict containing the configurable parameters.
-    config_file: str or Path
-        Path of the yaml file containing the configurable parameters.
-    test: bool
-        Test flag.
+    The light emission package is used to simulate an artificial light source, used for calibration.
     """
 
     def __init__(
@@ -55,7 +29,7 @@ class SimulatorLightEmission(SimtelRunner):
         telescope_model,
         calibration_model,
         site_model,
-        default_le_config,
+        light_emission_config,
         le_application,
         simtel_path,
         light_source_type,
@@ -86,8 +60,8 @@ class SimulatorLightEmission(SimtelRunner):
         )
 
         self.le_application = le_application
-        self.default_le_config = default_le_config
-        self.distance = self.telescope_calibration_device_distance()
+        self.light_emission_config = light_emission_config
+        self.distance = None
         self.light_source_type = light_source_type
         self._telescope_model.export_config_file()
         self.test = test
@@ -179,36 +153,6 @@ class SimulatorLightEmission(SimtelRunner):
         )
         return pointing_vector.tolist(), [tel_theta, tel_phi, laser_theta, laser_phi]
 
-    def telescope_calibration_device_distance(self):
-        """
-        Calculate the distance between the telescope and the calibration device.
-
-        Returns
-        -------
-        astropy Quantity
-            The distance between the telescope and the calibration device.
-        """
-        if not self.default_le_config:
-            x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value(
-                "array_element_position_ground"
-            )
-            x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value(
-                "array_element_position_ground"
-            )
-
-        else:
-            x_tel = self.default_le_config["x_pos"]["default"].to(u.m).value
-            y_tel = self.default_le_config["y_pos"]["default"].to(u.m).value
-            z_tel = self.default_le_config["z_pos"]["default"].to(u.m).value
-
-            x_cal, y_cal, z_cal = 0, 0, 0
-
-        tel_vect = np.array([x_tel, y_tel, z_tel])
-        cal_vect = np.array([x_cal, y_cal, z_cal])
-        distance = np.linalg.norm(cal_vect - tel_vect)
-
-        return distance * u.m
-
     def _make_light_emission_script(self):
         """
         Create the light emission script to run the light emission package.
@@ -235,11 +179,11 @@ class SimulatorLightEmission(SimtelRunner):
 
         if self.light_source_type == "led":
             if self.le_application[1] == "variable":
-                command += f" -x {self.default_le_config['x_pos']['default'].to(u.cm).value}"
-                command += f" -y {self.default_le_config['y_pos']['default'].to(u.cm).value}"
-                command += f" -z {self.default_le_config['z_pos']['default'].to(u.cm).value}"
+                command += f" -x {self.light_emission_config['x_pos']['default'].to(u.cm).value}"
+                command += f" -y {self.light_emission_config['y_pos']['default'].to(u.cm).value}"
+                command += f" -z {self.light_emission_config['z_pos']['default'].to(u.cm).value}"
                 command += (
-                    f" -d {','.join(map(str, self.default_le_config['direction']['default']))}"
+                    f" -d {','.join(map(str, self.light_emission_config['direction']['default']))}"
                 )
                 command += f" -n {self.photons_per_run}"
 
@@ -470,3 +414,173 @@ class SimulatorLightEmission(SimtelRunner):
 
         _script_file.chmod(_script_file.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
         return _script_file
+
+    def process_simulation_output(self, args_dict, figures):
+        """Process the simulation output, including plotting and saving figures."""
+        try:
+            filename = self._get_simulation_output_filename()
+            distance = self._get_distance_for_plotting()
+
+            fig = self._plot_simulation_output(
+                filename,
+                args_dict["boundary_thresh"],
+                args_dict["picture_thresh"],
+                args_dict["min_neighbors"],
+                distance,
+                args_dict["return_cleaned"],
+            )
+            figures.append(fig)
+
+        except AttributeError:
+            msg = (
+                f"Telescope not triggered at distance of "
+                f"{self.light_emission_config['z_pos']['default']}"
+            )
+            self._logger.warning(msg)
+
+    def _get_simulation_output_filename(self):
+        """Get the filename of the simulation output."""
+        return (
+            f"{self.output_directory}/{self.le_application[0]}_{self.le_application[1]}.simtel.gz"
+        )
+
+    def _get_distance_for_plotting(self):
+        """Get the distance to be used for plotting."""
+        try:
+            return self.light_emission_config["z_pos"]["default"]
+        except KeyError:
+            return round(self.distance, 2)
+
+    def _plot_simulation_output(
+        self, filename, boundary_thresh, picture_thresh, min_neighbors, distance, return_cleaned
+    ):
+        """Plot the simulation output."""
+        return plot_simtel_ctapipe(
+            filename,
+            cleaning_args=[boundary_thresh, picture_thresh, min_neighbors],
+            distance=distance,
+            return_cleaned=return_cleaned,
+        )
+
+    def save_figures_to_pdf(self, figures, telescope):
+        """Save the generated figures to a PDF file."""
+        save_figs_to_pdf(
+            figures,
+            f"{self.output_directory}/"
+            f"{telescope}_{self.le_application[0]}_{self.le_application[1]}.pdf",
+        )
+
+    def run_simulation(self, args_dict, figures):
+        """Run the light emission simulation."""
+        run_script = self.prepare_script(generate_postscript=True, **args_dict)
+        log_file = Path(self.output_directory) / "logfile.log"
+        with open(log_file, "w", encoding="utf-8") as log_file:
+            subprocess.run(
+                run_script,
+                shell=False,
+                check=False,
+                text=True,
+                stdout=log_file,
+                stderr=log_file,
+            )
+        self.process_simulation_output(args_dict, figures)
+
+    def distance_list(self, arg):
+        """
+        Convert distance list to astropy quantities.
+
+        Parameters
+        ----------
+        arg: list
+            List of distances.
+
+        Returns
+        -------
+        values: list
+            List of distances as astropy quantities.
+        """
+        try:
+            return [float(x) * u.m for x in arg]
+        except ValueError as exc:
+            raise ValueError("Distances must be numeric values") from exc
+
+    def update_light_emission_config(self, key: str, value):
+        """
+        Update the light emission configuration.
+
+        Parameters
+        ----------
+        key : str
+            The key in the configuration to update.
+        value : Any
+            The new value to set for the key.
+        """
+        if key in self.light_emission_config:
+            self.light_emission_config[key]["default"] = value
+        else:
+            raise KeyError(f"Key '{key}' not found in light emission configuration.")
+
+    def calculate_distance_telescope_calibration_device(self):
+        """
+        Calculate the distance(s) between the telescope and the calibration device.
+
+        Returns
+        -------
+        list of astropy.Quantity
+            A list of distances for variable positions or a single distance for layout positions.
+        """
+        if not self.light_emission_config:
+            # Layout positions: Use DB coordinates
+            x_cal, y_cal, z_cal = self._calibration_model.get_parameter_value(
+                "array_element_position_ground"
+            )
+            x_tel, y_tel, z_tel = self._telescope_model.get_parameter_value(
+                "array_element_position_ground"
+            )
+            tel_vect = np.array([x_tel, y_tel, z_tel])
+            cal_vect = np.array([x_cal, y_cal, z_cal])
+            distance = np.linalg.norm(cal_vect - tel_vect)
+            return [distance * u.m]
+
+        # Variable positions: Calculate distances for all positions
+        x_tel = self.light_emission_config["x_pos"]["default"].to(u.m).value
+        y_tel = self.light_emission_config["y_pos"]["default"].to(u.m).value
+        z_positions = self.light_emission_config["z_pos"]["default"]
+
+        distances = []
+        for z in z_positions:
+            tel_vect = np.array([x_tel, y_tel, z.to(u.m).value])
+            cal_vect = np.array([0, 0, 0])  # Calibration device at origin
+            distances.append(np.linalg.norm(cal_vect - tel_vect) * u.m)
+        return distances
+
+    def simulate_variable_distances(self, args_dict):
+        """Simulate light emission for variable distances."""
+        if args_dict["distances_ls"] is not None:
+            self.update_light_emission_config(
+                "z_pos", self.distance_list(args_dict["distances_ls"])
+            )
+        self._logger.info(
+            f"Simulating for distances: {self.light_emission_config['z_pos']['default']}"
+        )
+
+        figures = []
+        distances = self.calculate_distance_telescope_calibration_device()
+
+        for current_distance, z_pos in zip(
+            distances, self.light_emission_config["z_pos"]["default"]
+        ):
+            self.update_light_emission_config("z_pos", z_pos)
+            self.distance = current_distance
+            self.run_simulation(args_dict, figures)
+
+        self.save_figures_to_pdf(figures, args_dict["telescope"])
+
+    def simulate_layout_positions(self, args_dict):
+        """Simulate light emission for layout positions."""
+        figures = []
+        self.distance = self.calculate_distance_telescope_calibration_device()[
+            0
+        ]  # Single distance for layout
+        self.run_simulation(args_dict, figures)
+        self.save_figures_to_pdf(figures, args_dict["telescope"])

simtools/simulator.py

@@ -329,7 +329,7 @@ class Simulator:
             input_file_list = self._enforce_list_type(input_file_list)
             _runs_and_files = {self._guess_run_from_file(file): file for file in input_file_list}
         elif self.simulation_software in ["corsika", "corsika_simtel"]:
-            _runs_and_files = {run: None for run in self._get_runs_to_simulate()}
+            _runs_and_files = dict.fromkeys(self._get_runs_to_simulate())
         if len(_runs_and_files) == 0:
             raise ValueError("No runs to submit.")
         return _runs_and_files
@@ -392,7 +392,7 @@ class Simulator:
 
         """
         keys = ["output", "sub_out", "log", "input", "hist", "corsika_log"]
-        defaults = {key: None for key in keys}
+        defaults = dict.fromkeys(keys)
         results = {key: defaults[key] for key in keys}
         results["output"] = str(
             self._simulation_runner.get_file_name(file_type="output", run_number=run_number)

simtools/testing/configuration.py

@@ -1,6 +1,7 @@
 """Integration test configuration."""
 
 import logging
+import os
 from pathlib import Path
 
 import yaml
@@ -14,6 +15,10 @@ class VersionError(Exception):
     """Raise if model version requested is not supported."""
 
 
+class ProductionDBError(Exception):
+    """Raise if production db is used."""
+
+
 def get_list_of_test_configurations(config_files):
     """
     Return list of test configuration dictionaries or test names.
@@ -100,6 +105,7 @@ def configure(config, tmp_test_directory, request):
 
     if "CONFIGURATION" in config:
         _skip_test_for_model_version(config, model_version_requested)
+        _skip_test_for_production_db(config)
 
         config_file, config_string, config_file_model_version = _prepare_test_options(
             config["CONFIGURATION"],
@@ -130,6 +136,18 @@ def _skip_test_for_model_version(config, model_version_requested):
         )
 
 
+def _skip_test_for_production_db(config):
+    """Skip test if production db is used."""
+    if not config.get("SKIP_FOR_PRODUCTION_DB"):
+        return
+
+    if "db.zeuthen.desy.de" in os.getenv("SIMTOOLS_DB_SERVER", ""):
+        raise ProductionDBError("Production database used for this test")
+
+    if "simpipe" in os.getenv("SIMTOOLS_DB_API_USER", ""):
+        raise ProductionDBError("Production database used for this test")
+
+
 def _prepare_test_options(config, output_path, model_version=None):
     """
     Prepare test configuration.