gammasimtools 0.11.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

simtools/production_configuration/derive_corsika_limits.py

@@ -0,0 +1,260 @@
+"""Calculate the thresholds for energy, radial distance, and viewcone."""
+
+import astropy.units as u
+import matplotlib.pyplot as plt
+import numpy as np
+import tables
+from astropy.coordinates import AltAz
+from ctapipe.coordinates import GroundFrame, TiltedGroundFrame
+
+
+class LimitCalculator:
+    """
+    Compute thresholds/limits for energy, radial distance, and viewcone.
+
+    Event data is read from the reduced MC event data file.
+
+    Parameters
+    ----------
+    event_data_file : str
+        Path to the HDF5 file containing the event data.
+    telescope_list : list, optional
+        List of telescope IDs to filter the events (default is None).
+    """
+
+    def __init__(self, event_data_file, telescope_list=None):
+        """
+        Initialize the LimitCalculator with the given event data file.
+
+        Parameters
+        ----------
+        event_data_file : str
+            Path to the reduced MC event data file.
+        telescope_list : list, optional
+            List of telescope IDs to filter the events (default is None).
+        """
+        self.event_data_file = event_data_file
+        self.telescope_list = telescope_list
+        self.event_x_core = None
+        self.event_y_core = None
+        self.simulated = None
+        self.shower_id_triggered = None
+        self.list_of_files = None
+        self.shower_sim_azimuth = None
+        self.shower_sim_altitude = None
+        self.array_azimuth = None
+        self.array_altitude = None
+        self.trigger_telescope_list_list = None
+        self.units = {}
+        self._read_event_data()
+
+    def _read_event_data(self):
+        """Read the event data from the reduced MC event data file."""
+        with tables.open_file(self.event_data_file, mode="r") as f:
+            reduced_data = f.root.data.reduced_data
+            triggered_data = f.root.data.triggered_data
+            file_names = f.root.data.file_names
+            trigger_telescope_list_list = f.root.data.trigger_telescope_list_list
+
+            self.event_x_core = reduced_data.col("core_x")
+            self.event_y_core = reduced_data.col("core_y")
+            self.simulated = reduced_data.col("simulated")
+            self.shower_id_triggered = triggered_data.col("shower_id_triggered")
+            self.list_of_files = file_names.col("file_names")
+            self.shower_sim_azimuth = reduced_data.col("shower_sim_azimuth")
+            self.shower_sim_altitude = reduced_data.col("shower_sim_altitude")
+            self.array_altitude = reduced_data.col("array_altitudes")
+            self.array_azimuth = reduced_data.col("array_azimuths")
+
+            self.trigger_telescope_list_list = [
+                [np.int16(tel) for tel in event] for event in trigger_telescope_list_list
+            ]
+
+    def _compute_limits(self, hist, bin_edges, loss_fraction, limit_type="lower"):
+        """
+        Compute the limits based on the loss fraction.
+
+        Parameters
+        ----------
+        hist : np.ndarray
+            1D histogram array.
+        bin_edges : np.ndarray
+            Array of bin edges.
+        loss_fraction : float
+            Fraction of events to be lost.
+        limit_type : str, optional
+            Type of limit ('lower' or 'upper'). Default is 'lower'.
+
+        Returns
+        -------
+        float
+            Bin edge value corresponding to the threshold.
+        """
+        cumulative_sum = np.cumsum(hist) if limit_type == "upper" else np.cumsum(hist[::-1])
+        total_events = np.sum(hist)
+        threshold = (1 - loss_fraction) * total_events
+        bin_index = np.searchsorted(cumulative_sum, threshold)
+        return bin_edges[bin_index] if limit_type == "upper" else bin_edges[-bin_index]
+
+    def _prepare_data_for_limits(self):
+        """
+        Prepare the data required for computing limits.
+
+        Returns
+        -------
+        tuple
+            Tuple containing core distances, triggered energies, core bins, and energy bins.
+        """
+        shower_id_triggered_masked = self.shower_id_triggered
+        if self.telescope_list is not None:
+            mask = np.array(
+                [
+                    all(tel in event for tel in self.telescope_list)
+                    for event in self.trigger_telescope_list_list
+                ]
+            )
+            shower_id_triggered_masked = self.shower_id_triggered[mask]
+
+        triggered_energies = self.simulated[shower_id_triggered_masked]
+        energy_bins = np.logspace(
+            np.log10(triggered_energies.min()), np.log10(triggered_energies.max()), 1000
+        )
+        event_x_core_shower, event_y_core_shower = self._transform_to_shower_coordinates()
+        core_distances_all = np.sqrt(event_x_core_shower**2 + event_y_core_shower**2)
+        core_distances_triggered = core_distances_all[shower_id_triggered_masked]
+        core_bins = np.linspace(
+            core_distances_triggered.min(), core_distances_triggered.max(), 1000
+        )
+
+        return core_distances_triggered, triggered_energies, core_bins, energy_bins
+
+    def compute_lower_energy_limit(self, loss_fraction):
+        """
+        Compute the lower energy limit in TeV based on the event loss fraction.
+
+        Parameters
+        ----------
+        loss_fraction : float
+            Fraction of events to be lost.
+
+        Returns
+        -------
+        astropy.units.Quantity
+            Lower energy limit.
+        """
+        _, triggered_energies, _, energy_bins = self._prepare_data_for_limits()
+
+        hist, _ = np.histogram(triggered_energies, bins=energy_bins)
+        lower_bin_edge_value = self._compute_limits(
+            hist, energy_bins, loss_fraction, limit_type="lower"
+        )
+        return lower_bin_edge_value * u.TeV
+
+    def compute_upper_radial_distance(self, loss_fraction):
+        """
+        Compute the upper radial distance based on the event loss fraction.
+
+        Parameters
+        ----------
+        loss_fraction : float
+            Fraction of events to be lost.
+
+        Returns
+        -------
+        astropy.units.Quantity
+            Upper radial distance in m.
+        """
+        core_distances_triggered, _, core_bins, _ = self._prepare_data_for_limits()
+
+        hist, _ = np.histogram(core_distances_triggered, bins=core_bins)
+        upper_bin_edge_value = self._compute_limits(
+            hist, core_bins, loss_fraction, limit_type="upper"
+        )
+        return upper_bin_edge_value * u.m
+
+    def compute_viewcone(self, loss_fraction):
+        """
+        Compute the viewcone based on the event loss fraction.
+
+        Parameters
+        ----------
+        loss_fraction : float
+            Fraction of events to be lost.
+
+        Returns
+        -------
+        astropy.units.Quantity
+            Viewcone radius in degrees.
+        """
+        # already in radians
+        azimuth_diff = self.array_azimuth - self.shower_sim_azimuth  # * (np.pi / 180.0)
+        sim_altitude_rad = self.shower_sim_altitude  # * (np.pi / 180.0)
+        array_altitude_rad = self.array_altitude  # * (np.pi / 180.0)
+        x_1 = np.cos(azimuth_diff) * np.cos(sim_altitude_rad)
+        y_1 = np.sin(azimuth_diff) * np.cos(sim_altitude_rad)
+        z_1 = np.sin(sim_altitude_rad)
+        x_2 = x_1 * np.sin(array_altitude_rad) - z_1 * np.cos(array_altitude_rad)
+        y_2 = y_1
+        z_2 = x_1 * np.cos(array_altitude_rad) + z_1 * np.sin(array_altitude_rad)
+        off_angles = np.arctan2(np.sqrt(x_2**2 + y_2**2), z_2) * (180.0 / np.pi)
+        angle_bins = np.linspace(off_angles.min(), off_angles.max(), 400)
+        hist, _ = np.histogram(off_angles, bins=angle_bins)
+
+        upper_bin_edge_value = self._compute_limits(
+            hist, angle_bins, loss_fraction, limit_type="upper"
+        )
+        return upper_bin_edge_value * u.deg
+
+    def _transform_to_shower_coordinates(self):
+        """
+        Transform core positions from ground coordinates to shower coordinates.
+
+        Returns
+        -------
+        tuple
+            Core positions in shower coordinates (x, y).
+        """
+        pointing_az = self.shower_sim_azimuth * u.rad
+        pointing_alt = self.shower_sim_altitude * u.rad
+
+        pointing = AltAz(az=pointing_az, alt=pointing_alt)
+        ground = GroundFrame(x=self.event_x_core * u.m, y=self.event_y_core * u.m, z=0 * u.m)
+        shower_frame = ground.transform_to(TiltedGroundFrame(pointing_direction=pointing))
+
+        return shower_frame.x.value, shower_frame.y.value
+
+    def plot_data(self):
+        """Plot the core distances and energies of triggered events."""
+        shower_id_triggered_masked = self.shower_id_triggered
+        if self.telescope_list is not None:
+            mask = np.array(
+                [
+                    all(tel in event for tel in self.telescope_list)
+                    for event in self.trigger_telescope_list_list
+                ]
+            )
+            shower_id_triggered_masked = self.shower_id_triggered[mask]
+
+        core_distances_all = np.sqrt(self.event_x_core**2 + self.event_y_core**2)
+        core_distances_triggered = core_distances_all[shower_id_triggered_masked]
+        triggered_energies = self.simulated[shower_id_triggered_masked]
+
+        core_bins = np.linspace(core_distances_triggered.min(), core_distances_triggered.max(), 400)
+        energy_bins = np.logspace(
+            np.log10(triggered_energies.min()), np.log10(triggered_energies.max()), 400
+        )
+        plt.figure(figsize=(8, 6))
+        plt.hist2d(
+            core_distances_triggered,
+            triggered_energies,
+            bins=[core_bins, energy_bins],
+            norm="log",
+            cmap="viridis",
+        )
+
+        plt.colorbar(label="Event Count")
+        plt.xlabel("Core Distance [m]")
+        plt.ylabel("Energy [TeV]")
+        plt.yscale("log")
+        plt.title("2D Histogram of Triggered Core Distance vs Energy")
+        plt.show()

simtools/production_configuration/extract_mc_event_data.py

@@ -0,0 +1,253 @@
+"""Generate a reduced dataset from given simulation event list and save the output to file."""
+
+import logging
+
+import numpy as np
+import tables
+from ctapipe.core import Container, Field
+from ctapipe.io import HDF5TableWriter
+from eventio import EventIOFile
+from eventio.simtel import ArrayEvent, MCEvent, MCRunHeader, MCShower, TriggerInformation
+
+DEFAULT_FILTERS = tables.Filters(complevel=5, complib="zlib", shuffle=True, bitshuffle=False)
+
+
+class ReducedDatasetContainer(Container):
+    """Container for reduced dataset information."""
+
+    simulated = Field(None, "Simulated energy")
+    core_x = Field(None, "X-coordinate of the shower core")
+    core_y = Field(None, "Y-coordinate of the shower core")
+    shower_sim_azimuth = Field(None, "Simulated azimuth angle of the shower")
+    shower_sim_altitude = Field(None, "Simulated altitude angle of the shower")
+    array_altitudes = Field(None, "Altitudes for the array")
+    array_azimuths = Field(None, "Azimuths for the array")
+
+
+class TriggeredShowerContainer(Container):
+    """Container for triggered shower information."""
+
+    shower_id_triggered = Field(None, "Triggered shower ID")
+    triggered_energies = Field(None, "Triggered energies")
+
+
+class FileNamesContainer(Container):
+    """Container for file names."""
+
+    file_names = Field(None, "Input file names")
+
+
+class MCEventExtractor:
+    """
+    Generate a reduced dataset from given simulation event list and save the output to file.
+
+    Attributes
+    ----------
+    input_files : list
+        List of input file paths to process.
+    output_file : str
+        Path to the output HDF5 file.
+    max_files : int, optional
+        Maximum number of files to process.
+    """
+
+    def __init__(self, input_files, output_file, max_files=100):
+        """
+        Initialize the MCEventExtractor with input files, output file, and max file limit.
+
+        Parameters
+        ----------
+        input_files : list
+            List of input file paths to process.
+        output_file : str
+            Path to the output HDF5 file.
+        max_files : int, optional
+            Maximum number of files to process.
+        """
+        self._logger = logging.getLogger(__name__)
+        self.input_files = input_files
+        self.output_file = output_file
+        self.max_files = max_files
+        self.shower = None
+        self.n_use = None
+        self.shower_id_offset = 0
+
+    def process_files(self):
+        """Process the input files and store them in an HDF5 file."""
+        if not self.input_files:
+            self._logger.warning("No input files provided.")
+            return
+
+        data_lists = self._initialize_data_lists()
+        self.shower_id_offset = 0
+        # Process the first file in write mode
+        self._logger.info(f"Processing file 1/{self.max_files}: {self.input_files[0]}")
+        self._process_file(self.input_files[0], data_lists, str(self.input_files[0]))
+        self._write_all_data(data_lists, mode="w")
+        self.shower_id_offset = len(data_lists["simulated"])
+        self._reset_data_lists(data_lists)
+
+        # Process remaining files in append mode
+        for i_file, file in enumerate(self.input_files[1 : self.max_files], start=2):
+            self._logger.info(f"Processing file {i_file}/{self.max_files}: {file}")
+            self._process_file(file, data_lists, str(file))
+            if len(data_lists["simulated"]) >= 1e7:
+                self._write_all_data(data_lists, mode="a")
+                self.shower_id_offset += len(data_lists["simulated"])
+                self._reset_data_lists(data_lists)
+
+        # Final write for any remaining data
+        self._write_all_data(data_lists, mode="a")
+
+    def _write_all_data(self, data_lists, mode):
+        """Write all data sections at once helper method."""
+        self._write_data(data_lists, mode=mode)
+        self._write_variable_length_data(data_lists["trigger_telescope_list_list"], mode="a")
+        self._write_file_names(data_lists["file_names"], mode="a")
+
+    def _write_file_names(self, file_names, mode="a"):
+        """Write file names to HDF5 file."""
+        print("file_names", file_names)
+        with HDF5TableWriter(
+            self.output_file, group_name="data", mode=mode, filters=DEFAULT_FILTERS
+        ) as writer:
+            file_names_container = FileNamesContainer()
+            for file_name in file_names:
+                file_names_container.file_names = file_name
+                writer.write(table_name="file_names", containers=[file_names_container])
+
+    def _write_variable_length_data(self, trigger_telescope_list_list, mode="a"):
+        """Write variable-length array data to HDF5 file."""
+        with tables.open_file(self.output_file, mode=mode) as f:
+            if "trigger_telescope_list_list" in f.root.data:
+                vlarray = f.root.data.trigger_telescope_list_list
+            else:
+                vlarray = f.create_vlarray(
+                    f.root.data,
+                    "trigger_telescope_list_list",
+                    tables.Int16Atom(),
+                    "List of triggered telescope IDs",
+                )
+
+            for item in trigger_telescope_list_list:
+                vlarray.append(item)
+
+    def _initialize_data_lists(self):
+        """Initialize data lists."""
+        return {
+            "simulated": [],
+            "shower_id_triggered": [],
+            "triggered_energies": [],
+            "core_x": [],
+            "core_y": [],
+            "trigger_telescope_list_list": [],
+            "file_names": [],
+            "shower_sim_azimuth": [],
+            "shower_sim_altitude": [],
+            "array_altitudes": [],
+            "array_azimuths": [],
+        }
+
+    def _process_file(self, file, data_lists, file_name):
+        """Process a single file and update data lists."""
+        with EventIOFile(file) as f:
+            array_altitude = None
+            array_azimuth = None
+            for eventio_object in f:
+                if isinstance(eventio_object, MCRunHeader):
+                    self._process_mc_run_header(eventio_object, data_lists)
+                elif isinstance(eventio_object, MCShower):
+                    self._process_mc_shower(
+                        eventio_object, data_lists, array_altitude, array_azimuth
+                    )
+                elif isinstance(eventio_object, MCEvent):
+                    self._process_mc_event(eventio_object, data_lists)
+                elif isinstance(eventio_object, ArrayEvent):
+                    self._process_array_event(eventio_object, data_lists)
+            data_lists["file_names"].extend([file_name])
+
+    def _process_mc_run_header(self, eventio_object, data_lists):
+        """Process MC run header and update data lists."""
+        mc_head = eventio_object.parse()
+        self.n_use = mc_head["n_use"]  # reuse factor n_use needed to extend the values below
+        array_altitude = np.mean(mc_head["alt_range"])
+        array_azimuth = np.mean(mc_head["az_range"])
+        data_lists["array_altitudes"].extend(self.n_use * [array_altitude])
+        data_lists["array_azimuths"].extend(self.n_use * [array_azimuth])
+
+    def _process_mc_shower(self, eventio_object, data_lists, array_altitude, array_azimuth):
+        """Process MC shower and update data lists."""
+        self.shower = eventio_object.parse()
+        data_lists["simulated"].extend(self.n_use * [self.shower["energy"]])
+        data_lists["shower_sim_azimuth"].extend(self.n_use * [self.shower["azimuth"]])
+        data_lists["shower_sim_altitude"].extend(self.n_use * [self.shower["altitude"]])
+        data_lists["array_altitudes"].extend(self.n_use * [array_altitude])
+        data_lists["array_azimuths"].extend(self.n_use * [array_azimuth])
+
+    def _process_mc_event(self, eventio_object, data_lists):
+        """Process MC event and update data lists."""
+        event = eventio_object.parse()
+        data_lists["core_x"].append(event["xcore"])
+        data_lists["core_y"].append(event["ycore"])
+
+    def _process_array_event(self, eventio_object, data_lists):
+        """Process array event and update data lists."""
+        for i, obj in enumerate(eventio_object):
+            if i == 0 and isinstance(obj, TriggerInformation):
+                self._process_trigger_information(obj, data_lists)
+
+    def _process_trigger_information(self, trigger_info, data_lists):
+        """Process trigger information and update data lists."""
+        trigger_info = trigger_info.parse()
+        telescopes = trigger_info["telescopes_with_data"]
+        if len(telescopes) > 0:
+            data_lists["shower_id_triggered"].append(self.shower["shower"] + self.shower_id_offset)
+            data_lists["triggered_energies"].append(self.shower["energy"])
+            data_lists["trigger_telescope_list_list"].append(np.array(telescopes, dtype=np.int16))
+
+    def _write_data(self, data_lists, mode="a"):
+        """Write data to HDF5 file using HDF5TableWriter."""
+        with HDF5TableWriter(
+            self.output_file, group_name="data", mode=mode, filters=DEFAULT_FILTERS
+        ) as writer:
+            # Write reduced dataset container
+            reduced_container = ReducedDatasetContainer()
+            for i in range(len(data_lists["simulated"])):
+                reduced_container.simulated = data_lists["simulated"][i]
+                reduced_container.core_x = data_lists["core_x"][i]
+                reduced_container.core_y = data_lists["core_y"][i]
+                reduced_container.shower_sim_azimuth = data_lists["shower_sim_azimuth"][i]
+                reduced_container.shower_sim_altitude = data_lists["shower_sim_altitude"][i]
+                reduced_container.array_altitudes = data_lists["array_altitudes"][i]
+                reduced_container.array_azimuths = data_lists["array_azimuths"][i]
+                writer.write(table_name="reduced_data", containers=[reduced_container])
+
+            # Write triggered shower container
+            triggered_container = TriggeredShowerContainer()
+            for i in range(len(data_lists["shower_id_triggered"])):
+                triggered_container.shower_id_triggered = data_lists["shower_id_triggered"][i]
+                triggered_container.triggered_energies = data_lists["triggered_energies"][i]
+                writer.write(table_name="triggered_data", containers=[triggered_container])
+
+    def _reset_data_lists(self, data_lists):
+        """Reset data lists during batch processing."""
+        for key in data_lists:
+            data_lists[key] = []
+
+    def print_dataset_information(self):
+        """Print information about the datasets in the generated HDF5 file."""
+        try:
+            with tables.open_file(self.output_file, mode="r") as reader:
+                print("Datasets in file:")
+                for key in reader.root.data._v_children.keys():  # pylint: disable=protected-access
+                    dset = reader.root.data._v_children[key]  # pylint: disable=protected-access
+                    print(f"- {key}: shape={dset.shape}, dtype={dset.dtype}")
+
+                    # Print first 5 values each
+                    print(f"  First 5 values: {dset[:5]}")
+
+                    # Print units if available
+                    # units = dset.attrs.get("units", "N/A")
+                    # print(f"  Units: {units}")
+        except Exception as exc:
+            raise ValueError("An error occurred while reading the HDF5 file") from exc

simtools/ray_tracing/mirror_panel_psf.py

@@ -295,6 +295,6 @@ class MirrorPanelPSF:
         )
         writer.ModelDataWriter.dump(
             args_dict=self.args_dict,
-            metadata=MetadataCollector(args_dict=self.args_dict).get_top_level_metadata(),
+            metadata=MetadataCollector(args_dict=self.args_dict),
             product_data=result_table,
         )