gammasimtools 0.15.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/production_configuration/interpolation_handler.py

@@ -1,37 +1,88 @@
-"""Interpolates between instances of StatisticalErrorEvaluator using EventScaler."""
+"""Handle interpolation between multiple StatisticalUncertaintyEvaluator instances."""
+
+import logging
 
 import astropy.units as u
 import numpy as np
 from scipy.interpolate import griddata
 
-from simtools.production_configuration.event_scaler import EventScaler
+from simtools.production_configuration.derive_production_statistics import (
+    ProductionStatisticsDerivator,
+)
 
 __all__ = ["InterpolationHandler"]
 
 
 class InterpolationHandler:
-    """Handle interpolation between multiple StatisticalErrorEvaluator instances."""
+    """
+    Calculate the required events for production via interpolation from a grid.
+
+    This class provides methods to interpolate production statistics across a grid of
+    parameter values (azimuth, zenith, NSB, offset) and energy.
+    """
+
+    def __init__(self, evaluators, metrics: dict, grid_points_production: list):
+        """
+        Initialize the InterpolationHandler.
 
-    def __init__(self, evaluators, metrics: dict):
+        Parameters
+        ----------
+        evaluators : list
+            List of StatisticalUncertaintyEvaluator instances.
+        metrics : dict
+            Dictionary of metrics to use for production statistics.
+        grid_points_production : list
+            List of grid points for interpolation, each being a dictionary with keys
+            'azimuth', 'zenith_angle', 'nsb', 'offset' etc.
+        """
+        self._logger = logging.getLogger(__name__)
         self.evaluators = evaluators
         self.metrics = metrics
-        self.event_scalers = [EventScaler(e, self.metrics) for e in self.evaluators]
+        self.grid_points_production = grid_points_production
 
+        self._initialize_derivators()
+        self._extract_grid_properties()
+
+        self.data, self.grid_points = self._build_data_array()
+        self.interpolated_production_statistics = None
+        self.interpolated_production_statistics_with_energy = None
+        self._non_flat_mask = None
+
+    def _initialize_derivators(self):
+        """Initialize production statistics derivators for all evaluators."""
+        self.derive_production_statistics = [
+            ProductionStatisticsDerivator(e, self.metrics) for e in self.evaluators
+        ]
+
+        self.production_statistics = [
+            derivator.derive_statistics(return_sum=False)
+            for derivator in self.derive_production_statistics
+        ]
+        self.production_statistics_sum = [
+            derivator.derive_statistics(return_sum=True)
+            for derivator in self.derive_production_statistics
+        ]
+
+    def _extract_grid_properties(self):
+        """Extract grid properties from evaluators."""
         self.azimuths = [e.grid_point[1].to(u.deg).value for e in self.evaluators]
         self.zeniths = [e.grid_point[2].to(u.deg).value for e in self.evaluators]
         self.nsbs = [e.grid_point[3] for e in self.evaluators]
         self.offsets = [e.grid_point[4].to(u.deg).value for e in self.evaluators]
-
         self.energy_grids = [
             (e.data["bin_edges_low"][:-1] + e.data["bin_edges_high"][:-1]) / 2
             for e in self.evaluators
         ]
-        self.scaled_events = [
-            scaler.scale_events(return_sum=False) for scaler in self.event_scalers
-        ]
         self.energy_thresholds = np.array([e.energy_threshold for e in self.evaluators])
 
-        self.data, self.grid_points = self._build_data_array()
+        # Check if energy grids are consistent
+        if self.energy_grids and not all(
+            np.array_equal(self.energy_grids[0], grid) for grid in self.energy_grids
+        ):
+            self._logger.warning(
+                "Energy grids are not identical across evaluators. "
+                "Using the first evaluator's energy grid for interpolation."
+            )
 
     def _build_data_array(self):
         """
@@ -44,148 +95,304 @@ class InterpolationHandler:
         np.ndarray
             The corresponding grid points.
         """
-        # Flatten the energy grid and other dimensions into a combined array
+        if not self.evaluators:
+            return np.array([]), np.array([])
+
         flat_data_list = []
         flat_grid_points = []
 
-        for e, energy_grid, scaled_events in zip(
-            self.evaluators, self.energy_grids, self.scaled_events
+        for i, (energy_grid, production_statistics) in enumerate(
+            zip(self.energy_grids, self.production_statistics)
         ):
-            az = np.full(len(energy_grid), e.grid_point[1].to(u.deg).value)
-            zen = np.full(len(energy_grid), e.grid_point[2].to(u.deg).value)
-            nsb = np.full(len(energy_grid), e.grid_point[3])
-            offset = np.full(len(energy_grid), e.grid_point[4].to(u.deg).value)
+            az = self.azimuths[i]
+            zen = self.zeniths[i]
+            nsb = self.nsbs[i]
+            offset = self.offsets[i]
+
+            az_array = np.full(len(energy_grid), az)
+            zen_array = np.full(len(energy_grid), zen)
+            nsb_array = np.full(len(energy_grid), nsb)
+            offset_array = np.full(len(energy_grid), offset)
+
+            grid_points = np.column_stack(
+                [energy_grid.to(u.TeV).value, az_array, zen_array, nsb_array, offset_array]
+            )
 
-            # Combine grid points and data
-            grid_points = np.column_stack([energy_grid.to(u.TeV).value, az, zen, nsb, offset])
             flat_grid_points.append(grid_points)
-            flat_data_list.append(scaled_events)
+            flat_data_list.append(production_statistics)
 
-        # Flatten the list and convert to numpy arrays
         flat_grid_points = np.vstack(flat_grid_points)
         flat_data = np.hstack(flat_data_list)
 
-        # Sort the grid points and corresponding data by energy
         sorted_indices = np.argsort(flat_grid_points[:, 0])
         sorted_grid_points = flat_grid_points[sorted_indices]
         sorted_data = flat_data[sorted_indices]
 
         return sorted_data, sorted_grid_points
 
-    def _remove_flat_dimensions(self, grid_points):
-        """Identify and remove flat dimensions (dimensions with no variance)."""
+    def _remove_flat_dimensions(self, grid_points, threshold=1e-6):
+        """
+        Identify and remove flat dimensions (dimensions with no variance).
+
+        Parameters
+        ----------
+        grid_points : np.ndarray
+            Grid points to analyze.
+        threshold : float, optional
+            Threshold for determining flatness, by default 1e-6
+
+        Returns
+        -------
+        tuple
+            (reduced_grid_points, non_flat_mask)
+        """
+        if grid_points.size == 0:
+            return grid_points, np.array([], dtype=bool)
+
         variance = np.var(grid_points, axis=0)
-        non_flat_mask = variance > 1e-6  # Threshold for determining flatness
+        non_flat_mask = variance > threshold
+
+        if not np.any(non_flat_mask):
+            self._logger.warning(
+                "All dimensions are flat. Keeping all dimensions for interpolation."
+            )
+            return grid_points, np.ones_like(variance, dtype=bool)
+
         reduced_grid_points = grid_points[:, non_flat_mask]
         return reduced_grid_points, non_flat_mask
 
-    def interpolate(self, query_points: np.ndarray) -> np.ndarray:
+    def build_grid_points_no_energy(self):
+        """
+        Build grid points without energy dimension.
+
+        Returns
+        -------
+        tuple
+            (production_statistics, grid_points_no_energy)
+        """
+        if not self.evaluators:
+            self._logger.error("No evaluators available for grid point building.")
+            return np.array([]), np.array([])
+
+        flat_data_list = []
+        flat_grid_points = []
+
+        for i, production_statistics_sum in enumerate(self.production_statistics_sum):
+            az = self.azimuths[i]
+            zen = self.zeniths[i]
+            nsb = self.nsbs[i]
+            offset = self.offsets[i]
+
+            flat_data_list.append(float(production_statistics_sum.value))
+
+            grid_point = np.array([[az, zen, nsb, offset]])
+            flat_grid_points.append(grid_point)
+
+        flat_grid_points = np.vstack(flat_grid_points)
+        return flat_data_list, flat_grid_points
+
+    def _prepare_energy_independent_data(self):
+        """
+        Prepare data for energy-independent interpolation.
+
+        Returns
+        -------
+        tuple
+            (production_statistic, grid_points_no_energy)
         """
-        Interpolate the number of simulated events given query points.
+        production_statistic, grid_points_no_energy = self.build_grid_points_no_energy()
+        production_statistic = np.array(production_statistic, dtype=float)
+        grid_points_no_energy, non_flat_mask = self._remove_flat_dimensions(grid_points_no_energy)
+
+        self._non_flat_mask = non_flat_mask  # Store for later use
+        return production_statistic, grid_points_no_energy
+
+    def _prepare_production_grid_points(self):
+        """
+        Convert grid_points_production to a format suitable for interpolation.
+
+        Returns
+        -------
+        np.ndarray
+            Reduced production grid points.
+        """
+        production_grid_points = []
+
+        for point in self.grid_points_production:
+            production_grid_points.append(
+                [
+                    point["azimuth"]["value"],
+                    point["zenith_angle"]["value"],
+                    point["nsb"]["value"],
+                    point["offset"]["value"],
+                ]
+            )
+
+        production_grid_points = np.array(production_grid_points)
+
+        return production_grid_points[:, self._non_flat_mask]
+
+    def _perform_interpolation(self, grid_points, values, query_points, method="linear"):
+        """
+        Perform interpolation using griddata.
 
         Parameters
         ----------
+        grid_points : np.ndarray
+            Grid points for interpolation.
+        values : np.ndarray
+            Values at the grid points.
         query_points : np.ndarray
-            Array of query points with shape (n, 5), where n is the number of points,
-            and 5 represents (energy, azimuth, zenith, nsb, offset).
+            Query points for interpolation.
+        method : str, optional
+            Interpolation method, by default "linear".
 
         Returns
         -------
         np.ndarray
-            Interpolated values at the query points.
+            Interpolated values.
         """
-        reduced_grid_points, non_flat_mask = self._remove_flat_dimensions(self.grid_points)
-        reduced_query_points = query_points[:, non_flat_mask]
+        self._logger.debug(f"Grid points shape: {grid_points.shape}")
+        self._logger.debug(f"Values shape: {values.shape}")
+        self._logger.debug(f"Query points shape: {query_points.shape}")
 
-        # Interpolate using the reduced dimensions
         return griddata(
-            reduced_grid_points,
-            self.data,
-            reduced_query_points,
-            method="linear",
+            grid_points,
+            values,
+            query_points,
+            method=method,
             fill_value=np.nan,
             rescale=True,
         )
 
-    def interpolate_energy_threshold(self, query_point: np.ndarray) -> float:
+    def _perform_interpolation_with_energy(self):
         """
-        Interpolate the energy threshold for a given grid point.
+        Perform energy-dependent interpolation.
 
-        Parameters
-        ----------
-        query_point : np.ndarray
-            Array specifying the grid point (energy, azimuth, zenith, NSB, offset).
+        Returns
+        -------
+        np.ndarray
+            Energy-dependent interpolated values.
+        """
+        # Get grid points with energy dimension
+        grid_points_energy = self.grid_points
+        grid_points_energy, _ = self._remove_flat_dimensions(grid_points_energy)
+
+        # Build energy query grid
+        reduced_production_grid_points = self._prepare_production_grid_points()
+        energy_grid = self.energy_grids[0] if self.energy_grids else []
+
+        energy_query_grid = []
+        for energy in energy_grid:
+            for grid_point in reduced_production_grid_points:
+                energy_query_grid.append(np.hstack([energy.to(u.TeV).value, grid_point]))
+
+        energy_query_grid = np.array(energy_query_grid)
+
+        self._logger.debug(f"Grid points with energy shape: {grid_points_energy.shape}")
+        self._logger.debug(f"Data shape: {self.data.shape}")
+        self._logger.debug(f"Energy query grid shape: {energy_query_grid.shape}")
+
+        interpolated_values = self._perform_interpolation(
+            grid_points_energy, self.data, energy_query_grid
+        )
+
+        reshaped = interpolated_values.reshape(
+            len(reduced_production_grid_points), len(energy_grid)
+        )
+        return np.array([reshaped])
+
+    def interpolate(self) -> np.ndarray:
+        """
+        Interpolate production statistics at the grid points specified in grid_points_production.
+
+        This method performs two types of interpolation:
+        1. Energy-independent interpolation using the sum of production statistics
+        2. Energy-dependent interpolation for each energy bin
 
         Returns
         -------
-        float
-            Interpolated energy threshold.
+        np.ndarray
+            Interpolated values at the query points.
         """
-        flat_grid_points = []
-        flat_energy_thresholds = []
-
-        for e in self.evaluators:
-            az = e.grid_point[1].to(u.deg).value
-            zen = e.grid_point[2].to(u.deg).value
-            nsb = e.grid_point[3]
-            offset = e.grid_point[4].to(u.deg).value
-            grid_point = np.array([az, zen, nsb, offset])
-            flat_grid_points.append(grid_point)
-            flat_energy_thresholds.append(e.energy_threshold)
+        if not self.evaluators:
+            self._logger.error("No evaluators available for interpolation.")
+            return np.array([])
 
-        flat_grid_points = np.array(flat_grid_points)
-        flat_energy_thresholds = np.array(flat_energy_thresholds)
+        # Energy-independent interpolation
+        production_statistic, grid_points_no_energy = self._prepare_energy_independent_data()
+        reduced_production_grid_points = self._prepare_production_grid_points()
 
-        reduced_grid_points, non_flat_mask = self._remove_flat_dimensions(flat_grid_points)
-        full_non_flat_mask = np.concatenate(([False], non_flat_mask))
-        reduced_query_point = query_point[0][full_non_flat_mask]
+        self.interpolated_production_statistics = self._perform_interpolation(
+            grid_points_no_energy, production_statistic, reduced_production_grid_points
+        )
 
-        interpolated_threshold = griddata(
-            reduced_grid_points,
-            flat_energy_thresholds,
-            reduced_query_point,
-            method="linear",
-            fill_value=np.nan,
-            rescale=False,
+        # Energy-dependent interpolation
+        self.interpolated_production_statistics_with_energy = (
+            self._perform_interpolation_with_energy()
         )
 
-        return interpolated_threshold.item()
+        return self.interpolated_production_statistics
 
-    def plot_comparison(self, evaluator):
+    def plot_comparison(self, grid_point_index=0):
         """
-        Plot a comparison between the simulated, scaled, and reconstructed events.
+        Plot a comparison between interpolated production statistics and reconstructed events.
 
         Parameters
         ----------
-        evaluator : StatisticalErrorEvaluator
-            The evaluator for which to plot the comparison.
+        grid_point_index : int, optional
+            Index of the grid point to plot, by default 0
+
+        Returns
+        -------
+        matplotlib.axes.Axes
+            The Axes object containing the plot.
         """
-        import matplotlib.pyplot as plt  # pylint: disable=import-outside-toplevel
+        import matplotlib.pyplot as plt  # pylint: disable=C0415
+
+        if not self.evaluators:
+            self._logger.error("No evaluators available for plotting.")
+            _, ax = plt.subplots()
+            ax.text(0.5, 0.5, "No data available", ha="center", va="center")
+            return ax
+
+        # Use first evaluator for energy bins
+        bin_edges_low = self.evaluators[0].data["bin_edges_low"][:-1]
+        bin_edges_high = self.evaluators[0].data["bin_edges_high"][:-1]
+        midpoints = (bin_edges_low + bin_edges_high) / 2
+
+        if (
+            self.interpolated_production_statistics_with_energy is None
+            or len(self.interpolated_production_statistics_with_energy) == 0
+            or len(self.interpolated_production_statistics_with_energy[0]) <= grid_point_index
+        ):
+            self._logger.warning(
+                f"Invalid grid point index {grid_point_index}. Using index 0 instead."
+            )
+            grid_point_index = 0
 
-        midpoints = 0.5 * (evaluator.data["bin_edges_high"] + evaluator.data["bin_edges_low"])
+        _, ax = plt.subplots()
 
-        self.grid_points = np.column_stack(
-            [
-                midpoints,
-                np.full_like(midpoints, evaluator.grid_point[1]),
-                np.full_like(midpoints, evaluator.grid_point[2]),
-                np.full_like(midpoints, evaluator.grid_point[3]),
-                np.full_like(midpoints, evaluator.grid_point[4]),
+        if (
+            self.interpolated_production_statistics_with_energy is not None
+            and len(self.interpolated_production_statistics_with_energy) > 0
+        ):
+            interpolated_stats = self.interpolated_production_statistics_with_energy[0][
+                grid_point_index
             ]
-        )
-
-        self.interpolate(self.grid_points)
-
-        plt.plot(midpoints, evaluator.scaled_events, label="Scaled")
+            ax.plot(midpoints, interpolated_stats, label="Interpolated Production Statistics")
 
         reconstructed_event_histogram, _ = np.histogram(
-            evaluator.data["event_energies_reco"], bins=evaluator.data["bin_edges_low"]
+            self.evaluators[0].data["event_energies_reco"],
+            bins=self.evaluators[0].data["bin_edges_low"],
         )
-        plt.plot(midpoints[:-1], reconstructed_event_histogram, label="Reconstructed")
-
-        plt.legend()
-        plt.xscale("log")
-        plt.xlabel("Energy (Midpoint of Bin Edges)")
-        plt.ylabel("Event Count")
-        plt.title("Comparison of Simulated, scaled, and reconstructed events")
-        plt.show()
+        ax.plot(midpoints, reconstructed_event_histogram, label="Reconstructed Events")
+
+        ax.legend()
+        ax.set_xscale("log")
+        ax.set_yscale("log")
+        ax.set_xlabel("Energy (TeV)")
+        ax.set_ylabel("Event Count")
+        ax.set_title("Comparison of Interpolated and Reconstructed Events")
+
+        return ax

simtools/ray_tracing/mirror_panel_psf.py

@@ -9,7 +9,7 @@ from astropy.table import QTable, Table
 import simtools.data_model.model_data_writer as writer
 import simtools.utils.general as gen
 from simtools.data_model.metadata_collector import MetadataCollector
-from simtools.model.telescope_model import TelescopeModel
+from simtools.model.model_utils import initialize_simulation_models
 from simtools.ray_tracing.ray_tracing import RayTracing
 
 
@@ -38,7 +38,7 @@ class MirrorPanelPSF:
         self._logger.debug("Initializing MirrorPanelPSF")
 
         self.args_dict = args_dict
-        self.telescope_model = self._define_telescope_model(label, db_config)
+        self.telescope_model, self.site_model = self._define_telescope_model(label, db_config)
 
         if self.args_dict["test"]:
             self.args_dict["number_of_mirrors_to_test"] = 2
@@ -63,37 +63,32 @@ class MirrorPanelPSF:
         This includes updating the configuration with mirror list and/or random focal length given
         as input.
 
-        Attributes
-        ----------
-        label: str
-            Application label.
-        db_config:
-            Dictionary with database configuration.
-
         Returns
         -------
-        tel TelescopeModel
-            telescope model
-
+        tel : TelescopeModel
+            The telescope model.
+        site_model : SiteModel
+            The site model.
         """
-        tel = TelescopeModel(
+        tel_model, site_model = initialize_simulation_models(
+            label=label,
+            db_config=db_config,
             site=self.args_dict["site"],
             telescope_name=self.args_dict["telescope"],
             model_version=self.args_dict["model_version"],
-            mongo_db_config=db_config,
-            label=label,
         )
         if self.args_dict["mirror_list"] is not None:
             mirror_list_file = gen.find_file(
                 name=self.args_dict["mirror_list"], loc=self.args_dict["model_path"]
             )
-            tel.change_parameter("mirror_list", self.args_dict["mirror_list"])
-            tel.export_parameter_file("mirror_list", mirror_list_file)
+            tel_model.change_parameter("mirror_list", self.args_dict["mirror_list"])
+            tel_model.export_parameter_file("mirror_list", mirror_list_file)
         if self.args_dict["random_focal_length"] is not None:
-            tel.change_parameter("random_focal_length", str(self.args_dict["random_focal_length"]))
-        tel.export_model_files()
+            tel_model.change_parameter(
+                "random_focal_length", str(self.args_dict["random_focal_length"])
+            )
 
-        return tel
+        return tel_model, site_model
 
     def _get_psf_containment(self):
         """Read measured single-mirror point-spread function from file and return mean and sigma."""
@@ -229,6 +224,7 @@ class MirrorPanelPSF:
         self.telescope_model.change_parameter("mirror_reflection_random_angle", rnda)
         ray = RayTracing(
             telescope_model=self.telescope_model,
+            site_model=self.site_model,
             simtel_path=self.args_dict.get("simtel_path", None),
             single_mirror_mode=True,
             mirror_numbers=(

simtools/ray_tracing/psf_analysis.py

@@ -91,7 +91,7 @@ class PSFImage:
 
     def _process_simtel_file_using_rx(self, photon_file):
         """
-        Process a simtel file with photon lists using the RX method.
+        Process a sim_telarray file with photon lists using the RX method.
 
         Parameters
         ----------
@@ -152,7 +152,7 @@ class PSFImage:
                 self._process_simtel_line(line)
 
         if not self._is_photon_positions_ok():
-            msg = "Problems reading Simtel file - invalid data"
+            msg = "Problems reading sim_telarray file - invalid data"
             self._logger.error(msg)
             raise RuntimeError(msg)
 

simtools/ray_tracing/ray_tracing.py

@@ -33,6 +33,8 @@ class RayTracing:
     ----------
     telescope_model: TelescopeModel
         telescope model
+    site_model: SiteModel
+        site model
     simtel_path: str (or Path)
         Location of sim_telarray installation.
     label: str
@@ -61,6 +63,7 @@ class RayTracing:
     def __init__(
         self,
         telescope_model,
+        site_model,
         simtel_path,
         label=None,
         zenith_angle=20.0 * u.deg,
@@ -77,7 +80,7 @@ class RayTracing:
         self.simtel_path = Path(simtel_path)
         self._io_handler = io_handler.IOHandler()
 
-        self.telescope_model = telescope_model
+        self.telescope_model, self.site_model = telescope_model, site_model
         self.label = label if label is not None else self.telescope_model.label
 
         self.zenith_angle = zenith_angle.to("deg").value
@@ -86,11 +89,11 @@ class RayTracing:
         self.use_random_focal_length = use_random_focal_length
         self.mirrors = self._initialize_mirror_configuration(source_distance, mirror_numbers)
         self.output_directory = self._io_handler.get_output_directory(
-            label=self.label, sub_dir="ray-tracing"
+            label=self.label, sub_dir="ray_tracing"
         )
         self.output_directory.joinpath("results").mkdir(parents=True, exist_ok=True)
         self._file_results = self.output_directory.joinpath("results").joinpath(
-            self._generate_file_name(file_type="ray-tracing", suffix=".ecsv")
+            self._generate_file_name(file_type="ray_tracing", suffix=".ecsv")
         )
         self._psf_images = {}
         self._results = None
@@ -211,6 +214,7 @@ class RayTracing:
                 simtel = SimulatorRayTracing(
                     simtel_path=self.simtel_path,
                     telescope_model=self.telescope_model,
+                    site_model=self.site_model,
                     test=test,
                     config_data={
                         "zenith_angle": self.zenith_angle,
@@ -509,7 +513,7 @@ class RayTracing:
 
         if save:
             plot_file_name = self._generate_file_name(
-                file_type="ray-tracing",
+                file_type="ray_tracing",
                 suffix=".pdf",
                 extra_label=key,
             )
@@ -520,7 +524,7 @@ class RayTracing:
 
             for off_axis_key, image in self._psf_images.items():
                 image_file_name = self._generate_file_name(
-                    file_type="ray-tracing",
+                    file_type="ray_tracing",
                     off_axis_angle=off_axis_key,
                     suffix=".pdf",
                     extra_label=f"image_{key}",
@@ -530,7 +534,7 @@ class RayTracing:
                 image.plot_image(file_name=image_file)
 
                 image_cumulative_file_name = self._generate_file_name(
-                    file_type="ray-tracing",
+                    file_type="ray_tracing",
                     off_axis_angle=off_axis_key,
                     suffix=".pdf",
                     extra_label=f"cumulative_psf_{key}",
@@ -632,8 +636,9 @@ class RayTracing:
         self, file_type, suffix, off_axis_angle=None, mirror_number=None, extra_label=None
     ):
         """Generate file name for output files."""
+        file_type_prefix = file_type if file_type == "ray_tracing" else f"ray_tracing_{file_type}"
         return names.generate_file_name(
-            file_type=file_type,
+            file_type=file_type_prefix,
             suffix=suffix,
             site=self.telescope_model.site,
             telescope_model_name=self.telescope_model.name,