gammasimtools 0.16.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

simtools/production_configuration/derive_corsika_limits_grid.py

@@ -0,0 +1,232 @@
+"""Derive CORSIKA limits for a grid of parameters."""
+
+import datetime
+import logging
+
+import numpy as np
+from astropy.table import Column, Table
+
+import simtools.utils.general as gen
+from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.io_operations import io_handler
+from simtools.model.site_model import SiteModel
+from simtools.production_configuration.derive_corsika_limits import LimitCalculator
+
+_logger = logging.getLogger(__name__)
+
+
+def generate_corsika_limits_grid(args_dict, db_config=None):
+    """
+    Generate CORSIKA limits for a grid of parameters.
+
+    Requires at least one event data file per parameter set.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Dictionary containing command line arguments.
+    db_config : dict, optional
+        Database configuration dictionary.
+    """
+    event_data_files = gen.get_list_of_files_from_command_line(
+        args_dict["event_data_files"], [".hdf5", ".gz"]
+    )  # accept fits.gz files (.gz)
+    if args_dict.get("array_layout_name"):
+        telescope_configs = _read_array_layouts_from_db(
+            args_dict["array_layout_name"],
+            args_dict.get("site"),
+            args_dict.get("model_version"),
+            db_config,
+        )
+    else:
+        telescope_configs = gen.collect_data_from_file(args_dict["telescope_ids"])[
+            "telescope_configs"
+        ]
+
+    results = []
+    for file_path in event_data_files:
+        for array_name, telescope_ids in telescope_configs.items():
+            _logger.info(f"Processing file: {file_path} with telescope config: {array_name}")
+            result = _process_file(
+                file_path,
+                array_name,
+                telescope_ids,
+                args_dict["loss_fraction"],
+                args_dict["plot_histograms"],
+            )
+            result["layout"] = array_name
+            results.append(result)
+
+    write_results(results, args_dict)
+
+
+def _process_file(file_path, array_name, telescope_ids, loss_fraction, plot_histograms):
+    """
+    Compute limits for a single file.
+
+    Parameters
+    ----------
+    file_path : str
+        Path to the event data file.
+    array_name : str
+        Name of the telescope array configuration.
+    telescope_ids : list[int]
+        List of telescope IDs to filter the events.
+    loss_fraction : float
+        Fraction of events to be lost.
+    plot_histograms : bool
+        Whether to plot histograms.
+
+    Returns
+    -------
+    dict
+        Dictionary containing the computed limits and metadata.
+    """
+    calculator = LimitCalculator(file_path, array_name=array_name, telescope_list=telescope_ids)
+    limits = calculator.compute_limits(loss_fraction)
+
+    if plot_histograms:
+        calculator.plot_data(io_handler.IOHandler().get_output_directory())
+
+    return limits
+
+
+def write_results(results, args_dict):
+    """
+    Write the computed limits as astropy table to file.
+
+    Parameters
+    ----------
+    results : list[dict]
+        List of computed limits.
+    args_dict : dict
+        Dictionary containing command line arguments.
+    """
+    table = _create_results_table(results, args_dict["loss_fraction"])
+
+    output_dir = io_handler.IOHandler().get_output_directory("corsika_limits")
+    output_file = output_dir / args_dict["output_file"]
+
+    table.write(output_file, format="ascii.ecsv", overwrite=True)
+    _logger.info(f"Results saved to {output_file}")
+
+    MetadataCollector.dump(args_dict, output_file)
+
+
+def _create_results_table(results, loss_fraction):
+    """
+    Convert list of simulation results to an astropy Table with metadata.
+
+    Round values to appropriate precision and add metadata.
+
+    Parameters
+    ----------
+    results : list[dict]
+        Computed limits per file and telescope configuration.
+    loss_fraction : float
+        Fraction of lost events (added to metadata).
+
+    Returns
+    -------
+    astropy.table.Table
+        Table with computed limits.
+    """
+    cols = [
+        "primary_particle",
+        "array_name",
+        "telescope_ids",
+        "zenith",
+        "azimuth",
+        "nsb_level",
+        "lower_energy_limit",
+        "upper_radius_limit",
+        "viewcone_radius",
+    ]
+
+    columns = {name: [] for name in cols}
+    units = {}
+
+    for res in results:
+        _process_result_row(res, cols, columns, units)
+
+    table_cols = _create_table_columns(cols, columns, units)
+    table = Table(table_cols)
+
+    table.meta.update(
+        {
+            "created": datetime.datetime.now().isoformat(),
+            "description": "Lookup table for CORSIKA limits computed from simulations.",
+            "loss_fraction": loss_fraction,
+        }
+    )
+
+    return table
+
+
+def _process_result_row(res, cols, columns, units):
+    """Process a single result row and add values to columns."""
+    for k in cols:
+        val = res.get(k, None)
+        if val is not None:
+            val = _round_value(k, val)
+            _logger.debug(f"Adding {k}: {val} to column data")
+
+        if hasattr(val, "unit"):
+            columns[k].append(val.value)
+            units[k] = val.unit
+        else:
+            columns[k].append(val)
+            if k not in units:
+                units[k] = None
+
+
+def _round_value(key, val):
+    """Round value based on key type."""
+    if key == "lower_energy_limit":
+        return np.floor(val * 1e3) / 1e3
+    if key == "upper_radius_limit":
+        return np.ceil(val / 25) * 25
+    if key == "viewcone_radius":
+        return np.ceil(val / 0.25) * 0.25
+    return val
+
+
+def _create_table_columns(cols, columns, units):
+    """Create table columns with appropriate data types."""
+    table_cols = []
+    for k in cols:
+        col_data = columns[k]
+        if any(isinstance(v, list | tuple) for v in col_data):
+            col = Column(data=col_data, name=k, unit=units.get(k), dtype=object)
+        else:
+            col = Column(data=col_data, name=k, unit=units.get(k))
+        table_cols.append(col)
+    return table_cols
+
+
+def _read_array_layouts_from_db(layouts, site, model_version, db_config):
+    """
+    Read array layouts from the database.
+
+    Parameters
+    ----------
+    layouts : list[str]
+        List of layout names to read. If "all", read all available layouts.
+    site : str
+        Site name for the array layouts.
+    model_version : str
+        Model version for the array layouts.
+    db_config : dict
+        Database configuration dictionary.
+
+    Returns
+    -------
+    dict
+        Dictionary mapping layout names to telescope IDs.
+    """
+    site_model = SiteModel(site=site, model_version=model_version, mongo_db_config=db_config)
+    layout_names = site_model.get_list_of_array_layouts() if layouts == ["all"] else layouts
+    layout_dict = {}
+    for layout_name in layout_names:
+        layout_dict[layout_name] = site_model.get_array_elements_for_layout(layout_name)
+    return layout_dict

simtools/production_configuration/derive_production_statistics.py

@@ -27,7 +27,7 @@ class ProductionStatisticsDerivator:
 
         Parameters
         ----------
-        evaluator : StatisticalErrorEvaluator
+        evaluator : StatisticalUncertaintyEvaluator
             The evaluator responsible for calculating metrics and handling event data.
         metrics : dict
             Dictionary containing metrics, including target error for effective area.
@@ -35,6 +35,50 @@ class ProductionStatisticsDerivator:
         self.evaluator = evaluator
         self.metrics = metrics
 
+    def _compute_scaling_factor(self) -> np.ndarray:
+        """
+        Compute bin-wise scaling factors based on error metrics.
+
+        Takes into account the energy range specified in the metrics and
+        calculates a separate scaling factor for each energy bin.
+
+        Returns
+        -------
+        np.ndarray
+            Array of scaling factors for each energy bin.
+        """
+        uncertainty_effective_area = self.evaluator.metric_results.get("uncertainty_effective_area")
+        relative_uncertainties = uncertainty_effective_area.get("relative_uncertainties")
+        energy_range = (
+            self.metrics.get("uncertainty_effective_area").get("energy_range").get("value")
+        )
+        energy_unit = u.Unit(
+            self.metrics.get("uncertainty_effective_area").get("energy_range").get("unit")
+        )
+
+        energy_range_converted = np.array(energy_range) * energy_unit
+
+        bin_edges = self.evaluator.energy_bin_edges
+        bin_centers = (bin_edges[:-1] + bin_edges[1:]) / 2
+
+        # Mask for bins within the metric specified energy range
+        mask = (bin_centers >= energy_range_converted[0]) & (
+            bin_centers <= energy_range_converted[1]
+        )
+
+        scaling_factors = np.zeros_like(relative_uncertainties)
+
+        target_uncertainty = (
+            self.metrics.get("uncertainty_effective_area").get("target_uncertainty").get("value")
+        )
+
+        # Calculate scaling factor only for bins within the energy range
+        # For bins with zero events/uncertainty, use a scaling factor of 0
+        valid_bins = mask & (relative_uncertainties > 0)
+        scaling_factors[valid_bins] = (relative_uncertainties[valid_bins] / target_uncertainty) ** 2
+
+        return scaling_factors
+
     def derive_statistics(self, return_sum: bool = True) -> u.Quantity:
         """
         Derive the production statistics based on statistical error metrics.
@@ -54,31 +98,15 @@ class ProductionStatisticsDerivator:
             If 'return_sum' is False, returns an array of production statistics along the energy
             axis as a u.Quantity.
         """
-        scaling_factor = self._compute_scaling_factor()
-
+        scaling_factors = self._compute_scaling_factor()
         base_events = self._number_of_simulated_events()
+        # currently we use the maximum of the scaling factors to scale the events. This is a soft
+        # requirement if we want to keep the power law shape of the production statistics.
+        scaled_events = base_events * np.max(scaling_factors)
 
         if return_sum:
-            return np.sum(base_events * scaling_factor)
-        return base_events * scaling_factor
-
-    def _compute_scaling_factor(self) -> float:
-        """
-        Compute the scaling factor based on the error metrics.
-
-        Returns
-        -------
-        float
-            The scaling factor.
-        """
-        metric_results = self.evaluator.calculate_metrics()
-        uncertainty_effective_area = metric_results.get("uncertainty_effective_area")
-        current_max_error = uncertainty_effective_area.get("max_error")
-        target_max_error = self.metrics.get("uncertainty_effective_area").get("target_error")[
-            "value"
-        ]
-
-        return (current_max_error / target_max_error) ** 2
+            return np.sum(scaled_events)
+        return scaled_events
 
     def _number_of_simulated_events(self) -> u.Quantity:
         """
@@ -112,13 +140,16 @@ class ProductionStatisticsDerivator:
         bin_edges = self.evaluator.create_bin_edges()
         bin_idx = np.digitize(energy, bin_edges) - 1
 
-        scaling_factor = self._compute_scaling_factor()
+        # Get scaling factors for all bins
+        scaling_factors = self._compute_scaling_factor()
 
         simulated_event_histogram = self.evaluator.data.get("simulated_event_histogram", [])
 
         if bin_idx < 0 or bin_idx >= len(simulated_event_histogram):
-            raise ValueError(f"Energy {energy} is outside therange of the simulated events data.")
+            raise ValueError(f"Energy {energy} is outside the range of the simulated events data.")
 
         base_events = self._number_of_simulated_events()
         base_event_at_energy = base_events[bin_idx]
-        return base_event_at_energy * scaling_factor
+        scaling_factor_at_energy = scaling_factors[bin_idx]
+
+        return base_event_at_energy * scaling_factor_at_energy

simtools/production_configuration/derive_production_statistics_handler.py

@@ -1,11 +1,11 @@
 """
-Module to run the StatisticalErrorEvaluator and interpolate results.
+Derives the required statistics for a requested set of production parameters through interpolation.
 
 This module provides the `ProductionStatisticsHandler` class, which manages the workflow for
 derivation of required number of events for a simulation production using pre-defined metrics.
 
 The module includes functionality to:
-- Initialize evaluators for statistical error calculations based on input parameters.
+- Initialize evaluators for statistical uncertainty calculations based on input parameters.
 - Perform interpolation using the initialized evaluators to estimate production statistics at a
 query point.
 - Write the results of the interpolation to an output file.
@@ -17,10 +17,9 @@ import logging
 from pathlib import Path
 
 import astropy.units as u
-import numpy as np
 
-from simtools.production_configuration.calculate_statistical_errors_grid_point import (
-    StatisticalErrorEvaluator,
+from simtools.production_configuration.calculate_statistical_uncertainties_grid_point import (
+    StatisticalUncertaintyEvaluator,
 )
 from simtools.production_configuration.interpolation_handler import InterpolationHandler
 from simtools.utils.general import collect_data_from_file
@@ -35,7 +34,7 @@ class ProductionStatisticsHandler:
     production at a specified query point.
     """
 
-    def __init__(self, args_dict):
+    def __init__(self, args_dict, output_path):
         """
         Initialize the manager with the provided arguments.
 
@@ -43,35 +42,56 @@ class ProductionStatisticsHandler:
         ----------
         args_dict : dict
             Dictionary of command-line arguments.
+        output_path : Path
+            Path to the directory where the event statistics output file will be saved.
         """
         self.args = args_dict
         self.logger = logging.getLogger(__name__)
-        self.output_path = Path(self.args.get("output_path", "."))
-        self.output_filepath = self.output_path.joinpath(f"{self.args['output_file']}")
+        self.output_path = output_path
         self.metrics = collect_data_from_file(self.args["metrics_file"])
         self.evaluator_instances = []
+        self.interpolation_handler = None
+        self.grid_points_production = self._load_grid_points_production()
+
+    def _load_grid_points_production(self):
+        """Load grid points from the JSON file."""
+        grid_points_production_file = self.args["grid_points_production_file"]
+        return collect_data_from_file(grid_points_production_file)
 
     def initialize_evaluators(self):
-        """Initialize StatisticalErrorEvaluator instances for the given zeniths and offsets."""
-        if not (self.args["base_path"] and self.args["zeniths"] and self.args["camera_offsets"]):
+        """Initialize StatisticalUncertaintyEvaluator instances for the given grid point."""
+        if not (
+            self.args["base_path"]
+            and self.args["zeniths"]
+            and self.args["azimuths"]
+            and self.args["nsb"]
+            and self.args["offsets"]
+        ):
             self.logger.warning("No files read")
             self.logger.warning(f"Base Path: {self.args['base_path']}")
             self.logger.warning(f"Zeniths: {self.args['zeniths']}")
-            self.logger.warning(f"Camera offsets: {self.args['camera_offsets']}")
+            self.logger.warning(f"Camera offsets: {self.args['offsets']}")
             return
 
-        for zenith, offset in itertools.product(self.args["zeniths"], self.args["camera_offsets"]):
-            file_name = self.args["file_name_template"].format(zenith=int(zenith))
+        for zenith, azimuth, nsb, offset in itertools.product(
+            self.args["zeniths"], self.args["azimuths"], self.args["nsb"], self.args["offsets"]
+        ):
+            file_name = self.args["file_name_template"].format(
+                zenith=int(zenith),
+                azimuth=azimuth,
+                nsb=nsb,
+                offset=offset,
+            )
             file_path = Path(self.args["base_path"]).joinpath(file_name)
 
             if not file_path.exists():
                 self.logger.warning(f"File not found: {file_path}. Skipping.")
                 continue
 
-            evaluator = StatisticalErrorEvaluator(
+            evaluator = StatisticalUncertaintyEvaluator(
                 file_path,
                 metrics=self.metrics,
-                grid_point=(None, None, zenith, None, offset * u.deg),
+                grid_point=(None, azimuth, zenith, nsb, offset * u.deg),
             )
             evaluator.calculate_metrics()
             self.evaluator_instances.append(evaluator)
@@ -82,38 +102,51 @@ class ProductionStatisticsHandler:
             self.logger.error("No evaluators initialized. Cannot perform interpolation.")
             return None
 
-        interpolation_handler = InterpolationHandler(self.evaluator_instances, metrics=self.metrics)
-        query_point = self.args.get("query_point")
-        if not query_point or len(query_point) != 5:
-            raise ValueError(
-                "Invalid query point format. "
-                f"Expected 5 values, got {len(query_point) if query_point else 'None'}."
+        self.interpolation_handler = InterpolationHandler(
+            self.evaluator_instances,
+            metrics=self.metrics,
+            grid_points_production=self.grid_points_production,
+        )
+        qrid_points_with_statistics = []
+
+        interpolated_production_statistics = self.interpolation_handler.interpolate()
+        for grid_point, statistics in zip(
+            self.grid_points_production, interpolated_production_statistics
+        ):
+            qrid_points_with_statistics.append(
+                {
+                    "grid_point": grid_point,
+                    "interpolated_production_statistics": float(statistics),
+                }
             )
-        query_points = np.array([self.args["query_point"]])
-        return interpolation_handler.interpolate(query_points)
+        return qrid_points_with_statistics
 
     def write_output(self, production_statistics):
         """Write the derived event statistics to a file."""
-        output_data = {
-            "query_point": self.args["query_point"],
-            "production_statistics": production_statistics.tolist(),
-        }
-        self.output_filepath.parent.mkdir(parents=True, exist_ok=True)
-        with open(self.output_filepath, "w", encoding="utf-8") as f:
+        output_data = (production_statistics,)
+        output_filename = self.args["output_file"]
+        self.output_path.mkdir(parents=True, exist_ok=True)
+        output_file_path = self.output_path.joinpath(output_filename)
+        with open(output_file_path, "w", encoding="utf-8") as f:
             json.dump(output_data, f, indent=4)
-        self.logger.info(f"Output saved to {self.output_filepath}")
-        self.logger.info(
-            f"production statistics for grid point "
-            f"{self.args['query_point']}: {production_statistics}"
-        )
+        self.logger.info(f"Output saved to {self.output_path}")
+
+    def plot_production_statistics_comparison(self):
+        """Plot the derived event statistics."""
+        ax = self.interpolation_handler.plot_comparison()
+        plot_path = self.output_path.joinpath("production_statistics_comparison.png")
+        plot_path.parent.mkdir(parents=True, exist_ok=True)
+        ax.figure.savefig(plot_path)
+        self.logger.info(f"Plot saved to {plot_path}")
 
     def run(self):
         """Run the scaling and interpolation workflow."""
-        self.logger.info(f"Zeniths: {self.args['zeniths']}")
-        self.logger.info(f"Camera offsets: {self.args['camera_offsets']}")
-        self.logger.info(f"Query Point: {self.args['query_point']}")
+        self.logger.info(f"Grid Points File: {self.args['grid_points_production_file']}")
         self.logger.info(f"Metrics File: {self.args['metrics_file']}")
 
         self.initialize_evaluators()
         production_statistics = self.perform_interpolation()
+        if self.args.get("plot_production_statistics"):
+            self.plot_production_statistics_comparison()
+
         self.write_output(production_statistics)