gammasimtools 0.15.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/applications/production_derive_corsika_limits.py

@@ -1,18 +1,39 @@
 #!/usr/bin/python3
 
 r"""
-Derives the limits for energy, radial distance, and viewcone to be used in CORSIKA simulations.
+Derive CORSIKA configuration limits for energy, core distance, and viewcone radius.
 
-The limits are derived based on the event loss fraction specified by the user.
+This tool determines configuration limits based on triggered events from broad-range
+simulations. It supports setting:
+
+- **ERANGE**: Derives the lower energy limit; upper limit is user-defined.
+- **CSCAT**: Derives the upper core distance; lower limit is user-defined.
+- **VIEWCONE**: Derives the viewcone radius; lower limit is user-defined.
+
+Limits are computed based on a configurable maximum event loss fraction.
+Results are provided as a table with the following columns:
+
+- particle_type: Particle type (e.g., gamma, proton, electron).
+- telescope_ids: List of telescope IDs used in the simulation.
+- zenith: Zenith angle.
+- azimuth: Azimuth angle.
+- nsb: Night sky background level
+- layout: Layout of the telescope array used in the simulation.
+- lower_energy_limit: Derived lower energy limit.
+- upper_radius_limit: Derived upper radial distance limit.
+- viewcone_radius: Derived upper viewcone radius limit.
+
+The input event data files are generated using the application simtools-generate-simtel-event-data
+and are required for each point in the lookup table.
 
 Command line arguments
 ----------------------
 event_data_files (str, required)
-    Path to a file containing event data file paths.
+    Path to a file containing event data files derived with 'simtools-generate-simtel-event-data'.
 telescope_ids (str, required)
     Path to a file containing telescope configurations.
 loss_fraction (float, required)
-    Fraction of events to be lost.
+    Maximum event-loss fraction for limit computation.
 plot_histograms (bool, optional)
     Plot histograms of the event data.
 output_file (str, optional)
@@ -32,44 +53,17 @@ Derive limits for a given file with a specified loss fraction.
         --output_file corsika_simulation_limits_lookup.ecsv
 """
 
-import datetime
 import logging
 
-from astropy.table import Table
-
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.data_model.metadata_collector import MetadataCollector
-from simtools.io_operations import io_handler
-from simtools.production_configuration.derive_corsika_limits import LimitCalculator
-
-_logger = logging.getLogger(__name__)
+from simtools.production_configuration.derive_corsika_limits_grid import (
+    generate_corsika_limits_grid,
+)
 
 
 def _parse():
-    """
-    Parse command line configuration.
-
-    Parameters
-    ----------
-    event_data_files: str
-        Path to a file listing event data file paths. These files contain the
-        simulation data used for deriving the limits.
-    loss_fraction: float
-        Fraction of events to be excluded during limit computation. Determines
-        thresholds for energy, radial distance, and viewcone.
-    telescope_ids: str
-        Path to a file defining telescope configurations. Specifies telescope
-        arrays or IDs used to filter events during processing.
-    plot_histograms: bool
-        If True, generates and saves histograms of the event data to visualize
-        the computed limits and distributions.
-
-    Returns
-    -------
-    CommandLineParser
-        Command line parser object
-    """
+    """Parse command line configuration."""
     config = configurator.Configurator(
         description="Derive limits for energy, radial distance, and viewcone."
     )
@@ -86,7 +80,10 @@ def _parse():
         help="Path to a file containing telescope configurations.",
     )
     config.parser.add_argument(
-        "--loss_fraction", type=float, required=True, help="Fraction of events to be lost."
+        "--loss_fraction",
+        type=float,
+        required=True,
+        help="Maximum event-loss fraction for limit computation.",
     )
     config.parser.add_argument(
         "--plot_histograms",
@@ -94,110 +91,7 @@ def _parse():
         action="store_true",
         default=False,
     )
-    config.parser.add_argument(
-        "--output_file",
-        type=str,
-        default="corsika_simulation_limits_lookup.ecsv",
-        help="Output file for the derived limits (default: "
-        "'corsika_simulation_limits_lookup.ecsv').",
-    )
-    return config.initialize(db_config=False)
-
-
-def process_file(file_path, telescope_ids, loss_fraction, plot_histograms):
-    """
-    Process a single file and compute limits.
-
-    Parameters
-    ----------
-    file_path : str
-        Path to the event data file.
-    telescope_ids : list[int]
-        List of telescope IDs to filter the events.
-    loss_fraction : float
-        Fraction of events to be lost.
-    plot_histograms : bool
-        Whether to plot histograms.
-
-    Returns
-    -------
-    dict
-        Dictionary containing the computed limits.
-    """
-    calculator = LimitCalculator(file_path, telescope_list=telescope_ids)
-
-    lower_energy_limit = calculator.compute_lower_energy_limit(loss_fraction)
-    upper_radial_distance = calculator.compute_upper_radial_distance(loss_fraction)
-    viewcone = calculator.compute_viewcone(loss_fraction)
-
-    if plot_histograms:
-        _logger.info(
-            f"Plotting histograms written to {io_handler.IOHandler().get_output_directory()}"
-        )
-        calculator.plot_data(
-            lower_energy_limit,
-            upper_radial_distance,
-            viewcone,
-            io_handler.IOHandler().get_output_directory(),
-        )
-
-    return {
-        "file_path": file_path,
-        "telescope_ids": telescope_ids,
-        "lower_energy_threshold": lower_energy_limit,
-        "upper_radius_threshold": upper_radial_distance,
-        "viewcone_radius": viewcone,
-    }
-
-
-def create_results_table(results, loss_fraction):
-    """
-    Create an Astropy Table from the results.
-
-    Parameters
-    ----------
-    results : list[dict]
-        List of dictionaries containing the computed limits for each file
-          and telescope configuration.
-    loss_fraction : float
-        Fraction of events to be lost, added as metadata to the table.
-
-    Returns
-    -------
-    astropy.table.Table
-        An Astropy Table containing the results with appropriate units and metadata.
-    """
-    table = Table(
-        rows=[
-            (
-                res["file_path"],
-                res["telescope_ids"],
-                res["lower_energy_threshold"],
-                res["upper_radius_threshold"],
-                res["viewcone_radius"],
-            )
-            for res in results
-        ],
-        names=[
-            "file_path",
-            "telescope_ids",
-            "lower_energy_threshold",
-            "upper_radius_threshold",
-            "viewcone_radius",
-        ],
-    )
-
-    table["lower_energy_threshold"].unit = "TeV"
-    table["upper_radius_threshold"].unit = "m"
-    table["viewcone_radius"].unit = "deg"
-
-    table.meta["created"] = datetime.datetime.now().isoformat()
-    table.meta["description"] = (
-        "Lookup table for CORSIKA limits computed from gamma-ray shower simulations."
-    )
-    table.meta["loss_fraction"] = loss_fraction
-
-    return table
+    return config.initialize(db_config=False, output=True)
 
 
 def main():
@@ -207,33 +101,7 @@ def main():
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "info")))
 
-    event_data_files = gen.collect_data_from_file(args_dict["event_data_files"])["files"]
-    telescope_configs = gen.collect_data_from_file(args_dict["telescope_ids"])["telescope_configs"]
-
-    results = []
-    for file_path in event_data_files:
-        for array_name, telescope_ids in telescope_configs.items():
-            _logger.info(f"Processing file: {file_path} with telescope config: {array_name}")
-            result = process_file(
-                file_path,
-                telescope_ids,
-                args_dict["loss_fraction"],
-                args_dict["plot_histograms"],
-            )
-            result["layout"] = array_name
-            results.append(result)
-
-    table = create_results_table(results, args_dict["loss_fraction"])
-
-    output_dir = io_handler.IOHandler().get_output_directory("corsika_limits")
-    output_file = f"{output_dir}/{args_dict['output_file']}"
-
-    table.write(output_file, format="ascii.ecsv", overwrite=True)
-    _logger.info(f"Results saved to {output_file}")
-
-    metadata_file = f"{output_dir}/metadata.yml"
-    MetadataCollector.dump(args_dict, metadata_file)
-    _logger.info(f"Metadata saved to {metadata_file}")
+    generate_corsika_limits_grid(args_dict)
 
 
 if __name__ == "__main__":

simtools/applications/production_derive_statistics.py

@@ -0,0 +1,159 @@
+#!/usr/bin/python3
+
+r"""
+Derive the required number of events for MC productions for a grid of observational conditions.
+
+This application evaluates statistical uncertainties from the analysis of events after the
+application of loose gamma/hadron separation cuts, then interpolates the derived number of required
+events for the specified grid points provided in a file. The resulting grid points will have the
+derived number of required events added.
+
+The metric for the required uncertainty is pre-defined and must be configured via the metrics file.
+
+Command line arguments
+----------------------
+grid_points_production_file (str, required)
+    Path to the file containing grid points. Each grid point should include azimuth, zenith, NSB,
+    offset.
+metrics_file (str, optional)
+    Path to the metrics definition file. Default: 'production_simulation_config_metrics.yml'.
+base_path (str, required)
+    Path to the directory containing the event files for interpolation (after loose gamma/hadron
+    cuts).
+file_name_template (str, optional)
+    Template for the event file name. Default:
+    'prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits'.
+plot_production_statistics (flag, optional)
+    If provided, plots the production statistics. Default: False.
+
+Example
+-------
+To evaluate statistical uncertainties and perform interpolation, run the command line script:
+
+.. code-block:: console
+
+    simtools-production-derive-statistics \\
+        --grid_points_production_file path/to/grid_points_production.json \\
+        --metrics_file "path/to/metrics.yaml" \\
+        --base_path path/to/production_event_files/ \\
+        --file_name_template "prod6_LaPalma-{zenith}deg\\
+            _gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits" \\
+        --zeniths 20 40 52 60 \\
+        --offsets 0 \\
+        --azimuths 180 \\
+        --nsb 0.0 \\
+        --plot_production_statistics
+
+Output
+------
+The output will be a file containing the grid points with the derived number of required events
+added.
+"""
+
+import logging
+from pathlib import Path
+
+from simtools.configuration import configurator
+from simtools.io_operations import io_handler
+from simtools.production_configuration.derive_production_statistics_handler import (
+    ProductionStatisticsHandler,
+)
+from simtools.utils import general as gen
+
+
+def _parse(label, description):
+    """
+    Parse command line arguments for the statistical error evaluator application.
+
+    Returns
+    -------
+    argparse.Namespace
+        Parsed command line arguments.
+    """
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--grid_points_production_file",
+        type=str,
+        required=True,
+        help="Path to the JSON file containing grid points for a production.",
+    )
+    config.parser.add_argument(
+        "--metrics_file",
+        required=True,
+        type=str,
+        default=None,
+        help="Metrics definition file. (default: production_simulation_config_metrics.yml)",
+    )
+    config.parser.add_argument(
+        "--base_path",
+        type=str,
+        required=True,
+        help="Path to the DL2 MC event files for interpolation.",
+    )
+    config.parser.add_argument(
+        "--file_name_template",
+        required=False,
+        type=str,
+        default=("prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits"),
+        help=("Template for the DL2 MC event file name."),
+    )
+    config.parser.add_argument(
+        "--zeniths",
+        required=True,
+        nargs="+",
+        type=float,
+        help="List of zenith angles in deg that describe the supplied DL2 files.",
+    )
+    config.parser.add_argument(
+        "--azimuths",
+        required=True,
+        nargs="+",
+        type=float,
+        help="List of azimuth angles in deg that describe the supplied DL2 files.",
+    )
+    config.parser.add_argument(
+        "--nsb",
+        required=True,
+        nargs="+",
+        type=float,
+        help="List of nsb values that describe the supplied DL2 files.",
+    )
+    config.parser.add_argument(
+        "--offsets",
+        required=True,
+        nargs="+",
+        type=float,
+        help="List of camera offsets in deg that describe the supplied DL2 files.",
+    )
+    config.parser.add_argument(
+        "--plot_production_statistics",
+        required=False,
+        action="store_true",
+        default=False,
+        help="Plot production statistics.",
+    )
+
+    return config.initialize(db_config=False, output=True)
+
+
+def main():
+    """Run the ProductionStatisticsHandler."""
+    label = Path(__file__).stem
+
+    args_dict, _ = _parse(
+        label,
+        "Evaluate statistical uncertainties from DL2 MC event files and interpolate results.",
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    _io_handler = io_handler.IOHandler()
+    output_path = _io_handler.get_output_directory(label, sub_dir="")
+
+    manager = ProductionStatisticsHandler(args_dict, output_path=output_path)
+    manager.run()
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/production_generate_grid.py

@@ -0,0 +1,197 @@
+#!/usr/bin/python3
+
+r"""
+Generate a grid of simulation points using flexible axes definitions.
+
+This application generates a grid of simulation points based on the provided axes
+definitions. The axes definitions (range, binning) are specified in a file.
+The viewcone, radius and energy thresholds are provided as a lookup table and
+are interpolated based on the generated grid points. The generated grid points are
+filtered based on the specified telescope IDs and the limits from the lookup table.
+The generated grid points are saved to a file.
+It can also convert the generated points to RA/Dec coordinates if the selected
+coordinate system is 'ra_dec'.
+
+Command line arguments
+----------------------
+axes (str, required)
+    Path to a YAML or JSON file defining the axes of the grid.
+coordinate_system (str, optional, default='zenith_azimuth')
+    The coordinate system for the grid generation ('zenith_azimuth' or 'ra_dec').
+observing_time (str, optional, default=now)
+    Time of the observation (format: 'YYYY-MM-DD HH:MM:SS').
+lookup_table (str, required)
+    Path to the lookup table for simulation limits. The table should contain
+    varying azimuth and/or zenith angles.
+telescope_ids (list of int, optional)
+    List of telescope IDs as used in sim_telarray to filter the events.
+output_file (str, optional, default='grid_output.json')
+    Output file for the generated grid points (default: 'grid_output.json').
+
+
+Example
+-------
+To generate a grid of simulation points, execute the script as follows:
+
+.. code-block:: console
+
+    simtools-production-generate-grid --site North --model_version 6.0.0 \
+      --axes  tests/resources/production_grid_generation_axes_definition.yml \
+      --coordinate_system ra_dec --observing_time "2017-09-16 00:00:00" \
+      --lookup_table tests/resources/corsika_simulation_limits_lookup.ecsv \
+        --telescope_ids 1
+"""
+
+import logging
+from pathlib import Path
+
+from astropy.coordinates import EarthLocation
+from astropy.time import Time
+
+from simtools.configuration import configurator
+from simtools.io_operations import io_handler
+from simtools.model.site_model import SiteModel
+from simtools.production_configuration.generate_production_grid import GridGeneration
+from simtools.utils.general import collect_data_from_file
+
+
+def _parse(label, description):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    axes (str, required)
+        Path to a YAML or JSON file defining the axes of the grid.
+    coordinate_system (str, optional, default='zenith_azimuth')
+        The coordinate system for the grid generation ('zenith_azimuth' or 'ra_dec').
+    observing_time (str, optional, default=now)
+        Time of the observation (format: 'YYYY-MM-DD HH:MM:SS').
+    lookup_table (str, required)
+        Path to the lookup table for simulation limits. The table should contain
+        varying azimuth and/or zenith angles.
+    telescope_ids (list of int, optional)
+        List of telescope IDs as used in sim_telarray to filter the events.
+    output_file (str, optional, default='grid_output.json')
+        Output file for the generated grid points (default: 'grid_output.json').
+
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--axes",
+        type=str,
+        required=True,
+        help="Path to a file defining the grid axes.",
+    )
+    config.parser.add_argument(
+        "--coordinate_system",
+        type=str,
+        default="zenith_azimuth",
+        help="Coordinate system ('zenith_azimuth' or 'ra_dec').",
+    )
+    config.parser.add_argument(
+        "--observing_time",
+        type=str,
+        required=False,
+        help="Time of the observation (format: 'YYYY-MM-DD HH:MM:SS').",
+    )
+    config.parser.add_argument(
+        "--output_file",
+        type=str,
+        default="grid_output.json",
+        help="Output file for the generated grid points (default: 'grid_output.json').",
+    )
+    config.parser.add_argument(
+        "--telescope_ids",
+        type=int,
+        nargs="*",
+        default=None,
+        help="List of telescope IDs as used in sim_telarray to get the specific limits from the "
+        "lookup table.",
+    )
+    config.parser.add_argument(
+        "--lookup_table",
+        type=str,
+        required=True,
+        help="Path to the lookup table for simulation limits. "
+        "Table required with varying azimuth and or zenith angle. ",
+    )
+
+    return config.initialize(db_config=True, simulation_model=["version", "site", "model_version"])
+
+
+def load_axes(file_path: str):
+    """
+    Load axes definitions from a YAML or JSON file.
+
+    Parameters
+    ----------
+    file_path : str
+        Path to the axes YAML or JSON file.
+
+    Returns
+    -------
+    list[dict]
+        List of axes definitions with Quantity values.
+    """
+    if not Path(file_path).exists():
+        raise FileNotFoundError(f"Axes file {file_path} not found.")
+
+    return collect_data_from_file(file_path)
+
+
+def main():
+    """Run the Grid Generation application."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(
+        label,
+        "Generate a grid of simulation points using flexible axes definitions.",
+    )
+
+    _logger = logging.getLogger()
+    _logger.setLevel(logging.INFO)
+
+    output_path = io_handler.IOHandler().get_output_directory(label)
+    output_filepath = Path(output_path).joinpath(f"{args_dict['output_file']}")
+
+    axes = load_axes(args_dict["axes"])
+    site_model = SiteModel(
+        mongo_db_config=db_config,
+        model_version=args_dict["model_version"],
+        site=args_dict["site"],
+    )
+
+    ref_lat = site_model.get_parameter_value_with_unit("reference_point_latitude")
+    ref_long = site_model.get_parameter_value_with_unit("reference_point_longitude")
+    altitude = site_model.get_parameter_value_with_unit("reference_point_altitude")
+
+    observing_location = EarthLocation(lat=ref_lat, lon=ref_long, height=altitude)
+
+    observing_time = (
+        Time(args_dict["observing_time"]) if args_dict.get("observing_time") else Time.now()
+    )
+
+    grid_gen = GridGeneration(
+        axes=axes,
+        coordinate_system=args_dict["coordinate_system"],
+        observing_location=observing_location,
+        observing_time=observing_time,
+        lookup_table=args_dict["lookup_table"],
+        telescope_ids=args_dict["telescope_ids"],
+    )
+
+    grid_points = grid_gen.generate_grid()
+
+    if args_dict["coordinate_system"] == "ra_dec":
+        grid_points = grid_gen.convert_coordinates(grid_points)
+    grid_gen.serialize_grid_points(grid_points, output_file=output_filepath)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/simulate_light_emission.py

@@ -45,7 +45,7 @@ model_version (str, optional)
     Version of the simulation model.
 light_source_type (str, optional)
     Select calibration light source type: led (default) or laser.
-    This changes the pre-compiled (simtel_array) application that is used to run the
+    This changes the pre-compiled (sim_telarray) application that is used to run the
     light emission package with. Currently we use xyzls (laser), and ls-beam can be
     accessed by using the laser option.
 off_axis_angle (float, optional)
@@ -116,8 +116,7 @@ import astropy.units as u
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.model.calibration_model import CalibrationModel
-from simtools.model.site_model import SiteModel
-from simtools.model.telescope_model import TelescopeModel
+from simtools.model.model_utils import initialize_simulation_models
 from simtools.simtel.simulator_light_emission import SimulatorLightEmission
 
 
@@ -298,12 +297,12 @@ def main():
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    telescope_model = TelescopeModel(
+    telescope_model, site_model = initialize_simulation_models(
+        label=label,
+        db_config=db_config,
         site=args_dict["site"],
         telescope_name=args_dict["telescope"],
-        mongo_db_config=db_config,
         model_version=args_dict["model_version"],
-        label=label,
     )
 
     calibration_model = CalibrationModel(
@@ -314,13 +313,6 @@ def main():
         label=label,
     )
 
-    site_model = SiteModel(
-        site=args_dict["site"],
-        mongo_db_config=db_config,
-        model_version=args_dict["model_version"],
-        label=label,
-    )
-
     light_source = SimulatorLightEmission(
         telescope_model=telescope_model,
         calibration_model=calibration_model,
@@ -330,6 +322,7 @@ def main():
         simtel_path=args_dict["simtel_path"],
         light_source_type=args_dict["light_source_type"],
         label=label,
+        test=args_dict["test"],
     )
 
     if args_dict["light_source_setup"] == "variable":