gammasimtools 0.14.0__py3-none-any.whl → 0.16.0__py3-none-any.whl

simtools/applications/production_derive_statistics.py

@@ -0,0 +1,125 @@
+#!/usr/bin/python3
+
+r"""
+Application to run the StatisticalErrorEvaluator and interpolate results.
+
+This application evaluates statistical uncertainties from DL2 MC event files
+based on input parameters like zenith angles and camera offsets, and performs interpolation
+for a specified query point.
+
+Command line arguments
+----------------------
+base_path (str, required)
+    Path to the directory containing the DL2 MC event file for interpolation.
+zeniths (list of int, required)
+    List of zenith angles to consider.
+camera_offsets (list of int, required)
+    List of offsets in degrees.
+query_point (list of float, required)
+    Query point for interpolation. The query point must contain exactly 5 values:
+        - Energy (TeV)
+        - Azimuth (degrees)
+        - Zenith (degrees)
+        - NSB (MHz)
+        - Offset (degrees)
+output_file (str, optional)
+    Output file to store the results. Default: 'interpolated_production_statistics.json'.
+metrics_file (str, optional)
+    Path to the metrics definition file. Default: 'production_simulation_config_metrics.yml'.
+file_name_template (str, optional)
+    Template for the file name. Default:
+    'prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits'.
+
+Example
+-------
+To evaluate statistical uncertainties and perform interpolation, run the command line script:
+
+.. code-block:: console
+
+    simtools-production-derive-statistics --base_path tests/resources/production_dl2_fits/ \\
+        --zeniths 20 40 52 60 --camera_offsets 0 --query_point 1 180 30 0 0 \\
+        --metrics_file "path/to/metrics.yaml" \\
+        --output_path simtools-output/derived_events \\
+        --output_file derived_events.json
+
+The output will display the production statistics for the specified query point and save
+ the results to the specified output file.
+"""
+
+from pathlib import Path
+
+from simtools.configuration import configurator
+from simtools.production_configuration.derive_production_statistics_handler import (
+    ProductionStatisticsHandler,
+)
+
+
+def _parse(label, description):
+    """
+    Parse command line arguments for the statistical error evaluator application.
+
+    Returns
+    -------
+    argparse.Namespace
+        Parsed command line arguments.
+    """
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--base_path",
+        type=str,
+        required=True,
+        help="Path to the DL2 MC event files for interpolation.",
+    )
+    config.parser.add_argument(
+        "--zeniths",
+        required=True,
+        nargs="+",
+        type=float,
+        help="List of zenith angles.",
+    )
+    config.parser.add_argument(
+        "--camera_offsets",
+        required=True,
+        nargs="+",
+        type=float,
+        help="List of camera offsets in degrees.",
+    )
+    config.parser.add_argument(
+        "--query_point",
+        required=True,
+        nargs=5,
+        type=float,
+        help="Grid point for interpolation (energy, azimuth, zenith, NSB, offset).",
+    )
+    config.parser.add_argument(
+        "--metrics_file",
+        required=False,
+        type=str,
+        default="production_simulation_config_metrics.yml",
+        help="Metrics definition file. (default: production_simulation_config_metrics.yml)",
+    )
+    config.parser.add_argument(
+        "--file_name_template",
+        required=False,
+        type=str,
+        default=("prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits"),
+        help=("Template for the DL2 MC event file name."),
+    )
+    return config.initialize(db_config=False, output=True)
+
+
+def main():
+    """Run the ProductionStatisticsHandler."""
+    label = Path(__file__).stem
+    args_dict, _ = _parse(
+        label,
+        "Evaluate statistical uncertainties from DL2 MC event files and interpolate results.",
+    )
+
+    manager = ProductionStatisticsHandler(args_dict)
+    manager.run()
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/production_generate_grid.py

@@ -0,0 +1,197 @@
+#!/usr/bin/python3
+
+r"""
+Generate a grid of simulation points using flexible axes definitions.
+
+This application generates a grid of simulation points based on the provided axes
+definitions. The axes definitions (range, binning) are specified in a file.
+The viewcone, radius and energy thresholds are provided as a lookup table and
+are interpolated based on the generated grid points. The generated grid points are
+filtered based on the specified telescope IDs and the limits from the lookup table.
+The generated grid points are saved to a file.
+It can also convert the generated points to RA/Dec coordinates if the selected
+coordinate system is 'ra_dec'.
+
+Command line arguments
+----------------------
+axes (str, required)
+    Path to a YAML or JSON file defining the axes of the grid.
+coordinate_system (str, optional, default='zenith_azimuth')
+    The coordinate system for the grid generation ('zenith_azimuth' or 'ra_dec').
+observing_time (str, optional, default=now)
+    Time of the observation (format: 'YYYY-MM-DD HH:MM:SS').
+lookup_table (str, required)
+    Path to the lookup table for simulation limits. The table should contain
+    varying azimuth and/or zenith angles.
+telescope_ids (list of int, optional)
+    List of telescope IDs as used in sim_telarray to filter the events.
+output_file (str, optional, default='grid_output.json')
+    Output file for the generated grid points (default: 'grid_output.json').
+
+
+Example
+-------
+To generate a grid of simulation points, execute the script as follows:
+
+.. code-block:: console
+
+    simtools-production-generate-grid --site North --model_version 6.0.0 \
+      --axes  tests/resources/production_grid_generation_axes_definition.yml \
+      --coordinate_system ra_dec --observing_time "2017-09-16 00:00:00" \
+      --lookup_table tests/resources/corsika_simulation_limits_lookup.ecsv \
+        --telescope_ids 1
+"""
+
+import logging
+from pathlib import Path
+
+from astropy.coordinates import EarthLocation
+from astropy.time import Time
+
+from simtools.configuration import configurator
+from simtools.io_operations import io_handler
+from simtools.model.site_model import SiteModel
+from simtools.production_configuration.generate_production_grid import GridGeneration
+from simtools.utils.general import collect_data_from_file
+
+
+def _parse(label, description):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    axes (str, required)
+        Path to a YAML or JSON file defining the axes of the grid.
+    coordinate_system (str, optional, default='zenith_azimuth')
+        The coordinate system for the grid generation ('zenith_azimuth' or 'ra_dec').
+    observing_time (str, optional, default=now)
+        Time of the observation (format: 'YYYY-MM-DD HH:MM:SS').
+    lookup_table (str, required)
+        Path to the lookup table for simulation limits. The table should contain
+        varying azimuth and/or zenith angles.
+    telescope_ids (list of int, optional)
+        List of telescope IDs as used in sim_telarray to filter the events.
+    output_file (str, optional, default='grid_output.json')
+        Output file for the generated grid points (default: 'grid_output.json').
+
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--axes",
+        type=str,
+        required=True,
+        help="Path to a file defining the grid axes.",
+    )
+    config.parser.add_argument(
+        "--coordinate_system",
+        type=str,
+        default="zenith_azimuth",
+        help="Coordinate system ('zenith_azimuth' or 'ra_dec').",
+    )
+    config.parser.add_argument(
+        "--observing_time",
+        type=str,
+        required=False,
+        help="Time of the observation (format: 'YYYY-MM-DD HH:MM:SS').",
+    )
+    config.parser.add_argument(
+        "--output_file",
+        type=str,
+        default="grid_output.json",
+        help="Output file for the generated grid points (default: 'grid_output.json').",
+    )
+    config.parser.add_argument(
+        "--telescope_ids",
+        type=int,
+        nargs="*",
+        default=None,
+        help="List of telescope IDs as used in sim_telarray to get the specific limits from the "
+        "lookup table.",
+    )
+    config.parser.add_argument(
+        "--lookup_table",
+        type=str,
+        required=True,
+        help="Path to the lookup table for simulation limits. "
+        "Table required with varying azimuth and or zenith angle. ",
+    )
+
+    return config.initialize(db_config=True, simulation_model=["version", "site", "model_version"])
+
+
+def load_axes(file_path: str):
+    """
+    Load axes definitions from a YAML or JSON file.
+
+    Parameters
+    ----------
+    file_path : str
+        Path to the axes YAML or JSON file.
+
+    Returns
+    -------
+    list[dict]
+        List of axes definitions with Quantity values.
+    """
+    if not Path(file_path).exists():
+        raise FileNotFoundError(f"Axes file {file_path} not found.")
+
+    return collect_data_from_file(file_path)
+
+
+def main():
+    """Run the Grid Generation application."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(
+        label,
+        "Generate a grid of simulation points using flexible axes definitions.",
+    )
+
+    _logger = logging.getLogger()
+    _logger.setLevel(logging.INFO)
+
+    output_path = io_handler.IOHandler().get_output_directory(label)
+    output_filepath = Path(output_path).joinpath(f"{args_dict['output_file']}")
+
+    axes = load_axes(args_dict["axes"])
+    site_model = SiteModel(
+        mongo_db_config=db_config,
+        model_version=args_dict["model_version"],
+        site=args_dict["site"],
+    )
+
+    ref_lat = site_model.get_parameter_value_with_unit("reference_point_latitude")
+    ref_long = site_model.get_parameter_value_with_unit("reference_point_longitude")
+    altitude = site_model.get_parameter_value_with_unit("reference_point_altitude")
+
+    observing_location = EarthLocation(lat=ref_lat, lon=ref_long, height=altitude)
+
+    observing_time = (
+        Time(args_dict["observing_time"]) if args_dict.get("observing_time") else Time.now()
+    )
+
+    grid_gen = GridGeneration(
+        axes=axes,
+        coordinate_system=args_dict["coordinate_system"],
+        observing_location=observing_location,
+        observing_time=observing_time,
+        lookup_table=args_dict["lookup_table"],
+        telescope_ids=args_dict["telescope_ids"],
+    )
+
+    grid_points = grid_gen.generate_grid()
+
+    if args_dict["coordinate_system"] == "ra_dec":
+        grid_points = grid_gen.convert_coordinates(grid_points)
+    grid_gen.serialize_grid_points(grid_points, output_file=output_filepath)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/production_generate_simulation_config.py

@@ -31,8 +31,6 @@ nsb (float, required)
 file_path (str, required)
         Path to file with MC events at CTAO DL2 data level.
         Used for statistical uncertainty evaluation.
-file_type (str, required)
-        Type of the DL2 MC event file ('point-like' or 'cone').
 metrics (str, required)
         Path to a YAML file containing metrics for evaluation.
 site (str, required)
@@ -46,7 +44,6 @@ To run the simulation configuration, execute the script as follows:
 
         simtools-production-generate-simulation-config --azimuth 60.0 --elevation 45.0 \
                 --nsb 0.3 --file_path tests/resources/production_dl2_fits/dl2_mc_events_file.fits \
-                --file_type "point-like"    \
                 --metrics_file tests/resources/production_simulation_config_metrics.yml --site North
 
 The output will show the derived simulation parameters.
@@ -85,12 +82,6 @@ def _parse(label):
     config.parser.add_argument(
         "--file_path", type=str, required=True, help="Path to MC event file in DL2 format."
     )
-    config.parser.add_argument(
-        "--file_type",
-        type=str,
-        required=True,
-        help="Type of the DL2 MC event file ('point-like' or 'cone').",
-    )
     config.parser.add_argument(
         "--metrics_file",
         required=True,
@@ -138,7 +129,6 @@ def main():
     simulation_config = SimulationConfig(
         grid_point=grid_point_config,
         file_path=args_dict["file_path"],
-        file_type=args_dict["file_type"],
         metrics=metrics,
     )
 

simtools/applications/simulate_light_emission.py

@@ -45,7 +45,7 @@ model_version (str, optional)
     Version of the simulation model.
 light_source_type (str, optional)
     Select calibration light source type: led (default) or laser.
-    This changes the pre-compiled (simtel_array) application that is used to run the
+    This changes the pre-compiled (sim_telarray) application that is used to run the
     light emission package with. Currently we use xyzls (laser), and ls-beam can be
     accessed by using the laser option.
 off_axis_angle (float, optional)
@@ -116,8 +116,7 @@ import astropy.units as u
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.model.calibration_model import CalibrationModel
-from simtools.model.site_model import SiteModel
-from simtools.model.telescope_model import TelescopeModel
+from simtools.model.model_utils import initialize_simulation_models
 from simtools.simtel.simulator_light_emission import SimulatorLightEmission
 
 
@@ -298,12 +297,12 @@ def main():
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    telescope_model = TelescopeModel(
+    telescope_model, site_model = initialize_simulation_models(
+        label=label,
+        db_config=db_config,
         site=args_dict["site"],
         telescope_name=args_dict["telescope"],
-        mongo_db_config=db_config,
         model_version=args_dict["model_version"],
-        label=label,
     )
 
     calibration_model = CalibrationModel(
@@ -314,13 +313,6 @@ def main():
         label=label,
     )
 
-    site_model = SiteModel(
-        site=args_dict["site"],
-        mongo_db_config=db_config,
-        model_version=args_dict["model_version"],
-        label=label,
-    )
-
     light_source = SimulatorLightEmission(
         telescope_model=telescope_model,
         calibration_model=calibration_model,

simtools/applications/simulate_prod.py

@@ -122,15 +122,26 @@ def _parse(description=None):
         required=False,
         default=False,
     )
-    config.parser.add_argument(
-        "--sim_telarray_seeds",
+    sim_telarray_seed_group = config.parser.add_argument_group(
+        title="Random seeds for sim_telarray instrument setup",
+    )
+    sim_telarray_seed_group.add_argument(
+        "--sim_telarray_instrument_seeds",
         help=(
-            "Use the provided random seed for sim_telarray instrument setup."
-            "Used only for testing purposes for now."
+            "Random seed used for sim_telarray instrument setup. "
+            "If '--sim_telarray_random_instrument_instances is not set: use as sim_telarray seed "
+            " ('random_seed' parameter). "
+            "Otherwise: use as base seed for the generation of random instrument instance seeds."
         ),
         type=str,
         required=False,
     )
+    sim_telarray_seed_group.add_argument(
+        "--sim_telarray_random_instrument_instances",
+        help="Number of random instrument instances initialized in sim_telarray.",
+        type=int,
+        required=False,
+    )
     return config.initialize(
         db_config=True,
         job_submission=True,
@@ -150,6 +161,7 @@ def main():  # noqa: D103
     )
 
     simulator.simulate()
+    simulator.validate_metadata()
 
     if simulator.submit_engine == "local":
         logger.info(

simtools/applications/validate_cumulative_psf.py

@@ -80,7 +80,7 @@ import numpy as np
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
-from simtools.model.telescope_model import TelescopeModel
+from simtools.model.model_utils import initialize_simulation_models
 from simtools.ray_tracing.ray_tracing import RayTracing
 from simtools.visualization import visualize
 
@@ -136,18 +136,20 @@ def main():  # noqa: D103
     _io_handler = io_handler.IOHandler()
     output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
 
-    tel_model = TelescopeModel(
+    tel_model, site_model = initialize_simulation_models(
+        label=label,
+        db_config=db_config,
         site=args_dict["site"],
         telescope_name=args_dict["telescope"],
-        mongo_db_config=db_config,
         model_version=args_dict["model_version"],
-        label=label,
     )
+
     if args_dict.get("telescope_model_file"):
         tel_model.change_multiple_parameters_from_file(args_dict["telescope_model_file"])
 
     ray = RayTracing(
         telescope_model=tel_model,
+        site_model=site_model,
         simtel_path=args_dict["simtel_path"],
         zenith_angle=args_dict["zenith"] * u.deg,
         source_distance=args_dict["src_distance"] * u.km,

simtools/applications/validate_file_using_schema.py

@@ -131,6 +131,7 @@ def validate_dict_using_schema(args_dict, logger):
     """
     Validate a schema file (or several files) given in yaml or json format.
 
+    This function validate all documents in a multi-document YAML file.
     Schema is either given as command line argument, read from the meta_schema_url or from
     the metadata section of the data dictionary.
 
@@ -141,9 +142,12 @@ def validate_dict_using_schema(args_dict, logger):
         try:
             data = gen.collect_data_from_file(file_name=file_name)
         except FileNotFoundError as exc:
-            logger.error(f"Error reading schema file from {file_name}")
-            raise exc
-        schema.validate_dict_using_schema(data, _get_schema_file_name(args_dict, data))
+            raise FileNotFoundError(f"Error reading schema file from {file_name}") from exc
+        data = data if isinstance(data, list) else [data]
+        for data_dict in data:
+            schema.validate_dict_using_schema(
+                data_dict, _get_schema_file_name(args_dict, data_dict)
+            )
         logger.info(f"Successful validation of file {file_name}")
 
 

simtools/applications/validate_optics.py

@@ -76,7 +76,7 @@ from matplotlib.backends.backend_pdf import PdfPages
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
-from simtools.model.telescope_model import TelescopeModel
+from simtools.model.model_utils import initialize_simulation_models
 from simtools.ray_tracing.ray_tracing import RayTracing
 from simtools.visualization import visualize
 
@@ -128,12 +128,12 @@ def main():  # noqa: D103
     _io_handler = io_handler.IOHandler()
     output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
 
-    tel_model = TelescopeModel(
+    tel_model, site_model = initialize_simulation_models(
+        label=label,
+        db_config=db_config,
         site=args_dict["site"],
         telescope_name=args_dict["telescope"],
         model_version=args_dict["model_version"],
-        label=label,
-        mongo_db_config=db_config,
     )
 
     ######################################################################
@@ -151,6 +151,7 @@ def main():  # noqa: D103
 
     ray = RayTracing(
         telescope_model=tel_model,
+        site_model=site_model,
         simtel_path=args_dict["simtel_path"],
         zenith_angle=args_dict["zenith"] * u.deg,
         source_distance=args_dict["src_distance"] * u.km,

simtools/camera/camera_efficiency.py

@@ -9,8 +9,7 @@ import numpy as np
 from astropy.table import Table
 
 from simtools.io_operations import io_handler
-from simtools.model.site_model import SiteModel
-from simtools.model.telescope_model import TelescopeModel
+from simtools.model.model_utils import initialize_simulation_models
 from simtools.simtel.simulator_camera_efficiency import SimulatorCameraEfficiency
 from simtools.utils import names
 from simtools.visualization import visualize
@@ -52,8 +51,12 @@ class CameraEfficiency:
         self.test = test
 
         self.io_handler = io_handler.IOHandler()
-        self.telescope_model, self._site_model = self._initialize_simulation_models(
-            config_data, db_config
+        self.telescope_model, self.site_model = initialize_simulation_models(
+            self.label,
+            db_config,
+            config_data["site"],
+            config_data["telescope"],
+            config_data["model_version"],
         )
         self.output_dir = self.io_handler.get_output_directory(self.label, sub_dir="plots")
 
@@ -67,34 +70,6 @@ class CameraEfficiency:
         """Return string representation of the CameraEfficiency instance."""
         return f"CameraEfficiency(label={self.label})\n"
 
-    def _initialize_simulation_models(self, config_data, db_config):
-        """
-        Initialize site and telescope models.
-
-        Parameters
-        ----------
-        config_data: dict
-            Dict containing the configurable parameters.
-
-        Returns
-        -------
-        tuple
-            Tuple containing the site and telescope models.
-        """
-        tel_model = TelescopeModel(
-            site=config_data["site"],
-            telescope_name=config_data["telescope"],
-            mongo_db_config=db_config,
-            model_version=config_data["model_version"],
-            label=self.label,
-        )
-        site_model = SiteModel(
-            site=config_data["site"],
-            model_version=config_data["model_version"],
-            mongo_db_config=db_config,
-        )
-        return tel_model, site_model
-
     def _configuration_from_args_dict(self, config_data):
         """
         Extract the configuration data from the args_dict.
@@ -131,10 +106,10 @@ class CameraEfficiency:
         }
 
     def _load_files(self):
-        """Define the variables for the file names, including the results, simtel and log file."""
+        """Define variables used for file names, including results, sim_telarray and log files."""
         _file = {}
         for label, suffix in zip(
-            ["results", "simtel", "log"],
+            ["results", "sim_telarray", "log"],
             [".ecsv", ".dat", ".log"],
         ):
             file_name = names.generate_file_name(
@@ -164,8 +139,9 @@ class CameraEfficiency:
         simtel = SimulatorCameraEfficiency(
             simtel_path=self._simtel_path,
             telescope_model=self.telescope_model,
+            site_model=self.site_model,
             zenith_angle=self.config["zenith_angle"],
-            file_simtel=self._file["simtel"],
+            file_simtel=self._file["sim_telarray"],
             file_log=self._file["log"],
             label=self.label,
             nsb_spectrum=self.config["nsb_spectrum"],
@@ -177,8 +153,7 @@ class CameraEfficiency:
 
     def export_model_files(self):
         """Export model and config files to the output directory."""
-        self.telescope_model.export_config_file()
-        self.telescope_model.export_model_files()
+        self.telescope_model.write_sim_telarray_config_file()
         if not self.config.get("skip_correction_to_nsb_spectrum", False):
             self.telescope_model.export_nsb_spectrum_to_telescope_altitude_correction_file(
                 model_directory=self.telescope_model.config_file_directory
@@ -237,7 +212,7 @@ class CameraEfficiency:
 
         # Search for at least 5 consecutive numbers to see that we are in the table
         re_table = re.compile("{0}{0}{0}{0}{0}".format(r"[-+]?[0-9]*\.?[0-9]+\s+"))
-        with open(self._file["simtel"], encoding="utf-8") as file:
+        with open(self._file["sim_telarray"], encoding="utf-8") as file:
             for line in file:
                 if re_table.match(line):
                     words = line.split()
@@ -442,7 +417,7 @@ class CameraEfficiency:
         nsb_integral = 0.0001 * (n1_sum - 0.5 * n1_integral_edges_sum)
         nsb_rate_ref_conditions = (
             nsb_rate_provided_spectrum
-            * self._site_model.get_parameter_value("nsb_reference_value")
+            * self.site_model.get_parameter_value("nsb_reference_value")
             / nsb_integral
         )
         return nsb_rate_provided_spectrum, nsb_rate_ref_conditions

simtools/configuration/commandline_parser.py

@@ -50,7 +50,7 @@ class CommandLineParser(argparse.ArgumentParser):
             Add output file configuration to list of args.
         simulation_model: list
             List of simulation model configuration parameters to add to list of args
-            (use: 'version', 'telescope', 'site')
+            (use: 'model_version', 'telescope', 'site')
         simulation_configuration: dict
             Dict of simulation software configuration parameters to add to list of args.
         db_config: bool
@@ -271,6 +271,7 @@ class CommandLineParser(argparse.ArgumentParser):
                 help="production model version",
                 type=str,
                 default=None,
+                nargs="+",
             )
         if "parameter_version" in model_options:
             _job_group.add_argument(
@@ -341,9 +342,9 @@ class CommandLineParser(argparse.ArgumentParser):
             "--simulation_software",
             help="Simulation software steps.",
             type=str,
-            choices=["corsika", "simtel", "corsika_simtel"],
+            choices=["corsika", "sim_telarray", "corsika_sim_telarray"],
             required=True,
-            default="corsika_simtel",
+            default="corsika_sim_telarray",
         )
 
     @staticmethod