gammasimtools 0.13.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

simtools/applications/production_derive_corsika_limits.py

@@ -7,15 +7,16 @@ The limits are derived based on the event loss fraction specified by the user.
 
 Command line arguments
 ----------------------
-event_data_file (str, required)
-    Path to the file containing the event data.
+event_data_files (str, required)
+    Path to a file containing event data file paths.
+telescope_ids (str, required)
+    Path to a file containing telescope configurations.
 loss_fraction (float, required)
     Fraction of events to be lost.
-telescope_ids (list of int, optional)
-    List of telescope IDs to filter the events (default is None).
-    Definition of the telescope IDs can be found
-    in the telescope definition file used for simulations.
-
+plot_histograms (bool, optional)
+    Plot histograms of the event data.
+output_file (str, optional)
+    Path to the output file for the derived limits.
 
 Example
 -------
@@ -23,16 +24,23 @@ Derive limits for a given file with a specified loss fraction.
 
 .. code-block:: console
 
-    simtools-production-derive-corsika-limits\\
-        --event_data_file path/to/event_data_file.hdf5 \\
+    simtools-production-derive-corsika-limits \\
+        --event_data_files path/to/event_data_files.yaml \\
+        --telescope_ids path/to/telescope_configs.yaml \\
         --loss_fraction 1e-6 \\
-        --telescope_ids 1 2 3
+        --plot_histograms \\
+        --output_file corsika_simulation_limits_lookup.ecsv
 """
 
+import datetime
 import logging
 
+from astropy.table import Table
+
 import simtools.utils.general as gen
 from simtools.configuration import configurator
+from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.io_operations import io_handler
 from simtools.production_configuration.derive_corsika_limits import LimitCalculator
 
 _logger = logging.getLogger(__name__)
@@ -44,63 +52,188 @@ def _parse():
 
     Parameters
     ----------
-    event_data_file: str
-        The event data file.
+    event_data_files: str
+        Path to a file listing event data file paths. These files contain the
+        simulation data used for deriving the limits.
     loss_fraction: float
-        Loss fraction of events for limit derivation.
-    telescope_ids: list of int, optional
-        List of telescope IDs to filter the events (default is None).
-        Definition of the telescope IDs can be found in the telescope
-        definition file used for simulations.
+        Fraction of events to be excluded during limit computation. Determines
+        thresholds for energy, radial distance, and viewcone.
+    telescope_ids: str
+        Path to a file defining telescope configurations. Specifies telescope
+        arrays or IDs used to filter events during processing.
+    plot_histograms: bool
+        If True, generates and saves histograms of the event data to visualize
+        the computed limits and distributions.
 
     Returns
     -------
     CommandLineParser
         Command line parser object
-
     """
     config = configurator.Configurator(
         description="Derive limits for energy, radial distance, and viewcone."
     )
     config.parser.add_argument(
-        "--event_data_file",
+        "--event_data_files",
+        type=str,
+        required=True,
+        help="Path to a file containing event data file paths.",
+    )
+    config.parser.add_argument(
+        "--telescope_ids",
         type=str,
         required=True,
-        help="Path to the event data file containing the event data.",
+        help="Path to a file containing telescope configurations.",
     )
     config.parser.add_argument(
         "--loss_fraction", type=float, required=True, help="Fraction of events to be lost."
     )
     config.parser.add_argument(
-        "--telescope_ids",
-        type=int,
-        nargs="*",
-        help="List of telescope IDs to filter the events as used in sim_telarray.",
+        "--plot_histograms",
+        help="Plot histograms of the event data.",
+        action="store_true",
+        default=False,
+    )
+    config.parser.add_argument(
+        "--output_file",
+        type=str,
+        default="corsika_simulation_limits_lookup.ecsv",
+        help="Output file for the derived limits (default: "
+        "'corsika_simulation_limits_lookup.ecsv').",
     )
     return config.initialize(db_config=False)
 
 
-def main():
-    """Derive limits for energy, radial distance, and viewcone."""
-    args_dict, _ = _parse()
-
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+def process_file(file_path, telescope_ids, loss_fraction, plot_histograms):
+    """
+    Process a single file and compute limits.
 
-    event_data_file_path = args_dict["event_data_file"]
-    loss_fraction = args_dict["loss_fraction"]
-    telescope_ids = args_dict.get("telescope_ids")
+    Parameters
+    ----------
+    file_path : str
+        Path to the event data file.
+    telescope_ids : list[int]
+        List of telescope IDs to filter the events.
+    loss_fraction : float
+        Fraction of events to be lost.
+    plot_histograms : bool
+        Whether to plot histograms.
 
-    calculator = LimitCalculator(event_data_file_path, telescope_list=telescope_ids)
+    Returns
+    -------
+    dict
+        Dictionary containing the computed limits.
+    """
+    calculator = LimitCalculator(file_path, telescope_list=telescope_ids)
 
     lower_energy_limit = calculator.compute_lower_energy_limit(loss_fraction)
-    _logger.info(f"Lower energy threshold: {lower_energy_limit}")
-
     upper_radial_distance = calculator.compute_upper_radial_distance(loss_fraction)
-    _logger.info(f"Upper radius threshold: {upper_radial_distance}")
-
     viewcone = calculator.compute_viewcone(loss_fraction)
-    _logger.info(f"Viewcone radius: {viewcone}")
+
+    if plot_histograms:
+        _logger.info(
+            f"Plotting histograms written to {io_handler.IOHandler().get_output_directory()}"
+        )
+        calculator.plot_data(
+            lower_energy_limit,
+            upper_radial_distance,
+            viewcone,
+            io_handler.IOHandler().get_output_directory(),
+        )
+
+    return {
+        "file_path": file_path,
+        "telescope_ids": telescope_ids,
+        "lower_energy_threshold": lower_energy_limit,
+        "upper_radius_threshold": upper_radial_distance,
+        "viewcone_radius": viewcone,
+    }
+
+
+def create_results_table(results, loss_fraction):
+    """
+    Create an Astropy Table from the results.
+
+    Parameters
+    ----------
+    results : list[dict]
+        List of dictionaries containing the computed limits for each file
+          and telescope configuration.
+    loss_fraction : float
+        Fraction of events to be lost, added as metadata to the table.
+
+    Returns
+    -------
+    astropy.table.Table
+        An Astropy Table containing the results with appropriate units and metadata.
+    """
+    table = Table(
+        rows=[
+            (
+                res["file_path"],
+                res["telescope_ids"],
+                res["lower_energy_threshold"],
+                res["upper_radius_threshold"],
+                res["viewcone_radius"],
+            )
+            for res in results
+        ],
+        names=[
+            "file_path",
+            "telescope_ids",
+            "lower_energy_threshold",
+            "upper_radius_threshold",
+            "viewcone_radius",
+        ],
+    )
+
+    table["lower_energy_threshold"].unit = "TeV"
+    table["upper_radius_threshold"].unit = "m"
+    table["viewcone_radius"].unit = "deg"
+
+    table.meta["created"] = datetime.datetime.now().isoformat()
+    table.meta["description"] = (
+        "Lookup table for CORSIKA limits computed from gamma-ray shower simulations."
+    )
+    table.meta["loss_fraction"] = loss_fraction
+
+    return table
+
+
+def main():
+    """Derive limits for energy, radial distance, and viewcone."""
+    args_dict, _ = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "info")))
+
+    event_data_files = gen.collect_data_from_file(args_dict["event_data_files"])["files"]
+    telescope_configs = gen.collect_data_from_file(args_dict["telescope_ids"])["telescope_configs"]
+
+    results = []
+    for file_path in event_data_files:
+        for array_name, telescope_ids in telescope_configs.items():
+            _logger.info(f"Processing file: {file_path} with telescope config: {array_name}")
+            result = process_file(
+                file_path,
+                telescope_ids,
+                args_dict["loss_fraction"],
+                args_dict["plot_histograms"],
+            )
+            result["layout"] = array_name
+            results.append(result)
+
+    table = create_results_table(results, args_dict["loss_fraction"])
+
+    output_dir = io_handler.IOHandler().get_output_directory("corsika_limits")
+    output_file = f"{output_dir}/{args_dict['output_file']}"
+
+    table.write(output_file, format="ascii.ecsv", overwrite=True)
+    _logger.info(f"Results saved to {output_file}")
+
+    metadata_file = f"{output_dir}/metadata.yml"
+    MetadataCollector.dump(args_dict, metadata_file)
+    _logger.info(f"Metadata saved to {metadata_file}")
 
 
 if __name__ == "__main__":

simtools/applications/production_scale_events.py

@@ -4,8 +4,8 @@ r"""
 Application to run the StatisticalErrorEvaluator and interpolate results.
 
 This application evaluates statistical uncertainties from DL2 MC event files
-based on input parameters like zenith angles and offsets, and can perform interpolation
-for a specified grid point.
+based on input parameters like zenith angles and offsets, and performs interpolation
+for a specified query point.
 
 Command line arguments
 ----------------------
@@ -15,10 +15,20 @@ zeniths (list of int, required)
     List of zenith angles to consider.
 offsets (list of int, required)
     List of offsets in degrees.
-interpolate (bool, optional)
-    If set, performs interpolation for a specific grid point.
-query_point (list of int, optional)
-    Grid point for interpolation (energy, azimuth, zenith, NSB, offset).
+query_point (list of float, required)
+    Query point for interpolation. The query point must contain exactly 5 values:
+        - Energy (TeV)
+        - Azimuth (degrees)
+        - Zenith (degrees)
+        - NSB (MHz)
+        - Offset (degrees)
+output_file (str, optional)
+    Output file to store the results. Default: 'interpolated_scaled_events.json'.
+metrics_file (str, optional)
+    Path to the metrics definition file. Default: 'production_simulation_config_metrics.yml'.
+file_name_template (str, optional)
+    Template for the file name. Default:
+    'prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits'.
 
 Example
 -------
@@ -26,14 +36,16 @@ To evaluate statistical uncertainties and perform interpolation, run the command
 
 .. code-block:: console
 
-    simtools-production-scale-events --base_path tests/resources/production_dl2_fits/ \
-        --zeniths 20 52 40 60 --offsets 0 --interpolate --query_point 1 180 30 0 0 \
-        --metrics_file "path/to/metrics.yaml"
+    simtools-production-scale-events --base_path tests/resources/production_dl2_fits/ \\
+        --zeniths 20 40 52 60 --offsets 0 --query_point 1 180 30 0 0 \\
+        --metrics_file "path/to/metrics.yaml" \\
+        --output_file "output.json"
 
-
-The output will display the scaled events for the specified grid point.
+The output will display the scaled events for the specified query point and save
+ the results to the specified output file.
 """
 
+import itertools
 import json
 import logging
 from pathlib import Path
@@ -69,33 +81,47 @@ def _parse(label, description):
         help="Path to the DL2 MC event files for interpolation.",
     )
     config.parser.add_argument(
-        "--zeniths", nargs="+", type=CommandLineParser.zenith_angle, help="List of zenith angles."
+        "--zeniths",
+        required=True,
+        nargs="+",
+        type=CommandLineParser.zenith_angle,
+        help="List of zenith angles.",
     )
     config.parser.add_argument(
-        "--offsets", nargs="+", type=float, help="List of offsets in degrees."
+        "--offsets", required=True, nargs="+", type=float, help="List of offsets in degrees."
     )
 
-    config.parser.add_argument(
-        "--interpolate", action="store_true", help="Interpolate results for a specific grid point."
-    )
     config.parser.add_argument(
         "--query_point",
+        required=True,
         nargs=5,
         type=float,
         help="Grid point for interpolation (energy, azimuth, zenith, NSB, offset).",
     )
     config.parser.add_argument(
         "--output_file",
+        required=False,
         type=str,
         default="interpolated_scaled_events.json",
         help="Output file to store the results. (default: 'interpolated_scaled_events.json').",
     )
     config.parser.add_argument(
         "--metrics_file",
+        required=False,
         type=str,
         default="production_simulation_config_metrics.yml",
         help="Metrics definition file. (default: production_simulation_config_metrics.yml)",
     )
+    config.parser.add_argument(
+        "--file_name_template",
+        required=False,
+        type=str,
+        default=("prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits"),
+        help=(
+            "Template for the file name. (default: "
+            "'prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits')"
+        ),
+    )
     return config.initialize(db_config=False)
 
 
@@ -115,27 +141,25 @@ def main():
 
     evaluator_instances = []
 
-    metrics = (
-        gen.collect_data_from_file(args_dict["metrics_file"]) if "metrics_file" in args_dict else {}
-    )
+    metrics = gen.collect_data_from_file(args_dict["metrics_file"])
 
     if args_dict["base_path"] and args_dict["zeniths"] and args_dict["offsets"]:
-        for zenith in args_dict["zeniths"]:
-            for offset in args_dict["offsets"]:
-                # Build file path based on base_path, zenith, and offset
-                file_name = f"prod6_LaPalma-{int(zenith.value)}deg_"
-                file_name += "gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits"
-                file_path = Path(args_dict["base_path"]).joinpath(file_name)
-
-                evaluator = StatisticalErrorEvaluator(
-                    file_path,
-                    file_type="Gamma-cone",
-                    metrics=metrics,
-                    grid_point=(1 * u.TeV, 180 * u.deg, zenith, 0, offset * u.deg),
-                )
-
-                evaluator.calculate_metrics()
-                evaluator_instances.append(evaluator)
+        for zenith, offset in itertools.product(args_dict["zeniths"], args_dict["offsets"]):
+            file_name = args_dict["file_name_template"].format(zenith=int(zenith.value))
+            file_path = Path(args_dict["base_path"]).joinpath(file_name)
+
+            if not file_path.exists():
+                logger.warning(f"File not found: {file_path}. Skipping.")
+                continue
+
+            evaluator = StatisticalErrorEvaluator(
+                file_path,
+                file_type="Gamma-cone",
+                metrics=metrics,
+                grid_point=(1 * u.TeV, 180 * u.deg, zenith, 0, offset * u.deg),
+            )
+            evaluator.calculate_metrics()
+            evaluator_instances.append(evaluator)
 
     else:
         logger.warning("No files read")
@@ -143,7 +167,6 @@ def main():
         logger.warning(f"Zeniths: {args_dict['zeniths']}")
         logger.warning(f"Offsets: {args_dict['offsets']}")
 
-    # Perform interpolation for the given query point
     interpolation_handler = InterpolationHandler(evaluator_instances, metrics=metrics)
     query_points = np.array([args_dict["query_point"]])
     scaled_events = interpolation_handler.interpolate(query_points)

simtools/applications/run_application.py

@@ -1,12 +1,30 @@
 #!/usr/bin/python3
 
 """
-Run simtools applications from configuration files for setting workflows.
+Run several simtools applications using a configuration file.
 
 Allows to run several simtools applications with a single configuration file, which includes
 both the name of the simtools application and the configuration for the application.
 
-Strong assumption on the directory structure for input and output files of applications.
+This application is used for model parameter setting workflows.
+Strong assumptions are applied on the directory structure for input and output files of
+applications.
+
+Example
+-------
+
+Run the application with the configuration file 'config_file_name':
+
+.. code-block:: console
+
+    simtools-run-application --configuration_file config_file_name
+
+Run the application with the configuration file 'config_file_name', but skipping all steps except
+step 2 and 3 (useful for debugging):
+
+.. code-block:: console
+
+    simtools-run-application --configuration_file config_file_name --steps 2 3
 
 """
 
@@ -49,6 +67,12 @@ def _parse(label, description, usage):
         required=True,
         default=None,
     )
+    config.parser.add_argument(
+        "--steps",
+        type=int,
+        nargs="+",
+        help="List of steps to be execution (e.g., '--steps 7 8 9'; do not specify to run all).",
+    )
     return config.initialize(db_config=True)
 
 
@@ -81,7 +105,7 @@ def get_subdirectory_name(path):
         raise ValueError(f"Could not find subdirectory under 'input': {exc}") from exc
 
 
-def read_application_configuration(configuration_file, logger):
+def read_application_configuration(configuration_file, steps, logger):
     """
     Read application configuration from file and modify for setting workflows.
 
@@ -97,6 +121,8 @@ def read_application_configuration(configuration_file, logger):
     ----------
     configuration_file : str
         Configuration file name.
+    steps : list
+        List of steps to be executed (None: all steps).
     logger : Logger
         Logger object.
 
@@ -111,20 +137,20 @@ def read_application_configuration(configuration_file, logger):
     application_config = gen.collect_data_from_file(configuration_file).get("CTA_SIMPIPE")
     place_holder = "__SETTING_WORKFLOW__"
     workflow_dir, setting_workflow = get_subdirectory_name(configuration_file)
-    output_path = (
-        workflow_dir.with_name(workflow_dir.name.replace("input", "output")) / setting_workflow
-    )
+    output_path = Path(str(workflow_dir).replace("input", "output")) / Path(setting_workflow)
     output_path.mkdir(parents=True, exist_ok=True)
     logger.info(f"Setting workflow output path to {output_path}")
     log_file = output_path / "simtools.log"
     configurations = application_config.get("APPLICATIONS")
-    for config in configurations:
+    for step_count, config in enumerate(configurations, start=1):
+        config["RUN_APPLICATION"] = step_count in steps if steps else True
         for key, value in config.get("CONFIGURATION", {}).items():
             if isinstance(value, str):
                 config["CONFIGURATION"][key] = value.replace(place_holder, setting_workflow)
             if isinstance(value, list):
                 config["CONFIGURATION"][key] = [
-                    item.replace(place_holder, setting_workflow) for item in value
+                    item.replace(place_holder, setting_workflow) if isinstance(item, str) else item
+                    for item in value
                 ]
         config["CONFIGURATION"]["USE_PLAIN_OUTPUT_PATH"] = True
         config["CONFIGURATION"]["OUTPUT_PATH"] = str(output_path)
@@ -135,21 +161,25 @@ def read_application_configuration(configuration_file, logger):
 def main():  # noqa: D103
     args_dict, db_config = _parse(
         Path(__file__).stem,
-        description="Run simtools applications from configuration file.",
+        description="Run simtools applications using a configuration file.",
         usage="simtools-run-application --config_file config_file_name",
     )
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     configurations, log_file = read_application_configuration(
-        args_dict["configuration_file"], logger
+        args_dict["configuration_file"], args_dict["steps"], logger
     )
 
     with log_file.open("w", encoding="utf-8") as file:
         file.write("Running simtools applications\n")
         file.write(dependencies.get_version_string(db_config))
         for config in configurations:
-            logger.info(f"Running application: {config.get('APPLICATION')}")
+            if config.get("RUN_APPLICATION"):
+                logger.info(f"Running application: {config.get('APPLICATION')}")
+            else:
+                logger.info(f"Skipping application: {config.get('APPLICATION')}")
+                continue
             config = gen.change_dict_keys_case(config, False)
             stdout, stderr = run_application(
                 config.get("APPLICATION"), config.get("CONFIGURATION"), logger