gammasimtools 0.16.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

simtools/applications/production_derive_corsika_limits.py

@@ -1,18 +1,56 @@
 #!/usr/bin/python3
 
 r"""
-Derives the limits for energy, radial distance, and viewcone to be used in CORSIKA simulations.
-
-The limits are derived based on the event loss fraction specified by the user.
+Derive CORSIKA configuration limits for energy, core distance, and viewcone radius.
+
+This tool determines configuration limits based on triggered events from broad-range
+simulations. It supports the derivation of the following CORSIKA configuration parameters:
+
+- **ERANGE**: lower energy limit
+- **CSCAT**: upper core distance
+- **VIEWCONE**: viewcone radius
+
+Broad-range simulations in this context are simulation sets generated with wide-ranging
+definitions for above parameters.
+Limits are computed based on a configurable maximum event loss fraction.
+Results are provided as a table with the following columns:
+
++---------------------+-----------+--------+-----------------------------------------------+
+| Field               | Data Type | Units  | Description                                   |
++=====================+===========+========+===============================================+
+| primary_particle    | string    |        | Particle type (e.g., gamma, proton).          |
++---------------------+-----------+--------+-----------------------------------------------+
+| array_name          | string    |        | Array name (custom or as defined in           |
+|                     |           |        | 'array_layouts').                             |
++---------------------+-----------+--------+-----------------------------------------------+
+| telescope_ids       | string    |        | Comma-separated list of telescope IDs         |
+|                     |           |        | of this array.                                |
++---------------------+-----------+--------+-----------------------------------------------+
+| zenith              | float64   | deg    | Direction of array pointing zenith.           |
++---------------------+-----------+--------+-----------------------------------------------+
+| azimuth             | float64   | deg    | Direction of array pointing azimuth.          |
++---------------------+-----------+--------+-----------------------------------------------+
+| nsb_level           | float64   |        | Night sky background level.                   |
++---------------------+-----------+--------+-----------------------------------------------+
+| lower_energy_limit  | float64   | TeV    | Derived lower energy limit (**ERANGE**)       |
++---------------------+-----------+--------+-----------------------------------------------+
+| upper_radius_limit  | float64   | m      | Derived upper core distance limit (**CSCAT**) |
++---------------------+-----------+--------+-----------------------------------------------+
+| viewcone_radius     | float64   | deg    | Derived viewcone radius limit (**VIEWCONE**)  |
++---------------------+-----------+--------+-----------------------------------------------+
+
+The input event data files are generated using the application simtools-generate-simtel-event-data
+and are required for each point in the observational parameter space (e.g., array pointing
+directions, level of night sky background, etc.).
 
 Command line arguments
 ----------------------
 event_data_files (str, required)
-    Path to a file containing event data file paths.
-telescope_ids (str, required)
-    Path to a file containing telescope configurations.
+    Path to reduced event data file.
+telescope_ids (str, optional)
+    Custom array layout file containing telescope IDs.
 loss_fraction (float, required)
-    Fraction of events to be lost.
+    Maximum event-loss fraction for limit computation.
 plot_histograms (bool, optional)
     Plot histograms of the event data.
 output_file (str, optional)
@@ -20,7 +58,19 @@ output_file (str, optional)
 
 Example
 -------
-Derive limits for a given file with a specified loss fraction.
+
+Derive limits for a list of array layouts (use 'all' to derive limits for all layouts):
+
+.. code-block:: console
+
+    simtools-production-derive-corsika-limits \\
+        --event_data_files path/to/event_data_files.yaml \\
+        --array_layout_name alpha,beta \\
+        --loss_fraction 1e-6 \\
+        --plot_histograms \\
+        --output_file corsika_simulation_limits_lookup.ecsv
+
+Derive limits for a given file for custom defined array layouts:
 
 .. code-block:: console
 
@@ -32,63 +82,38 @@ Derive limits for a given file with a specified loss fraction.
         --output_file corsika_simulation_limits_lookup.ecsv
 """
 
-import datetime
 import logging
-import re
-
-import numpy as np
-from astropy.table import Table
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.data_model.metadata_collector import MetadataCollector
-from simtools.io_operations import io_handler
-from simtools.production_configuration.derive_corsika_limits import LimitCalculator
-
-_logger = logging.getLogger(__name__)
+from simtools.production_configuration.derive_corsika_limits_grid import (
+    generate_corsika_limits_grid,
+)
 
 
 def _parse():
-    """
-    Parse command line configuration.
-
-    Parameters
-    ----------
-    event_data_files: str
-        Path to a file listing event data file paths. These files contain the
-        simulation data used for deriving the limits.
-    loss_fraction: float
-        Fraction of events to be excluded during limit computation. Determines
-        thresholds for energy, radial distance, and viewcone.
-    telescope_ids: str
-        Path to a file defining telescope configurations. Specifies telescope
-        arrays or IDs used to filter events during processing.
-    plot_histograms: bool
-        If True, generates and saves histograms of the event data to visualize
-        the computed limits and distributions.
-
-    Returns
-    -------
-    CommandLineParser
-        Command line parser object
-    """
+    """Parse command line configuration."""
     config = configurator.Configurator(
         description="Derive limits for energy, radial distance, and viewcone."
     )
     config.parser.add_argument(
         "--event_data_files",
         type=str,
+        nargs="+",
         required=True,
-        help="Path to a file containing event data file paths.",
+        help="List of event data files or ascii file listing data files ",
     )
     config.parser.add_argument(
         "--telescope_ids",
         type=str,
-        required=True,
+        required=False,
         help="Path to a file containing telescope configurations.",
     )
     config.parser.add_argument(
-        "--loss_fraction", type=float, required=True, help="Fraction of events to be lost."
+        "--loss_fraction",
+        type=float,
+        required=True,
+        help="Maximum event-loss fraction for limit computation.",
     )
     config.parser.add_argument(
         "--plot_histograms",
@@ -96,197 +121,25 @@ def _parse():
         action="store_true",
         default=False,
     )
-    config.parser.add_argument(
-        "--output_file",
-        type=str,
-        default="corsika_simulation_limits_lookup.ecsv",
-        help="Output file for the derived limits (default: "
-        "'corsika_simulation_limits_lookup.ecsv').",
-    )
-    return config.initialize(db_config=False)
-
-
-def process_file(file_path, telescope_ids, loss_fraction, plot_histograms):
-    """
-    Process a single file and compute limits.
-
-    Parameters
-    ----------
-    file_path : str
-        Path to the event data file.
-    telescope_ids : list[int]
-        List of telescope IDs to filter the events.
-    loss_fraction : float
-        Fraction of events to be lost.
-    plot_histograms : bool
-        Whether to plot histograms.
-
-    Returns
-    -------
-    dict
-        Dictionary containing the computed limits and metadata.
-    """
-    # Extract zenith and azimuth from the file path
-    match = re.search(r"za(\d+)deg.*azm(\d+)deg", file_path)
-    if not match:
-        raise ValueError(f"Could not extract zenith and azimuth from file path: {file_path}")
-    zenith = int(match.group(1))
-    azimuth = int(match.group(2))
-
-    if "dark" in file_path:
-        nsb = 1
-    elif "halfmoon" in file_path:
-        nsb = 5
-    elif "moon" in file_path:
-        nsb = 19
-    else:
-        _logger.warning(f"Could not determine NSB from file path: {file_path}")
-        nsb = None
-
-    if "gamma-diffuse" in file_path:
-        particle_type = "gamma-diffuse"
-    elif "gamma" in file_path:
-        particle_type = "gamma"
-    elif "proton" in file_path:
-        particle_type = "proton"
-    elif "electron" in file_path:
-        particle_type = "electron"
-    else:
-        _logger.warning(f"Could not determine particle type from file path: {file_path}")
-        particle_type = "unknown"
-
-    calculator = LimitCalculator(file_path, telescope_list=telescope_ids)
-
-    lower_energy_limit = calculator.compute_lower_energy_limit(loss_fraction)
-    upper_radial_distance = calculator.compute_upper_radial_distance(loss_fraction)
-    viewcone = calculator.compute_viewcone(loss_fraction)
-
-    if plot_histograms:
-        _logger.info(
-            f"Plotting histograms written to {io_handler.IOHandler().get_output_directory()}"
-        )
-        calculator.plot_data(
-            lower_energy_limit,
-            upper_radial_distance,
-            viewcone,
-            io_handler.IOHandler().get_output_directory(),
-        )
-
-    return {
-        "particle_type": particle_type,
-        "telescope_ids": telescope_ids,
-        "zenith": zenith,
-        "azimuth": azimuth,
-        "nsb": nsb,
-        "lower_energy_threshold": lower_energy_limit,
-        "upper_radius_threshold": upper_radial_distance,
-        "viewcone_radius": viewcone,
-    }
-
-
-def create_results_table(results, loss_fraction):
-    """
-    Create an Astropy Table from the results.
-
-    Parameters
-    ----------
-    results : list[dict]
-        List of dictionaries containing the computed limits for each file
-        and telescope configuration.
-    loss_fraction : float
-        Fraction of events to be lost, added as metadata to the table.
-
-    Returns
-    -------
-    astropy.table.Table
-        An Astropy Table containing the results with appropriate units and metadata.
-    """
-    print("results", results)
-    table = Table(
-        rows=[
-            (
-                res["particle_type"],
-                res["telescope_ids"],
-                res["zenith"],
-                res["azimuth"],
-                res["nsb"],
-                res["lower_energy_threshold"].value,
-                res["upper_radius_threshold"].value,
-                res["viewcone_radius"].value,
-            )
-            for res in results
-        ],
-        names=[
-            "particle_type",
-            "telescope_ids",
-            "zenith",
-            "azimuth",
-            "nsb",
-            "lower_energy_threshold",
-            "upper_radius_threshold",
-            "viewcone_radius",
-        ],
-        dtype=[
-            "S64",
-            object,
-            np.float64,
-            np.float64,
-            object,
-            np.float64,
-            np.float64,
-            np.float64,
+    return config.initialize(
+        db_config=True,
+        output=True,
+        simulation_model=[
+            "site",
+            "model_version",
+            "layout",
         ],
     )
-    table["zenith"].unit = "deg"
-    table["azimuth"].unit = "deg"
-    table["lower_energy_threshold"].unit = "TeV"
-    table["upper_radius_threshold"].unit = "m"
-    table["viewcone_radius"].unit = "deg"
-
-    table.meta["created"] = datetime.datetime.now().isoformat()
-    table.meta["description"] = (
-        "Lookup table for CORSIKA limits computed from gamma-ray shower simulations "
-        "using simtool production_derive_corsika_limits"
-    )
-    table.meta["loss_fraction"] = loss_fraction
-
-    return table
 
 
 def main():
     """Derive limits for energy, radial distance, and viewcone."""
-    args_dict, _ = _parse()
+    args_dict, db_config = _parse()
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "info")))
 
-    event_data_files = gen.collect_data_from_file(args_dict["event_data_files"])["files"]
-    telescope_configs = gen.collect_data_from_file(args_dict["telescope_ids"])["telescope_configs"]
-
-    results = []
-    for file_path in event_data_files:
-        for array_name, telescope_ids in telescope_configs.items():
-            _logger.info(f"Processing file: {file_path} with telescope config: {array_name}")
-            result = process_file(
-                file_path,
-                telescope_ids,
-                args_dict["loss_fraction"],
-                args_dict["plot_histograms"],
-            )
-            result["layout"] = array_name
-            results.append(result)
-
-    table = create_results_table(results, args_dict["loss_fraction"])
-
-    output_dir = io_handler.IOHandler().get_output_directory("corsika_limits")
-    output_file = f"{output_dir}/{args_dict['output_file']}"
-
-    table.write(output_file, format="ascii.ecsv", overwrite=True)
-    _logger.info(f"Results saved to {output_file}")
-
-    metadata_file = f"{output_dir}/metadata.yml"
-    MetadataCollector.dump(args_dict, metadata_file)
-    _logger.info(f"Metadata saved to {metadata_file}")
+    generate_corsika_limits_grid(args_dict, db_config)
 
 
 if __name__ == "__main__":

simtools/applications/production_derive_statistics.py

@@ -1,34 +1,30 @@
 #!/usr/bin/python3
 
 r"""
-Application to run the StatisticalErrorEvaluator and interpolate results.
+Derive the required number of events for MC productions for a grid of observational conditions.
 
-This application evaluates statistical uncertainties from DL2 MC event files
-based on input parameters like zenith angles and camera offsets, and performs interpolation
-for a specified query point.
+This application evaluates statistical uncertainties from the analysis of events after the
+application of loose gamma/hadron separation cuts, then interpolates the derived number of required
+events for the specified grid points provided in a file. The resulting grid points will have the
+derived number of required events added.
+
+The metric for the required uncertainty is pre-defined and must be configured via the metrics file.
 
 Command line arguments
 ----------------------
-base_path (str, required)
-    Path to the directory containing the DL2 MC event file for interpolation.
-zeniths (list of int, required)
-    List of zenith angles to consider.
-camera_offsets (list of int, required)
-    List of offsets in degrees.
-query_point (list of float, required)
-    Query point for interpolation. The query point must contain exactly 5 values:
-        - Energy (TeV)
-        - Azimuth (degrees)
-        - Zenith (degrees)
-        - NSB (MHz)
-        - Offset (degrees)
-output_file (str, optional)
-    Output file to store the results. Default: 'interpolated_production_statistics.json'.
+grid_points_production_file (str, required)
+    Path to the file containing grid points. Each grid point should include azimuth, zenith, NSB,
+    offset.
 metrics_file (str, optional)
     Path to the metrics definition file. Default: 'production_simulation_config_metrics.yml'.
+base_path (str, required)
+    Path to the directory containing the event files for interpolation (after loose gamma/hadron
+    cuts).
 file_name_template (str, optional)
-    Template for the file name. Default:
+    Template for the event file name. Default:
     'prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits'.
+plot_production_statistics (flag, optional)
+    If provided, plots the production statistics. Default: False.
 
 Example
 -------
@@ -36,22 +32,33 @@ To evaluate statistical uncertainties and perform interpolation, run the command
 
 .. code-block:: console
 
-    simtools-production-derive-statistics --base_path tests/resources/production_dl2_fits/ \\
-        --zeniths 20 40 52 60 --camera_offsets 0 --query_point 1 180 30 0 0 \\
+    simtools-production-derive-statistics \\
+        --grid_points_production_file path/to/grid_points_production.json \\
         --metrics_file "path/to/metrics.yaml" \\
-        --output_path simtools-output/derived_events \\
-        --output_file derived_events.json
-
-The output will display the production statistics for the specified query point and save
- the results to the specified output file.
+        --base_path path/to/production_event_files/ \\
+        --file_name_template "prod6_LaPalma-{zenith}deg\\
+            _gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits" \\
+        --zeniths 20 40 52 60 \\
+        --offsets 0 \\
+        --azimuths 180 \\
+        --nsb 0.0 \\
+        --plot_production_statistics
+
+Output
+------
+The output will be a file containing the grid points with the derived number of required events
+added.
 """
 
+import logging
 from pathlib import Path
 
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.production_configuration.derive_production_statistics_handler import (
     ProductionStatisticsHandler,
 )
+from simtools.utils import general as gen
 
 
 def _parse(label, description):
@@ -65,59 +72,86 @@ def _parse(label, description):
     """
     config = configurator.Configurator(label=label, description=description)
 
+    config.parser.add_argument(
+        "--grid_points_production_file",
+        type=str,
+        required=True,
+        help="Path to the JSON file containing grid points for a production.",
+    )
+    config.parser.add_argument(
+        "--metrics_file",
+        required=True,
+        type=str,
+        default=None,
+        help="Metrics definition file. (default: production_simulation_config_metrics.yml)",
+    )
     config.parser.add_argument(
         "--base_path",
         type=str,
         required=True,
         help="Path to the DL2 MC event files for interpolation.",
     )
+    config.parser.add_argument(
+        "--file_name_template",
+        required=False,
+        type=str,
+        default=("prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits"),
+        help=("Template for the DL2 MC event file name."),
+    )
     config.parser.add_argument(
         "--zeniths",
         required=True,
         nargs="+",
         type=float,
-        help="List of zenith angles.",
+        help="List of zenith angles in deg that describe the supplied DL2 files.",
     )
     config.parser.add_argument(
-        "--camera_offsets",
+        "--azimuths",
         required=True,
         nargs="+",
         type=float,
-        help="List of camera offsets in degrees.",
+        help="List of azimuth angles in deg that describe the supplied DL2 files.",
     )
     config.parser.add_argument(
-        "--query_point",
+        "--nsb",
         required=True,
-        nargs=5,
+        nargs="+",
         type=float,
-        help="Grid point for interpolation (energy, azimuth, zenith, NSB, offset).",
+        help="List of nsb values that describe the supplied DL2 files.",
     )
     config.parser.add_argument(
-        "--metrics_file",
-        required=False,
-        type=str,
-        default="production_simulation_config_metrics.yml",
-        help="Metrics definition file. (default: production_simulation_config_metrics.yml)",
+        "--offsets",
+        required=True,
+        nargs="+",
+        type=float,
+        help="List of camera offsets in deg that describe the supplied DL2 files.",
     )
     config.parser.add_argument(
-        "--file_name_template",
+        "--plot_production_statistics",
         required=False,
-        type=str,
-        default=("prod6_LaPalma-{zenith}deg_gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits"),
-        help=("Template for the DL2 MC event file name."),
+        action="store_true",
+        default=False,
+        help="Plot production statistics.",
     )
+
     return config.initialize(db_config=False, output=True)
 
 
 def main():
     """Run the ProductionStatisticsHandler."""
     label = Path(__file__).stem
+
     args_dict, _ = _parse(
         label,
         "Evaluate statistical uncertainties from DL2 MC event files and interpolate results.",
     )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    _io_handler = io_handler.IOHandler()
+    output_path = _io_handler.get_output_directory(label, sub_dir="")
 
-    manager = ProductionStatisticsHandler(args_dict)
+    manager = ProductionStatisticsHandler(args_dict, output_path=output_path)
     manager.run()
 
 

simtools/applications/simulate_light_emission.py

@@ -322,6 +322,7 @@ def main():
         simtel_path=args_dict["simtel_path"],
         light_source_type=args_dict["light_source_type"],
         label=label,
+        test=args_dict["test"],
     )
 
     if args_dict["light_source_setup"] == "variable":

simtools/applications/simulate_prod.py

@@ -115,6 +115,16 @@ def _parse(description=None):
         required=False,
         default=False,
     )
+    config.parser.add_argument(
+        "--save_reduced_event_lists",
+        help=(
+            "Save reduced event lists with event data on simulated and triggered events. "
+            "Saved with the same name as the sim_telarray output file (different extension). "
+        ),
+        action="store_true",
+        required=False,
+        default=False,
+    )
     config.parser.add_argument(
         "--corsika_test_seeds",
         help="Use predefined random seeds for CORSIKA for testing purposes.",
@@ -122,6 +132,15 @@ def _parse(description=None):
         required=False,
         default=False,
     )
+    config.parser.add_argument(
+        "--sequential",
+        help=(
+            "Enables single-core mode (as far as possible); "
+            "otherwise, CORSIKA and sim_telarray run in parallel."
+        ),
+        action="store_true",
+        default=False,
+    )
     sim_telarray_seed_group = config.parser.add_argument_group(
         title="Random seeds for sim_telarray instrument setup",
     )
@@ -144,7 +163,6 @@ def _parse(description=None):
     )
     return config.initialize(
         db_config=True,
-        job_submission=True,
         simulation_model=["site", "layout", "telescope", "model_version"],
         simulation_configuration={"software": None, "corsika_configuration": ["all"]},
     )
@@ -163,23 +181,17 @@ def main():  # noqa: D103
     simulator.simulate()
     simulator.validate_metadata()
 
-    if simulator.submit_engine == "local":
-        logger.info(
-            f"Production run complete for primary {args_dict['primary']} showers "
-            f"from {args_dict['azimuth_angle']} azimuth and {args_dict['zenith_angle']} zenith "
-            f"at {args_dict['site']} site, using {args_dict['model_version']} model."
-        )
-        if args_dict.get("pack_for_grid_register"):
-            simulator.pack_for_register(args_dict["pack_for_grid_register"])
-        if args_dict["save_file_lists"]:
-            simulator.save_file_lists()
-    else:
-        logger.info("Production run submitted to the workload manager")
-        if args_dict["pack_for_grid_register"] or args_dict["save_file_lists"]:
-            logger.warning(
-                "Packing for grid register or saving file lists not supported for "
-                f"{simulator.submit_engine}."
-            )
+    logger.info(
+        f"Production run complete for primary {args_dict['primary']} showers "
+        f"from {args_dict['azimuth_angle']} azimuth and {args_dict['zenith_angle']} zenith "
+        f"at {args_dict['site']} site, using {args_dict['model_version']} model."
+    )
+    if args_dict["save_reduced_event_lists"]:
+        simulator.save_reduced_event_lists()
+    if args_dict.get("pack_for_grid_register"):
+        simulator.pack_for_register(args_dict["pack_for_grid_register"])
+    if args_dict["save_file_lists"]:
+        simulator.save_file_lists()
 
 
 if __name__ == "__main__":

simtools/applications/simulate_prod_htcondor_generator.py

@@ -76,7 +76,6 @@ def _parse(description=None):
     )
     return config.initialize(
         db_config=False,
-        job_submission=False,
         simulation_model=["site", "layout", "telescope", "model_version"],
         simulation_configuration={"software": None, "corsika_configuration": ["all"]},
     )