gammasimtools 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/applications/print_version.py

@@ -6,13 +6,12 @@ The versions of simtools, the DB, sim_telarray, and CORSIKA are printed.
 
 """
 
-import json
 import logging
 from pathlib import Path
 
 from simtools import dependencies, version
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 from simtools.utils import general as gen
 
 
@@ -66,15 +65,15 @@ def main():
 
     version_list = version_string.strip().split("\n")
     for version_entry in version_list:
-        key, value = version_entry.split(": ")
+        key, value = version_entry.split(": ", 1)
         version_dict[key] = value
 
-    with open(
-        io_handler_instance.get_output_file(args_dict["output_file"], label=label),
-        "w",
-        encoding="utf-8",
-    ) as f:
-        json.dump(version_dict, f, indent=4)
+    ascii_handler.write_data_to_file(
+        data=version_dict,
+        output_file=io_handler_instance.get_output_file(
+            args_dict.get("output_file", "simtools_version.json"), label=label
+        ),
+    )
 
 
 if __name__ == "__main__":

simtools/applications/production_derive_corsika_limits.py

@@ -45,7 +45,7 @@ directions, level of night sky background, etc.).
 
 Command line arguments
 ----------------------
-event_data_files (str, required)
+event_data_file (str, required)
     Path to reduced event data file.
 telescope_ids (str, optional)
     Custom array layout file containing telescope IDs.
@@ -64,7 +64,7 @@ Derive limits for a list of array layouts (use 'all' to derive limits for all la
 .. code-block:: console
 
     simtools-production-derive-corsika-limits \\
-        --event_data_files path/to/event_data_files.yaml \\
+        --event_data_file event_dat_file.hdf5 \\
         --array_layout_name alpha,beta \\
         --loss_fraction 1e-6 \\
         --plot_histograms \\
@@ -75,7 +75,7 @@ Derive limits for a given file for custom defined array layouts:
 .. code-block:: console
 
     simtools-production-derive-corsika-limits \\
-        --event_data_files path/to/event_data_files.yaml \\
+        --event_data_file event_dat_file.hdf5 \\
         --telescope_ids path/to/telescope_configs.yaml \\
         --loss_fraction 1e-6 \\
         --plot_histograms \\
@@ -86,7 +86,7 @@ import logging
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.production_configuration.derive_corsika_limits_grid import (
+from simtools.production_configuration.derive_corsika_limits import (
     generate_corsika_limits_grid,
 )
 
@@ -97,11 +97,10 @@ def _parse():
         description="Derive limits for energy, radial distance, and viewcone."
     )
     config.parser.add_argument(
-        "--event_data_files",
+        "--event_data_file",
         type=str,
-        nargs="+",
         required=True,
-        help="List of event data files or ascii file listing data files ",
+        help="Event data file containing reduced event data.",
     )
     config.parser.add_argument(
         "--telescope_ids",

simtools/applications/production_derive_statistics.py

@@ -54,7 +54,7 @@ import logging
 from pathlib import Path
 
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.production_configuration.derive_production_statistics_handler import (
     ProductionStatisticsHandler,
 )

simtools/applications/production_generate_grid.py

@@ -49,10 +49,10 @@ from astropy.coordinates import EarthLocation
 from astropy.time import Time
 
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
+from simtools.io.ascii_handler import collect_data_from_file
 from simtools.model.site_model import SiteModel
 from simtools.production_configuration.generate_production_grid import GridGeneration
-from simtools.utils.general import collect_data_from_file
 
 
 def _parse(label, description):

simtools/applications/production_merge_corsika_limits.py

@@ -0,0 +1,214 @@
+#!/usr/bin/python3
+
+r"""
+Merge CORSIKA limit tables from multiple grid points and check grid completeness.
+
+The corsika limit tables are first generated by the application
+'simtools-production-derive-corsika-limits' and contain derived limits for
+energy, radial distance, and viewcone for different observational conditions.
+The individual tables are merged into a single table for each grid point by
+the application 'merge_tables'. This tool finalizes the merging process and
+checks the completeness of the grid against a provided grid definition file.
+
+This tool supports three main use cases:
+
+1. Merge multiple CORSIKA limit tables into a single file and optionally generate
+   plots of the derived limits.
+2. Merge tables and also check for grid completeness against a provided grid
+   definition file. This requires the --grid_definition parameter. Coverage plots
+   can also be generated.
+3. Check grid completeness of an already merged table file. This requires both
+   the --merged_table and --grid_definition parameters.
+
+Command line arguments
+----------------------
+input_files (str)
+    Directory containing corsika_simulation_limits_lookup*.ecsv files to be merged,
+    or a list of specific files to merge. Not used if --merged_table is provided.
+input_files_list (str)
+    Path to a text file containing a list of input files (one file path per line)
+    to be merged. Not used if --merged_table is provided.
+merged_table (str)
+    Path to an already merged table file. Used for checking grid completeness.
+grid_definition (str)
+    Path to a YAML file defining the expected grid points. Required for grid
+    completeness checks and coverage plots.
+output_file (str, optional)
+    Name of the output file for the merged limits table.
+    Default is "merged_corsika_limits.ecsv".
+plot_grid_coverage (bool, optional)
+    Flag to generate plots showing grid coverage. Requires --grid_definition.
+plot_limits (bool, optional)
+    Flag to generate plots showing the derived limits.
+
+Examples
+--------
+1. Merge CORSIKA limit tables from a directory:
+
+   .. code-block:: console
+
+      simtools-production-merge-corsika-limits \\
+          --input_files "simtools-output/corsika_limits/" \\
+          --output_file merged_limits.ecsv --plot_limits
+
+2. Merge tables and check grid completeness:
+
+   .. code-block:: console
+
+      simtools-production-merge-corsika-limits \\
+          --input_files "simtools-output/corsika_limits/" \\
+          --grid_definition grid_definition.yaml \\
+          --output_file merged_limits.ecsv --plot_grid_coverage
+
+3. Check grid completeness of an existing merged table:
+
+   .. code-block:: console
+
+      simtools-production-merge-corsika-limits \\
+          --merged_table merged_limits.ecsv \\
+          --grid_definition grid_definition.yaml --plot_grid_coverage
+
+4. Merge tables using a list of files from a text file:
+
+   .. code-block:: console
+
+      simtools-production-merge-corsika-limits \\
+          --input_files_list file_list.txt \\
+          --output_file merged_limits.ecsv
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.data_model import data_reader
+from simtools.io import ascii_handler
+from simtools.production_configuration.merge_corsika_limits import CorsikaMergeLimits
+
+_logger = logging.getLogger(__name__)
+
+
+def _parse():
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        description="Merge CORSIKA limit tables and check grid completeness."
+    )
+    config.parser.add_argument(
+        "--input_files",
+        type=str,
+        default=None,
+        nargs="+",
+        help=(
+            "A list of input files to be merged, or a single directory "
+            "containing the files (*.ecsv)."
+        ),
+    )
+    config.parser.add_argument(
+        "--input_files_list",
+        type=str,
+        default=None,
+        help=(
+            "Path to a text file containing a list of input files (one file path per line) "
+            "to be merged."
+        ),
+    )
+    config.parser.add_argument(
+        "--merged_table",
+        type=str,
+        default=None,
+        help="Path to an already merged table file.",
+    )
+    config.parser.add_argument(
+        "--grid_definition",
+        type=str,
+        default=None,
+        help="Path to YAML file defining the expected grid points.",
+    )
+    config.parser.add_argument(
+        "--plot_grid_coverage",
+        help="Generate plots showing grid coverage.",
+        action="store_true",
+        default=False,
+    )
+    config.parser.add_argument(
+        "--plot_limits",
+        help="Generate plots showing the derived limits.",
+        action="store_true",
+        default=False,
+    )
+    return config.initialize(output=True)
+
+
+def main():
+    """Merge CORSIKA limit tables and check grid completeness."""
+    args_dict, _ = _parse()
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "info")))
+
+    merger = CorsikaMergeLimits()
+    grid_definition = (
+        ascii_handler.collect_data_from_file(args_dict["grid_definition"])
+        if args_dict.get("grid_definition")
+        else None
+    )
+
+    if args_dict.get("merged_table"):
+        # Case 3: Check coverage on an existing merged table
+        merged_table_path = Path(args_dict["merged_table"]).expanduser()
+        merged_table = data_reader.read_table_from_file(merged_table_path)
+        input_files = [merged_table_path]
+    elif args_dict.get("input_files") or args_dict.get("input_files_list"):
+        # Case 1 & 2: Merge files
+        input_files = []
+
+        # Process input_files argument
+        if args_dict.get("input_files"):
+            raw_paths = args_dict.get("input_files")
+            if len(raw_paths) == 1 and Path(raw_paths[0]).expanduser().is_dir():
+                input_dir = Path(raw_paths[0]).expanduser()
+                input_files.extend(input_dir.glob("*.ecsv"))
+            else:
+                input_files.extend(Path(f).expanduser() for f in raw_paths)
+
+        # Process input_files_list argument
+        if args_dict.get("input_files_list"):
+            files_from_list = merger.read_file_list(args_dict["input_files_list"])
+            input_files.extend(files_from_list)
+
+        if not input_files:
+            raise FileNotFoundError(
+                "No input files found. Check --input_files or --input_files_list arguments."
+            )
+        merged_table = merger.merge_tables(input_files)
+    else:
+        raise ValueError(
+            "Either --input_files, --input_files_list, or --merged_table must be provided."
+        )
+
+    is_complete, grid_completeness = merger.check_grid_completeness(merged_table, grid_definition)
+
+    if args_dict.get("plot_grid_coverage"):
+        merger.plot_grid_coverage(merged_table, grid_definition)
+
+    if args_dict.get("plot_limits"):
+        merger.plot_limits(merged_table)
+
+    if not args_dict.get("merged_table"):
+        # Write output file only when merging
+        output_file = merger.output_dir / args_dict["output_file"]
+        merger.write_merged_table(
+            merged_table,
+            output_file,
+            input_files,
+            {
+                "is_complete": is_complete,
+                "expected": grid_completeness.get("expected", 0),
+                "found": grid_completeness.get("found", 0),
+                "missing": grid_completeness.get("missing", []),
+            },
+        )
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/run_application.py

@@ -8,36 +8,66 @@ both the name of the simtools application and the configuration for the applicat
 
 This application is used for model parameter setting workflows.
 Strong assumptions are applied on the directory structure for input and output files of
-applications.
+applications, for details see the CTAO Simulation Model Parameter Setting
+`repository <https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/\
+simulation-model-parameter-setting>`_.
+
+.. tip::
+   Browse the CTAO Simulation Model Parameter Setting
+   `repository <https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/\
+simulation-model-parameter-setting>`_
+   for a list of examples and templates for configuration files.
+
+For simplified configuration, a placeholder called ``__SETTING_WORKFLOW__`` can be used in the
+configuration file. This placeholder will be replaced with the directory below ``input``
+(example: configuration file is in ``input/LSTN-design/num_gains/20250214T134800/config.yml``,
+then the placeholder will be replaced with ``LSTN-design/num_gains/20250214T134800``).
+This will also be the directory for any output generated by the application.
+
+Run time environments can be defined in the configuration file using the ``runtime_environment``
+block. The following example shows how to define a runtime environment:
+
+.. code-block:: yaml
+
+     runtime_environment:
+     image: ghcr.io/gammasim/simtools-prod-sim-telarray-240927-corsika-77550-bernlohr-1.68-\
+prod6-baseline-qgs2-no_opt:20250716-122341
+     network: simtools-mongo-network
+     environment_file: ./.env
+     container_engine: podman
+     options:
+     - '--arch amd64'
+
+If the ``ignore_runtime_environment`` flag is set, the application will run in the current
+environment, ignoring any definitions in the configuration file.
+
+The database configuration and setting of other environment variables is done as
+described in :ref:`environment-variables`.
 
 Example
 -------
 
-Run the application with the configuration file 'config_file_name':
+Run the application with the configuration file ``config_file_name``:
 
 .. code-block:: console
 
-    simtools-run-application --configuration_file config_file_name
+     simtools-run-application --configuration_file config_file_name
 
-Run the application with the configuration file 'config_file_name', but skipping all steps except
+Run the application with the configuration file ``config_file_name``, but skipping all steps except
 step 2 and 3 (useful for debugging):
 
 .. code-block:: console
 
-    simtools-run-application --configuration_file config_file_name --steps 2 3
+     simtools-run-application --configuration_file config_file_name --steps 2 3
 
 """
 
 import logging
-import subprocess
-import tempfile
 from pathlib import Path
 
-import yaml
-
 import simtools.utils.general as gen
-from simtools import dependencies
 from simtools.configuration import configurator
+from simtools.runners import simtools_runner
 
 
 def _parse(label, description, usage):
@@ -73,91 +103,15 @@ def _parse(label, description, usage):
         nargs="+",
         help="List of steps to be execution (e.g., '--steps 7 8 9'; do not specify to run all).",
     )
+    config.parser.add_argument(
+        "--ignore_runtime_environment",
+        action="store_true",
+        help="Ignore the runtime environment and run the application in the current environment.",
+        default=False,
+    )
     return config.initialize(db_config=True)
 
 
-def run_application(application, configuration, logger):
-    """Run a simtools application and return stdout and stderr."""
-    with tempfile.NamedTemporaryFile(mode="w", delete=True, suffix=".yml") as temp_config:
-        yaml.dump(configuration, temp_config, default_flow_style=False)
-        temp_config.flush()
-        configuration_file = Path(temp_config.name)
-        try:
-            result = subprocess.run(
-                [application, "--config", configuration_file],
-                check=True,
-                capture_output=True,
-                text=True,
-            )
-        except subprocess.CalledProcessError as exc:
-            logger.error(f"Error running application {application}: {exc.stderr}")
-            raise exc
-        return result.stdout, result.stderr
-
-
-def get_subdirectory_name(path):
-    """Get the first subdirectory name under 'input'."""
-    path = Path(path).resolve()
-    try:
-        input_index = path.parts.index("input")
-        return path.parts[input_index], path.parts[input_index + 1]
-    except (ValueError, IndexError) as exc:
-        raise ValueError(f"Could not find subdirectory under 'input': {exc}") from exc
-
-
-def read_application_configuration(configuration_file, steps, logger):
-    """
-    Read application configuration from file and modify for setting workflows.
-
-    Strong assumptions on the structure of input and output files:
-
-    - configuration file is expected to be in './input/<workflow directory>/<yaml file>'
-    - output files will be written out to './output/<workflow directory>/'
-
-    Replaces the placeholders in the configuration file with the actual values.
-    Sets 'USE_PLAIN_OUTPUT_PATH' to True for all applications.
-
-    Parameters
-    ----------
-    configuration_file : str
-        Configuration file name.
-    steps : list
-        List of steps to be executed (None: all steps).
-    logger : Logger
-        Logger object.
-
-    Returns
-    -------
-    dict
-        Application configuration.
-    Path
-        Path to the log file.
-
-    """
-    application_config = gen.collect_data_from_file(configuration_file).get("CTA_SIMPIPE")
-    place_holder = "__SETTING_WORKFLOW__"
-    workflow_dir, setting_workflow = get_subdirectory_name(configuration_file)
-    output_path = Path(str(workflow_dir).replace("input", "output")) / Path(setting_workflow)
-    output_path.mkdir(parents=True, exist_ok=True)
-    logger.info(f"Setting workflow output path to {output_path}")
-    log_file = output_path / "simtools.log"
-    configurations = application_config.get("APPLICATIONS")
-    for step_count, config in enumerate(configurations, start=1):
-        config["RUN_APPLICATION"] = step_count in steps if steps else True
-        for key, value in config.get("CONFIGURATION", {}).items():
-            if isinstance(value, str):
-                config["CONFIGURATION"][key] = value.replace(place_holder, setting_workflow)
-            if isinstance(value, list):
-                config["CONFIGURATION"][key] = [
-                    item.replace(place_holder, setting_workflow) if isinstance(item, str) else item
-                    for item in value
-                ]
-        config["CONFIGURATION"]["USE_PLAIN_OUTPUT_PATH"] = True
-        config["CONFIGURATION"]["OUTPUT_PATH"] = str(output_path)
-
-    return configurations, log_file
-
-
 def main():  # noqa: D103
     args_dict, db_config = _parse(
         Path(__file__).stem,
@@ -167,27 +121,7 @@ def main():  # noqa: D103
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    configurations, log_file = read_application_configuration(
-        args_dict["configuration_file"], args_dict["steps"], logger
-    )
-
-    with log_file.open("w", encoding="utf-8") as file:
-        file.write("Running simtools applications\n")
-        file.write(dependencies.get_version_string(db_config))
-        for config in configurations:
-            if config.get("RUN_APPLICATION"):
-                logger.info(f"Running application: {config.get('APPLICATION')}")
-            else:
-                logger.info(f"Skipping application: {config.get('APPLICATION')}")
-                continue
-            config = gen.change_dict_keys_case(config, False)
-            stdout, stderr = run_application(
-                config.get("APPLICATION"), config.get("CONFIGURATION"), logger
-            )
-            file.write("=" * 80 + "\n")
-            file.write(f"Application: {config.get('APPLICATION')}\n")
-            file.write("STDOUT:\n" + stdout)
-            file.write("STDERR:\n" + stderr)
+    simtools_runner.run_applications(args_dict, db_config, logger)
 
 
 if __name__ == "__main__":

simtools/applications/simulate_calibration_events.py

@@ -0,0 +1,166 @@
+#!/usr/bin/python3
+
+r"""
+Simulate calibration events like pedestal or flasher events.
+
+Use sim_telarray to simulate calibration events for an array of telescopes.
+The following types of calibration events are supported:
+
+* Pedestal events (includes night-sky background and camera noise)
+* Dark pedestal events (closed camera lid, camera noise only)
+* Flasher events (simulated flasher light source)
+
+Example
+-------
+
+Simulate pedestal events for alpha North. The assumed level night-sky background is 2.0 times the
+nominal value. A list of stars can be provided to simulate additional contributions.
+
+.. code-block:: console
+
+    simtools-simulate-calibration-events --run_mode=pedestals \\
+        --run_number 10 --number_of_events 1000 \\
+        --array_layout_name alpha --site North \\
+        --model_version 6.0.0 \\
+        --zenith_angle 20 --azimuth_angle 0 \\
+        --nsb_scaling_factor 2.0
+
+Simulate flasher events for alpha South. Note that the same flasher configuration is used
+for all telescopes.
+
+.. code-block:: console
+
+    simtools-simulate-calibration-events --run_mode=flasher \\
+        --run_number 10 --number_of_events 1000 \\
+        --array_layout_name subsystem_msts --site South \\
+        --model_version 6.0.0 \\
+        --zenith_angle 20 --azimuth_angle 0 \\
+        --flasher_photons 500 --flasher_var_photons 0.05 \\
+        --flasher_exp_time 1.59 --flasher_sig_time 0.4
+
+Command Line Arguments
+----------------------
+run_mode (str, required)
+    Run mode, e.g. "pedestals" or "flasher".
+run_number (int, required)
+    Run number for the simulation.
+number_of_events (int, required)
+    Number of calibration events to simulate.
+array_layout_name (str, required)
+    Array layout name, e.g. "alpha".
+site (str, required)
+    Site name.
+model_version (str, optional)
+    Version of the simulation model.
+nsb_scaling_factor (float, optional)
+    Scaling factor for the night-sky background rate. Default is 1.0, which
+    corresponds to the nominal (dark sky) NSB rate.
+stars (str, optional)
+    Path to a file containing a list of stars (azimuth, zenith, weighting factor
+    separated by whitespace). If provided, the stars will be used to simulate
+    additional contributions to the night-sky background. For details on the
+    parameters, see the sim_telarray manual.
+zenith_angle (float, optional)
+    Zenith angle in degrees.
+azimuth_angle (float, optional)
+    Azimuth angle in degrees.
+flasher_photons (float, optional)
+    Number of photons in the flasher pulse at each photodetector.
+flasher_var_photons (float, optional)
+    Relative variance of the number of photons in the flasher pulse.
+flasher_exp_time (float, optional)
+    Exponential decay time of the flasher pulse in nano-seconds.
+flasher_sig_time (float, optional)
+    Sigma of Gaussian-shaped flasher pulse in nano-seconds.
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.simulator import Simulator
+
+
+def _parse(label):
+    """Parse command line configuration."""
+    config = configurator.Configurator(label=label, description="Simulate calibration events.")
+    config.parser.add_argument(
+        "--run_mode",
+        help="Calibration run mode",
+        type=str,
+        required=True,
+        choices=["pedestals", "dark_pedestals", "nsb_only_pedestals", "flasher"],
+    )
+    config.parser.add_argument(
+        "--number_of_events",
+        help="Number of calibration events to simulate",
+        type=int,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--nsb_scaling_factor",
+        help=(
+            "Scaling factor for the NSB rate. "
+            "Default is 1.0, which corresponds to the nominal (dark sky) NSB rate."
+        ),
+        type=float,
+        required=False,
+        default=1.0,
+    )
+    config.parser.add_argument(
+        "--stars",
+        help="List of stars (azimuth, zenith, weighting factor).",
+        type=str,
+        default=None,
+    )
+    flasher_args = config.parser.add_argument_group("Flasher configuration")
+    flasher_args.add_argument(
+        "--flasher_photons",
+        help="Number of photons in the flasher pulse at each photodetector.",
+        type=float,
+        default=500.0,
+    )
+    flasher_args.add_argument(
+        "--flasher_var_photons",
+        help="Relative variance of the number of photons in the flasher pulse.",
+        type=float,
+        default=0.05,
+    )
+    flasher_args.add_argument(
+        "--flasher_exp_time",
+        help="Exponential decay time of the flasher pulse in nanoseconds.",
+        type=float,
+        default=0.0,
+    )
+    flasher_args.add_argument(
+        "--flasher_sig_time",
+        help="Sigma of Gaussian-shaped flasher pulse in nanoseconds.",
+        type=float,
+        default=0.0,
+    )
+
+    return config.initialize(
+        db_config=True,
+        simulation_model=["site", "layout", "telescope", "model_version"],
+        simulation_configuration={
+            "corsika_configuration": ["run_number", "azimuth_angle", "zenith_angle"],
+            "sim_telarray_configuration": ["all"],
+        },
+    )
+
+
+def main():  # noqa: D103
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(label)
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    simulator = Simulator(label=args_dict.get("label"), args_dict=args_dict, db_config=db_config)
+    simulator.simulate()
+
+
+if __name__ == "__main__":
+    main()