gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/applications/production_derive_limits.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python3
+
+r"""
+Derives the limits for energy, radial distance, and viewcone to be used in CORSIKA simulations.
+
+The limits are derived based on the event loss fraction specified by the user.
+
+Command line arguments
+----------------------
+event_data_file (str, required)
+    Path to the file containing the event data.
+loss_fraction (float, required)
+    Fraction of events to be lost.
+
+
+Example
+-------
+Derive limits for a given file with a specified loss fraction.
+
+.. code-block:: console
+
+    simtools-production-derive-limits\\
+        --event_data_file path/to/event_data_file.hdf5 \\
+        --loss_fraction 1e-6
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.io_operations.hdf5_handler import read_hdf5
+from simtools.production_configuration.limits_calculation import LimitCalculator
+
+_logger = logging.getLogger(__name__)
+
+
+def _parse():
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    event_data_file: str
+        The event data file.
+    loss_fraction: float
+        Loss fraction of events for limit derivation.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object
+
+    """
+    config = configurator.Configurator(
+        description="Derive limits for energy, radial distance, and viewcone."
+    )
+    config.parser.add_argument(
+        "--event_data_file",
+        type=str,
+        required=True,
+        help="Path to the event data file containing the event data.",
+    )
+    config.parser.add_argument(
+        "--loss_fraction", type=float, required=True, help="Fraction of events to be lost."
+    )
+    return config.initialize(db_config=False)
+
+
+def main():
+    """Derive limits for energy, radial distance, and viewcone."""
+    args_dict, _ = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    event_data_file_path = args_dict["event_data_file"]
+    loss_fraction = args_dict["loss_fraction"]
+
+    _logger.info(f"Loading event data file: {event_data_file_path}")
+    tables = read_hdf5(event_data_file_path)
+
+    calculator = LimitCalculator(tables)
+
+    lower_energy_limit = calculator.compute_lower_energy_limit(loss_fraction)
+    _logger.info(f"Lower energy threshold: {lower_energy_limit}")
+
+    upper_radial_distance = calculator.compute_upper_radial_distance(loss_fraction)
+    _logger.info(f"Upper radius threshold: {upper_radial_distance}")
+
+    viewcone = calculator.compute_viewcone(loss_fraction)
+    _logger.info(f"Viewcone radius: {viewcone}")
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/production_generate_simulation_config.py

@@ -1,32 +1,42 @@
 #!/usr/bin/python3
 
 r"""
-Configure a simulation based on command-line arguments.
+Derive simulation configuration parameters for a simulation production.
 
-This application configures and
-generates simulation parameters for a specific grid point in a statistical uncertainty
-evaluation setup.
+Derived simulation configuration parameters include:
+
+* energy range
+* shower core scatter radius
+* view cone radius
+* total number of events to be simulated
+
+Configuration parameters depend on characteristics of the observations, especially elevation,
+azimuth, and night sky background.
+
+The configuration parameters are derived according to the required precision. The metrics are:
+
+* statistical uncertainty on the determination of the effective area as function of primary energy
+* fraction of lost events to the selected core scatter and view cone radius (to be implemented)
+* statistical uncertainty of the energy migration matrix as function of primary energy
+    (to be implemented)
 
 Command line arguments
 ----------------------
 azimuth (float, required)
-    Azimuth angle in degrees.
+        Azimuth angle in degrees.
 elevation (float, required)
-    Elevation angle in degrees.
+        Elevation angle in degrees.
 nsb (float, required)
-    Night sky background value.
-ctao_data_level (str, required)
-    The data level for the simulation (e.g., 'A', 'B', 'C').
-science_case (str, required)
-    The science case for the simulation.
+        Night sky background value.
 file_path (str, required)
-    Path to file with MC events at CTAO DL2 data level. Used for statistical uncertainty evaluation.
+        Path to file with MC events at CTAO DL2 data level.
+        Used for statistical uncertainty evaluation.
 file_type (str, required)
-    Type of the DL2 MC event file ('point-like' or 'cone').
-metrics (str, optional)
-    Path to a YAML file containing metrics for evaluation.
+        Type of the DL2 MC event file ('point-like' or 'cone').
+metrics (str, required)
+        Path to a YAML file containing metrics for evaluation.
 site (str, required)
-    The observatory site (North or South).
+        The observatory site (North or South).
 
 Example
 -------
@@ -34,13 +44,12 @@ To run the simulation configuration, execute the script as follows:
 
 .. code-block:: console
 
-    simtools-production-generate-simulation-config --azimuth 60.0 --elevation 45.0 \
-        --nsb 0.3 --ctao_data_level "A" --science_case "high_precision" \
-        --file_path tests/resources/production_dl2_fits/dl2_mc_events_file.fits \
-        --file_type "point-like"    \
-        --metrics_file tests/resources/production_simulation_config_metrics.yaml --site North
+        simtools-production-generate-simulation-config --azimuth 60.0 --elevation 45.0 \
+                --nsb 0.3 --file_path tests/resources/production_dl2_fits/dl2_mc_events_file.fits \
+                --file_type "point-like"    \
+                --metrics_file tests/resources/production_simulation_config_metrics.yml --site North
 
-The output will show the configured simulation parameters.
+The output will show the derived simulation parameters.
 """
 
 import json
@@ -51,6 +60,7 @@ import astropy.units as u
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
+from simtools.data_model import schema
 from simtools.io_operations import io_handler
 from simtools.production_configuration.generate_simulation_config import (
     SimulationConfig,
@@ -60,7 +70,8 @@ from simtools.production_configuration.generate_simulation_config import (
 def _parse(label):
     """Parse command-line arguments."""
     config = configurator.Configurator(
-        label=label, description="Configure and run a simulation based on input parameters."
+        label=label,
+        description="Derive simulation configuration parameters for a simulation production.",
     )
     config.parser.add_argument(
         "--azimuth", type=float, required=True, help="Azimuth angle in degrees."
@@ -71,12 +82,6 @@ def _parse(label):
     config.parser.add_argument(
         "--nsb", type=float, required=True, help="Night sky background in units of 1/(sr*ns*cm**2)."
     )
-    config.parser.add_argument(
-        "--ctao_data_level", type=str, required=True, help="Data level (e.g., 'A', 'B', 'C')."
-    )
-    config.parser.add_argument(
-        "--science_case", type=str, required=True, help="Science case for the simulation."
-    )
     config.parser.add_argument(
         "--file_path", type=str, required=True, help="Path to MC event file in DL2 format."
     )
@@ -125,14 +130,13 @@ def main():
         "night_sky_background": args_dict["nsb"],
     }
 
-    metrics = (
-        gen.collect_data_from_file(args_dict["metrics_file"]) if "metrics_file" in args_dict else {}
+    metrics = gen.collect_data_from_file(args_dict["metrics_file"])
+    schema.validate_dict_using_schema(
+        data=metrics, schema_file="production_configuration_metrics.schema.yml"
     )
 
     simulation_config = SimulationConfig(
         grid_point=grid_point_config,
-        ctao_data_level=args_dict["ctao_data_level"],
-        science_case=args_dict["science_case"],
         file_path=args_dict["file_path"],
         file_type=args_dict["file_type"],
         metrics=metrics,

simtools/applications/production_scale_events.py

@@ -28,7 +28,7 @@ To evaluate statistical uncertainties and perform interpolation, run the command
 
     simtools-production-scale-events --base_path tests/resources/production_dl2_fits/ \
         --zeniths 20 52 40 60 --offsets 0 --interpolate --query_point 1 180 30 0 0 \
-        --science_case 1 --metrics_file "path/to/metrics.yaml"
+        --metrics_file "path/to/metrics.yaml"
 
 
 The output will display the scaled events for the specified grid point.
@@ -93,11 +93,8 @@ def _parse(label, description):
     config.parser.add_argument(
         "--metrics_file",
         type=str,
-        default="production_simulation_config_metrics.yaml",
-        help="Metrics definition file. (default: production_simulation_config_metrics.yaml)",
-    )
-    config.parser.add_argument(
-        "--science_case", type=str, required=True, help="Science case for the simulation."
+        default="production_simulation_config_metrics.yml",
+        help="Metrics definition file. (default: production_simulation_config_metrics.yml)",
     )
     return config.initialize(db_config=False)
 
@@ -147,9 +144,7 @@ def main():
         logger.warning(f"Offsets: {args_dict['offsets']}")
 
     # Perform interpolation for the given query point
-    interpolation_handler = InterpolationHandler(
-        evaluator_instances, science_case=args_dict["science_case"], metrics=metrics
-    )
+    interpolation_handler = InterpolationHandler(evaluator_instances, metrics=metrics)
     query_points = np.array([args_dict["query_point"]])
     scaled_events = interpolation_handler.interpolate(query_points)
 

simtools/applications/run_application.py

@@ -0,0 +1,165 @@
+#!/usr/bin/python3
+
+"""
+Run simtools applications from configuration files for setting workflows.
+
+Allows to run several simtools applications with a single configuration file, which includes
+both the name of the simtools application and the configuration for the application.
+
+Strong assumption on the directory structure for input and output files of applications.
+
+"""
+
+import logging
+import subprocess
+import tempfile
+from pathlib import Path
+
+import yaml
+
+import simtools.utils.general as gen
+from simtools import dependencies
+from simtools.configuration import configurator
+
+
+def _parse(label, description, usage):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing the application.
+    description : str
+        Description of the application.
+    usage : str
+        Example on how to use the application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description, usage=usage)
+
+    config.parser.add_argument(
+        "--configuration_file",
+        help="Application configuration.",
+        type=str,
+        required=True,
+        default=None,
+    )
+    return config.initialize(db_config=False)
+
+
+def run_application(application, configuration, logger):
+    """Run a simtools application and return stdout and stderr."""
+    with tempfile.NamedTemporaryFile(mode="w", delete=True, suffix=".yml") as temp_config:
+        yaml.dump(configuration, temp_config, default_flow_style=False)
+        temp_config.flush()
+        configuration_file = Path(temp_config.name)
+        try:
+            result = subprocess.run(
+                [application, "--config", configuration_file],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+        except subprocess.CalledProcessError as exc:
+            logger.error(f"Error running application {application}: {exc.stderr}")
+            raise exc
+        return result.stdout, result.stderr
+
+
+def get_subdirectory_name(path):
+    """Get the first subdirectory name under 'input'."""
+    path = Path(path).resolve()
+    try:
+        input_index = path.parts.index("input")
+        return path.parts[input_index], path.parts[input_index + 1]
+    except (ValueError, IndexError) as exc:
+        raise ValueError(f"Could not find subdirectory under 'input': {exc}") from exc
+
+
+def read_application_configuration(configuration_file, logger):
+    """
+    Read application configuration from file and modify for setting workflows.
+
+    Strong assumptions on the structure of input and output files:
+
+    - configuration file is expected to be in './input/<workflow directory>/<yaml file>'
+    - output files will be written out to './output/<workflow directory>/'
+
+    Replaces the placeholders in the configuration file with the actual values.
+    Sets 'USE_PLAIN_OUTPUT_PATH' to True for all applications.
+
+    Parameters
+    ----------
+    configuration_file : str
+        Configuration file name.
+    logger : Logger
+        Logger object.
+
+    Returns
+    -------
+    dict
+        Application configuration.
+    Path
+        Path to the log file.
+
+    """
+    application_config = gen.collect_data_from_file(configuration_file).get("CTA_SIMPIPE")
+    place_holder = "__SETTING_WORKFLOW__"
+    workflow_dir, setting_workflow = get_subdirectory_name(configuration_file)
+    output_path = str(workflow_dir).replace("input", "output") + setting_workflow
+    logger.info(f"Setting workflow output path to {output_path}")
+    log_file = (
+        Path(application_config.get("LOG_PATH", "./").replace(place_holder, setting_workflow))
+        / "simtools.log"
+    )
+    log_file.parent.mkdir(parents=True, exist_ok=True)
+    configurations = application_config.get("APPLICATIONS")
+    for config in configurations:
+        for key, value in config.get("CONFIGURATION", {}).items():
+            if isinstance(value, str):
+                config["CONFIGURATION"][key] = value.replace(place_holder, setting_workflow)
+            if isinstance(value, list):
+                config["CONFIGURATION"][key] = [
+                    item.replace(place_holder, setting_workflow) for item in value
+                ]
+        config["CONFIGURATION"]["USE_PLAIN_OUTPUT_PATH"] = True
+        config["OUTPUT_PATH"] = output_path
+
+    return configurations, log_file
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse(
+        Path(__file__).stem,
+        description="Run simtools applications from configuration file.",
+        usage="simtools-run-application --config_file config_file_name",
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    configurations, log_file = read_application_configuration(
+        args_dict["configuration_file"], logger
+    )
+
+    with log_file.open("w", encoding="utf-8") as file:
+        file.write("Running simtools applications\n")
+        file.write(dependencies.get_version_string())
+        for config in configurations:
+            logger.info(f"Running application: {config.get('APPLICATION')}")
+            config = gen.change_dict_keys_case(config, False)
+            stdout, stderr = run_application(
+                config.get("APPLICATION"), config.get("CONFIGURATION"), logger
+            )
+            file.write("=" * 80 + "\n")
+            file.write(f"Application: {config.get('APPLICATION')}\n")
+            file.write("STDOUT:\n" + stdout)
+            file.write("STDERR:\n" + stderr)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/simulate_light_emission.py

@@ -117,7 +117,6 @@ r"""
 
 """
 
-
 import logging
 import subprocess
 from pathlib import Path
@@ -390,7 +389,6 @@ def main():
             run_script = light_source.prepare_script(generate_postscript=True, **args_dict)
             log_file = f"{light_source.output_directory}/logfile.log"
             with open(log_file, "w", encoding="utf-8") as log_file:
-
                 subprocess.run(
                     run_script,
                     shell=False,
@@ -430,7 +428,6 @@ def main():
                 )
 
     elif args_dict["light_source_setup"] == "layout":
-
         light_source = SimulatorLightEmission(
             telescope_model=telescope_model,
             calibration_model=calibration_model,
@@ -443,7 +440,6 @@ def main():
         run_script = light_source.prepare_script(generate_postscript=True, **args_dict)
         log_file = f"{light_source.output_directory}/logfile.log"
         with open(log_file, "w", encoding="utf-8") as log_file:
-
             subprocess.run(
                 run_script, shell=False, check=False, text=True, stdout=log_file, stderr=log_file
             )

simtools/applications/simulate_prod.py

@@ -31,7 +31,7 @@ r"""
         Zenith angle in degrees.
     nshow (int, optional)
         Number of showers to simulate.
-        The Number of simulated events depends on the number of times a shower is re-used in the
+        The Number of simulated events depends on the number of times a shower is reused in the
         telescope simulation. The number provided here is before any reuse factors.
     start_run (int, required)
         Start run number such that the actual run number will be 'start_run' + 'run'.

simtools/applications/simulate_prod_htcondor_generator.py

@@ -1,40 +1,40 @@
 #!/usr/bin/python3
 
 r"""
-    Generate a run script and submit file for HT Condor job submission of a simulation production.
+Generate a run script and submit file for HT Condor job submission of a simulation production.
 
-    This tool facilitates the submission of multiple simulations to the HT Condor batch system,
-    enabling:
+This tool facilitates the submission of multiple simulations to the HT Condor batch system,
+enabling:
 
-    - Execution of simulations using the "simtools-simulate-prod" application.
-    - 'number_of_runs' jobs are submitted to the HT Condor batch system.
-    - Utilization of an Apptainer image containing the SimPipe simulation software and tools.
-    - Packaging of data and histogram files, and writing them to a specified directory.
+- Execution of simulations using the "simtools-simulate-prod" application.
+- 'number_of_runs' jobs are submitted to the HT Condor batch system.
+- Utilization of an Apptainer image containing the SimPipe simulation software and tools.
+- Packaging of data and histogram files, and writing them to a specified directory.
 
-    This tool is intended for use in an HT Condor environment. Jobs run in a container universe
-    using the Apptainer image specified in the command line ('--apptainer_image'). Output is written
-    to the 'output_path' directory, with 'simtools-output' and 'logs' subdirectories.
+This tool is intended for use in an HT Condor environment. Jobs run in a container universe
+using the Apptainer image specified in the command line ('--apptainer_image'). Output is written
+to the 'output_path' directory, with 'simtools-output' and 'logs' subdirectories.
 
-    Requirements for the 'simtools-simulate-prod-htcondor-generator' application:
+Requirements for the 'simtools-simulate-prod-htcondor-generator' application:
 
-    - Availability of an Apptainer image 'simtools-prod' (obtainable from the package registry on
-      GitHub, e.g., via 'apptainer pull --force docker://ghcr.io/gammasim/simtools-prod:latest').
-    - Environment parameters required to run CORSIKA and sim_telarray, as well as DB access
-      credentials.  These should be listed similarly to a '.env' file and copied to
-      'output_path/env.txt'.  Ensure that the path to the simulation software is correctly set to
-      'SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray'.
+- Availability of an Apptainer image 'simtools-prod' (obtainable from the package registry on
+  GitHub, e.g., via 'apptainer pull --force docker://ghcr.io/gammasim/simtools-prod:latest').
+- Environment parameters required to run CORSIKA and sim_telarray, as well as DB access
+  credentials.  These should be listed similarly to a '.env' file and copied to
+  'output_path/env.txt'.  Ensure that the path to the simulation software is correctly set to
+  'SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray'.
 
 
-    Command line arguments
-    ----------------------
-    output_path (str, required)
-        Directory where the output and the simulation data files will be written.
-    apptainer_image (str, optional)
-        Apptainer image to use for the simulation (full path).
-    priority (int, optional)
-        Job priority (default: 1).
+Command line arguments
+----------------------
+output_path (str, required)
+    Directory where the output and the simulation data files will be written.
+apptainer_image (str, optional)
+    Apptainer image to use for the simulation (full path).
+priority (int, optional)
+    Job priority (default: 1).
 
-    (all other command line arguments are identical to those of :ref:`simulate_prod`).
+(all other command line arguments are identical to those of :ref:`simulate_prod`).
 
 """
 

simtools/applications/submit_data_from_external.py

@@ -24,7 +24,7 @@ r"""
         simtools-submit-data-from-external \\
             --input_meta ./tests/resources/MLTdata-preproduction.meta.yml \\
             --input ./tests/resources/MLTdata-preproduction.ecsv \\
-            --schema ./tests/resources/schema_MST_mirror_2f_measurements.yml \\
+            --schema src/simtools/schemas/input/MST_mirror_2f_measurements.schema.yml \\
             --output_file TEST-submit_data_from_external.ecsv
 
     Expected final print-out message:
@@ -83,6 +83,12 @@ def _parse(label, description):
         type=str,
         required=False,
     )
+    config.parser.add_argument(
+        "--ignore_metadata",
+        help="Ignore metadata",
+        action="store_true",
+        required=False,
+    )
     return config.initialize(output=True)
 
 
@@ -95,16 +101,18 @@ def main():  # noqa: D103
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    _metadata = MetadataCollector(args_dict=args_dict)
+    _metadata = None if args_dict.get("ignore_metadata") else MetadataCollector(args_dict)
 
     data_validator = validate_data.DataValidator(
-        schema_file=_metadata.get_data_model_schema_file_name(),
+        schema_file=(
+            _metadata.get_data_model_schema_file_name() if _metadata else args_dict.get("schema")
+        ),
         data_file=args_dict["input"],
     )
 
     writer.ModelDataWriter.dump(
         args_dict=args_dict,
-        metadata=_metadata.get_top_level_metadata(),
+        metadata=_metadata.get_top_level_metadata() if _metadata else None,
         product_data=data_validator.validate_and_transform(),
     )
 

simtools/applications/submit_model_parameter_from_external.py

@@ -16,8 +16,8 @@ r"""
         instrument name.
     site (str)
         site location.
-    model_version (str)
-        Model version.
+    parameter_version (str)
+        Parameter version.
     input_meta (str, optional)
         input meta data file (yml format)
 
@@ -33,7 +33,7 @@ r"""
             --value 2 \\
             --instrument LSTN-design \\
             --site North \\
-            --model_version 6.0.0 \\
+            --parameter_version 0.1.0 \\
             --input_meta num_gains.metadata.yml
 
 """
@@ -70,8 +70,9 @@ def _parse(label, description):
     )
     config.parser.add_argument("--instrument", type=str, required=True, help="Instrument name")
     config.parser.add_argument("--site", type=str, required=True, help="Site location")
-    config.parser.add_argument("--model_version", type=str, required=True, help="Model version")
-
+    config.parser.add_argument(
+        "--parameter_version", type=str, required=True, help="Parameter version"
+    )
     config.parser.add_argument(
         "--value",
         type=str,
@@ -82,18 +83,22 @@ def _parse(label, description):
             'Examples: "--value=5", "--value=\'5 km\'", "--value=\'5 cm, 0.5 deg\'"'
         ),
     )
-
     config.parser.add_argument(
         "--input_meta",
         help="meta data file associated to input data",
         type=str,
         required=False,
     )
-    return config.initialize(output=True)
+    config.parser.add_argument(
+        "--check_parameter_version",
+        help="Check if the parameter version exists in the database",
+        action="store_true",
+    )
+    return config.initialize(output=True, db_config=True)
 
 
 def main():  # noqa: D103
-    args_dict, _ = _parse(
+    args_dict, db_config = _parse(
         label=Path(__file__).stem,
         description="Submit and validate a model parameters).",
     )
@@ -102,19 +107,21 @@ def main():  # noqa: D103
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     output_path = (
-        Path(args_dict["output_path"]) / args_dict["model_version"] / args_dict["instrument"]
+        Path(args_dict["output_path"]) / args_dict["instrument"] / args_dict["parameter"]
         if args_dict.get("output_path")
         else None
     )
+
     writer.ModelDataWriter.dump_model_parameter(
         parameter_name=args_dict["parameter"],
         value=args_dict["value"],
         instrument=args_dict["instrument"],
-        model_version=args_dict["model_version"],
-        output_file=Path(args_dict["parameter"]).with_suffix(".json"),
+        parameter_version=args_dict["parameter_version"],
+        output_file=Path(args_dict["parameter"] + "-" + args_dict["parameter_version"] + ".json"),
         output_path=output_path,
         use_plain_output_path=args_dict.get("use_plain_output_path"),
         metadata_input_dict=args_dict,
+        db_config=db_config if args_dict.get("check_parameter_version") else None,
     )
 
 

simtools/applications/validate_camera_efficiency.py

@@ -47,7 +47,7 @@ import logging
 from pathlib import Path
 
 import simtools.utils.general as gen
-from simtools.camera_efficiency import CameraEfficiency
+from simtools.camera.camera_efficiency import CameraEfficiency
 from simtools.configuration import configurator
 
 
@@ -106,7 +106,7 @@ def main():  # noqa: D103
     ce = CameraEfficiency(
         db_config=_db_config,
         simtel_path=args_dict["simtel_path"],
-        label=label,
+        label=args_dict.get("label", label),
         config_data=args_dict,
     )
     ce.simulate()