gammasimtools 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/applications/generate_corsika_histograms.py

@@ -152,7 +152,7 @@ r"""
      .. code-block:: console
 
         simtools-generate-corsika-histograms --iact_file /workdir/external/simtools/\\
-        testsresources/tel_output_10GeV-2-gamma-20deg-CTAO-South.corsikaio \\
+        tests/resources/tel_output_10GeV-2-gamma-20deg-CTAO-South.corsikaio \\
             --pdf --hdf5 \\
             --event_2d_histograms zenith azimuth --event_1d_histograms total_energy
 
@@ -208,7 +208,7 @@ def _parse(label, description):
 
     config.parser.add_argument(
         "--telescope_indices",
-        help="Name of the CORSIKA IACT file from which to generate the histograms.",
+        help="List of telescope indices to be considered in the generation of the histograms",
         type=str,
         required=False,
         nargs="+",

simtools/applications/generate_regular_arrays.py

@@ -77,7 +77,9 @@ def main():
         # Single telescope at the center
         if array_name[0] == "1":
             tel_name.append(
-                names.get_array_element_name_from_type_site_id(tel_size, args_dict["site"], "01")
+                names.generate_array_element_name_from_type_site_id(
+                    tel_size, args_dict["site"], "01"
+                )
             )
             pos_x.append(0 * u.m)
             pos_y.append(0 * u.m)
@@ -86,7 +88,7 @@ def main():
         else:
             for i in range(1, 5):
                 tel_name.append(
-                    names.get_array_element_name_from_type_site_id(
+                    names.generate_array_element_name_from_type_site_id(
                         tel_size, args_dict["site"], f"0{i}"
                     )
                 )

simtools/applications/production_derive_limits.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python3
+
+r"""
+Derives the limits for energy, radial distance, and viewcone to be used in CORSIKA simulations.
+
+The limits are derived based on the event loss fraction specified by the user.
+
+Command line arguments
+----------------------
+event_data_file (str, required)
+    Path to the file containing the event data.
+loss_fraction (float, required)
+    Fraction of events to be lost.
+
+
+Example
+-------
+Derive limits for a given file with a specified loss fraction.
+
+.. code-block:: console
+
+    simtools-production-derive-limits\\
+        --event_data_file path/to/event_data_file.hdf5 \\
+        --loss_fraction 1e-6
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.io_operations.hdf5_handler import read_hdf5
+from simtools.production_configuration.limits_calculation import LimitCalculator
+
+_logger = logging.getLogger(__name__)
+
+
+def _parse():
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    event_data_file: str
+        The event data file.
+    loss_fraction: float
+        Loss fraction of events for limit derivation.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object
+
+    """
+    config = configurator.Configurator(
+        description="Derive limits for energy, radial distance, and viewcone."
+    )
+    config.parser.add_argument(
+        "--event_data_file",
+        type=str,
+        required=True,
+        help="Path to the event data file containing the event data.",
+    )
+    config.parser.add_argument(
+        "--loss_fraction", type=float, required=True, help="Fraction of events to be lost."
+    )
+    return config.initialize(db_config=False)
+
+
+def main():
+    """Derive limits for energy, radial distance, and viewcone."""
+    args_dict, _ = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    event_data_file_path = args_dict["event_data_file"]
+    loss_fraction = args_dict["loss_fraction"]
+
+    _logger.info(f"Loading event data file: {event_data_file_path}")
+    tables = read_hdf5(event_data_file_path)
+
+    calculator = LimitCalculator(tables)
+
+    lower_energy_limit = calculator.compute_lower_energy_limit(loss_fraction)
+    _logger.info(f"Lower energy threshold: {lower_energy_limit}")
+
+    upper_radial_distance = calculator.compute_upper_radial_distance(loss_fraction)
+    _logger.info(f"Upper radius threshold: {upper_radial_distance}")
+
+    viewcone = calculator.compute_viewcone(loss_fraction)
+    _logger.info(f"Viewcone radius: {viewcone}")
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/production_generate_simulation_config.py

@@ -18,28 +18,25 @@ The configuration parameters are derived according to the required precision. Th
 * statistical uncertainty on the determination of the effective area as function of primary energy
 * fraction of lost events to the selected core scatter and view cone radius (to be implemented)
 * statistical uncertainty of the energy migration matrix as function of primary energy
-  (to be implemented)
+    (to be implemented)
 
 Command line arguments
 ----------------------
 azimuth (float, required)
-    Azimuth angle in degrees.
+        Azimuth angle in degrees.
 elevation (float, required)
-    Elevation angle in degrees.
+        Elevation angle in degrees.
 nsb (float, required)
-    Night sky background value.
-ctao_data_level (str, required)
-    The data level for the simulation (e.g., 'A', 'B', 'C').
-science_case (str, required)
-    The science case for the simulation.
+        Night sky background value.
 file_path (str, required)
-    Path to file with MC events at CTAO DL2 data level. Used for statistical uncertainty evaluation.
+        Path to file with MC events at CTAO DL2 data level.
+        Used for statistical uncertainty evaluation.
 file_type (str, required)
-    Type of the DL2 MC event file ('point-like' or 'cone').
-metrics (str, optional)
-    Path to a YAML file containing metrics for evaluation.
+        Type of the DL2 MC event file ('point-like' or 'cone').
+metrics (str, required)
+        Path to a YAML file containing metrics for evaluation.
 site (str, required)
-    The observatory site (North or South).
+        The observatory site (North or South).
 
 Example
 -------
@@ -47,11 +44,10 @@ To run the simulation configuration, execute the script as follows:
 
 .. code-block:: console
 
-    simtools-production-generate-simulation-config --azimuth 60.0 --elevation 45.0 \
-        --nsb 0.3 --ctao_data_level "A" --science_case "high_precision" \
-        --file_path tests/resources/production_dl2_fits/dl2_mc_events_file.fits \
-        --file_type "point-like"    \
-        --metrics_file tests/resources/production_simulation_config_metrics.yml --site North
+        simtools-production-generate-simulation-config --azimuth 60.0 --elevation 45.0 \
+                --nsb 0.3 --file_path tests/resources/production_dl2_fits/dl2_mc_events_file.fits \
+                --file_type "point-like"    \
+                --metrics_file tests/resources/production_simulation_config_metrics.yml --site North
 
 The output will show the derived simulation parameters.
 """
@@ -86,12 +82,6 @@ def _parse(label):
     config.parser.add_argument(
         "--nsb", type=float, required=True, help="Night sky background in units of 1/(sr*ns*cm**2)."
     )
-    config.parser.add_argument(
-        "--ctao_data_level", type=str, required=True, help="Data level (e.g., 'A', 'B', 'C')."
-    )
-    config.parser.add_argument(
-        "--science_case", type=str, required=True, help="Science case for the simulation."
-    )
     config.parser.add_argument(
         "--file_path", type=str, required=True, help="Path to MC event file in DL2 format."
     )
@@ -140,17 +130,13 @@ def main():
         "night_sky_background": args_dict["nsb"],
     }
 
-    metrics = (
-        gen.collect_data_from_file(args_dict["metrics_file"]) if "metrics_file" in args_dict else {}
-    )
+    metrics = gen.collect_data_from_file(args_dict["metrics_file"])
     schema.validate_dict_using_schema(
         data=metrics, schema_file="production_configuration_metrics.schema.yml"
     )
 
     simulation_config = SimulationConfig(
         grid_point=grid_point_config,
-        ctao_data_level=args_dict["ctao_data_level"],
-        science_case=args_dict["science_case"],
         file_path=args_dict["file_path"],
         file_type=args_dict["file_type"],
         metrics=metrics,

simtools/applications/production_scale_events.py

@@ -28,7 +28,7 @@ To evaluate statistical uncertainties and perform interpolation, run the command
 
     simtools-production-scale-events --base_path tests/resources/production_dl2_fits/ \
         --zeniths 20 52 40 60 --offsets 0 --interpolate --query_point 1 180 30 0 0 \
-        --science_case 1 --metrics_file "path/to/metrics.yaml"
+        --metrics_file "path/to/metrics.yaml"
 
 
 The output will display the scaled events for the specified grid point.
@@ -96,9 +96,6 @@ def _parse(label, description):
         default="production_simulation_config_metrics.yml",
         help="Metrics definition file. (default: production_simulation_config_metrics.yml)",
     )
-    config.parser.add_argument(
-        "--science_case", type=str, required=True, help="Science case for the simulation."
-    )
     return config.initialize(db_config=False)
 
 
@@ -147,9 +144,7 @@ def main():
         logger.warning(f"Offsets: {args_dict['offsets']}")
 
     # Perform interpolation for the given query point
-    interpolation_handler = InterpolationHandler(
-        evaluator_instances, science_case=args_dict["science_case"], metrics=metrics
-    )
+    interpolation_handler = InterpolationHandler(evaluator_instances, metrics=metrics)
     query_points = np.array([args_dict["query_point"]])
     scaled_events = interpolation_handler.interpolate(query_points)
 

simtools/applications/run_application.py

@@ -0,0 +1,165 @@
+#!/usr/bin/python3
+
+"""
+Run simtools applications from configuration files for setting workflows.
+
+Allows to run several simtools applications with a single configuration file, which includes
+both the name of the simtools application and the configuration for the application.
+
+Strong assumption on the directory structure for input and output files of applications.
+
+"""
+
+import logging
+import subprocess
+import tempfile
+from pathlib import Path
+
+import yaml
+
+import simtools.utils.general as gen
+from simtools import dependencies
+from simtools.configuration import configurator
+
+
+def _parse(label, description, usage):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing the application.
+    description : str
+        Description of the application.
+    usage : str
+        Example on how to use the application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description, usage=usage)
+
+    config.parser.add_argument(
+        "--configuration_file",
+        help="Application configuration.",
+        type=str,
+        required=True,
+        default=None,
+    )
+    return config.initialize(db_config=False)
+
+
+def run_application(application, configuration, logger):
+    """Run a simtools application and return stdout and stderr."""
+    with tempfile.NamedTemporaryFile(mode="w", delete=True, suffix=".yml") as temp_config:
+        yaml.dump(configuration, temp_config, default_flow_style=False)
+        temp_config.flush()
+        configuration_file = Path(temp_config.name)
+        try:
+            result = subprocess.run(
+                [application, "--config", configuration_file],
+                check=True,
+                capture_output=True,
+                text=True,
+            )
+        except subprocess.CalledProcessError as exc:
+            logger.error(f"Error running application {application}: {exc.stderr}")
+            raise exc
+        return result.stdout, result.stderr
+
+
+def get_subdirectory_name(path):
+    """Get the first subdirectory name under 'input'."""
+    path = Path(path).resolve()
+    try:
+        input_index = path.parts.index("input")
+        return path.parts[input_index], path.parts[input_index + 1]
+    except (ValueError, IndexError) as exc:
+        raise ValueError(f"Could not find subdirectory under 'input': {exc}") from exc
+
+
+def read_application_configuration(configuration_file, logger):
+    """
+    Read application configuration from file and modify for setting workflows.
+
+    Strong assumptions on the structure of input and output files:
+
+    - configuration file is expected to be in './input/<workflow directory>/<yaml file>'
+    - output files will be written out to './output/<workflow directory>/'
+
+    Replaces the placeholders in the configuration file with the actual values.
+    Sets 'USE_PLAIN_OUTPUT_PATH' to True for all applications.
+
+    Parameters
+    ----------
+    configuration_file : str
+        Configuration file name.
+    logger : Logger
+        Logger object.
+
+    Returns
+    -------
+    dict
+        Application configuration.
+    Path
+        Path to the log file.
+
+    """
+    application_config = gen.collect_data_from_file(configuration_file).get("CTA_SIMPIPE")
+    place_holder = "__SETTING_WORKFLOW__"
+    workflow_dir, setting_workflow = get_subdirectory_name(configuration_file)
+    output_path = str(workflow_dir).replace("input", "output") + setting_workflow
+    logger.info(f"Setting workflow output path to {output_path}")
+    log_file = (
+        Path(application_config.get("LOG_PATH", "./").replace(place_holder, setting_workflow))
+        / "simtools.log"
+    )
+    log_file.parent.mkdir(parents=True, exist_ok=True)
+    configurations = application_config.get("APPLICATIONS")
+    for config in configurations:
+        for key, value in config.get("CONFIGURATION", {}).items():
+            if isinstance(value, str):
+                config["CONFIGURATION"][key] = value.replace(place_holder, setting_workflow)
+            if isinstance(value, list):
+                config["CONFIGURATION"][key] = [
+                    item.replace(place_holder, setting_workflow) for item in value
+                ]
+        config["CONFIGURATION"]["USE_PLAIN_OUTPUT_PATH"] = True
+        config["OUTPUT_PATH"] = output_path
+
+    return configurations, log_file
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse(
+        Path(__file__).stem,
+        description="Run simtools applications from configuration file.",
+        usage="simtools-run-application --config_file config_file_name",
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    configurations, log_file = read_application_configuration(
+        args_dict["configuration_file"], logger
+    )
+
+    with log_file.open("w", encoding="utf-8") as file:
+        file.write("Running simtools applications\n")
+        file.write(dependencies.get_version_string())
+        for config in configurations:
+            logger.info(f"Running application: {config.get('APPLICATION')}")
+            config = gen.change_dict_keys_case(config, False)
+            stdout, stderr = run_application(
+                config.get("APPLICATION"), config.get("CONFIGURATION"), logger
+            )
+            file.write("=" * 80 + "\n")
+            file.write(f"Application: {config.get('APPLICATION')}\n")
+            file.write("STDOUT:\n" + stdout)
+            file.write("STDERR:\n" + stderr)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/simulate_light_emission.py

@@ -117,7 +117,6 @@ r"""
 
 """
 
-
 import logging
 import subprocess
 from pathlib import Path
@@ -390,7 +389,6 @@ def main():
             run_script = light_source.prepare_script(generate_postscript=True, **args_dict)
             log_file = f"{light_source.output_directory}/logfile.log"
             with open(log_file, "w", encoding="utf-8") as log_file:
-
                 subprocess.run(
                     run_script,
                     shell=False,
@@ -430,7 +428,6 @@ def main():
                 )
 
     elif args_dict["light_source_setup"] == "layout":
-
         light_source = SimulatorLightEmission(
             telescope_model=telescope_model,
             calibration_model=calibration_model,
@@ -443,7 +440,6 @@ def main():
         run_script = light_source.prepare_script(generate_postscript=True, **args_dict)
         log_file = f"{light_source.output_directory}/logfile.log"
         with open(log_file, "w", encoding="utf-8") as log_file:
-
             subprocess.run(
                 run_script, shell=False, check=False, text=True, stdout=log_file, stderr=log_file
             )

simtools/applications/submit_model_parameter_from_external.py

@@ -73,7 +73,6 @@ def _parse(label, description):
     config.parser.add_argument(
         "--parameter_version", type=str, required=True, help="Parameter version"
     )
-
     config.parser.add_argument(
         "--value",
         type=str,
@@ -84,18 +83,22 @@ def _parse(label, description):
             'Examples: "--value=5", "--value=\'5 km\'", "--value=\'5 cm, 0.5 deg\'"'
         ),
     )
-
     config.parser.add_argument(
         "--input_meta",
         help="meta data file associated to input data",
         type=str,
         required=False,
     )
-    return config.initialize(output=True)
+    config.parser.add_argument(
+        "--check_parameter_version",
+        help="Check if the parameter version exists in the database",
+        action="store_true",
+    )
+    return config.initialize(output=True, db_config=True)
 
 
 def main():  # noqa: D103
-    args_dict, _ = _parse(
+    args_dict, db_config = _parse(
         label=Path(__file__).stem,
         description="Submit and validate a model parameters).",
     )
@@ -104,19 +107,21 @@ def main():  # noqa: D103
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     output_path = (
-        Path(args_dict["output_path"]) / args_dict["parameter_version"] / args_dict["instrument"]
+        Path(args_dict["output_path"]) / args_dict["instrument"] / args_dict["parameter"]
         if args_dict.get("output_path")
         else None
     )
+
     writer.ModelDataWriter.dump_model_parameter(
         parameter_name=args_dict["parameter"],
         value=args_dict["value"],
         instrument=args_dict["instrument"],
         parameter_version=args_dict["parameter_version"],
-        output_file=Path(args_dict["parameter"]).with_suffix(".json"),
+        output_file=Path(args_dict["parameter"] + "-" + args_dict["parameter_version"] + ".json"),
         output_path=output_path,
         use_plain_output_path=args_dict.get("use_plain_output_path"),
         metadata_input_dict=args_dict,
+        db_config=db_config if args_dict.get("check_parameter_version") else None,
     )
 
 

simtools/applications/validate_file_using_schema.py

@@ -170,9 +170,9 @@ def validate_data_file(args_dict, logger):
         data_file=args_dict["file_name"],
         check_exact_data_type=args_dict["require_exact_data_type"],
     )
-    data_validator.validate_and_transform(is_model_parameter=True)
-    if args_dict["data_type"].lower() == "model_parameter":
-        data_validator.validate_parameter_and_file_name()
+    data_validator.validate_and_transform(
+        is_model_parameter=(args_dict["data_type"].lower() == "model_parameter")
+    )
 
     logger.info(f"Successful validation of data file {args_dict['file_name']}")
 

simtools/configuration/commandline_parser.py

@@ -70,6 +70,7 @@ class CommandLineParser(argparse.ArgumentParser):
             self.initialize_output_arguments()
         self.initialize_config_files()
         self.initialize_application_execution_arguments()
+        self.initialize_user_arguments()
 
     def initialize_config_files(self):
         """Initialize configuration files."""
@@ -175,6 +176,34 @@ class CommandLineParser(argparse.ArgumentParser):
             "--version", action="version", version=f"%(prog)s {simtools.version.__version__}"
         )
 
+    def initialize_user_arguments(self):
+        """Initialize user arguments."""
+        _job_group = self.add_argument_group("user")
+        _job_group.add_argument(
+            "--user_name",
+            help="user name",
+            type=str,
+            required=False,
+        )
+        _job_group.add_argument(
+            "--user_organization",
+            help="user organization",
+            type=str,
+            required=False,
+        )
+        _job_group.add_argument(
+            "--user_email",
+            help="user email",
+            type=str,
+            required=False,
+        )
+        _job_group.add_argument(
+            "--user_orcid",
+            help="user ORCID",
+            type=str,
+            required=False,
+        )
+
     def initialize_db_config_arguments(self):
         """Initialize DB configuration parameters."""
         _job_group = self.add_argument_group("database configuration")

simtools/configuration/configurator.py

@@ -287,16 +287,14 @@ class Configurator:
             )
             # yaml parser adds \n in multiline strings, remove them
             _config_dict = gen.remove_substring_recursively_from_dict(_config_dict, substring="\n")
-            if "CTA_SIMPIPE" in _config_dict:
-                try:
-                    self._fill_from_config_dict(
-                        input_dict=gen.change_dict_keys_case(
-                            _config_dict["CTA_SIMPIPE"]["CONFIGURATION"],
-                        ),
-                        overwrite=True,
-                    )
-                except KeyError:
-                    self._logger.info(f"No CTA_SIMPIPE:CONFIGURATION dict found in {config_file}.")
+            # read configuration for first application
+            if "CONFIGURATION" in _config_dict.get("CTA_SIMPIPE", {}).get("APPLICATIONS", [{}])[0]:
+                self._fill_from_config_dict(
+                    input_dict=gen.change_dict_keys_case(
+                        _config_dict["CTA_SIMPIPE"]["APPLICATIONS"][0]["CONFIGURATION"],
+                    ),
+                    overwrite=True,
+                )
             else:
                 self._fill_from_config_dict(
                     input_dict=gen.change_dict_keys_case(_config_dict), overwrite=True

simtools/corsika/corsika_config.py

@@ -232,25 +232,25 @@ class CorsikaConfig:
 
     def _input_config_first_interaction_height(self, entry):
         """Return FIXHEI parameter CORSIKA format."""
-        return [f"{entry['value']*u.Unit(entry['unit']).to('cm'):.2f}", "0"]
+        return [f"{entry['value'] * u.Unit(entry['unit']).to('cm'):.2f}", "0"]
 
     def _input_config_corsika_starting_grammage(self, entry):
         """Return FIXCHI parameter CORSIKA format."""
-        return f"{entry['value']*u.Unit(entry['unit']).to('g/cm2')}"
+        return f"{entry['value'] * u.Unit(entry['unit']).to('g/cm2')}"
 
     def _input_config_corsika_particle_kinetic_energy_cutoff(self, entry):
         """Return ECUTS parameter CORSIKA format."""
         e_cuts = entry["value"]
         return [
-            f"{e_cuts[0]*u.Unit(entry['unit']).to('GeV')} "
-            f"{e_cuts[1]*u.Unit(entry['unit']).to('GeV')} "
-            f"{e_cuts[2]*u.Unit(entry['unit']).to('GeV')} "
-            f"{e_cuts[3]*u.Unit(entry['unit']).to('GeV')}"
+            f"{e_cuts[0] * u.Unit(entry['unit']).to('GeV')} "
+            f"{e_cuts[1] * u.Unit(entry['unit']).to('GeV')} "
+            f"{e_cuts[2] * u.Unit(entry['unit']).to('GeV')} "
+            f"{e_cuts[3] * u.Unit(entry['unit']).to('GeV')}"
         ]
 
     def _input_config_corsika_longitudinal_parameters(self, entry):
         """Return LONGI parameter CORSIKA format."""
-        return ["T", f"{entry['value']*u.Unit(entry['unit']).to('g/cm2')}", "F", "F"]
+        return ["T", f"{entry['value'] * u.Unit(entry['unit']).to('g/cm2')}", "F", "F"]
 
     def _corsika_configuration_cherenkov_parameters(self, parameters_from_db):
         """
@@ -279,8 +279,8 @@ class CorsikaConfig:
         """Return CWAVLG parameter CORSIKA format."""
         wavelength_range = entry["value"]
         return [
-            f"{wavelength_range[0]*u.Unit(entry['unit']).to('nm')}",
-            f"{wavelength_range[1]*u.Unit(entry['unit']).to('nm')}",
+            f"{wavelength_range[0] * u.Unit(entry['unit']).to('nm')}",
+            f"{wavelength_range[1] * u.Unit(entry['unit']).to('nm')}",
         ]
 
     def _corsika_configuration_iact_parameters(self, parameters_from_db):
@@ -311,7 +311,8 @@ class CorsikaConfig:
         return {
             "DEBUG": ["F", 6, "F", 1000000],
             "DATBAS": ["yes"],
-            "DIRECT": ["/dev/null"],
+            "DIRECT": ["./"],
+            "PAROUT": ["F", "F"],
         }
 
     def _input_config_io_buff(self, entry):