gammasimtools 0.15.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/applications/simulate_prod.py

@@ -115,6 +115,16 @@ def _parse(description=None):
         required=False,
         default=False,
     )
+    config.parser.add_argument(
+        "--save_reduced_event_lists",
+        help=(
+            "Save reduced event lists with event data on simulated and triggered events. "
+            "Saved with the same name as the sim_telarray output file (different extension). "
+        ),
+        action="store_true",
+        required=False,
+        default=False,
+    )
     config.parser.add_argument(
         "--corsika_test_seeds",
         help="Use predefined random seeds for CORSIKA for testing purposes.",
@@ -123,17 +133,36 @@ def _parse(description=None):
         default=False,
     )
     config.parser.add_argument(
-        "--sim_telarray_seeds",
+        "--sequential",
+        help=(
+            "Enables single-core mode (as far as possible); "
+            "otherwise, CORSIKA and sim_telarray run in parallel."
+        ),
+        action="store_true",
+        default=False,
+    )
+    sim_telarray_seed_group = config.parser.add_argument_group(
+        title="Random seeds for sim_telarray instrument setup",
+    )
+    sim_telarray_seed_group.add_argument(
+        "--sim_telarray_instrument_seeds",
         help=(
-            "Use the provided random seed for sim_telarray instrument setup."
-            "Used only for testing purposes for now."
+            "Random seed used for sim_telarray instrument setup. "
+            "If '--sim_telarray_random_instrument_instances is not set: use as sim_telarray seed "
+            " ('random_seed' parameter). "
+            "Otherwise: use as base seed for the generation of random instrument instance seeds."
         ),
         type=str,
         required=False,
     )
+    sim_telarray_seed_group.add_argument(
+        "--sim_telarray_random_instrument_instances",
+        help="Number of random instrument instances initialized in sim_telarray.",
+        type=int,
+        required=False,
+    )
     return config.initialize(
         db_config=True,
-        job_submission=True,
         simulation_model=["site", "layout", "telescope", "model_version"],
         simulation_configuration={"software": None, "corsika_configuration": ["all"]},
     )
@@ -150,24 +179,19 @@ def main():  # noqa: D103
     )
 
     simulator.simulate()
+    simulator.validate_metadata()
 
-    if simulator.submit_engine == "local":
-        logger.info(
-            f"Production run complete for primary {args_dict['primary']} showers "
-            f"from {args_dict['azimuth_angle']} azimuth and {args_dict['zenith_angle']} zenith "
-            f"at {args_dict['site']} site, using {args_dict['model_version']} model."
-        )
-        if args_dict.get("pack_for_grid_register"):
-            simulator.pack_for_register(args_dict["pack_for_grid_register"])
-        if args_dict["save_file_lists"]:
-            simulator.save_file_lists()
-    else:
-        logger.info("Production run submitted to the workload manager")
-        if args_dict["pack_for_grid_register"] or args_dict["save_file_lists"]:
-            logger.warning(
-                "Packing for grid register or saving file lists not supported for "
-                f"{simulator.submit_engine}."
-            )
+    logger.info(
+        f"Production run complete for primary {args_dict['primary']} showers "
+        f"from {args_dict['azimuth_angle']} azimuth and {args_dict['zenith_angle']} zenith "
+        f"at {args_dict['site']} site, using {args_dict['model_version']} model."
+    )
+    if args_dict["save_reduced_event_lists"]:
+        simulator.save_reduced_event_lists()
+    if args_dict.get("pack_for_grid_register"):
+        simulator.pack_for_register(args_dict["pack_for_grid_register"])
+    if args_dict["save_file_lists"]:
+        simulator.save_file_lists()
 
 
 if __name__ == "__main__":

simtools/applications/simulate_prod_htcondor_generator.py

@@ -76,7 +76,6 @@ def _parse(description=None):
     )
     return config.initialize(
         db_config=False,
-        job_submission=False,
         simulation_model=["site", "layout", "telescope", "model_version"],
         simulation_configuration={"software": None, "corsika_configuration": ["all"]},
     )

simtools/applications/submit_array_layouts.py

@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+
+r"""
+    Submit array-layouts definition and corresponding metadata and validate list of telescopes.
+
+    Includes validation that all defined telescope exists.
+
+    Command line arguments
+    ----------------------
+    array_layouts (str, required)
+        Array layouts file.
+    updated_parameter_version (str, optional)
+        Updated parameter version.
+    input_meta (str, optional)
+        Input meta data file(s) associated to input data (wildcards or list of files allowed).
+    model_version (str, required)
+        Model version.
+
+    Example
+    -------
+
+    Submit a new array layout dictionary:
+
+    .. code-block:: console
+
+        simtools-submit-array-layouts \
+            --array_layouts array_layouts.json \\
+            --model_version 6.0.0 \\
+            --updated_parameter_version 0.1.0 \\
+            --input_meta array_layouts.metadata.yml
+
+
+    """
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler
+from simtools.layout.array_layout_utils import validate_array_layouts_with_db, write_array_layouts
+
+
+def _parse(label, description):
+    """Parse command line configuration."""
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--array_layouts",
+        type=str,
+        required=True,
+        help="Array layout dictionary file.",
+    )
+    config.parser.add_argument(
+        "--updated_parameter_version",
+        help="Updated parameter version.",
+        type=str,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--input_meta",
+        help="meta data file(s) associated to input data (wildcards or list of files allowed)",
+        type=str,
+        nargs="+",
+        required=False,
+    )
+
+    return config.initialize(output=True, db_config=True, simulation_model=["model_version"])
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse(
+        label=Path(__file__).stem,
+        description="Submit and validate array layouts.",
+    )
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    array_layouts = validate_array_layouts_with_db(
+        production_table=db.read_production_table_from_mongo_db(
+            collection_name="telescopes", model_version=args_dict["model_version"]
+        ),
+        array_layouts=gen.collect_data_from_file(args_dict["array_layouts"]),
+    )
+
+    write_array_layouts(array_layouts=array_layouts, args_dict=args_dict, db_config=db_config)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/validate_cumulative_psf.py

@@ -80,7 +80,7 @@ import numpy as np
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
-from simtools.model.telescope_model import TelescopeModel
+from simtools.model.model_utils import initialize_simulation_models
 from simtools.ray_tracing.ray_tracing import RayTracing
 from simtools.visualization import visualize
 
@@ -136,18 +136,20 @@ def main():  # noqa: D103
     _io_handler = io_handler.IOHandler()
     output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
 
-    tel_model = TelescopeModel(
+    tel_model, site_model = initialize_simulation_models(
+        label=label,
+        db_config=db_config,
         site=args_dict["site"],
         telescope_name=args_dict["telescope"],
-        mongo_db_config=db_config,
         model_version=args_dict["model_version"],
-        label=label,
     )
+
     if args_dict.get("telescope_model_file"):
         tel_model.change_multiple_parameters_from_file(args_dict["telescope_model_file"])
 
     ray = RayTracing(
         telescope_model=tel_model,
+        site_model=site_model,
         simtel_path=args_dict["simtel_path"],
         zenith_angle=args_dict["zenith"] * u.deg,
         source_distance=args_dict["src_distance"] * u.km,

simtools/applications/validate_file_using_schema.py

@@ -131,6 +131,7 @@ def validate_dict_using_schema(args_dict, logger):
     """
     Validate a schema file (or several files) given in yaml or json format.
 
+    This function validate all documents in a multi-document YAML file.
     Schema is either given as command line argument, read from the meta_schema_url or from
     the metadata section of the data dictionary.
 
@@ -141,9 +142,12 @@ def validate_dict_using_schema(args_dict, logger):
         try:
             data = gen.collect_data_from_file(file_name=file_name)
         except FileNotFoundError as exc:
-            logger.error(f"Error reading schema file from {file_name}")
-            raise exc
-        schema.validate_dict_using_schema(data, _get_schema_file_name(args_dict, data))
+            raise FileNotFoundError(f"Error reading schema file from {file_name}") from exc
+        data = data if isinstance(data, list) else [data]
+        for data_dict in data:
+            schema.validate_dict_using_schema(
+                data_dict, _get_schema_file_name(args_dict, data_dict)
+            )
         logger.info(f"Successful validation of file {file_name}")
 
 

simtools/applications/validate_optics.py

@@ -76,7 +76,7 @@ from matplotlib.backends.backend_pdf import PdfPages
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
-from simtools.model.telescope_model import TelescopeModel
+from simtools.model.model_utils import initialize_simulation_models
 from simtools.ray_tracing.ray_tracing import RayTracing
 from simtools.visualization import visualize
 
@@ -128,12 +128,12 @@ def main():  # noqa: D103
     _io_handler = io_handler.IOHandler()
     output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
 
-    tel_model = TelescopeModel(
+    tel_model, site_model = initialize_simulation_models(
+        label=label,
+        db_config=db_config,
         site=args_dict["site"],
         telescope_name=args_dict["telescope"],
         model_version=args_dict["model_version"],
-        label=label,
-        mongo_db_config=db_config,
     )
 
     ######################################################################
@@ -151,6 +151,7 @@ def main():  # noqa: D103
 
     ray = RayTracing(
         telescope_model=tel_model,
+        site_model=site_model,
         simtel_path=args_dict["simtel_path"],
         zenith_angle=args_dict["zenith"] * u.deg,
         source_distance=args_dict["src_distance"] * u.km,

simtools/applications/verify_simulation_model_production_tables.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python3
+
+"""
+Verify simulation model production tables and model parameters for completeness.
+
+This application is a utility to be used in the CI pipeline of the SimulationModels
+repository. It checks that all model parameters defined in the production tables
+exist in the simulation models repository.
+
+"""
+
+import logging
+
+from simtools.configuration import configurator
+from simtools.model import model_repository
+from simtools.utils import general as gen
+
+
+def _parse():
+    """Parse command line arguments."""
+    config = configurator.Configurator(
+        description=(
+            "Verify simulation model production tables and model parameters for completeness. "
+            "This application checks that all model parameters defined in the production tables "
+            "exist in the simulation models repository."
+        )
+    )
+    config.parser.add_argument(
+        "--simulation_models_path",
+        help="Path to the simulation models repository.",
+        type=str,
+        required=True,
+    )
+    return config.initialize(db_config=False, output=False, paths=False)
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    if not model_repository.verify_simulation_model_production_tables(
+        simulation_models_path=args_dict["simulation_models_path"]
+    ):
+        raise RuntimeError(
+            "Verification failed: Some model parameters are missing in the repository."
+        )
+
+
+if __name__ == "__main__":
+    main()

simtools/camera/camera_efficiency.py

@@ -9,8 +9,7 @@ import numpy as np
 from astropy.table import Table
 
 from simtools.io_operations import io_handler
-from simtools.model.site_model import SiteModel
-from simtools.model.telescope_model import TelescopeModel
+from simtools.model.model_utils import initialize_simulation_models
 from simtools.simtel.simulator_camera_efficiency import SimulatorCameraEfficiency
 from simtools.utils import names
 from simtools.visualization import visualize
@@ -52,8 +51,12 @@ class CameraEfficiency:
         self.test = test
 
         self.io_handler = io_handler.IOHandler()
-        self.telescope_model, self._site_model = self._initialize_simulation_models(
-            config_data, db_config
+        self.telescope_model, self.site_model = initialize_simulation_models(
+            self.label,
+            db_config,
+            config_data["site"],
+            config_data["telescope"],
+            config_data["model_version"],
         )
         self.output_dir = self.io_handler.get_output_directory(self.label, sub_dir="plots")
 
@@ -67,34 +70,6 @@ class CameraEfficiency:
         """Return string representation of the CameraEfficiency instance."""
         return f"CameraEfficiency(label={self.label})\n"
 
-    def _initialize_simulation_models(self, config_data, db_config):
-        """
-        Initialize site and telescope models.
-
-        Parameters
-        ----------
-        config_data: dict
-            Dict containing the configurable parameters.
-
-        Returns
-        -------
-        tuple
-            Tuple containing the site and telescope models.
-        """
-        tel_model = TelescopeModel(
-            site=config_data["site"],
-            telescope_name=config_data["telescope"],
-            mongo_db_config=db_config,
-            model_version=config_data["model_version"],
-            label=self.label,
-        )
-        site_model = SiteModel(
-            site=config_data["site"],
-            model_version=config_data["model_version"],
-            mongo_db_config=db_config,
-        )
-        return tel_model, site_model
-
     def _configuration_from_args_dict(self, config_data):
         """
         Extract the configuration data from the args_dict.
@@ -131,15 +106,15 @@ class CameraEfficiency:
         }
 
     def _load_files(self):
-        """Define the variables for the file names, including the results, simtel and log file."""
+        """Define variables used for file names, including results, sim_telarray and log files."""
         _file = {}
         for label, suffix in zip(
-            ["results", "simtel", "log"],
+            ["results", "sim_telarray", "log"],
             [".ecsv", ".dat", ".log"],
         ):
             file_name = names.generate_file_name(
                 file_type=(
-                    "camera-efficiency-table" if label == "results" else "camera-efficiency"
+                    "camera_efficiency_table" if label == "results" else "camera_efficiency"
                 ),
                 suffix=suffix,
                 site=self.telescope_model.site,
@@ -151,7 +126,7 @@ class CameraEfficiency:
 
             _file[label] = self.io_handler.get_output_directory(
                 label=self.label,
-                sub_dir="camera-efficiency",
+                sub_dir="camera_efficiency",
             ).joinpath(file_name)
         return _file
 
@@ -164,8 +139,9 @@ class CameraEfficiency:
         simtel = SimulatorCameraEfficiency(
             simtel_path=self._simtel_path,
             telescope_model=self.telescope_model,
+            site_model=self.site_model,
             zenith_angle=self.config["zenith_angle"],
-            file_simtel=self._file["simtel"],
+            file_simtel=self._file["sim_telarray"],
             file_log=self._file["log"],
             label=self.label,
             nsb_spectrum=self.config["nsb_spectrum"],
@@ -177,8 +153,7 @@ class CameraEfficiency:
 
     def export_model_files(self):
         """Export model and config files to the output directory."""
-        self.telescope_model.export_config_file()
-        self.telescope_model.export_model_files()
+        self.telescope_model.write_sim_telarray_config_file()
         if not self.config.get("skip_correction_to_nsb_spectrum", False):
             self.telescope_model.export_nsb_spectrum_to_telescope_altitude_correction_file(
                 model_directory=self.telescope_model.config_file_directory
@@ -237,7 +212,7 @@ class CameraEfficiency:
 
         # Search for at least 5 consecutive numbers to see that we are in the table
         re_table = re.compile("{0}{0}{0}{0}{0}".format(r"[-+]?[0-9]*\.?[0-9]+\s+"))
-        with open(self._file["simtel"], encoding="utf-8") as file:
+        with open(self._file["sim_telarray"], encoding="utf-8") as file:
             for line in file:
                 if re_table.match(line):
                     words = line.split()
@@ -317,7 +292,7 @@ class CameraEfficiency:
                 self._results, self._file["results"], format="basic", overwrite=True
             )
             _results_summary_file = (
-                str(self._file["results"]).replace(".ecsv", ".txt").replace("-table-", "-summary-")
+                str(self._file["results"]).replace(".ecsv", ".txt").replace("_table_", "_summary_")
             )
             self._logger.info(f"Exporting summary results to {_results_summary_file}")
             with open(_results_summary_file, "w", encoding="utf-8") as file:
@@ -442,7 +417,7 @@ class CameraEfficiency:
         nsb_integral = 0.0001 * (n1_sum - 0.5 * n1_integral_edges_sum)
         nsb_rate_ref_conditions = (
             nsb_rate_provided_spectrum
-            * self._site_model.get_parameter_value("nsb_reference_value")
+            * self.site_model.get_parameter_value("nsb_reference_value")
             / nsb_integral
         )
         return nsb_rate_provided_spectrum, nsb_rate_ref_conditions

simtools/configuration/commandline_parser.py

@@ -37,7 +37,6 @@ class CommandLineParser(argparse.ArgumentParser):
         simulation_model=None,
         simulation_configuration=None,
         db_config=False,
-        job_submission=False,
     ):
         """
         Initialize default arguments used by all applications (e.g., log level or test flag).
@@ -50,18 +49,14 @@ class CommandLineParser(argparse.ArgumentParser):
             Add output file configuration to list of args.
         simulation_model: list
             List of simulation model configuration parameters to add to list of args
-            (use: 'version', 'telescope', 'site')
+            (use: 'model_version', 'telescope', 'site')
         simulation_configuration: dict
             Dict of simulation software configuration parameters to add to list of args.
         db_config: bool
             Add database configuration parameters to list of args.
-        job_submission: bool
-            Add job submission configuration parameters to list of args.
         """
         self.initialize_simulation_model_arguments(simulation_model)
         self.initialize_simulation_configuration_arguments(simulation_configuration)
-        if job_submission:
-            self.initialize_job_submission_arguments()
         if db_config:
             self.initialize_db_config_arguments()
         if paths:
@@ -228,28 +223,6 @@ class CommandLineParser(argparse.ArgumentParser):
             default=None,
         )
 
-    def initialize_job_submission_arguments(self):
-        """Initialize job submission arguments for simulator."""
-        _job_group = self.add_argument_group("job submission")
-        _job_group.add_argument(
-            "--submit_engine",
-            help="job submission command",
-            type=str,
-            required=True,
-            choices=[
-                "qsub",
-                "htcondor",
-                "local",
-            ],
-            default="local",
-        )
-        _job_group.add_argument(
-            "--submit_options",
-            help="additional options (comma separated) for submission command",
-            type=str,
-            required=False,
-        )
-
     def initialize_simulation_model_arguments(self, model_options):
         """
         Initialize default arguments for simulation model definition.
@@ -271,6 +244,7 @@ class CommandLineParser(argparse.ArgumentParser):
                 help="production model version",
                 type=str,
                 default=None,
+                nargs="+",
             )
         if "parameter_version" in model_options:
             _job_group.add_argument(
@@ -302,7 +276,7 @@ class CommandLineParser(argparse.ArgumentParser):
         if "layout" in model_options or "layout_file" in model_options:
             _job_group = self._add_model_option_layout(
                 job_group=_job_group,
-                add_layout_file="layout_file" in model_options,
+                model_options=model_options,
                 # layout info is always required for layout related tasks with the exception
                 # of listing the available layouts in the DB
                 required="--list_available_layouts" not in self._option_string_actions,
@@ -341,9 +315,9 @@ class CommandLineParser(argparse.ArgumentParser):
             "--simulation_software",
             help="Simulation software steps.",
             type=str,
-            choices=["corsika", "simtel", "corsika_simtel"],
+            choices=["corsika", "sim_telarray", "corsika_sim_telarray"],
             required=True,
-            default="corsika_simtel",
+            default="corsika_sim_telarray",
         )
 
     @staticmethod
@@ -388,8 +362,14 @@ class CommandLineParser(argparse.ArgumentParser):
                 "type": int,
                 "required": False,
             },
-            "run_number_start": {
-                "help": "Run number for the first run.",
+            "run_number_offset": {
+                "help": "An offset for the run number to be simulated.",
+                "type": int,
+                "required": False,
+                "default": 0,
+            },
+            "run_number": {
+                "help": "Run number to be simulated.",
                 "type": int,
                 "required": True,
                 "default": 1,
@@ -476,7 +456,7 @@ class CommandLineParser(argparse.ArgumentParser):
                 pass
 
     @staticmethod
-    def _add_model_option_layout(job_group, add_layout_file, required=True):
+    def _add_model_option_layout(job_group, model_options, required=True):
         """
         Add layout option to the job group.
 
@@ -484,8 +464,8 @@ class CommandLineParser(argparse.ArgumentParser):
         ----------
         job_group: argparse.ArgumentParser
             Job group
-        add_layout_file: bool
-            Add layout file option
+        model_options: list
+            List of model options.
 
         Returns
         -------
@@ -506,7 +486,7 @@ class CommandLineParser(argparse.ArgumentParser):
             required=False,
             default=None,
         )
-        if add_layout_file:
+        if "layout_file" in model_options:
             _layout_group.add_argument(
                 "--array_layout_file",
                 help="file(s) with the list of array elements (astropy table format).",
@@ -515,6 +495,21 @@ class CommandLineParser(argparse.ArgumentParser):
                 required=False,
                 default=None,
             )
+        if "layout_parameter_file" in model_options:
+            _layout_group.add_argument(
+                "--array_layout_parameter_file",
+                help="Array layout model parameter file (typically in JSON format).",
+                type=str,
+                required=False,
+                default=None,
+            )
+        if "plot_all_layouts" in model_options:
+            _layout_group.add_argument(
+                "--plot_all_layouts",
+                help="plot all available layouts",
+                action="store_true",
+                required=False,
+            )
         return job_group
 
     def _add_model_option_site(self, job_group):

simtools/configuration/configurator.py

@@ -106,7 +106,6 @@ class Configurator:
         simulation_model=None,
         simulation_configuration=None,
         db_config=False,
-        job_submission=False,
     ):
         """
         Initialize application configuration.
@@ -134,8 +133,6 @@ class Configurator:
             Dict of simulation software configuration parameters to add to list of args.
         db_config: bool
             Add database configuration parameters to list of args.
-        job_submission: bool
-            Add job submission configuration to list of args.
 
         Returns
         -------
@@ -151,7 +148,6 @@ class Configurator:
             simulation_model=simulation_model,
             simulation_configuration=simulation_configuration,
             db_config=db_config,
-            job_submission=job_submission,
         )
 
         self._fill_from_command_line(require_command_line=require_command_line)
@@ -163,6 +159,7 @@ class Configurator:
             self.config["activity_id"] = str(uuid.uuid4())
         if self.config["label"] is None:
             self.config["label"] = self.label
+        self._initialize_model_versions()
 
         self._initialize_io_handler()
         if output:
@@ -329,6 +326,15 @@ class Configurator:
 
         self._fill_from_config_dict(_env_dict)
 
+    def _initialize_model_versions(self):
+        """Initialize model versions."""
+        if (
+            self.config.get("model_version", None)
+            and isinstance(self.config["model_version"], list)
+            and len(self.config["model_version"]) == 1
+        ):
+            self.config["model_version"] = self.config["model_version"][0]
+
     def _initialize_io_handler(self):
         """Initialize IOHandler with input and output paths."""
         _io_handler = io_handler.IOHandler()