gammasimtools 0.16.0__py3-none-any.whl → 0.18.0__py3-none-any.whl

simtools/applications/submit_array_layouts.py

@@ -0,0 +1,93 @@
+#!/usr/bin/python3
+
+r"""
+    Submit array-layouts definition and corresponding metadata and validate list of telescopes.
+
+    Includes validation that all defined telescope exists.
+
+    Command line arguments
+    ----------------------
+    array_layouts (str, required)
+        Array layouts file.
+    updated_parameter_version (str, optional)
+        Updated parameter version.
+    input_meta (str, optional)
+        Input meta data file(s) associated to input data (wildcards or list of files allowed).
+    model_version (str, required)
+        Model version.
+
+    Example
+    -------
+
+    Submit a new array layout dictionary:
+
+    .. code-block:: console
+
+        simtools-submit-array-layouts \
+            --array_layouts array_layouts.json \\
+            --model_version 6.0.0 \\
+            --updated_parameter_version 0.1.0 \\
+            --input_meta array_layouts.metadata.yml
+
+
+    """
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler
+from simtools.layout.array_layout_utils import validate_array_layouts_with_db, write_array_layouts
+
+
+def _parse(label, description):
+    """Parse command line configuration."""
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--array_layouts",
+        type=str,
+        required=True,
+        help="Array layout dictionary file.",
+    )
+    config.parser.add_argument(
+        "--updated_parameter_version",
+        help="Updated parameter version.",
+        type=str,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--input_meta",
+        help="meta data file(s) associated to input data (wildcards or list of files allowed)",
+        type=str,
+        nargs="+",
+        required=False,
+    )
+
+    return config.initialize(output=True, db_config=True, simulation_model=["model_version"])
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse(
+        label=Path(__file__).stem,
+        description="Submit and validate array layouts.",
+    )
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    array_layouts = validate_array_layouts_with_db(
+        production_table=db.read_production_table_from_mongo_db(
+            collection_name="telescopes", model_version=args_dict["model_version"]
+        ),
+        array_layouts=gen.collect_data_from_file(args_dict["array_layouts"]),
+    )
+
+    write_array_layouts(array_layouts=array_layouts, args_dict=args_dict, db_config=db_config)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/verify_simulation_model_production_tables.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python3
+
+"""
+Verify simulation model production tables and model parameters for completeness.
+
+This application is a utility to be used in the CI pipeline of the SimulationModels
+repository. It checks that all model parameters defined in the production tables
+exist in the simulation models repository.
+
+"""
+
+import logging
+
+from simtools.configuration import configurator
+from simtools.model import model_repository
+from simtools.utils import general as gen
+
+
+def _parse():
+    """Parse command line arguments."""
+    config = configurator.Configurator(
+        description=(
+            "Verify simulation model production tables and model parameters for completeness. "
+            "This application checks that all model parameters defined in the production tables "
+            "exist in the simulation models repository."
+        )
+    )
+    config.parser.add_argument(
+        "--simulation_models_path",
+        help="Path to the simulation models repository.",
+        type=str,
+        required=True,
+    )
+    return config.initialize(db_config=False, output=False, paths=False)
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    if not model_repository.verify_simulation_model_production_tables(
+        simulation_models_path=args_dict["simulation_models_path"]
+    ):
+        raise RuntimeError(
+            "Verification failed: Some model parameters are missing in the repository."
+        )
+
+
+if __name__ == "__main__":
+    main()

simtools/camera/camera_efficiency.py

@@ -114,7 +114,7 @@ class CameraEfficiency:
         ):
             file_name = names.generate_file_name(
                 file_type=(
-                    "camera-efficiency-table" if label == "results" else "camera-efficiency"
+                    "camera_efficiency_table" if label == "results" else "camera_efficiency"
                 ),
                 suffix=suffix,
                 site=self.telescope_model.site,
@@ -126,7 +126,7 @@ class CameraEfficiency:
 
             _file[label] = self.io_handler.get_output_directory(
                 label=self.label,
-                sub_dir="camera-efficiency",
+                sub_dir="camera_efficiency",
             ).joinpath(file_name)
         return _file
 
@@ -292,7 +292,7 @@ class CameraEfficiency:
                 self._results, self._file["results"], format="basic", overwrite=True
             )
             _results_summary_file = (
-                str(self._file["results"]).replace(".ecsv", ".txt").replace("-table-", "-summary-")
+                str(self._file["results"]).replace(".ecsv", ".txt").replace("_table_", "_summary_")
             )
             self._logger.info(f"Exporting summary results to {_results_summary_file}")
             with open(_results_summary_file, "w", encoding="utf-8") as file:

simtools/configuration/commandline_parser.py

@@ -37,7 +37,6 @@ class CommandLineParser(argparse.ArgumentParser):
         simulation_model=None,
         simulation_configuration=None,
         db_config=False,
-        job_submission=False,
     ):
         """
         Initialize default arguments used by all applications (e.g., log level or test flag).
@@ -55,13 +54,9 @@ class CommandLineParser(argparse.ArgumentParser):
             Dict of simulation software configuration parameters to add to list of args.
         db_config: bool
             Add database configuration parameters to list of args.
-        job_submission: bool
-            Add job submission configuration parameters to list of args.
         """
         self.initialize_simulation_model_arguments(simulation_model)
         self.initialize_simulation_configuration_arguments(simulation_configuration)
-        if job_submission:
-            self.initialize_job_submission_arguments()
         if db_config:
             self.initialize_db_config_arguments()
         if paths:
@@ -228,28 +223,6 @@ class CommandLineParser(argparse.ArgumentParser):
             default=None,
         )
 
-    def initialize_job_submission_arguments(self):
-        """Initialize job submission arguments for simulator."""
-        _job_group = self.add_argument_group("job submission")
-        _job_group.add_argument(
-            "--submit_engine",
-            help="job submission command",
-            type=str,
-            required=True,
-            choices=[
-                "qsub",
-                "htcondor",
-                "local",
-            ],
-            default="local",
-        )
-        _job_group.add_argument(
-            "--submit_options",
-            help="additional options (comma separated) for submission command",
-            type=str,
-            required=False,
-        )
-
     def initialize_simulation_model_arguments(self, model_options):
         """
         Initialize default arguments for simulation model definition.
@@ -303,7 +276,7 @@ class CommandLineParser(argparse.ArgumentParser):
         if "layout" in model_options or "layout_file" in model_options:
             _job_group = self._add_model_option_layout(
                 job_group=_job_group,
-                add_layout_file="layout_file" in model_options,
+                model_options=model_options,
                 # layout info is always required for layout related tasks with the exception
                 # of listing the available layouts in the DB
                 required="--list_available_layouts" not in self._option_string_actions,
@@ -389,8 +362,14 @@ class CommandLineParser(argparse.ArgumentParser):
                 "type": int,
                 "required": False,
             },
-            "run_number_start": {
-                "help": "Run number for the first run.",
+            "run_number_offset": {
+                "help": "An offset for the run number to be simulated.",
+                "type": int,
+                "required": False,
+                "default": 0,
+            },
+            "run_number": {
+                "help": "Run number to be simulated.",
                 "type": int,
                 "required": True,
                 "default": 1,
@@ -477,7 +456,7 @@ class CommandLineParser(argparse.ArgumentParser):
                 pass
 
     @staticmethod
-    def _add_model_option_layout(job_group, add_layout_file, required=True):
+    def _add_model_option_layout(job_group, model_options, required=True):
         """
         Add layout option to the job group.
 
@@ -485,8 +464,8 @@ class CommandLineParser(argparse.ArgumentParser):
         ----------
         job_group: argparse.ArgumentParser
             Job group
-        add_layout_file: bool
-            Add layout file option
+        model_options: list
+            List of model options.
 
         Returns
         -------
@@ -495,7 +474,8 @@ class CommandLineParser(argparse.ArgumentParser):
         _layout_group = job_group.add_mutually_exclusive_group(required=required)
         _layout_group.add_argument(
             "--array_layout_name",
-            help="array layout name (e.g., alpha, subsystem_msts)",
+            help="array layout name(s) (e.g., alpha, subsystem_msts)",
+            nargs="+",
             type=str,
             required=False,
         )
@@ -507,7 +487,7 @@ class CommandLineParser(argparse.ArgumentParser):
             required=False,
             default=None,
         )
-        if add_layout_file:
+        if "layout_file" in model_options:
             _layout_group.add_argument(
                 "--array_layout_file",
                 help="file(s) with the list of array elements (astropy table format).",
@@ -516,6 +496,21 @@ class CommandLineParser(argparse.ArgumentParser):
                 required=False,
                 default=None,
             )
+        if "layout_parameter_file" in model_options:
+            _layout_group.add_argument(
+                "--array_layout_parameter_file",
+                help="Array layout model parameter file (typically in JSON format).",
+                type=str,
+                required=False,
+                default=None,
+            )
+        if "plot_all_layouts" in model_options:
+            _layout_group.add_argument(
+                "--plot_all_layouts",
+                help="plot all available layouts",
+                action="store_true",
+                required=False,
+            )
         return job_group
 
     def _add_model_option_site(self, job_group):

simtools/configuration/configurator.py

@@ -106,7 +106,6 @@ class Configurator:
         simulation_model=None,
         simulation_configuration=None,
         db_config=False,
-        job_submission=False,
     ):
         """
         Initialize application configuration.
@@ -134,8 +133,6 @@ class Configurator:
             Dict of simulation software configuration parameters to add to list of args.
         db_config: bool
             Add database configuration parameters to list of args.
-        job_submission: bool
-            Add job submission configuration to list of args.
 
         Returns
         -------
@@ -151,7 +148,6 @@ class Configurator:
             simulation_model=simulation_model,
             simulation_configuration=simulation_configuration,
             db_config=db_config,
-            job_submission=job_submission,
         )
 
         self._fill_from_command_line(require_command_line=require_command_line)

simtools/constants.py

@@ -21,3 +21,5 @@ MODEL_PARAMETER_SCHEMA_URL = (
     "https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/"
     "/model_parameters"
 )
+# Path to resource files
+RESOURCE_PATH = files("simtools") / "resources"

simtools/corsika/corsika_config.py

@@ -638,6 +638,9 @@ class CorsikaConfig:
         ValueError
             If file_type is unknown or if the run number is not given for file_type==config_tmp.
         """
+        if file_type == "config_tmp" and run_number is None:
+            raise ValueError("Must provide a run number for a temporary CORSIKA config file")
+
         file_label = f"_{self.label}" if self.label is not None else ""
 
         _vc_low = self.get_config_parameter("VIEWCONE")[0]
@@ -646,25 +649,27 @@ class CorsikaConfig:
             f"_cone{int(_vc_low):d}-{int(_vc_high):d}" if _vc_low != 0 or _vc_high != 0 else ""
         )
 
+        run_number_in_file_name = ""
+        if file_type == "output_generic":
+            # The XXXXXX will be replaced by the run number after the pfp step with sed
+            run_number_in_file_name = "runXXXXXX_"
+        if file_type == "config_tmp":
+            run_number_in_file_name = f"run{run_number:06}_"
+
         base_name = (
-            f"{self.primary_particle.name}_{self.array_model.site}_{self.array_model.layout_name}_"
-            f"za{int(self.get_config_parameter('THETAP')[0]):03}-"
-            f"azm{self.azimuth_angle:03}deg"
-            f"{view_cone}{file_label}"
+            f"{self.primary_particle.name}_{run_number_in_file_name}"
+            f"za{int(self.get_config_parameter('THETAP')[0]):03}deg_"
+            f"azm{self.azimuth_angle:03}deg{view_cone}_"
+            f"{self.array_model.site}_{self.array_model.layout_name}_"
+            f"{self.array_model.model_version}{file_label}"
         )
 
         if file_type == "config_tmp":
-            if run_number is None:
-                raise ValueError("Must provide a run number for a temporary CORSIKA config file")
-            return f"corsika_config_run{run_number:06}_{base_name}.txt"
+            return f"corsika_config_{base_name}.txt"
         if file_type == "config":
             return f"corsika_config_{base_name}.input"
         if file_type == "output_generic":
-            # The XXXXXX will be replaced by the run number after the pfp step with sed
-            return (
-                f"runXXXXXX_{base_name}_{self.array_model.site}_"
-                f"{self.array_model.layout_name}{file_label}.zst"
-            )
+            return f"{base_name}.zst"
         if file_type == "multipipe":
             return f"multi_cta-{self.array_model.site}-{self.array_model.layout_name}.cfg"
 

simtools/corsika/primary_particle.py

@@ -7,7 +7,7 @@ from particle import Corsika7ID, InvalidParticle, Particle
 
 class PrimaryParticle:
     """
-    Primary particle definition using CORSIKA7 or PDG ID.
+    Primary particle definition using CORSIKA7, eventio, or PDG ID.
 
     Uses a dictionary to map particle common names to CORSIKA7 IDs.
     Particles not found in the dictionary are searched in the PDG particle database.
@@ -20,27 +20,31 @@ class PrimaryParticle:
         The actual ID of the primary particle.
     """
 
+    _valid_types = {
+        "corsika7_id": "corsika7_id",
+        "common_name": "name",
+        "pdg_id": "pdg_id",
+        "eventio_id": "eventio_id",
+    }
+
     def __init__(self, particle_id_type=None, particle_id=None):
         self._logger = logging.getLogger(__name__)
-
         self._corsika7_id = None
         self._name = None
         self._pdg_id = None
         self._pdg_name = None
+        self._eventio_id = None
 
-        if bool(particle_id_type) != bool(particle_id):
+        if (particle_id_type is None) != (particle_id is None):
             raise ValueError("Both 'particle_id_type' and 'particle_id' must be provided together.")
 
-        valid_id_types = {"corsika7_id", "common_name", "pdg_id"}
-        if particle_id_type and particle_id_type not in valid_id_types:
-            raise ValueError(f"Particle ID type must be one of {valid_id_types}")
-
-        if particle_id_type == "corsika7_id":
-            self.corsika7_id = particle_id
-        elif particle_id_type == "common_name":
-            self.name = particle_id
-        elif particle_id_type == "pdg_id":
-            self.pdg_id = particle_id
+        if particle_id_type:
+            try:
+                setattr(self, self._valid_types[particle_id_type], particle_id)
+            except KeyError as exc:
+                raise ValueError(
+                    f"Particle ID type must be one of {set(self._valid_types)}"
+                ) from exc
 
     def __str__(self):
         """Return a string representation of the primary particle."""
@@ -70,6 +74,35 @@ class PrimaryParticle:
             raise ValueError(f"Invalid CORSIKA7 ID: {value}") from exc
         self._corsika7_id = int(value)
 
+    @property
+    def eventio_id(self):
+        """
+        EventIO ID of the primary particle.
+
+        0 (gamma), 1(e-), 2(mu-), 100*A+Z for nucleons and nuclei, negative for antimatter.
+        """
+        return self._eventio_id
+
+    @eventio_id.setter
+    def eventio_id(self, value):
+        """Set EventIO ID of the primary particle."""
+        mapping = {
+            0: 1,
+            1: 3,
+            -1: 2,
+            2: 6,
+            -2: 5,
+            100: 13,
+            101: 14,
+            -101: 15,
+        }
+
+        try:
+            self.corsika7_id = mapping.get(value, value)
+        except (ValueError, InvalidParticle) as exc:
+            raise ValueError(f"Invalid EventIO ID: {value}") from exc
+        self._eventio_id = value
+
     @property
     def name(self):
         """Common name of the primary particle."""

simtools/data_model/metadata_collector.py

@@ -13,7 +13,6 @@ from pathlib import Path
 
 import yaml
 
-import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
 from simtools.constants import METADATA_JSON_SCHEMA
@@ -209,7 +208,7 @@ class MetadataCollector:
             url = self.input_metadata[0][self.observatory]["product"]["data"]["model"]["url"]
             self._logger.debug(f"Schema file from input metadata: {url}")
             return url
-        except (KeyError, TypeError):
+        except (KeyError, TypeError, IndexError):
             pass
 
         self._logger.warning("No schema file found.")
@@ -254,7 +253,7 @@ class MetadataCollector:
             )
             if _site is not None:
                 return names.validate_site_name(_site)
-        except (KeyError, TypeError):
+        except (KeyError, TypeError, IndexError):
             pass
         return None
 
@@ -360,6 +359,11 @@ class MetadataCollector:
                 _input_metadata = self._read_input_metadata_from_yml_or_json(metadata_file)
             elif Path(metadata_file).suffix == ".ecsv":
                 _input_metadata = self._read_input_metadata_from_ecsv(metadata_file)
+            elif Path(metadata_file).name.endswith((".simtel.zst", ".simtel")):
+                self._logger.warning(
+                    "Metadata extraction from sim_telarray files is not supported yet."
+                )
+                continue
             else:
                 raise gen.InvalidConfigDataError(f"Unknown metadata file format: {metadata_file}")
 

simtools/data_model/schema.py

@@ -4,6 +4,7 @@ import logging
 from pathlib import Path
 
 import jsonschema
+from referencing import Registry, Resource
 
 import simtools.utils.general as gen
 from simtools.constants import (
@@ -119,8 +120,14 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
             ),  # default version to ensure backward compatibility
         )
 
+    validator = jsonschema.Draft6Validator(
+        schema=json_schema,
+        format_checker=format_checkers.format_checker,
+        registry=Registry(retrieve=_retrieve_yaml_schema_from_uri),
+    )
+
     try:
-        jsonschema.validate(data, schema=json_schema, format_checker=format_checkers.format_checker)
+        validator.validate(instance=data)
     except jsonschema.exceptions.ValidationError as exc:
         _logger.error(f"Validation failed using schema: {json_schema} for data: {data}")
         raise exc
@@ -135,6 +142,13 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     return data
 
 
+def _retrieve_yaml_schema_from_uri(uri):
+    """Load schema from a file URI."""
+    path = SCHEMA_PATH / Path(uri.removeprefix("file:/"))
+    contents = gen.collect_data_from_file(file_name=path)
+    return Resource.from_contents(contents)
+
+
 def load_schema(schema_file=None, schema_version=None):
     """
     Load parameter schema from file.

simtools/db/db_handler.py

@@ -217,7 +217,7 @@ class DatabaseHandler:
                 raise ValueError(
                     "Only one model version can be passed to get_model_parameter, not a list."
                 )
-            production_table = self._read_production_table_from_mongo_db(
+            production_table = self.read_production_table_from_mongo_db(
                 collection_name, model_version
             )
             array_element_list = self._get_array_element_list(
@@ -263,7 +263,7 @@ class DatabaseHandler:
         dict containing the parameters
         """
         pars = {}
-        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        production_table = self.read_production_table_from_mongo_db(collection, model_version)
         array_element_list = self._get_array_element_list(
             array_element_name, site, production_table, collection
         )
@@ -409,6 +409,11 @@ class DatabaseHandler:
             Version of the model.
         export_file_as_table: bool
             If True, export the file as an astropy table (ecsv format).
+
+        Returns
+        -------
+        astropy.table.Table or None
+            If export_file_as_table is True
         """
         parameters = self.get_model_parameter(
             parameter,
@@ -518,9 +523,9 @@ class DatabaseHandler:
             parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time
         return {k: parameters[k] for k in sorted(parameters)}
 
-    def _read_production_table_from_mongo_db(self, collection_name, model_version):
+    def read_production_table_from_mongo_db(self, collection_name, model_version):
         """
-        Read production table from MongoDB.
+        Read production table for the given collection from MongoDB.
 
         Parameters
         ----------
@@ -591,7 +596,7 @@ class DatabaseHandler:
         list
             Sorted list of all array elements found in collection
         """
-        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        production_table = self.read_production_table_from_mongo_db(collection, model_version)
         return sorted([entry for entry in production_table["parameters"] if "-design" not in entry])
 
     def get_design_model(self, model_version, array_element_name, collection="telescopes"):
@@ -613,7 +618,7 @@ class DatabaseHandler:
         str
             Design model for a given array element.
         """
-        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        production_table = self.read_production_table_from_mongo_db(collection, model_version)
         try:
             return production_table["design_model"][array_element_name]
         except KeyError:
@@ -641,7 +646,7 @@ class DatabaseHandler:
         list
             Sorted list of all array element names found in collection
         """
-        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        production_table = self.read_production_table_from_mongo_db(collection, model_version)
         all_array_elements = production_table["parameters"]
         return sorted(
             [
@@ -759,7 +764,7 @@ class DatabaseHandler:
         """
         db_name = self._get_db_name(db_name)
         collection = self.get_collection(db_name, "production_tables")
-        self._logger.info(f"Adding production for {production_table.get('collection')} to to DB")
+        self._logger.debug(f"Adding production for {production_table.get('collection')} to to DB")
         collection.insert_one(production_table)
         DatabaseHandler.production_table_cached.clear()
 
@@ -809,13 +814,13 @@ class DatabaseHandler:
                 raise ValueError(f"File is not UTF-8 encoded: {file_path}")
             files_to_add_to_db.add(f"{file_path}")
 
-        self._logger.info(
+        self._logger.debug(
             f"Adding a new entry to DB {db_name} and collection {collection_name}:\n{par_dict}"
         )
         collection.insert_one(par_dict)
 
         for file_to_insert_now in files_to_add_to_db:
-            self._logger.info(f"Will also add the file {file_to_insert_now} to the DB")
+            self._logger.debug(f"Will also add the file {file_to_insert_now} to the DB")
             self.insert_file_to_db(file_to_insert_now, db_name)
 
         self._reset_parameter_cache()
@@ -964,7 +969,7 @@ class DatabaseHandler:
             return [array_element_name]
         if collection == "configuration_sim_telarray":
             # get design model from 'telescope' or 'calibration_device' production tables
-            production_table = self._read_production_table_from_mongo_db(
+            production_table = self.read_production_table_from_mongo_db(
                 names.get_collection_name_from_array_element_name(array_element_name),
                 production_table["model_version"],
             )

simtools/db/db_model_upload.py

@@ -27,7 +27,7 @@ def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
         Path to location of all additional files to be uploaded.
     """
     par_dict = gen.collect_data_from_file(file_name=file)
-    logger.info(
+    logger.debug(
         f"Adding the following parameter to the DB: {par_dict['parameter']} "
         f"version {par_dict['parameter_version']} "
         f"(collection {collection} in database {db_name})"
@@ -120,7 +120,7 @@ def _read_production_table(model_dict, file, model_name):
         },
     )
     parameter_dict = gen.collect_data_from_file(file_name=file)
-    logger.info(f"Reading production table for {array_element} (collection {collection})")
+    logger.debug(f"Reading production table for {array_element} (collection {collection})")
     try:
         if array_element in ("configuration_corsika", "configuration_sim_telarray"):
             model_dict[collection]["parameters"] = parameter_dict["parameters"]