gammasimtools 0.15.0__py3-none-any.whl → 0.17.0__py3-none-any.whl

simtools/corsika/corsika_config.py

@@ -124,9 +124,15 @@ class CorsikaConfig:
         if db_config is None:  # all following parameter require DB
             return config
 
-        db_model_parameters = ModelParameter(
-            mongo_db_config=db_config, model_version=args_dict["model_version"]
-        )
+        # If the user provided multiple model versions, we take the first one
+        # because for CORSIKA config we need only one and it doesn't matter which
+        model_versions = args_dict.get("model_version", None)
+        if not isinstance(model_versions, list):
+            model_versions = [model_versions]
+        self.assert_corsika_configurations_match(model_versions, db_config=db_config)
+        model_version = model_versions[0]
+        self._logger.debug(f"Using model version {model_version} for CORSIKA parameters")
+        db_model_parameters = ModelParameter(mongo_db_config=db_config, model_version=model_version)
         parameters_from_db = db_model_parameters.get_simulation_software_parameters("corsika")
 
         config["INTERACTION_FLAGS"] = self._corsika_configuration_interaction_flags(
@@ -140,6 +146,60 @@ class CorsikaConfig:
 
         return config
 
+    def assert_corsika_configurations_match(self, model_versions, db_config=None):
+        """
+        Assert that CORSIKA configurations match across all model versions.
+
+        Parameters
+        ----------
+        model_versions : list
+            List of model versions to check.
+        db_config : dict, optional
+            Database configuration.
+
+        Raises
+        ------
+        InvalidCorsikaInputError
+            If CORSIKA parameters are not identical across all model versions.
+        """
+        if len(model_versions) < 2:
+            return
+
+        parameters_from_db_list = []
+
+        # Get parameters for all model versions
+        for model_version in model_versions:
+            db_model_parameters = ModelParameter(
+                mongo_db_config=db_config, model_version=model_version
+            )
+            parameters_from_db_list.append(
+                db_model_parameters.get_simulation_software_parameters("corsika")
+            )
+
+        # Parameters that can differ between model versions (e.g., i/o buffer size)
+        skip_parameters = ["corsika_iact_io_buffer", "corsika_iact_split_auto"]
+
+        # Check if all parameters match
+        for i in range(len(parameters_from_db_list) - 1):
+            for key in parameters_from_db_list[i]:
+                if key in skip_parameters:
+                    continue
+
+                current_value = parameters_from_db_list[i][key]["value"]
+                next_value = parameters_from_db_list[i + 1][key]["value"]
+
+                if current_value != next_value:
+                    self._logger.warning(
+                        f"Parameter '{key}' mismatch between model versions:\n"
+                        f"  {model_versions[i]}: {current_value}\n"
+                        f"  {model_versions[i + 1]}: {next_value}"
+                    )
+                    raise InvalidCorsikaInputError(
+                        f"CORSIKA parameter '{key}' differs between model versions "
+                        f"{model_versions[i]} and {model_versions[i + 1]}. "
+                        f"Values are {current_value} and {next_value} respectively."
+                    )
+
     def _corsika_configuration_from_user_input(self, args_dict):
         """
         Get CORSIKA configuration from user input.
@@ -236,7 +296,45 @@ class CorsikaConfig:
 
     def _input_config_corsika_starting_grammage(self, entry):
         """Return FIXCHI parameter CORSIKA format."""
-        return f"{entry['value'] * u.Unit(entry['unit']).to('g/cm2')}"
+        value = self._get_starting_grammage_value(entry["value"])
+        return f"{value * u.Unit(entry['unit']).to('g/cm2')}"
+
+    def _get_starting_grammage_value(self, value_entry):
+        """
+        Get appropriate starting grammage value from entry values.
+
+        Parameters
+        ----------
+        value_entry : float or list
+            Value or list of grammage configurations
+
+        Returns
+        -------
+        float
+            Selected grammage value
+        """
+        if not isinstance(value_entry, list):
+            return value_entry
+
+        tel_types = {tel.design_model for tel in self.array_model.telescope_model.values()}
+        particle = self.primary_particle.name
+        matched_values = self._get_matching_grammage_values(value_entry, tel_types, particle)
+
+        return min(matched_values) if matched_values else 0
+
+    def _get_matching_grammage_values(self, configs, tel_types, particle):
+        """Get list of matching grammage values for particle and telescope types."""
+        matched = []
+        defaults = []
+
+        for config in configs:
+            if config.get("instrument") is None or config.get("instrument") in tel_types:
+                if config["primary_particle"] == particle:
+                    matched.append(config["value"])
+                elif config["primary_particle"] == "default":
+                    defaults.append(config["value"])
+
+        return matched if matched else defaults
 
     def _input_config_corsika_particle_kinetic_energy_cutoff(self, entry):
         """Return ECUTS parameter CORSIKA format."""
@@ -540,6 +638,9 @@ class CorsikaConfig:
         ValueError
             If file_type is unknown or if the run number is not given for file_type==config_tmp.
         """
+        if file_type == "config_tmp" and run_number is None:
+            raise ValueError("Must provide a run number for a temporary CORSIKA config file")
+
         file_label = f"_{self.label}" if self.label is not None else ""
 
         _vc_low = self.get_config_parameter("VIEWCONE")[0]
@@ -548,25 +649,27 @@ class CorsikaConfig:
             f"_cone{int(_vc_low):d}-{int(_vc_high):d}" if _vc_low != 0 or _vc_high != 0 else ""
         )
 
+        run_number_in_file_name = ""
+        if file_type == "output_generic":
+            # The XXXXXX will be replaced by the run number after the pfp step with sed
+            run_number_in_file_name = "runXXXXXX_"
+        if file_type == "config_tmp":
+            run_number_in_file_name = f"run{run_number:06}_"
+
         base_name = (
-            f"{self.primary_particle.name}_{self.array_model.site}_{self.array_model.layout_name}_"
-            f"za{int(self.get_config_parameter('THETAP')[0]):03}-"
-            f"azm{self.azimuth_angle:03}deg"
-            f"{view_cone}{file_label}"
+            f"{self.primary_particle.name}_{run_number_in_file_name}"
+            f"za{int(self.get_config_parameter('THETAP')[0]):03}deg_"
+            f"azm{self.azimuth_angle:03}deg{view_cone}_"
+            f"{self.array_model.site}_{self.array_model.layout_name}_"
+            f"{self.array_model.model_version}{file_label}"
         )
 
         if file_type == "config_tmp":
-            if run_number is None:
-                raise ValueError("Must provide a run number for a temporary CORSIKA config file")
-            return f"corsika_config_run{run_number:06}_{base_name}.txt"
+            return f"corsika_config_{base_name}.txt"
         if file_type == "config":
             return f"corsika_config_{base_name}.input"
         if file_type == "output_generic":
-            # The XXXXXX will be replaced by the run number after the pfp step with sed
-            return (
-                f"runXXXXXX_{base_name}_{self.array_model.site}_"
-                f"{self.array_model.layout_name}{file_label}.zst"
-            )
+            return f"{base_name}.zst"
         if file_type == "multipipe":
             return f"multi_cta-{self.array_model.site}-{self.array_model.layout_name}.cfg"
 
@@ -587,7 +690,7 @@ class CorsikaConfig:
         str
             Output file name.
         """
-        sub_dir = "corsika_simtel" if use_multipipe else "corsika"
+        sub_dir = "corsika_sim_telarray" if use_multipipe else "corsika"
         config_file_name = self.get_corsika_config_file_name(file_type="config")
         file_directory = self.io_handler.get_output_directory(label=self.label, sub_dir=sub_dir)
         self._logger.debug(f"Creating directory {file_directory}")

simtools/corsika/primary_particle.py

@@ -7,7 +7,7 @@ from particle import Corsika7ID, InvalidParticle, Particle
 
 class PrimaryParticle:
     """
-    Primary particle definition using CORSIKA7 or PDG ID.
+    Primary particle definition using CORSIKA7, eventio, or PDG ID.
 
     Uses a dictionary to map particle common names to CORSIKA7 IDs.
     Particles not found in the dictionary are searched in the PDG particle database.
@@ -20,27 +20,31 @@ class PrimaryParticle:
         The actual ID of the primary particle.
     """
 
+    _valid_types = {
+        "corsika7_id": "corsika7_id",
+        "common_name": "name",
+        "pdg_id": "pdg_id",
+        "eventio_id": "eventio_id",
+    }
+
     def __init__(self, particle_id_type=None, particle_id=None):
         self._logger = logging.getLogger(__name__)
-
         self._corsika7_id = None
         self._name = None
         self._pdg_id = None
         self._pdg_name = None
+        self._eventio_id = None
 
-        if bool(particle_id_type) != bool(particle_id):
+        if (particle_id_type is None) != (particle_id is None):
             raise ValueError("Both 'particle_id_type' and 'particle_id' must be provided together.")
 
-        valid_id_types = {"corsika7_id", "common_name", "pdg_id"}
-        if particle_id_type and particle_id_type not in valid_id_types:
-            raise ValueError(f"Particle ID type must be one of {valid_id_types}")
-
-        if particle_id_type == "corsika7_id":
-            self.corsika7_id = particle_id
-        elif particle_id_type == "common_name":
-            self.name = particle_id
-        elif particle_id_type == "pdg_id":
-            self.pdg_id = particle_id
+        if particle_id_type:
+            try:
+                setattr(self, self._valid_types[particle_id_type], particle_id)
+            except KeyError as exc:
+                raise ValueError(
+                    f"Particle ID type must be one of {set(self._valid_types)}"
+                ) from exc
 
     def __str__(self):
         """Return a string representation of the primary particle."""
@@ -70,6 +74,35 @@ class PrimaryParticle:
             raise ValueError(f"Invalid CORSIKA7 ID: {value}") from exc
         self._corsika7_id = int(value)
 
+    @property
+    def eventio_id(self):
+        """
+        EventIO ID of the primary particle.
+
+        0 (gamma), 1(e-), 2(mu-), 100*A+Z for nucleons and nuclei, negative for antimatter.
+        """
+        return self._eventio_id
+
+    @eventio_id.setter
+    def eventio_id(self, value):
+        """Set EventIO ID of the primary particle."""
+        mapping = {
+            0: 1,
+            1: 3,
+            -1: 2,
+            2: 6,
+            -2: 5,
+            100: 13,
+            101: 14,
+            -101: 15,
+        }
+
+        try:
+            self.corsika7_id = mapping.get(value, value)
+        except (ValueError, InvalidParticle) as exc:
+            raise ValueError(f"Invalid EventIO ID: {value}") from exc
+        self._eventio_id = value
+
     @property
     def name(self):
         """Common name of the primary particle."""

simtools/data_model/format_checkers.py

@@ -5,6 +5,7 @@ import re
 import astropy.units as u
 import jsonschema
 
+from simtools.corsika.primary_particle import PrimaryParticle
 from simtools.utils import names
 
 format_checker = jsonschema.FormatChecker()
@@ -50,3 +51,11 @@ def check_array_triggers_name(name):
         raise ValueError(f"Array trigger name '{name}' does not match pattern '{pattern}'")
     names.validate_array_element_type(re.match(pattern, name).group(1))
     return True
+
+
+@format_checker.checks("common_particle_name")
+def check_common_particle_name(name):
+    """Validate common particle names for jsonschema."""
+    if name in PrimaryParticle.particle_names() or name == "default":
+        return True
+    raise ValueError(f"Invalid common particle name: '{name}'")

simtools/data_model/metadata_collector.py

@@ -13,7 +13,6 @@ from pathlib import Path
 
 import yaml
 
-import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
 from simtools.constants import METADATA_JSON_SCHEMA
@@ -209,7 +208,7 @@ class MetadataCollector:
             url = self.input_metadata[0][self.observatory]["product"]["data"]["model"]["url"]
             self._logger.debug(f"Schema file from input metadata: {url}")
             return url
-        except (KeyError, TypeError):
+        except (KeyError, TypeError, IndexError):
             pass
 
         self._logger.warning("No schema file found.")
@@ -254,7 +253,7 @@ class MetadataCollector:
             )
             if _site is not None:
                 return names.validate_site_name(_site)
-        except (KeyError, TypeError):
+        except (KeyError, TypeError, IndexError):
             pass
         return None
 
@@ -360,6 +359,11 @@ class MetadataCollector:
                 _input_metadata = self._read_input_metadata_from_yml_or_json(metadata_file)
             elif Path(metadata_file).suffix == ".ecsv":
                 _input_metadata = self._read_input_metadata_from_ecsv(metadata_file)
+            elif Path(metadata_file).name.endswith((".simtel.zst", ".simtel")):
+                self._logger.warning(
+                    "Metadata extraction from sim_telarray files is not supported yet."
+                )
+                continue
             else:
                 raise gen.InvalidConfigDataError(f"Unknown metadata file format: {metadata_file}")
 

simtools/data_model/model_data_writer.py

@@ -270,6 +270,9 @@ class ModelDataWriter:
             "type": self._get_parameter_type(),
             "file": self._parameter_is_a_file(),
             "meta_parameter": False,
+            "model_parameter_schema_version": self.schema_dict.get(
+                "model_parameter_schema_version", "0.1.0"
+            ),
         }
         return self.validate_and_transform(
             product_data_dict=data_dict,

simtools/data_model/schema.py

@@ -18,7 +18,7 @@ from simtools.utils import names
 _logger = logging.getLogger(__name__)
 
 
-def get_get_model_parameter_schema_files(schema_directory=MODEL_PARAMETER_SCHEMA_PATH):
+def get_model_parameter_schema_files(schema_directory=MODEL_PARAMETER_SCHEMA_PATH):
     """
     Return list of parameters and schema files located in schema file directory.
 
@@ -35,7 +35,8 @@ def get_get_model_parameter_schema_files(schema_directory=MODEL_PARAMETER_SCHEMA
         raise FileNotFoundError(f"No schema files found in {schema_directory}")
     parameters = []
     for schema_file in schema_files:
-        schema_dict = gen.collect_data_from_file(file_name=schema_file)
+        # reading parameter 'name' only - first document in schema file should be ok
+        schema_dict = gen.collect_data_from_file(file_name=schema_file, yaml_document=0)
         parameters.append(schema_dict.get("name"))
     return parameters, schema_files
 
@@ -182,6 +183,15 @@ def load_schema(schema_file=None, schema_version=None):
     return schema
 
 
+def _get_array_element_list():
+    """Build complete list of array elements including design types."""
+    elements = set(names.array_elements().keys())
+    for array_element in names.array_elements():
+        for design_type in names.array_element_design_types(array_element):
+            elements.add(f"{array_element}-{design_type}")
+    return sorted(elements)
+
+
 def _add_array_elements(key, schema):
     """
     Add list of array elements to schema.
@@ -200,22 +210,23 @@ def _add_array_elements(key, schema):
     -------
     dict
         Schema dictionary with added array elements.
-
     """
-    _list_of_array_elements = sorted(names.array_elements().keys())
-
-    def recursive_search(sub_schema, key):
-        if key in sub_schema:
-            if "enum" in sub_schema[key] and isinstance(sub_schema[key]["enum"], list):
-                sub_schema[key]["enum"] = list(
-                    set(sub_schema[key]["enum"] + _list_of_array_elements)
-                )
-            else:
-                sub_schema[key]["enum"] = _list_of_array_elements
+    array_elements = _get_array_element_list()
+
+    def update_enum(sub_schema):
+        if "enum" in sub_schema and isinstance(sub_schema["enum"], list):
+            sub_schema["enum"] = list(set(sub_schema["enum"] + array_elements))
         else:
-            for _, v in sub_schema.items():
-                if isinstance(v, dict):
-                    recursive_search(v, key)
+            sub_schema["enum"] = array_elements
+
+    def recursive_search(sub_schema, target_key):
+        if target_key in sub_schema:
+            update_enum(sub_schema[target_key])
+            return
+
+        for v in sub_schema.values():
+            if isinstance(v, dict):
+                recursive_search(v, target_key)
 
     recursive_search(schema, key)
     return schema

simtools/data_model/validate_data.py

@@ -177,7 +177,11 @@ class DataValidator:
         if is_model_parameter:
             self._prepare_model_parameter()
 
-        self._data_description = self._read_validation_schema(self.schema_file_name)
+        self._data_description = self._read_validation_schema(
+            self.schema_file_name,
+            # use 0.1.0 as default; this corresponds to the first definition of the schema
+            self.data_dict.get("model_parameter_schema_version", "0.1.0"),
+        )
 
         value_as_list, unit_as_list = self._get_value_and_units_as_lists()
 
@@ -701,14 +705,20 @@ class DataValidator:
 
         return False
 
-    def _read_validation_schema(self, schema_file):
+    def _read_validation_schema(self, schema_file, schema_version=None):
         """
         Read validation schema from file.
 
+        The schema file can be a yaml file with several documents, each document
+        describing a different schema version. Returns first document if no
+        schema version is requested.
+
         Parameters
         ----------
         schema_file: Path
             Schema file describing input data.
+        schema_version: str
+            Version of the schema to be used.
 
         Returns
         -------
@@ -719,11 +729,21 @@ class DataValidator:
         ------
         KeyError
             if 'data' can not be read from dict in schema file
-        """
-        try:
-            return gen.collect_data_from_file(file_name=schema_file)["data"]
-        except KeyError as exc:
-            raise KeyError(f"Error reading validation schema from {schema_file}") from exc
+        ValueError
+            if schema version is not found in schema file
+        """
+        schema_data = gen.collect_data_from_file(file_name=schema_file)
+        entries = schema_data if isinstance(schema_data, list) else [schema_data]
+
+        for entry in entries:
+            if not schema_version or entry.get("version") == schema_version:
+                try:
+                    return entry["data"]
+                except KeyError as exc:
+                    raise KeyError(f"Error reading validation schema from {schema_file}") from exc
+        raise ValueError(
+            f"Schema version {schema_version} not found in schema file {schema_file}. "
+        )
 
     def _get_data_description(self, column_name=None, status_test=False):
         """

simtools/db/db_handler.py

@@ -122,9 +122,11 @@ class DatabaseHandler:
             port=self.mongo_db_config["db_api_port"],
             username=self.mongo_db_config["db_api_user"],
             password=self.mongo_db_config["db_api_pw"],
-            authSource=self.mongo_db_config.get("db_api_authentication_database")
-            if self.mongo_db_config.get("db_api_authentication_database")
-            else "admin",
+            authSource=(
+                self.mongo_db_config.get("db_api_authentication_database")
+                if self.mongo_db_config.get("db_api_authentication_database")
+                else "admin"
+            ),
             directConnection=direct_connection,
             ssl=not direct_connection,
             tlsallowinvalidhostnames=True,
@@ -211,7 +213,11 @@ class DatabaseHandler:
         """
         collection_name = names.get_collection_name_from_parameter_name(parameter)
         if model_version:
-            production_table = self._read_production_table_from_mongo_db(
+            if isinstance(model_version, list):
+                raise ValueError(
+                    "Only one model version can be passed to get_model_parameter, not a list."
+                )
+            production_table = self.read_production_table_from_mongo_db(
                 collection_name, model_version
             )
             array_element_list = self._get_array_element_list(
@@ -257,7 +263,7 @@ class DatabaseHandler:
         dict containing the parameters
         """
         pars = {}
-        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        production_table = self.read_production_table_from_mongo_db(collection, model_version)
         array_element_list = self._get_array_element_list(
             array_element_name, site, production_table, collection
         )
@@ -512,9 +518,9 @@ class DatabaseHandler:
             parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time
         return {k: parameters[k] for k in sorted(parameters)}
 
-    def _read_production_table_from_mongo_db(self, collection_name, model_version):
+    def read_production_table_from_mongo_db(self, collection_name, model_version):
         """
-        Read production table from MongoDB.
+        Read production table for the given collection from MongoDB.
 
         Parameters
         ----------
@@ -585,7 +591,7 @@ class DatabaseHandler:
         list
             Sorted list of all array elements found in collection
         """
-        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        production_table = self.read_production_table_from_mongo_db(collection, model_version)
         return sorted([entry for entry in production_table["parameters"] if "-design" not in entry])
 
     def get_design_model(self, model_version, array_element_name, collection="telescopes"):
@@ -607,7 +613,7 @@ class DatabaseHandler:
         str
             Design model for a given array element.
         """
-        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        production_table = self.read_production_table_from_mongo_db(collection, model_version)
         try:
             return production_table["design_model"][array_element_name]
         except KeyError:
@@ -635,7 +641,7 @@ class DatabaseHandler:
         list
             Sorted list of all array element names found in collection
         """
-        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        production_table = self.read_production_table_from_mongo_db(collection, model_version)
         all_array_elements = production_table["parameters"]
         return sorted(
             [
@@ -678,7 +684,7 @@ class DatabaseHandler:
                 model_version=model_version,
                 collection="configuration_corsika",
             )
-        if simulation_software == "simtel":
+        if simulation_software == "sim_telarray":
             return (
                 self.get_model_parameters(
                     site,
@@ -753,7 +759,7 @@ class DatabaseHandler:
         """
         db_name = self._get_db_name(db_name)
         collection = self.get_collection(db_name, "production_tables")
-        self._logger.info(f"Adding production for {production_table.get('collection')} to to DB")
+        self._logger.debug(f"Adding production for {production_table.get('collection')} to to DB")
         collection.insert_one(production_table)
         DatabaseHandler.production_table_cached.clear()
 
@@ -803,13 +809,13 @@ class DatabaseHandler:
                 raise ValueError(f"File is not UTF-8 encoded: {file_path}")
             files_to_add_to_db.add(f"{file_path}")
 
-        self._logger.info(
+        self._logger.debug(
             f"Adding a new entry to DB {db_name} and collection {collection_name}:\n{par_dict}"
         )
         collection.insert_one(par_dict)
 
         for file_to_insert_now in files_to_add_to_db:
-            self._logger.info(f"Will also add the file {file_to_insert_now} to the DB")
+            self._logger.debug(f"Will also add the file {file_to_insert_now} to the DB")
             self.insert_file_to_db(file_to_insert_now, db_name)
 
         self._reset_parameter_cache()
@@ -958,7 +964,7 @@ class DatabaseHandler:
             return [array_element_name]
         if collection == "configuration_sim_telarray":
             # get design model from 'telescope' or 'calibration_device' production tables
-            production_table = self._read_production_table_from_mongo_db(
+            production_table = self.read_production_table_from_mongo_db(
                 names.get_collection_name_from_array_element_name(array_element_name),
                 production_table["model_version"],
             )

simtools/db/db_model_upload.py

@@ -27,7 +27,7 @@ def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
         Path to location of all additional files to be uploaded.
     """
     par_dict = gen.collect_data_from_file(file_name=file)
-    logger.info(
+    logger.debug(
         f"Adding the following parameter to the DB: {par_dict['parameter']} "
         f"version {par_dict['parameter_version']} "
         f"(collection {collection} in database {db_name})"
@@ -120,7 +120,7 @@ def _read_production_table(model_dict, file, model_name):
         },
     )
     parameter_dict = gen.collect_data_from_file(file_name=file)
-    logger.info(f"Reading production table for {array_element} (collection {collection})")
+    logger.debug(f"Reading production table for {array_element} (collection {collection})")
     try:
         if array_element in ("configuration_corsika", "configuration_sim_telarray"):
             model_dict[collection]["parameters"] = parameter_dict["parameters"]

simtools/io_operations/io_handler.py

@@ -72,7 +72,7 @@ class IOHandler(metaclass=IOHandlerSingleton):
         label: str
             Instance label.
         sub_dir: str
-            Name of the subdirectory (ray-tracing, model etc)
+            Name of the subdirectory (ray_tracing, model etc)
 
         Returns
         -------
@@ -116,7 +116,7 @@ class IOHandler(metaclass=IOHandlerSingleton):
         label: str
             Instance label.
         sub_dir: str
-            Name of the subdirectory (ray-tracing, model etc)
+            Name of the subdirectory (ray_tracing, model etc)
 
         Returns
         -------