gammasimtools 0.21.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

simtools/corsika/corsika_config.py

@@ -114,8 +114,8 @@ class CorsikaConfig:
             return {}
 
         self.is_file_updated = False
-        self.azimuth_angle = int(args_dict["azimuth_angle"].to("deg").value)
-        self.zenith_angle = args_dict["zenith_angle"].to("deg").value
+        self.azimuth_angle = int(args_dict.get("azimuth_angle", 0.0 * u.deg).to("deg").value)
+        self.zenith_angle = int(args_dict.get("zenith_angle", 0.0 * u.deg).to("deg").value)
 
         self._logger.debug(
             f"Setting CORSIKA parameters from database ({args_dict['model_version']})"
@@ -563,8 +563,9 @@ class CorsikaConfig:
         if self.is_file_updated:
             self._logger.debug(f"CORSIKA input file already updated: {self.config_file_path}")
             return self.config_file_path
-        _output_generic_file_name = self.set_output_file_and_directory(use_multipipe=use_multipipe)
         self._logger.info(f"Exporting CORSIKA input file to {self.config_file_path}")
+        _output_generic_file_name = self.set_output_file_and_directory(use_multipipe=use_multipipe)
+        self._logger.info(f"Output generic file name: {_output_generic_file_name}")
 
         with open(self.config_file_path, "w", encoding="utf-8") as file:
             file.write("\n* [ RUN PARAMETERS ]\n")
@@ -712,13 +713,10 @@ class CorsikaConfig:
         str
             Output file name.
         """
-        sub_dir = "corsika_sim_telarray" if use_multipipe else "corsika"
-        config_file_name = self.get_corsika_config_file_name(file_type="config")
-        file_directory = self.io_handler.get_output_directory(label=self.label, sub_dir=sub_dir)
-        self._logger.debug(f"Creating directory {file_directory}")
-        file_directory.mkdir(parents=True, exist_ok=True)
-        self.config_file_path = file_directory.joinpath(config_file_name)
-
+        self.config_file_path = self.io_handler.get_output_file(
+            file_name=self.get_corsika_config_file_name(file_type="config"),
+            sub_dir="corsika_sim_telarray" if use_multipipe else "corsika",
+        )
         return self.get_corsika_config_file_name(file_type="output_generic")
 
     def _write_seeds(self, file, use_test_seeds=False):
@@ -748,7 +746,7 @@ class CorsikaConfig:
             Piece of text to be added to the CORSIKA input file.
         """
         corsika_input_list = ""
-        for telescope_name, telescope in self.array_model.telescope_model.items():
+        for telescope_name, telescope in self.array_model.telescope_models.items():
             positions = telescope.get_parameter_value_with_unit("array_element_position_ground")
             corsika_input_list += "TELESCOPE"
             for pos in positions:

simtools/corsika/corsika_histograms.py

@@ -67,7 +67,7 @@ class CorsikaHistograms:
             raise FileNotFoundError
 
         self.io_handler = io_handler.IOHandler()
-        _default_output_path = self.io_handler.get_output_directory(self.label, "corsika")
+        _default_output_path = self.io_handler.get_output_directory("corsika")
 
         if output_path is None:
             self.output_path = _default_output_path

simtools/data_model/model_data_writer.py

@@ -3,6 +3,7 @@
 import logging
 from pathlib import Path
 
+import packaging.version
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
@@ -29,8 +30,6 @@ class ModelDataWriter:
         Dictionary with configuration parameters.
     output_path: str or Path
         Path to output file.
-    use_plain_output_path: bool
-        Use plain output path.
     args_dict: dict
         Dictionary with configuration parameters.
 
@@ -41,7 +40,6 @@ class ModelDataWriter:
         product_data_file=None,
         product_data_format=None,
         output_path=None,
-        use_plain_output_path=True,
         args_dict=None,
     ):
         """Initialize model data writer."""
@@ -50,11 +48,8 @@ class ModelDataWriter:
         self.schema_dict = {}
         if args_dict is not None:
             output_path = args_dict.get("output_path", output_path)
-            use_plain_output_path = args_dict.get("use_plain_output_path", use_plain_output_path)
         if output_path is not None:
-            self.io_handler.set_paths(
-                output_path=output_path, use_plain_output_path=use_plain_output_path
-            )
+            self.io_handler.set_paths(output_path=output_path)
         try:
             self.product_data_file = self.io_handler.get_output_file(file_name=product_data_file)
         except TypeError:
@@ -104,9 +99,10 @@ class ModelDataWriter:
         parameter_version,
         output_file,
         output_path=None,
-        use_plain_output_path=False,
         metadata_input_dict=None,
         db_config=None,
+        unit=None,
+        meta_parameter=False,
     ):
         """
         Generate DB-style model parameter dict and write it to json file.
@@ -125,12 +121,12 @@ class ModelDataWriter:
             Name of output file.
         output_path: str or Path
             Path to output file.
-        use_plain_output_path: bool
-            Use plain output path.
         metadata_input_dict: dict
             Input to metadata collector.
         db_config: dict
             Database configuration. If not None, check if parameter with the same version exists.
+        unit: str
+            Unit of the parameter value (if applicable and value is not of type astropy Quantity).
 
         Returns
         -------
@@ -142,7 +138,6 @@ class ModelDataWriter:
             product_data_format="json",
             args_dict=None,
             output_path=output_path,
-            use_plain_output_path=use_plain_output_path,
         )
         if db_config is not None:
             writer.check_db_for_existing_parameter(
@@ -160,7 +155,13 @@ class ModelDataWriter:
             )
 
         _json_dict = writer.get_validated_parameter_dict(
-            parameter_name, value, instrument, parameter_version, unique_id
+            parameter_name,
+            value,
+            instrument,
+            parameter_version,
+            unique_id,
+            unit=unit,
+            meta_parameter=meta_parameter,
         )
         writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         return _json_dict
@@ -210,6 +211,8 @@ class ModelDataWriter:
         parameter_version,
         unique_id=None,
         schema_version=None,
+        unit=None,
+        meta_parameter=False,
     ):
         """
         Get validated parameter dictionary.
@@ -226,6 +229,12 @@ class ModelDataWriter:
             Version of the parameter.
         schema_version: str
             Version of the schema.
+        unique_id: str
+            Unique ID of the parameter set (from metadata).
+        unit: str
+            Unit of the parameter value (if applicable and value is not an astropy Quantity).
+        meta_parameter: bool
+            Setting for meta parameter flag.
 
         Returns
         -------
@@ -233,10 +242,10 @@ class ModelDataWriter:
             Validated parameter dictionary.
         """
         self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
-        schema_file = schema.get_model_parameter_schema_file(parameter_name)
-        self.schema_dict = ascii_handler.collect_data_from_file(schema_file)
+        self.schema_dict, schema_file = self._read_schema_dict(parameter_name, schema_version)
 
-        value, unit = value_conversion.split_value_and_unit(value)
+        if unit is None:
+            value, unit = value_conversion.split_value_and_unit(value)
 
         data_dict = {
             "schema_version": schema.get_model_parameter_schema_version(schema_version),
@@ -249,10 +258,8 @@ class ModelDataWriter:
             "unit": unit,
             "type": self._get_parameter_type(),
             "file": self._parameter_is_a_file(),
-            "meta_parameter": False,
-            "model_parameter_schema_version": self.schema_dict.get(
-                "model_parameter_schema_version", "0.1.0"
-            ),
+            "meta_parameter": meta_parameter,
+            "model_parameter_schema_version": self.schema_dict.get("schema_version", "0.1.0"),
         }
         return self.validate_and_transform(
             product_data_dict=data_dict,
@@ -260,19 +267,74 @@ class ModelDataWriter:
             is_model_parameter=True,
         )
 
+    def _read_schema_dict(self, parameter_name, schema_version):
+        """
+        Read schema dict for given parameter name and version.
+
+        Use newest schema version if schema_version is None.
+
+        Parameters
+        ----------
+        parameter_name: str
+            Name of the parameter.
+        schema_version: str
+            Schema version.
+
+        Returns
+        -------
+        dict
+            Schema dictionary.
+        """
+        schema_file = schema.get_model_parameter_schema_file(parameter_name)
+        schemas = ascii_handler.collect_data_from_file(schema_file)
+        if isinstance(schemas, list):
+            if schema_version is None:
+                return self._find_highest_schema_version(schemas), schema_file
+            for entry in schemas:
+                if entry.get("schema_version") == schema_version:
+                    return entry, schema_file
+        else:
+            return schemas, schema_file
+
+        raise ValueError(f"Schema version {schema_version} not found for {parameter_name}")
+
+    def _find_highest_schema_version(self, schema_list):
+        """
+        Find entry with highest schema_version in a list of schema dicts.
+
+        Parameters
+        ----------
+        schema_list: list
+            List of schema dictionaries.
+
+        Returns
+        -------
+        dict
+            Schema dictionary with highest schema_version.
+        """
+        try:
+            valid_entries = [entry for entry in schema_list if "schema_version" in entry]
+        except TypeError as exc:
+            raise TypeError("No valid schema versions found in the list.") from exc
+        return max(valid_entries, key=lambda e: packaging.version.Version(e["schema_version"]))
+
     def _get_parameter_type(self):
         """
         Return parameter type from schema.
 
+        Reduce list of types to single type if all types are the same.
+
         Returns
         -------
-        str
+        str or list[str]
             Parameter type
         """
-        _parameter_type = []
-        for data in self.schema_dict["data"]:
-            _parameter_type.append(data["type"])
-        return _parameter_type if len(_parameter_type) > 1 else _parameter_type[0]
+        _parameter_type = [data["type"] for data in self.schema_dict["data"]]
+        return (
+            _parameter_type[0]
+            if all(t == _parameter_type[0] for t in _parameter_type)
+            else _parameter_type
+        )
 
     def _parameter_is_a_file(self):
         """
@@ -399,7 +461,7 @@ class ModelDataWriter:
         ascii_handler.write_data_to_file(
             data=data_dict,
             output_file=self.io_handler.get_output_file(file_name),
-            sort_keys=False,
+            sort_keys=True,
             numpy_types=True,
         )
 

simtools/data_model/schema.py

@@ -4,9 +4,12 @@ import logging
 from pathlib import Path
 
 import jsonschema
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
 from referencing import Registry, Resource
 
 import simtools.utils.general as gen
+from simtools import version
 from simtools.constants import (
     METADATA_JSON_SCHEMA,
     MODEL_PARAMETER_METASCHEMA,
@@ -66,7 +69,7 @@ def get_model_parameter_schema_file(parameter):
 
 def get_model_parameter_schema_version(schema_version=None):
     """
-    Validate  and return schema versions.
+    Validate and return schema versions.
 
     If no schema_version is given, the most recent version is provided.
 
@@ -92,7 +95,9 @@ def get_model_parameter_schema_version(schema_version=None):
     raise ValueError(f"Schema version {schema_version} not found in {MODEL_PARAMETER_METASCHEMA}.")
 
 
-def validate_dict_using_schema(data, schema_file=None, json_schema=None):
+def validate_dict_using_schema(
+    data, schema_file=None, json_schema=None, ignore_software_version=False
+):
     """
     Validate a data dictionary against a schema.
 
@@ -102,6 +107,10 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
         dictionary to be validated
     schema_file (dict)
         schema used for validation
+    json_schema (dict)
+        schema used for validation
+    ignore_software_version: bool
+        If True, ignore software version check.
 
     Raises
     ------
@@ -115,6 +124,8 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     if json_schema is None:
         json_schema = load_schema(schema_file, get_schema_version_from_data(data))
 
+    _validate_deprecation_and_version(data, ignore_software_version)
+
     validator = jsonschema.Draft6Validator(
         schema=json_schema,
         format_checker=format_checkers.format_checker,
@@ -294,3 +305,51 @@ def _add_array_elements(key, schema):
 
     recursive_search(schema, key)
     return schema
+
+
+def _validate_deprecation_and_version(
+    data, software_name="simtools", ignore_software_version=False
+):
+    """
+    Check if data contains deprecated parameters or version mismatches.
+
+    Parameters
+    ----------
+    data: dict
+        Data dictionary to check.
+    software_name: str
+        Name of the software to check version against.
+    ignore_software_version: bool
+        If True, ignore software version check.
+    """
+    if not isinstance(data, dict):
+        return
+
+    if data.get("deprecated", False):
+        note = data.get("deprecation_note", "(no deprecation note provided)")
+        _logger.warning(f"Data is deprecated. Note: {note}")
+
+    def check_version(sw):
+        constraint = sw.get("version")
+        if constraint is None:
+            return
+        constraint = constraint.strip()
+        spec = SpecifierSet(constraint, prereleases=True)
+        if Version(version.__version__) in spec:
+            _logger.debug(
+                f"Version {version.__version__} of {software_name} matches constraint {constraint}."
+            )
+        else:
+            msg = (
+                f"Version {version.__version__} of {software_name} "
+                f"does not match constraint {constraint}."
+            )
+            if ignore_software_version:
+                _logger.warning(f"{msg}, but version check is ignored.")
+                return
+            raise ValueError(msg)
+
+    for sw in data.get("simulation_software", []):
+        if sw.get("name") == software_name:
+            check_version(sw)
+            break

simtools/data_model/validate_data.py

@@ -232,7 +232,7 @@ class DataValidator:
         else:
             self._check_data_type(np.array(value).dtype, index, value)
 
-        if self.data_dict.get("type") not in ("string", "dict", "file"):
+        if self._get_data_description(index).get("type", None) not in ("string", "dict", "file"):
             self._check_for_not_a_number(value, index)
             value, unit = self._check_and_convert_units(value, unit, index)
             for range_type in ("allowed_range", "required_range"):

simtools/db/db_handler.py

@@ -11,7 +11,6 @@ import gridfs
 import jsonschema
 from astropy.table import Table
 from bson.objectid import ObjectId
-from packaging.version import Version
 from pymongo import MongoClient
 
 from simtools.data_model import validate_data
@@ -100,7 +99,6 @@ class DatabaseHandler:
         self.list_of_collections = {}
 
         self._set_up_connection()
-        self._find_latest_simulation_model_db()
         self.db_name = (
             self.get_db_name(
                 db_simulation_model_version=self.mongo_db_config.get("db_simulation_model_version"),
@@ -127,6 +125,34 @@ class DatabaseHandler:
             return None
         return None if (db_simulation_model_version or model_name) else self.db_name
 
+    def print_connection_info(self):
+        """Print the connection information."""
+        if self.mongo_db_config:
+            self._logger.info(
+                f"Connected to MongoDB at {self.mongo_db_config['db_server']}:"
+                f"{self.mongo_db_config['db_api_port']} "
+                f"using database: {self.db_name}"
+            )
+        else:
+            self._logger.info("No MongoDB configuration provided.")
+
+    def is_remote_database(self):
+        """
+        Check if the database is remote.
+
+        Check for domain pattern like "cta-simpipe-protodb.zeuthen.desy.de"
+
+        Returns
+        -------
+        bool
+            True if the database is remote, False otherwise.
+        """
+        if self.mongo_db_config:
+            db_server = self.mongo_db_config["db_server"]
+            domain_pattern = r"^([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$"
+            return bool(re.match(domain_pattern, db_server))
+        return False
+
     def _validate_mongo_db_config(self, mongo_db_config):
         """Validate the MongoDB configuration."""
         if mongo_db_config is None or all(value is None for value in mongo_db_config.values()):
@@ -171,50 +197,45 @@ class DatabaseHandler:
             tlsallowinvalidcertificates=True,
         )
 
-    def _find_latest_simulation_model_db(self):
+    def generate_compound_indexes_for_databases(
+        self, db_name, db_simulation_model, db_simulation_model_version
+    ):
         """
-        Find the latest released version of the simulation model and update the DB config.
+        Generate compound indexes for several databases.
 
-        This is indicated by "LATEST" to the simulation model data base version.
-        Only released versions are considered, pre-releases are ignored.
+        Parameters
+        ----------
+        db_name: str
+            Name of the database.
+        db_simulation_model: str
+            Name of the simulation model.
+        db_simulation_model_version: str
+            Version of the simulation model.
 
         Raises
         ------
         ValueError
-            If the "LATEST" version is requested but no versions are found in the DB.
+            If the requested database is not found.
 
         """
-        try:
-            db_simulation_model_version = self.mongo_db_config["db_simulation_model_version"]
-            db_simulation_model = self.mongo_db_config["db_simulation_model"]
-            if db_simulation_model_version != "LATEST":
-                return
-        except TypeError:  # db_simulation_model_version is None
-            return
-
-        list_of_db_names = self.db_client.list_database_names()
-        filtered_list_of_db_names = [
-            s for s in list_of_db_names if s.startswith(db_simulation_model)
+        databases = [
+            d for d in self.db_client.list_database_names() if d not in ("config", "admin", "local")
         ]
-        pattern = re.compile(rf"{re.escape(db_simulation_model)}-v(\d+)-(\d+)-(\d+)(?:-(.+))?$")
-
-        versioned_strings = []
-        for s in filtered_list_of_db_names:
-            m = pattern.match(s)
-            if m:
-                # skip pre-releases (have suffix)
-                if m.group(4) is None:
-                    version_str = f"{m.group(1)}.{m.group(2)}.{m.group(3)}"
-                    versioned_strings.append((s, Version(version_str)))
-
-        if versioned_strings:
-            latest_string, _ = max(versioned_strings, key=lambda x: x[1])
-            self.mongo_db_config["db_simulation_model"] = latest_string
-            self._logger.info(
-                f"Updated the DB simulation model to the latest version {latest_string}"
+        requested = self.get_db_name(
+            db_name=db_name,
+            db_simulation_model_version=db_simulation_model_version,
+            model_name=db_simulation_model,
+        )
+        if requested != "all" and requested not in databases:
+            raise ValueError(
+                f"Requested database '{requested}' not found. "
+                f"Following databases are available: {', '.join(databases)}"
             )
-        else:
-            raise ValueError("LATEST requested but no released versions found in DB.")
+
+        databases = databases if requested == "all" else [requested]
+        for dbs in databases:
+            self._logger.info(f"Generating compound indexes for database: {dbs}")
+            self.generate_compound_indexes(db_name=dbs)
 
     def generate_compound_indexes(self, db_name=None):
         """
@@ -875,7 +896,7 @@ class DatabaseHandler:
         """
         db_name = db_name or self.db_name
         collection = self.get_collection("production_tables", db_name=db_name or self.db_name)
-        self._logger.debug(f"Adding production for {production_table.get('collection')} to to DB")
+        self._logger.debug(f"Adding production for {production_table.get('collection')} to the DB")
         collection.insert_one(production_table)
         DatabaseHandler.production_table_cached.clear()
         DatabaseHandler.model_versions_cached.clear()