gammasimtools 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/configuration/commandline_parser.py

@@ -198,13 +198,6 @@ class CommandLineParser(argparse.ArgumentParser):
             required=False,
             default=None,
         )
-        _job_group.add_argument(
-            "--db_simulation_model_url",
-            help="simulation model repository URL",
-            type=str,
-            required=False,
-            default=None,
-        )
 
     def initialize_job_submission_arguments(self):
         """Initialize job submission arguments for simulator."""
@@ -243,12 +236,13 @@ class CommandLineParser(argparse.ArgumentParser):
             return
 
         _job_group = self.add_argument_group("simulation model")
-        _job_group.add_argument(
-            "--model_version",
-            help="model version",
-            type=str,
-            default=None,
-        )
+        if "model_version" in model_options:
+            _job_group.add_argument(
+                "--model_version",
+                help="model version",
+                type=str,
+                default=None,
+            )
         if any(
             option in model_options for option in ["site", "telescope", "layout", "layout_file"]
         ):
@@ -263,8 +257,8 @@ class CommandLineParser(argparse.ArgumentParser):
             _job_group.add_argument(
                 "--telescope_model_file",
                 help=(
-                    "File with changes to telescope model "
-                    " (yaml format; experimental with insufficient validation steps)."
+                    "Path to a YAML file containing modifications to the telescope model. "
+                    "This feature is intended for developers and lacks validation."
                 ),
                 type=Path,
                 required=False,

simtools/configuration/configurator.py

@@ -10,6 +10,7 @@ import astropy.units as u
 from dotenv import load_dotenv
 
 import simtools.configuration.commandline_parser as argparser
+from simtools.db.db_handler import jsonschema_db_dict
 from simtools.io_operations import io_handler
 from simtools.utils import general as gen
 
@@ -438,24 +439,10 @@ class Configurator:
         """
         Return parameters for DB configuration.
 
-        Parameters
-        ----------
+        Returns
+        -------
         dict
-            Dictionary with DB parameters
+            Dictionary with DB parameters.
         """
-        _db_dict = {}
-        _db_para = (
-            "db_api_user",
-            "db_api_pw",
-            "db_api_port",
-            "db_server",
-            "db_simulation_model",
-            "db_simulation_model_url",
-        )
-        try:
-            for _para in _db_para:
-                _db_dict[_para] = self.config[_para]
-        except KeyError:
-            pass
-
-        return _db_dict
+        db_params = jsonschema_db_dict["properties"].keys()
+        return {param: self.config.get(param) for param in db_params if param in self.config}

simtools/constants.py

@@ -1,9 +1,9 @@
 """Project wide constants."""
 
-from pathlib import Path
+from importlib.resources import files
 
 # Path to metadata jsonschema
-METADATA_JSON_SCHEMA = Path(__file__).parent / "../simtools/schemas/metadata.metaschema.yml"
+METADATA_JSON_SCHEMA = files("simtools") / "schemas/metadata.metaschema.yml"
 
 # Path to model parameter schema files
-MODEL_PARAMETER_SCHEMA_PATH = Path(__file__).parent / "../simtools/schemas/model_parameters"
+MODEL_PARAMETER_SCHEMA_PATH = files("simtools") / "schemas/model_parameters"

simtools/data_model/metadata_collector.py

@@ -86,6 +86,24 @@ class MetadataCollector:
             except AttributeError:
                 self._logger.debug(f"Method _fill_{meta_type}_meta not implemented")
 
+    def get_top_level_metadata(self):
+        """
+        Return top level metadata dictionary (with updated activity end time).
+
+        Returns
+        -------
+        dict
+            Top level metadata dictionary.
+
+        """
+        try:
+            self.top_level_meta[self.observatory]["activity"][
+                "end"
+            ] = datetime.datetime.now().isoformat(timespec="seconds")
+        except KeyError:
+            pass
+        return self.top_level_meta
+
     def get_data_model_schema_file_name(self):
         """
         Return data model schema file name.
@@ -523,7 +541,7 @@ class MetadataCollector:
         Returns
         -------
         bool
-            True if all entries are None, False otherwise.
+            True if all entries are None.
 
         """
         if not isinstance(input_dict, dict):

simtools/data_model/metadata_model.py

@@ -38,7 +38,10 @@ def validate_schema(data, schema_file):
         if validation fails
 
     """
-    schema, schema_file = _load_schema(schema_file)
+    schema, schema_file = _load_schema(
+        schema_file,
+        data.get("schema_version", "0.1.0"),  # default version to ensure backward compatibility
+    )
 
     try:
         jsonschema.validate(data, schema=schema, format_checker=format_checkers.format_checker)
@@ -72,17 +75,17 @@ def get_default_metadata_dict(schema_file=None, observatory="CTA"):
     return _fill_defaults(schema["definitions"], observatory)
 
 
-def _load_schema(schema_file=None):
+def _load_schema(schema_file=None, schema_version=None):
     """
     Load parameter schema from file from simpipe metadata schema.
 
     Returns
     -------
-    schema_file dict
-        Schema used for validation.
-    schema_file str
+    schema_file: str
         File name schema is loaded from. If schema_file is not given,
         the default schema file name is returned.
+    schema_version: str
+        Schema version.
 
     Raises
     ------
@@ -98,6 +101,16 @@ def _load_schema(schema_file=None):
     except FileNotFoundError:
         schema_file = files("simtools").joinpath("schemas") / schema_file
         schema = gen.collect_data_from_file(file_name=schema_file)
+
+    if isinstance(schema, list):  # schema file with several schemas defined
+        if schema_version is None:
+            raise ValueError(f"Schema version not given in {schema_file}.")
+        schema = next((doc for doc in schema if doc.get("version") == schema_version), None)
+        if schema is None:
+            raise ValueError(f"Schema version {schema_version} not found in {schema_file}.")
+    elif schema_version is not None and schema_version != schema.get("version"):
+        _logger.warning(f"Schema version {schema_version} does not match {schema.get('version')}")
+
     _logger.debug(f"Loading schema from {schema_file}")
     _add_array_elements("InstrumentTypeElement", schema)
 

simtools/data_model/model_data_writer.py

@@ -170,7 +170,7 @@ class ModelDataWriter:
             metadata_input_dict["output_file"] = output_file
             metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
             writer.write_metadata_to_yml(
-                metadata=MetadataCollector(args_dict=metadata_input_dict).top_level_meta,
+                metadata=MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata(),
                 yml_file=output_path / f"{Path(output_file).stem}",
             )
         return _json_dict

simtools/data_model/validate_data.py

@@ -12,6 +12,7 @@ from astropy.table import Column, Table, unique
 from astropy.utils.diff import report_diff_values
 
 import simtools.utils.general as gen
+from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
 from simtools.data_model import format_checkers
 from simtools.utils import value_conversion
 
@@ -57,7 +58,7 @@ class DataValidator:
         self.data_table = data_table
         self.check_exact_data_type = check_exact_data_type
 
-    def validate_and_transform(self, is_model_parameter=False):
+    def validate_and_transform(self, is_model_parameter=False, lists_as_strings=False):
         """
         Validate data and data file.
 
@@ -65,6 +66,8 @@ class DataValidator:
         ----------
         is_model_parameter: bool
             This is a model parameter (add some data preparation)
+        lists_as_strings: bool
+            Convert lists to strings (as needed for model parameters)
 
         Returns
         -------
@@ -80,13 +83,9 @@ class DataValidator:
         if self.data_file_name:
             self.validate_data_file()
         if isinstance(self.data_dict, dict):
-            if is_model_parameter:
-                self._prepare_model_parameter()
-            self._validate_data_dict()
-            return self.data_dict
+            return self._validate_data_dict(is_model_parameter, lists_as_strings)
         if isinstance(self.data_table, Table):
-            self._validate_data_table()
-            return self.data_table
+            return self._validate_data_table()
         self._logger.error("No data or data table to validate")
         raise TypeError
 
@@ -108,25 +107,57 @@ class DataValidator:
 
     def validate_parameter_and_file_name(self):
         """Validate that file name and key 'parameter_name' in data dict are the same."""
-        if self.data_dict.get("parameter") != Path(self.data_file_name).stem:
+        if not str(Path(self.data_file_name).stem).startswith(self.data_dict.get("parameter")):
             raise ValueError(
                 f"Parameter name in data dict {self.data_dict.get('parameter')} and "
                 f"file name {Path(self.data_file_name).stem} do not match."
             )
 
-    def _validate_data_dict(self):
+    @staticmethod
+    def validate_model_parameter(par_dict):
+        """
+        Validate a simulation model parameter (static method).
+
+        Parameters
+        ----------
+        par_dict: dict
+            Data dictionary
+
+        Returns
+        -------
+        dict
+            Validated data dictionary
+        """
+        data_validator = DataValidator(
+            schema_file=MODEL_PARAMETER_SCHEMA_PATH / "f{par_dict['parameter']}.schema.yml",
+            data_dict=par_dict,
+            check_exact_data_type=False,
+        )
+        return data_validator.validate_and_transform(is_model_parameter=True)
+
+    def _validate_data_dict(self, is_model_parameter=False, lists_as_strings=False):
         """
         Validate values in a dictionary.
 
         Handles different types of naming in data dicts (using 'name' or 'parameter'
         keys for name fields).
 
+        Parameters
+        ----------
+        is_model_parameter: bool
+            This is a model parameter (add some data preparation)
+        lists_as_strings: bool
+            Convert lists to strings (as needed for model parameters)
+
         Raises
         ------
         KeyError
             if data dict does not contain a 'name' or 'parameter' key.
 
         """
+        if is_model_parameter:
+            self._prepare_model_parameter()
+
         if not (_name := self.data_dict.get("name") or self.data_dict.get("parameter")):
             raise KeyError("Data dict does not contain a 'name' or 'parameter' key.")
         self._data_description = self._read_validation_schema(self.schema_file_name, _name)
@@ -145,6 +176,11 @@ class DataValidator:
 
         self._check_version_string(self.data_dict.get("version"))
 
+        if lists_as_strings:
+            self._convert_results_to_model_format()
+
+        return self.data_dict
+
     def _validate_value_and_unit(self, value, unit, index):
         """
         Validate value, unit, and perform type checking and conversions.
@@ -191,7 +227,13 @@ class DataValidator:
         conversion_factor = [
             1 if v is None else u.Unit(v).to(u.Unit(t)) for v, t in zip(unit, target_unit)
         ]
-        return [v * c for v, c in zip(value, conversion_factor)], target_unit
+        try:
+            return [
+                v * c if not isinstance(v, bool) and not isinstance(v, dict) else v
+                for v, c in zip(value, conversion_factor)
+            ], target_unit
+        except TypeError:
+            return [None], target_unit
 
     def _validate_data_dict_using_json_schema(self, data, json_schema):
         """
@@ -228,6 +270,7 @@ class DataValidator:
             self._validate_data_columns()
             self._check_data_for_duplicates()
             self._sort_data()
+        return self.data_table
 
     def _validate_data_columns(self):
         """
@@ -726,6 +769,8 @@ class DataValidator:
                 )
             )
         except IndexError as exc:
+            if len(self._data_description) == 1:  # all columns are described by the same schema
+                return self._data_description[0]
             self._logger.error(
                 f"Data column '{column_name}' not found in reference column definition"
             )
@@ -771,6 +816,18 @@ class DataValidator:
         if self.data_dict["unit"] is not None:
             self.data_dict["unit"] = gen.convert_string_to_list(self.data_dict["unit"])
 
+    def _convert_results_to_model_format(self):
+        """
+        Convert results to model format.
+
+        Convert lists to strings (as needed for model parameters).
+        """
+        value = self.data_dict["value"]
+        if isinstance(value, list):
+            self.data_dict["value"] = gen.convert_list_to_string(value)
+        if isinstance(self.data_dict["unit"], list):
+            self.data_dict["unit"] = gen.convert_list_to_string(self.data_dict["unit"])
+
     def _check_version_string(self, version):
         """
         Check that version string follows semantic versioning.