gammasimtools 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/configuration/commandline_parser.py

@@ -239,7 +239,14 @@ class CommandLineParser(argparse.ArgumentParser):
         if "model_version" in model_options:
             _job_group.add_argument(
                 "--model_version",
-                help="model version",
+                help="production model version",
+                type=str,
+                default=None,
+            )
+        if "parameter_version" in model_options:
+            _job_group.add_argument(
+                "--parameter_version",
+                help="model parameter version",
                 type=str,
                 default=None,
             )

simtools/constants.py

@@ -2,8 +2,15 @@
 
 from importlib.resources import files
 
+# Schema path
+SCHEMA_PATH = files("simtools") / "schemas"
 # Path to metadata jsonschema
-METADATA_JSON_SCHEMA = files("simtools") / "schemas/metadata.metaschema.yml"
-
+METADATA_JSON_SCHEMA = SCHEMA_PATH / "metadata.metaschema.yml"
+# Path to model parameter metaschema
+MODEL_PARAMETER_METASCHEMA = SCHEMA_PATH / "model_parameter.metaschema.yml"
+# Path to model parameter description metaschema
+MODEL_PARAMETER_DESCRIPTION_METASCHEMA = (
+    SCHEMA_PATH / "model_parameter_and_data_schema.metaschema.yml"
+)
 # Path to model parameter schema files
-MODEL_PARAMETER_SCHEMA_PATH = files("simtools") / "schemas/model_parameters"
+MODEL_PARAMETER_SCHEMA_PATH = SCHEMA_PATH / "model_parameters"

simtools/corsika/corsika_config.py

@@ -6,7 +6,6 @@ from pathlib import Path
 import numpy as np
 from astropy import units as u
 
-import simtools.utils.general as gen
 from simtools.corsika.primary_particle import PrimaryParticle
 from simtools.io_operations import io_handler
 from simtools.model.model_parameter import ModelParameter
@@ -111,8 +110,6 @@ class CorsikaConfig:
         if args_dict is None:
             return {}
 
-        self._logger.debug("Setting CORSIKA parameters ")
-
         self._is_file_updated = False
         self.azimuth_angle = int(args_dict["azimuth_angle"].to("deg").value)
         self.zenith_angle = args_dict["zenith_angle"].to("deg").value
@@ -243,7 +240,7 @@ class CorsikaConfig:
 
     def _input_config_corsika_particle_kinetic_energy_cutoff(self, entry):
         """Return ECUTS parameter CORSIKA format."""
-        e_cuts = gen.convert_string_to_list(entry["value"])
+        e_cuts = entry["value"]
         return [
             f"{e_cuts[0]*u.Unit(entry['unit']).to('GeV')} "
             f"{e_cuts[1]*u.Unit(entry['unit']).to('GeV')} "
@@ -280,7 +277,7 @@ class CorsikaConfig:
 
     def _input_config_corsika_cherenkov_wavelength(self, entry):
         """Return CWAVLG parameter CORSIKA format."""
-        wavelength_range = gen.convert_string_to_list(entry["value"])
+        wavelength_range = entry["value"]
         return [
             f"{wavelength_range[0]*u.Unit(entry['unit']).to('nm')}",
             f"{wavelength_range[1]*u.Unit(entry['unit']).to('nm')}",
@@ -318,8 +315,12 @@ class CorsikaConfig:
         }
 
     def _input_config_io_buff(self, entry):
-        """Return IO_BUFFER parameter CORSIKA format."""
-        return f"{entry['value']}{entry['unit']}"
+        """Return IO_BUFFER parameter CORSIKA format (Byte or MB required)."""
+        value = entry["value"] * u.Unit(entry["unit"]).to("Mbyte")
+        # check if value is integer-like
+        if value.is_integer():
+            return f"{int(value)}MB"
+        return f"{int(entry['value'] * u.Unit(entry['unit']).to('byte'))}"
 
     def _rotate_azimuth_by_180deg(self, az, correct_for_geomagnetic_field_alignment=True):
         """

simtools/corsika/corsika_histograms.py

@@ -678,7 +678,7 @@ class CorsikaHistograms:
         ----------
         new_individual_telescopes: bool
             if False, the histograms are supposed to be filled for all telescopes.
-            if True, one histogram is set for each telescope sepparately.
+            if True, one histogram is set for each telescope separately.
         """
         if new_individual_telescopes is None:
             self._individual_telescopes = False

simtools/data_model/data_reader.py

@@ -112,9 +112,6 @@ def read_value_from_file(file_name, schema_file=None, validate=False):
     _logger.info("Reading data from %s", file_name)
 
     if validate:
-        if schema_file is None and "meta_schema_url" in data:
-            schema_file = data["meta_schema_url"]
-            _logger.debug(f"Using schema from meta_schema_url: {schema_file}")
         if schema_file is None:
             _collector = MetadataCollector(None, metadata_file_name=file_name)
             schema_file = _collector.get_data_model_schema_file_name()

simtools/data_model/metadata_collector.py

@@ -10,13 +10,12 @@ import datetime
 import getpass
 import logging
 import uuid
-from importlib.resources import files
 from pathlib import Path
 
 import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
-from simtools.data_model import metadata_model
+from simtools.data_model import metadata_model, schema
 from simtools.io_operations import io_handler
 from simtools.utils import names
 
@@ -135,7 +134,7 @@ class MetadataCollector:
         # from data model name
         if self.data_model_name:
             self._logger.debug(f"Schema file from data model name: {self.data_model_name}")
-            return f"{files('simtools')}/schemas/model_parameters/{self.data_model_name}.schema.yml"
+            return str(schema.get_model_parameter_schema_file(self.data_model_name))
 
         # from input metadata
         try:
@@ -267,7 +266,7 @@ class MetadataCollector:
             self._logger.error("Unknown metadata file format: %s", metadata_file_name)
             raise gen.InvalidConfigDataError
 
-        metadata_model.validate_schema(_input_metadata, None)
+        schema.validate_dict_using_schema(_input_metadata, None)
 
         return gen.change_dict_keys_case(
             self._process_metadata_from_file(_input_metadata),

simtools/data_model/metadata_model.py

@@ -9,48 +9,12 @@ Follows CTAO top-level data model definition.
 """
 
 import logging
-from importlib.resources import files
 
-import jsonschema
-
-import simtools.constants
-import simtools.utils.general as gen
-from simtools.data_model import format_checkers
-from simtools.utils import names
+import simtools.data_model.schema
 
 _logger = logging.getLogger(__name__)
 
 
-def validate_schema(data, schema_file):
-    """
-    Validate dictionary against schema.
-
-    Parameters
-    ----------
-    data
-        dictionary to be validated
-    schema_file (dict)
-        schema used for validation
-
-    Raises
-    ------
-    jsonschema.exceptions.ValidationError
-        if validation fails
-
-    """
-    schema, schema_file = _load_schema(
-        schema_file,
-        data.get("schema_version", "0.1.0"),  # default version to ensure backward compatibility
-    )
-
-    try:
-        jsonschema.validate(data, schema=schema, format_checker=format_checkers.format_checker)
-    except jsonschema.exceptions.ValidationError:
-        _logger.error(f"Failed using {schema}")
-        raise
-    _logger.debug(f"Successful validation of data using schema from {schema_file}")
-
-
 def get_default_metadata_dict(schema_file=None, observatory="CTA"):
     """
     Return metadata schema with default values.
@@ -71,90 +35,10 @@ def get_default_metadata_dict(schema_file=None, observatory="CTA"):
 
 
     """
-    schema, _ = _load_schema(schema_file)
+    schema = simtools.data_model.schema.load_schema(schema_file)
     return _fill_defaults(schema["definitions"], observatory)
 
 
-def _load_schema(schema_file=None, schema_version=None):
-    """
-    Load parameter schema from file from simpipe metadata schema.
-
-    Returns
-    -------
-    schema_file: str
-        File name schema is loaded from. If schema_file is not given,
-        the default schema file name is returned.
-    schema_version: str
-        Schema version.
-
-    Raises
-    ------
-    FileNotFoundError
-        if schema file is not found
-
-    """
-    if schema_file is None:
-        schema_file = files("simtools").joinpath(simtools.constants.METADATA_JSON_SCHEMA)
-
-    try:
-        schema = gen.collect_data_from_file(file_name=schema_file)
-    except FileNotFoundError:
-        schema_file = files("simtools").joinpath("schemas") / schema_file
-        schema = gen.collect_data_from_file(file_name=schema_file)
-
-    if isinstance(schema, list):  # schema file with several schemas defined
-        if schema_version is None:
-            raise ValueError(f"Schema version not given in {schema_file}.")
-        schema = next((doc for doc in schema if doc.get("version") == schema_version), None)
-        if schema is None:
-            raise ValueError(f"Schema version {schema_version} not found in {schema_file}.")
-    elif schema_version is not None and schema_version != schema.get("version"):
-        _logger.warning(f"Schema version {schema_version} does not match {schema.get('version')}")
-
-    _logger.debug(f"Loading schema from {schema_file}")
-    _add_array_elements("InstrumentTypeElement", schema)
-
-    return schema, schema_file
-
-
-def _add_array_elements(key, schema):
-    """
-    Add list of array elements to schema.
-
-    This assumes an element [key]['enum'] is a list of elements.
-
-    Parameters
-    ----------
-    key: str
-        Key in schema dictionary
-    schema: dict
-        Schema dictionary
-
-    Returns
-    -------
-    dict
-        Schema dictionary with added array elements.
-
-    """
-    _list_of_array_elements = sorted(names.array_elements().keys())
-
-    def recursive_search(sub_schema, key):
-        if key in sub_schema:
-            if "enum" in sub_schema[key] and isinstance(sub_schema[key]["enum"], list):
-                sub_schema[key]["enum"] = list(
-                    set(sub_schema[key]["enum"] + _list_of_array_elements)
-                )
-            else:
-                sub_schema[key]["enum"] = _list_of_array_elements
-        else:
-            for _, v in sub_schema.items():
-                if isinstance(v, dict):
-                    recursive_search(v, key)
-
-    recursive_search(schema, key)
-    return schema
-
-
 def _resolve_references(yaml_data, observatory="CTA"):
     """
     Resolve references in yaml data and expand the received dictionary accordingly.
@@ -227,21 +111,21 @@ def _fill_defaults(schema, observatory="CTA"):
     return defaults
 
 
-def _fill_defaults_recursive(subschema, current_dict):
+def _fill_defaults_recursive(sub_schema, current_dict):
     """
-    Recursively fill default values from the subschema into the current dictionary.
+    Recursively fill default values from the sub_schema into the current dictionary.
 
     Parameters
     ----------
-    subschema: dict
-        Subschema describing part of the input data.
+    sub_schema: dict
+        Sub schema describing part of the input data.
     current_dict: dict
         Current dictionary to fill with default values.
     """
-    if "properties" not in subschema:
+    if "properties" not in sub_schema:
         _raise_missing_properties_error()
 
-    for prop, prop_schema in subschema["properties"].items():
+    for prop, prop_schema in sub_schema["properties"].items():
         _process_property(prop, prop_schema, current_dict)
 
 

simtools/data_model/model_data_writer.py

@@ -10,8 +10,7 @@ import yaml
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
-from simtools.data_model import validate_data
+from simtools.data_model import schema, validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io_operations import io_handler
 from simtools.utils import names, value_conversion
@@ -122,7 +121,7 @@ class ModelDataWriter:
         parameter_name,
         value,
         instrument,
-        model_version,
+        parameter_version,
         output_file,
         output_path=None,
         use_plain_output_path=False,
@@ -139,8 +138,8 @@ class ModelDataWriter:
             Value of the parameter.
         instrument: str
             Name of the instrument.
-        model_version: str
-            Version of the model.
+        parameter_version: str
+            Version of the parameter.
         output_file: str
             Name of output file.
         output_path: str or Path
@@ -163,7 +162,7 @@ class ModelDataWriter:
             use_plain_output_path=use_plain_output_path,
         )
         _json_dict = writer.get_validated_parameter_dict(
-            parameter_name, value, instrument, model_version
+            parameter_name, value, instrument, parameter_version
         )
         writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         if metadata_input_dict is not None:
@@ -175,7 +174,9 @@ class ModelDataWriter:
             )
         return _json_dict
 
-    def get_validated_parameter_dict(self, parameter_name, value, instrument, model_version):
+    def get_validated_parameter_dict(
+        self, parameter_name, value, instrument, parameter_version, schema_version=None
+    ):
         """
         Get validated parameter dictionary.
 
@@ -187,8 +188,10 @@ class ModelDataWriter:
             Value of the parameter.
         instrument: str
             Name of the instrument.
-        model_version: str
-            Version of the model.
+        parameter_version: str
+            Version of the parameter.
+        schema_version: str
+            Version of the schema.
 
         Returns
         -------
@@ -196,29 +199,26 @@ class ModelDataWriter:
             Validated parameter dictionary.
         """
         self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
-        schema_file = self._read_model_parameter_schema(parameter_name)
+        schema_file = schema.get_model_parameter_schema_file(parameter_name)
+        self.schema_dict = gen.collect_data_from_file(schema_file)
 
         try:  # e.g. instrument is 'North"
             site = names.validate_site_name(instrument)
         except ValueError:  # e.g. instrument is 'LSTN-01'
             site = names.get_site_from_array_element_name(instrument)
 
-        try:
-            applicable = self._get_parameter_applicability(instrument)
-        except ValueError:
-            applicable = True  # Default to True (expect that this field goes in future)
-
         value, unit = value_conversion.split_value_and_unit(value)
 
         data_dict = {
+            "schema_version": schema.get_model_parameter_schema_version(schema_version),
             "parameter": parameter_name,
             "instrument": instrument,
             "site": site,
-            "version": model_version,
+            "parameter_version": parameter_version,
+            "unique_id": None,
             "value": value,
             "unit": unit,
             "type": self._get_parameter_type(),
-            "applicable": applicable,
             "file": self._parameter_is_a_file(),
         }
         return self.validate_and_transform(
@@ -227,22 +227,6 @@ class ModelDataWriter:
             is_model_parameter=True,
         )
 
-    def _read_model_parameter_schema(self, parameter_name):
-        """
-        Read model parameter schema.
-
-        Parameters
-        ----------
-        parameter_name: str
-            Name of the parameter.
-        """
-        schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml"
-        try:
-            self.schema_dict = gen.collect_data_from_file(file_name=schema_file)
-        except FileNotFoundError as exc:
-            raise FileNotFoundError(f"Schema file not found: {schema_file}") from exc
-        return schema_file
-
     def _get_parameter_type(self):
         """
         Return parameter type from schema.
@@ -273,36 +257,6 @@ class ModelDataWriter:
             pass
         return False
 
-    def _get_parameter_applicability(self, telescope_name):
-        """
-        Check if a parameter is applicable for a given telescope using schema files.
-
-        First check for exact telescope name (e.g., LSTN-01), if not listed in the schema
-        use telescope type (LSTN).
-
-        Parameters
-        ----------
-        telescope_name: str
-            Telescope name (e.g., LSTN-01)
-
-        Returns
-        -------
-        bool
-            True if parameter is applicable to telescope.
-
-        """
-        try:
-            if telescope_name in self.schema_dict["instrument"]["type"]:
-                return True
-        except KeyError as exc:
-            self._logger.error("Schema file does not contain 'instrument:type' key.")
-            raise exc
-
-        return (
-            names.get_array_element_type_from_name(telescope_name)
-            in self.schema_dict["instrument"]["type"]
-        )
-
     def _get_unit_from_schema(self):
         """
         Return unit(s) from schema dict.

simtools/data_model/schema.py

@@ -0,0 +1,213 @@
+"""Module providing functionality to read and validate dictionaries using schema."""
+
+import logging
+from pathlib import Path
+
+import jsonschema
+
+import simtools.utils.general as gen
+from simtools.constants import (
+    METADATA_JSON_SCHEMA,
+    MODEL_PARAMETER_METASCHEMA,
+    MODEL_PARAMETER_SCHEMA_PATH,
+    SCHEMA_PATH,
+)
+from simtools.data_model import format_checkers
+from simtools.utils import names
+
+_logger = logging.getLogger(__name__)
+
+
+def get_get_model_parameter_schema_files(schema_directory=MODEL_PARAMETER_SCHEMA_PATH):
+    """
+    Return list of parameters and schema files located in schema file directory.
+
+    Returns
+    -------
+    list
+        List of parameters found in schema file directory.
+    list
+        List of schema files found in schema file directory.
+
+    """
+    schema_files = sorted(Path(schema_directory).rglob("*.schema.yml"))
+    if not schema_files:
+        raise FileNotFoundError(f"No schema files found in {schema_directory}")
+    parameters = []
+    for schema_file in schema_files:
+        schema_dict = gen.collect_data_from_file(file_name=schema_file)
+        parameters.append(schema_dict.get("name"))
+    return parameters, schema_files
+
+
+def get_model_parameter_schema_file(parameter):
+    """
+    Return schema file path for a given model parameter.
+
+    Parameters
+    ----------
+    parameter: str
+        Model parameter name.
+
+    Returns
+    -------
+    Path
+        Schema file path.
+
+    """
+    schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter}.schema.yml"
+    if not schema_file.exists():
+        raise FileNotFoundError(f"Schema file not found: {schema_file}")
+    return schema_file
+
+
+def get_model_parameter_schema_version(schema_version=None):
+    """
+    Validate  and return schema versions.
+
+    If no schema_version is given, the most recent version is provided.
+
+    Parameters
+    ----------
+    schema_version: str
+        Schema version.
+
+    Returns
+    -------
+    str
+        Schema version.
+
+    """
+    schemas = gen.collect_data_from_file(MODEL_PARAMETER_METASCHEMA)
+
+    if schema_version is None and schemas:
+        return schemas[0].get("version")
+
+    if any(schema.get("version") == schema_version for schema in schemas):
+        return schema_version
+
+    raise ValueError(f"Schema version {schema_version} not found in {MODEL_PARAMETER_METASCHEMA}.")
+
+
+def validate_dict_using_schema(data, schema_file=None, json_schema=None):
+    """
+    Validate a data dictionary against a schema.
+
+    Parameters
+    ----------
+    data
+        dictionary to be validated
+    schema_file (dict)
+        schema used for validation
+
+    Raises
+    ------
+    jsonschema.exceptions.ValidationError
+        if validation fails
+
+    """
+    if json_schema is None and schema_file is None:
+        _logger.warning(f"No schema provided for validation of {data}")
+        return
+    if json_schema is None:
+        json_schema = load_schema(
+            schema_file,
+            data.get("schema_version", "0.1.0"),  # default version to ensure backward compatibility
+        )
+
+    try:
+        jsonschema.validate(data, schema=json_schema, format_checker=format_checkers.format_checker)
+    except jsonschema.exceptions.ValidationError as exc:
+        _logger.error(f"Validation failed using schema: {json_schema}")
+        raise exc
+    if data.get("meta_schema_url") and not gen.url_exists(data["meta_schema_url"]):
+        raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
+
+    _logger.debug(f"Successful validation of data using schema ({json_schema.get('name')})")
+
+
+def load_schema(schema_file=None, schema_version=None):
+    """
+    Load parameter schema from file.
+
+    Parameters
+    ----------
+    schema_file: str
+        Path to schema file.
+    schema_version: str
+        Schema version.
+
+    Returns
+    -------
+    schema: dict
+        Schema dictionary.
+
+    Raises
+    ------
+    FileNotFoundError
+        if schema file is not found
+
+    """
+    schema_file = schema_file or METADATA_JSON_SCHEMA
+
+    for path in (schema_file, SCHEMA_PATH / schema_file):
+        try:
+            schema = gen.collect_data_from_file(file_name=path)
+            break
+        except FileNotFoundError:
+            continue
+    else:
+        raise FileNotFoundError(f"Schema file not found: {schema_file}")
+
+    if isinstance(schema, list):  # schema file with several schemas defined
+        if schema_version is None:
+            raise ValueError(f"Schema version not given in {schema_file}.")
+        schema = next((doc for doc in schema if doc.get("version") == schema_version), None)
+        if schema is None:
+            raise ValueError(f"Schema version {schema_version} not found in {schema_file}.")
+    elif schema_version is not None and schema_version != schema.get("version"):
+        _logger.warning(f"Schema version {schema_version} does not match {schema.get('version')}")
+
+    _logger.debug(f"Loading schema from {schema_file}")
+    _add_array_elements("InstrumentTypeElement", schema)
+
+    return schema
+
+
+def _add_array_elements(key, schema):
+    """
+    Add list of array elements to schema.
+
+    Avoids having to list all array elements in multiple schema.
+    Assumes an element [key]['enum'] is a list of elements.
+
+    Parameters
+    ----------
+    key: str
+        Key in schema dictionary
+    schema: dict
+        Schema dictionary
+
+    Returns
+    -------
+    dict
+        Schema dictionary with added array elements.
+
+    """
+    _list_of_array_elements = sorted(names.array_elements().keys())
+
+    def recursive_search(sub_schema, key):
+        if key in sub_schema:
+            if "enum" in sub_schema[key] and isinstance(sub_schema[key]["enum"], list):
+                sub_schema[key]["enum"] = list(
+                    set(sub_schema[key]["enum"] + _list_of_array_elements)
+                )
+            else:
+                sub_schema[key]["enum"] = _list_of_array_elements
+        else:
+            for _, v in sub_schema.items():
+                if isinstance(v, dict):
+                    recursive_search(v, key)
+
+    recursive_search(schema, key)
+    return schema