gammasimtools 0.24.0__py3-none-any.whl → 0.26.0__py3-none-any.whl

simtools/corsika/primary_particle.py

@@ -185,7 +185,7 @@ class PrimaryParticle:
             "silicon": {"corsika7_id": 2814},
             "iron": {"corsika7_id": 5626},
         }
-        for _, ids in particles.items():
+        for ids in particles.values():
             ids["pdg_id"] = Corsika7ID(ids["corsika7_id"]).to_pdgid().numerator
             ids["pdg_name"] = Particle.findall(pdgid=ids["pdg_id"])[0].name
 

simtools/data_model/metadata_collector.py

@@ -360,6 +360,9 @@ class MetadataCollector:
                     "Metadata extraction from sim_telarray files is not supported yet."
                 )
                 continue
+            elif Path(metadata_file).name.endswith((".corsika.zst", ".corsika")):
+                self._logger.warning("Metadata extraction from CORSIKA files is not supported yet.")
+                continue
             else:
                 raise ValueError(f"Unknown metadata file format: {metadata_file}")
 
@@ -422,7 +425,7 @@ class MetadataCollector:
         product_dict["creation_time"] = gen.now_date_time_in_isoformat()
         product_dict["description"] = self.schema_dict.get("description", None)
 
-        # DATA:CATEGORY
+        # metadata DATA:CATEGORY
         product_dict["data"]["category"] = "SIM"
         product_dict["data"]["level"] = "R1"
         product_dict["data"]["type"] = "Service"
@@ -431,7 +434,7 @@ class MetadataCollector:
         except KeyError:
             pass
 
-        # DATA:MODEL
+        # metadata DATA:MODEL
         product_dict["data"]["model"]["name"] = (
             self.schema_dict.get("name")
             or self.args_dict.get("metadata_product_data_name")

simtools/data_model/metadata_model.py

@@ -8,13 +8,9 @@ Follows CTAO top-level data model definition.
 
 """
 
-import logging
-
 import simtools.data_model.schema
 import simtools.utils.general as gen
 
-_logger = logging.getLogger(__name__)
-
 
 def get_default_metadata_dict(
     schema_file=None, observatory="CTA", schema_version="latest", lower_case=True

simtools/data_model/model_data_writer.py

@@ -98,9 +98,10 @@ class ModelDataWriter:
         output_file,
         output_path=None,
         metadata_input_dict=None,
-        db_config=None,
         unit=None,
         meta_parameter=False,
+        model_parameter_schema_version=None,
+        check_db_for_existing_parameter=True,
     ):
         """
         Generate DB-style model parameter dict and write it to json file.
@@ -121,10 +122,14 @@ class ModelDataWriter:
             Path to output file.
         metadata_input_dict: dict
             Input to metadata collector.
-        db_config: dict
-            Database configuration. If not None, check if parameter with the same version exists.
         unit: str
             Unit of the parameter value (if applicable and value is not of type astropy Quantity).
+        meta_parameter: bool
+            Setting for meta parameter flag.
+        model_parameter_schema_version: str, None
+            Version of the model parameter schema (if None, use schema version from schema dict).
+        check_db_for_existing_parameter: bool
+            If True, check if parameter with same version exists in DB before writing.
 
         Returns
         -------
@@ -137,10 +142,8 @@ class ModelDataWriter:
             args_dict=None,
             output_path=output_path,
         )
-        if db_config is not None:
-            writer.check_db_for_existing_parameter(
-                parameter_name, instrument, parameter_version, db_config
-            )
+        if check_db_for_existing_parameter:
+            writer.check_db_for_existing_parameter(parameter_name, instrument, parameter_version)
 
         unique_id = None
         if metadata_input_dict is not None:
@@ -158,15 +161,14 @@ class ModelDataWriter:
             instrument,
             parameter_version,
             unique_id,
+            model_parameter_schema_version=model_parameter_schema_version,
             unit=unit,
             meta_parameter=meta_parameter,
         )
         writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         return _json_dict
 
-    def check_db_for_existing_parameter(
-        self, parameter_name, instrument, parameter_version, db_config
-    ):
+    def check_db_for_existing_parameter(self, parameter_name, instrument, parameter_version):
         """
         Check if a parameter with the same version exists in the simulation model database.
 
@@ -178,15 +180,15 @@ class ModelDataWriter:
             Name of the instrument.
         parameter_version: str
             Version of the parameter.
-        db_config: dict
-            Database configuration.
 
         Raises
         ------
         ValueError
             If parameter with the same version exists in the database.
         """
-        db = db_handler.DatabaseHandler(db_config=db_config)
+        db = db_handler.DatabaseHandler()
+        if not db.is_configured():
+            return
         try:
             db.get_model_parameter(
                 parameter=parameter_name,
@@ -211,6 +213,7 @@ class ModelDataWriter:
         schema_version=None,
         unit=None,
         meta_parameter=False,
+        model_parameter_schema_version=None,
     ):
         """
         Get validated parameter dictionary.
@@ -233,6 +236,8 @@ class ModelDataWriter:
             Unit of the parameter value (if applicable and value is not an astropy Quantity).
         meta_parameter: bool
             Setting for meta parameter flag.
+        model_parameter_schema_version: str, None
+            Version of the model parameter schema (if None, use schema version from schema dict).
 
         Returns
         -------
@@ -240,7 +245,9 @@ class ModelDataWriter:
             Validated parameter dictionary.
         """
         self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
-        self.schema_dict, schema_file = self._read_schema_dict(parameter_name, schema_version)
+        self.schema_dict, schema_file = self._read_schema_dict(
+            parameter_name, model_parameter_schema_version
+        )
 
         if unit is None:
             value, unit = value_conversion.split_value_and_unit(value)
@@ -257,7 +264,8 @@ class ModelDataWriter:
             "type": self._get_parameter_type(),
             "file": self._parameter_is_a_file(),
             "meta_parameter": meta_parameter,
-            "model_parameter_schema_version": self.schema_dict.get("schema_version", "0.1.0"),
+            "model_parameter_schema_version": model_parameter_schema_version
+            or self.schema_dict.get("schema_version", "0.1.0"),
         }
         return self.validate_and_transform(
             product_data_dict=data_dict,
@@ -361,8 +369,10 @@ class ModelDataWriter:
         """
         try:
             unit_list = []
-            for data in self.schema_dict["data"]:
-                unit_list.append(data["unit"] if data["unit"] != "dimensionless" else None)
+            unit_list = [
+                data["unit"] if data["unit"] != "dimensionless" else None
+                for data in self.schema_dict["data"]
+            ]
             return unit_list if len(unit_list) > 1 else unit_list[0]
         except (KeyError, IndexError):
             pass

simtools/data_model/schema.py

@@ -146,11 +146,10 @@ def validate_dict_using_schema(
 
 def _validate_meta_schema_url(data):
     """Validate meta_schema_url if present in data."""
-    if (
-        isinstance(data, dict)
-        and data.get("meta_schema_url")
-        and not gen.url_exists(data["meta_schema_url"])
-    ):
+    if not isinstance(data, dict):
+        return
+
+    if data.get("meta_schema_url") is not None and not gen.url_exists(data["meta_schema_url"]):
         raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
 
 
@@ -355,3 +354,111 @@ def validate_deprecation_and_version(data, software_name=None, ignore_software_v
             _logger.warning(f"{msg}, but version check is ignored.")
         else:
             raise ValueError(msg)
+
+
+def validate_schema_from_files(
+    file_directory, file_name=None, schema_file=None, ignore_software_version=False
+):
+    """
+    Validate a schema file or several files in a directory.
+
+    Files to be validated are taken from file_directory and file_name pattern.
+    The schema is either given as command line argument, read from the meta_schema_url or from
+    the metadata section of the data dictionary.
+
+    Parameters
+    ----------
+    file_directory : str or Path, optional
+        Directory with files to be validated.
+    file_name : str or Path, optional
+        File name pattern to be validated.
+    schema_file : str, optional
+        Schema file name provided directly.
+    ignore_software_version : bool
+        If True, ignore software version check.
+    """
+    if file_directory and file_name:
+        file_list = sorted(Path(file_directory).rglob(file_name))
+    else:
+        file_list = [Path(file_name)] if file_name else []
+
+    for _file_name in file_list:
+        try:
+            data = ascii_handler.collect_data_from_file(file_name=_file_name)
+        except FileNotFoundError as exc:
+            raise FileNotFoundError(f"Error reading schema file from {_file_name}") from exc
+        data = data if isinstance(data, list) else [data]
+        try:
+            for data_dict in data:
+                validate_dict_using_schema(
+                    data_dict,
+                    _get_schema_file_name(schema_file, _file_name, data_dict),
+                    ignore_software_version=ignore_software_version,
+                )
+        except Exception as exc:
+            raise ValueError(f"Validation of file {_file_name} failed") from exc
+        _logger.info(f"Successful validation of file {_file_name}")
+
+
+def _get_schema_file_name(schema_file=None, file_name=None, data_dict=None):
+    """
+    Get schema file name from metadata, data dict, or from file.
+
+    Parameters
+    ----------
+    schema_file : str, optional
+        Schema file name provided directly.
+    file_name : str or Path, optional
+        File name to extract schema information from.
+    data_dict : dict, optional
+        Dictionary with metaschema information.
+
+    Returns
+    -------
+    str or None
+        Schema file name.
+    """
+    if schema_file is not None:
+        return schema_file
+
+    if data_dict and (url := data_dict.get("meta_schema_url")):
+        return url
+
+    if file_name:
+        return _extract_schema_from_file(file_name)
+
+    return None
+
+
+def _extract_schema_url_from_metadata_dict(metadata, observatory="cta"):
+    """Extract schema URL from metadata dictionary."""
+    for key in (observatory, observatory.lower()):
+        url = metadata.get(key, {}).get("product", {}).get("data", {}).get("model", {}).get("url")
+        if url:
+            return url
+    return None
+
+
+def _extract_schema_from_file(file_name, observatory="cta"):
+    """
+    Extract schema file name from a metadata or data file.
+
+    Parameters
+    ----------
+    file_name : str or Path
+        File name to extract schema information from.
+    observatory : str
+        Observatory name (default: "cta").
+
+    Returns
+    -------
+    str or None
+        Schema file name or None if not found.
+
+    """
+    try:
+        metadata = ascii_handler.collect_data_from_file(file_name=file_name, yaml_document=0)
+    except FileNotFoundError:
+        return None
+
+    return _extract_schema_url_from_metadata_dict(metadata, observatory)

simtools/data_model/validate_data.py

@@ -14,6 +14,7 @@ import simtools.utils.general as gen
 from simtools.data_model import schema
 from simtools.io import ascii_handler
 from simtools.utils import names, value_conversion
+from simtools.version import is_valid_semantic_version
 
 
 class DataValidator:
@@ -46,7 +47,7 @@ class DataValidator:
         check_exact_data_type=True,
     ):
         """Initialize validation class and read required reference data columns."""
-        self._logger = logging.getLogger(__name__)
+        self.logger = logging.getLogger(__name__)
 
         self.data_file_name = data_file
         self.schema_file_name = schema_file
@@ -83,7 +84,7 @@ class DataValidator:
             return self._validate_data_dict(is_model_parameter, lists_as_strings)
         if isinstance(self.data_table, Table):
             return self._validate_data_table()
-        self._logger.error("No data or data table to validate")
+        self.logger.error("No data or data table to validate")
         raise TypeError
 
     def validate_data_file(self, is_model_parameter=None):
@@ -100,10 +101,10 @@ class DataValidator:
         try:
             if Path(self.data_file_name).suffix in (".yml", ".yaml", ".json"):
                 self.data_dict = ascii_handler.collect_data_from_file(self.data_file_name)
-                self._logger.info(f"Validating data from: {self.data_file_name}")
+                self.logger.info(f"Validating data from: {self.data_file_name}")
             else:
                 self.data_table = Table.read(self.data_file_name, guess=True, delimiter=r"\s")
-                self._logger.info(f"Validating tabled data from: {self.data_file_name}")
+                self.logger.info(f"Validating tabled data from: {self.data_file_name}")
         except (AttributeError, TypeError):
             pass
         if is_model_parameter:
@@ -129,7 +130,7 @@ class DataValidator:
             raise ValueError(f"Mismatch: version '{param_version}' vs. file '{file_stem}'.")
 
         if param_version is None:
-            self._logger.warning(f"File '{file_stem}' has no parameter version defined.")
+            self.logger.warning(f"File '{file_stem}' has no parameter version defined.")
 
     @staticmethod
     def validate_model_parameter(par_dict):
@@ -153,6 +154,46 @@ class DataValidator:
         )
         return data_validator.validate_and_transform(is_model_parameter=True)
 
+    @staticmethod
+    def validate_data_files(
+        file_directory=None,
+        file_name=None,
+        is_model_parameter=True,
+        check_exact_data_type=False,
+        schema_file=None,
+    ):
+        """
+        Validate data or model parameters in files in a directory or a single file.
+
+        Parameters
+        ----------
+        file_directory: str or Path
+            Directory with files to be validated.
+        file_name: str or Path
+            Name of the file to be validated.
+        is_model_parameter: bool
+            This is a model parameter (add some data preparation).
+        check_exact_data_type: bool
+            Require exact data type for validation.
+        """
+        if file_directory:
+            file_list = sorted(Path(file_directory).rglob("*.json"))
+        elif file_name:
+            file_list = [Path(file_name)]
+        else:
+            return
+
+        for data_file in file_list:
+            parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", data_file.stem)
+            schema_path = schema_file or schema.get_model_parameter_schema_file(f"{parameter_name}")
+            data_validator = DataValidator(
+                schema_file=schema_path,
+                data_file=data_file,
+                check_exact_data_type=check_exact_data_type,
+            )
+            data_validator.validate_and_transform(is_model_parameter)
+            data_validator.logger.info(f"Validated data file {data_file} with schema {schema_path}")
+
     def _validate_data_dict(self, is_model_parameter=False, lists_as_strings=False):
         """
         Validate values in a dictionary.
@@ -208,8 +249,10 @@ class DataValidator:
                     self.data_dict.get("instrument"), self.data_dict.get("site")
                 )
 
-        for version_string in ("version", "parameter_version", "model_version"):
-            self._check_version_string(self.data_dict.get(version_string))
+        for version_type in ("version", "parameter_version", "model_version"):
+            version_string = self.data_dict.get(version_type, "0.0.0")
+            if not is_valid_semantic_version(version_string):
+                raise ValueError(f"Invalid version string '{version_string}'")
 
         if lists_as_strings:
             self._convert_results_to_model_format()
@@ -278,7 +321,7 @@ class DataValidator:
                 "table_columns", None
             )
         except IndexError:
-            self._logger.error(f"Error reading validation schema from {self.schema_file_name}")
+            self.logger.error(f"Error reading validation schema from {self.schema_file_name}")
             raise
 
         if self._data_description is not None:
@@ -327,7 +370,7 @@ class DataValidator:
         for entry in self._data_description:
             if entry.get("required", False):
                 if entry["name"] in self.data_table.columns:
-                    self._logger.debug(f"Found required data column {entry['name']}")
+                    self.logger.debug(f"Found required data column {entry['name']}")
                 else:
                     raise KeyError(f"Missing required column {entry['name']}")
 
@@ -353,18 +396,18 @@ class DataValidator:
                     _columns_by_which_to_reverse_sort.append(entry["name"])
 
         if len(_columns_by_which_to_sort) > 0:
-            self._logger.debug(f"Sorting data columns: {_columns_by_which_to_sort}")
+            self.logger.debug(f"Sorting data columns: {_columns_by_which_to_sort}")
             try:
                 self.data_table.sort(_columns_by_which_to_sort)
             except AttributeError:
-                self._logger.error("No data table defined for sorting")
+                self.logger.error("No data table defined for sorting")
                 raise
         elif len(_columns_by_which_to_reverse_sort) > 0:
-            self._logger.debug(f"Reverse sorting data columns: {_columns_by_which_to_reverse_sort}")
+            self.logger.debug(f"Reverse sorting data columns: {_columns_by_which_to_reverse_sort}")
             try:
                 self.data_table.sort(_columns_by_which_to_reverse_sort, reverse=True)
             except AttributeError:
-                self._logger.error("No data table defined for reverse sorting")
+                self.logger.error("No data table defined for reverse sorting")
                 raise
 
     def _check_data_for_duplicates(self):
@@ -379,7 +422,7 @@ class DataValidator:
         """
         _column_with_unique_requirement = self._get_unique_column_requirement()
         if len(_column_with_unique_requirement) == 0:
-            self._logger.debug("No data columns with unique value requirement")
+            self.logger.debug("No data columns with unique value requirement")
             return
         _data_table_unique_for_key_column = unique(
             self.data_table, keys=_column_with_unique_requirement
@@ -412,10 +455,10 @@ class DataValidator:
 
         for entry in self._data_description:
             if "input_processing" in entry and "remove_duplicates" in entry["input_processing"]:
-                self._logger.debug(f"Removing duplicates for column {entry['name']}")
+                self.logger.debug(f"Removing duplicates for column {entry['name']}")
                 _unique_required_column.append(entry["name"])
 
-        self._logger.debug(f"Unique required columns: {_unique_required_column}")
+        self.logger.debug(f"Unique required columns: {_unique_required_column}")
         return _unique_required_column
 
     def _get_reference_unit(self, column_name):
@@ -470,7 +513,7 @@ class DataValidator:
             dtype=dtype,
             allow_subtypes=(not self.check_exact_data_type),
         ):
-            self._logger.error(
+            self.logger.error(
                 f"Invalid data type in column '{column_name}'. "
                 f"Expected type '{reference_dtype}', found '{dtype}' "
                 f"(exact type: {self.check_exact_data_type})"
@@ -505,9 +548,9 @@ class DataValidator:
             data = np.array(data)
 
         if np.isnan(data).any():
-            self._logger.info(f"Column {col_name} contains NaN.")
+            self.logger.info(f"Column {col_name} contains NaN.")
         if np.isinf(data).any():
-            self._logger.info(f"Column {col_name} contains infinite value.")
+            self.logger.info(f"Column {col_name} contains infinite value.")
 
         entry = self._get_data_description(col_name)
         if "allow_nan" in entry.get("input_processing", {}):
@@ -593,7 +636,7 @@ class DataValidator:
             # ensure that the data type is preserved (e.g., integers)
             return (type(data)(u.Unit(column_unit).to(reference_unit) * data), reference_unit)
         except (u.core.UnitConversionError, ValueError) as exc:
-            self._logger.error(
+            self.logger.error(
                 f"Invalid unit in data column '{col_name}'. "
                 f"Expected type '{reference_unit}', found '{column_unit}'"
             )
@@ -696,9 +739,9 @@ class DataValidator:
         try:
             col_index = int(col_name)
             if col_index < max_logs:
-                self._logger.debug(message)
+                self.logger.debug(message)
         except (ValueError, TypeError):
-            self._logger.debug(message)
+            self.logger.debug(message)
 
     @staticmethod
     def _interval_check(data, axis_range, range_type):
@@ -817,7 +860,7 @@ class DataValidator:
         except IndexError as exc:
             if len(self._data_description) == 1:  # all columns are described by the same schema
                 return self._data_description[0]
-            self._logger.error(
+            self.logger.error(
                 f"Data column '{column_name}' not found in reference column definition"
             )
             raise exc
@@ -835,7 +878,7 @@ class DataValidator:
         try:
             return _entry[_index]
         except IndexError:
-            self._logger.error(
+            self.logger.error(
                 f"Data column '{column_name}' not found in reference column definition"
             )
             raise
@@ -868,42 +911,31 @@ class DataValidator:
         if isinstance(self.data_dict["unit"], list):
             self.data_dict["unit"] = gen.convert_list_to_string(self.data_dict["unit"])
 
-    def _check_version_string(self, version):
-        """
-        Check that version string follows semantic versioning.
-
-        Parameters
-        ----------
-        version: str
-            version string
-
-        Raises
-        ------
-        ValueError
-            if version string does not follow semantic versioning
-
-        """
-        if version is None:
-            return
-        semver_regex = r"^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$"
-        if not re.match(semver_regex, version):
-            raise ValueError(f"Invalid version string '{version}'")
-        self._logger.debug(f"Valid version string '{version}'")
-
     def _check_site_and_array_element_consistency(self, instrument, site):
         """
         Check that site and array element names are consistent.
 
         An example for an inconsistency is 'LSTN' at site 'South'
         """
-        if not all([instrument, site]) or "OBS" in instrument:
+        if not (instrument and site):
+            return
+
+        instruments = [instrument] if isinstance(instrument, str) else instrument
+        if any(inst.startswith("OBS") for inst in instruments):
             return
 
         def to_sorted_list(value):
             """Return value as sorted list."""
             return [value] if isinstance(value, str) else sorted(value)
 
-        instrument_site = to_sorted_list(names.get_site_from_array_element_name(instrument))
+        instrument_sites = [names.get_site_from_array_element_name(inst) for inst in instruments]
+        # names.get_site_from_array_element_name might return a list
+        flat_sites = [
+            s
+            for sublist in instrument_sites
+            for s in (sublist if isinstance(sublist, list) else [sublist])
+        ]
+        instrument_site = to_sorted_list(set(flat_sites))
         site = to_sorted_list(site)
 
         if instrument_site != site:

simtools/db/db_handler.py

@@ -4,6 +4,7 @@ import logging
 from collections import defaultdict
 from pathlib import Path
 
+from simtools import settings
 from simtools.data_model import validate_data
 from simtools.db.mongo_db import MongoDBHandler
 from simtools.io import io_handler
@@ -20,11 +21,6 @@ class DatabaseHandler:
 
     - db_simulation_model_version (from db_config): version of the simulation model database
     - model_version (from production_tables): version of the model contained in the database
-
-    Parameters
-    ----------
-    db_config: dict
-        Dictionary with the DB configuration.
     """
 
     ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"]
@@ -33,13 +29,17 @@ class DatabaseHandler:
     model_parameters_cached = {}
     model_versions_cached = {}
 
-    def __init__(self, db_config=None):
+    def __init__(self):
         """Initialize the DatabaseHandler class."""
         self._logger = logging.getLogger(__name__)
 
-        self.db_config = MongoDBHandler.validate_db_config(db_config)
+        self.db_config = (
+            MongoDBHandler.validate_db_config(dict(settings.config.db_config))
+            if settings.config.db_config
+            else None
+        )
         self.io_handler = io_handler.IOHandler()
-        self.mongo_db_handler = MongoDBHandler(db_config) if self.db_config else None
+        self.mongo_db_handler = MongoDBHandler(self.db_config) if self.db_config else None
 
         self.db_name = (
             MongoDBHandler.get_db_name(
@@ -50,6 +50,17 @@ class DatabaseHandler:
             else None
         )
 
+    def is_configured(self):
+        """
+        Check if the DatabaseHandler is configured.
+
+        Returns
+        -------
+        bool
+            True if the DatabaseHandler is configured, False otherwise.
+        """
+        return self.mongo_db_handler is not None
+
     def get_db_name(self, db_name=None, db_simulation_model_version=None, model_name=None):
         """Build DB name from configuration."""
         if db_name:

simtools/db/db_model_upload.py

@@ -186,7 +186,8 @@ def _read_production_tables(model_path):
     models = [model_path.name]
     if (model_path / "info.yml").exists():
         info = ascii_handler.collect_data_from_file(file_name=model_path / "info.yml")
-        models.extend(info.get("model_version_history", []))
+        if info.get("model_update") == "patch_update":
+            models.extend(info.get("model_version_history", []))
     # sort oldest --> newest
     models = sorted(set(models), key=Version, reverse=False)
     for model in models: