gammasimtools 0.23.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

simtools/data_model/schema.py

@@ -4,12 +4,9 @@ import logging
 from pathlib import Path
 
 import jsonschema
-from packaging.specifiers import SpecifierSet
-from packaging.version import Version
 from referencing import Registry, Resource
 
 import simtools.utils.general as gen
-from simtools import version
 from simtools.constants import (
     METADATA_JSON_SCHEMA,
     MODEL_PARAMETER_METASCHEMA,
@@ -17,8 +14,10 @@ from simtools.constants import (
     SCHEMA_PATH,
 )
 from simtools.data_model import format_checkers
+from simtools.dependencies import get_software_version
 from simtools.io import ascii_handler
 from simtools.utils import names
+from simtools.version import check_version_constraint
 
 _logger = logging.getLogger(__name__)
 
@@ -96,7 +95,7 @@ def get_model_parameter_schema_version(schema_version=None):
 
 
 def validate_dict_using_schema(
-    data, schema_file=None, json_schema=None, ignore_software_version=False
+    data, schema_file=None, json_schema=None, ignore_software_version=False, offline=False
 ):
     """
     Validate a data dictionary against a schema.
@@ -124,7 +123,7 @@ def validate_dict_using_schema(
     if json_schema is None:
         json_schema = load_schema(schema_file, get_schema_version_from_data(data))
 
-    _validate_deprecation_and_version(data, ignore_software_version)
+    validate_deprecation_and_version(data, ignore_software_version=ignore_software_version)
 
     validator = jsonschema.Draft6Validator(
         schema=json_schema,
@@ -137,17 +136,23 @@ def validate_dict_using_schema(
     except jsonschema.exceptions.ValidationError as exc:
         _logger.error(f"Validation failed using schema: {json_schema} for data: {data}")
         raise exc
-    if (
-        isinstance(data, dict)
-        and data.get("meta_schema_url")
-        and not gen.url_exists(data["meta_schema_url"])
-    ):
-        raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
+
+    if not offline:
+        _validate_meta_schema_url(data)
 
     _logger.debug(f"Successful validation of data using schema ({json_schema.get('name')})")
     return data
 
 
+def _validate_meta_schema_url(data):
+    """Validate meta_schema_url if present in data."""
+    if not isinstance(data, dict):
+        return
+
+    if data.get("meta_schema_url") is not None and not gen.url_exists(data["meta_schema_url"]):
+        raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
+
+
 def _retrieve_yaml_schema_from_uri(uri):
     """Load schema from a file URI."""
     path = SCHEMA_PATH / Path(uri.removeprefix("file:/"))
@@ -307,9 +312,7 @@ def _add_array_elements(key, schema):
     return schema
 
 
-def _validate_deprecation_and_version(
-    data, software_name="simtools", ignore_software_version=False
-):
+def validate_deprecation_and_version(data, software_name=None, ignore_software_version=False):
     """
     Check if data contains deprecated parameters or version mismatches.
 
@@ -317,39 +320,145 @@ def _validate_deprecation_and_version(
     ----------
     data: dict
         Data dictionary to check.
-    software_name: str
-        Name of the software to check version against.
+    software_name: str or None
+        Name of the software to check version against. If None, use complete list
     ignore_software_version: bool
         If True, ignore software version check.
     """
     if not isinstance(data, dict):
         return
 
+    data_name = data.get("name", "<unknown>")
+
     if data.get("deprecated", False):
         note = data.get("deprecation_note", "(no deprecation note provided)")
-        _logger.warning(f"Data is deprecated. Note: {note}")
+        _logger.warning(f"Data for {data_name} is deprecated. Note: {note}")
 
-    def check_version(sw):
-        constraint = sw.get("version")
-        if constraint is None:
-            return
-        constraint = constraint.strip()
-        spec = SpecifierSet(constraint, prereleases=True)
-        if Version(version.__version__) in spec:
+    for sw in data.get("simulation_software", []):
+        name, constraint = sw.get("name"), sw.get("version")
+        if not name or not constraint:
+            continue
+        if software_name is not None and name.lower() != software_name.lower():
+            continue
+
+        software_version = get_software_version(name)
+        if check_version_constraint(software_version, constraint):
             _logger.debug(
-                f"Version {version.__version__} of {software_name} matches constraint {constraint}."
+                f"{data_name}: version {software_version} of {name} matches "
+                f"constraint {constraint}."
             )
+            continue
+
+        msg = f"{data_name}: version {software_version} of {name} does not match {constraint}."
+        if ignore_software_version:
+            _logger.warning(f"{msg}, but version check is ignored.")
         else:
-            msg = (
-                f"Version {version.__version__} of {software_name} "
-                f"does not match constraint {constraint}."
-            )
-            if ignore_software_version:
-                _logger.warning(f"{msg}, but version check is ignored.")
-                return
             raise ValueError(msg)
 
-    for sw in data.get("simulation_software", []):
-        if sw.get("name") == software_name:
-            check_version(sw)
-            break
+
+def validate_schema_from_files(
+    file_directory, file_name=None, schema_file=None, ignore_software_version=False
+):
+    """
+    Validate a schema file or several files in a directory.
+
+    Files to be validated are taken from file_directory and file_name pattern.
+    The schema is either given as command line argument, read from the meta_schema_url or from
+    the metadata section of the data dictionary.
+
+    Parameters
+    ----------
+    file_directory : str or Path, optional
+        Directory with files to be validated.
+    file_name : str or Path, optional
+        File name pattern to be validated.
+    schema_file : str, optional
+        Schema file name provided directly.
+    ignore_software_version : bool
+        If True, ignore software version check.
+    """
+    if file_directory and file_name:
+        file_list = sorted(Path(file_directory).rglob(file_name))
+    else:
+        file_list = [Path(file_name)] if file_name else []
+
+    for _file_name in file_list:
+        try:
+            data = ascii_handler.collect_data_from_file(file_name=_file_name)
+        except FileNotFoundError as exc:
+            raise FileNotFoundError(f"Error reading schema file from {_file_name}") from exc
+        data = data if isinstance(data, list) else [data]
+        try:
+            for data_dict in data:
+                validate_dict_using_schema(
+                    data_dict,
+                    _get_schema_file_name(schema_file, _file_name, data_dict),
+                    ignore_software_version=ignore_software_version,
+                )
+        except Exception as exc:
+            raise ValueError(f"Validation of file {_file_name} failed") from exc
+        _logger.info(f"Successful validation of file {_file_name}")
+
+
+def _get_schema_file_name(schema_file=None, file_name=None, data_dict=None):
+    """
+    Get schema file name from metadata, data dict, or from file.
+
+    Parameters
+    ----------
+    schema_file : str, optional
+        Schema file name provided directly.
+    file_name : str or Path, optional
+        File name to extract schema information from.
+    data_dict : dict, optional
+        Dictionary with metaschema information.
+
+    Returns
+    -------
+    str or None
+        Schema file name.
+    """
+    if schema_file is not None:
+        return schema_file
+
+    if data_dict and (url := data_dict.get("meta_schema_url")):
+        return url
+
+    if file_name:
+        return _extract_schema_from_file(file_name)
+
+    return None
+
+
+def _extract_schema_url_from_metadata_dict(metadata, observatory="cta"):
+    """Extract schema URL from metadata dictionary."""
+    for key in (observatory, observatory.lower()):
+        url = metadata.get(key, {}).get("product", {}).get("data", {}).get("model", {}).get("url")
+        if url:
+            return url
+    return None
+
+
+def _extract_schema_from_file(file_name, observatory="cta"):
+    """
+    Extract schema file name from a metadata or data file.
+
+    Parameters
+    ----------
+    file_name : str or Path
+        File name to extract schema information from.
+    observatory : str
+        Observatory name (default: "cta").
+
+    Returns
+    -------
+    str or None
+        Schema file name or None if not found.
+
+    """
+    try:
+        metadata = ascii_handler.collect_data_from_file(file_name=file_name, yaml_document=0)
+    except FileNotFoundError:
+        return None
+
+    return _extract_schema_url_from_metadata_dict(metadata, observatory)

simtools/data_model/validate_data.py

@@ -14,6 +14,7 @@ import simtools.utils.general as gen
 from simtools.data_model import schema
 from simtools.io import ascii_handler
 from simtools.utils import names, value_conversion
+from simtools.version import is_valid_semantic_version
 
 
 class DataValidator:
@@ -46,7 +47,7 @@ class DataValidator:
         check_exact_data_type=True,
     ):
         """Initialize validation class and read required reference data columns."""
-        self._logger = logging.getLogger(__name__)
+        self.logger = logging.getLogger(__name__)
 
         self.data_file_name = data_file
         self.schema_file_name = schema_file
@@ -83,7 +84,7 @@ class DataValidator:
             return self._validate_data_dict(is_model_parameter, lists_as_strings)
         if isinstance(self.data_table, Table):
             return self._validate_data_table()
-        self._logger.error("No data or data table to validate")
+        self.logger.error("No data or data table to validate")
         raise TypeError
 
     def validate_data_file(self, is_model_parameter=None):
@@ -100,10 +101,10 @@ class DataValidator:
         try:
             if Path(self.data_file_name).suffix in (".yml", ".yaml", ".json"):
                 self.data_dict = ascii_handler.collect_data_from_file(self.data_file_name)
-                self._logger.info(f"Validating data from: {self.data_file_name}")
+                self.logger.info(f"Validating data from: {self.data_file_name}")
             else:
                 self.data_table = Table.read(self.data_file_name, guess=True, delimiter=r"\s")
-                self._logger.info(f"Validating tabled data from: {self.data_file_name}")
+                self.logger.info(f"Validating tabled data from: {self.data_file_name}")
         except (AttributeError, TypeError):
             pass
         if is_model_parameter:
@@ -129,7 +130,7 @@ class DataValidator:
             raise ValueError(f"Mismatch: version '{param_version}' vs. file '{file_stem}'.")
 
         if param_version is None:
-            self._logger.warning(f"File '{file_stem}' has no parameter version defined.")
+            self.logger.warning(f"File '{file_stem}' has no parameter version defined.")
 
     @staticmethod
     def validate_model_parameter(par_dict):
@@ -153,6 +154,46 @@ class DataValidator:
         )
         return data_validator.validate_and_transform(is_model_parameter=True)
 
+    @staticmethod
+    def validate_data_files(
+        file_directory=None,
+        file_name=None,
+        is_model_parameter=True,
+        check_exact_data_type=False,
+        schema_file=None,
+    ):
+        """
+        Validate data or model parameters in files in a directory or a single file.
+
+        Parameters
+        ----------
+        file_directory: str or Path
+            Directory with files to be validated.
+        file_name: str or Path
+            Name of the file to be validated.
+        is_model_parameter: bool
+            This is a model parameter (add some data preparation).
+        check_exact_data_type: bool
+            Require exact data type for validation.
+        """
+        if file_directory:
+            file_list = sorted(Path(file_directory).rglob("*.json"))
+        elif file_name:
+            file_list = [Path(file_name)]
+        else:
+            return
+
+        for data_file in file_list:
+            parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", data_file.stem)
+            schema_path = schema_file or schema.get_model_parameter_schema_file(f"{parameter_name}")
+            data_validator = DataValidator(
+                schema_file=schema_path,
+                data_file=data_file,
+                check_exact_data_type=check_exact_data_type,
+            )
+            data_validator.validate_and_transform(is_model_parameter)
+            data_validator.logger.info(f"Validated data file {data_file} with schema {schema_path}")
+
     def _validate_data_dict(self, is_model_parameter=False, lists_as_strings=False):
         """
         Validate values in a dictionary.
@@ -208,8 +249,10 @@ class DataValidator:
                     self.data_dict.get("instrument"), self.data_dict.get("site")
                 )
 
-        for version_string in ("version", "parameter_version", "model_version"):
-            self._check_version_string(self.data_dict.get(version_string))
+        for version_type in ("version", "parameter_version", "model_version"):
+            version_string = self.data_dict.get(version_type, "0.0.0")
+            if not is_valid_semantic_version(version_string):
+                raise ValueError(f"Invalid version string '{version_string}'")
 
         if lists_as_strings:
             self._convert_results_to_model_format()
@@ -278,7 +321,7 @@ class DataValidator:
                 "table_columns", None
             )
         except IndexError:
-            self._logger.error(f"Error reading validation schema from {self.schema_file_name}")
+            self.logger.error(f"Error reading validation schema from {self.schema_file_name}")
             raise
 
         if self._data_description is not None:
@@ -327,7 +370,7 @@ class DataValidator:
         for entry in self._data_description:
             if entry.get("required", False):
                 if entry["name"] in self.data_table.columns:
-                    self._logger.debug(f"Found required data column {entry['name']}")
+                    self.logger.debug(f"Found required data column {entry['name']}")
                 else:
                     raise KeyError(f"Missing required column {entry['name']}")
 
@@ -353,18 +396,18 @@ class DataValidator:
                     _columns_by_which_to_reverse_sort.append(entry["name"])
 
         if len(_columns_by_which_to_sort) > 0:
-            self._logger.debug(f"Sorting data columns: {_columns_by_which_to_sort}")
+            self.logger.debug(f"Sorting data columns: {_columns_by_which_to_sort}")
             try:
                 self.data_table.sort(_columns_by_which_to_sort)
             except AttributeError:
-                self._logger.error("No data table defined for sorting")
+                self.logger.error("No data table defined for sorting")
                 raise
         elif len(_columns_by_which_to_reverse_sort) > 0:
-            self._logger.debug(f"Reverse sorting data columns: {_columns_by_which_to_reverse_sort}")
+            self.logger.debug(f"Reverse sorting data columns: {_columns_by_which_to_reverse_sort}")
             try:
                 self.data_table.sort(_columns_by_which_to_reverse_sort, reverse=True)
             except AttributeError:
-                self._logger.error("No data table defined for reverse sorting")
+                self.logger.error("No data table defined for reverse sorting")
                 raise
 
     def _check_data_for_duplicates(self):
@@ -379,7 +422,7 @@ class DataValidator:
         """
         _column_with_unique_requirement = self._get_unique_column_requirement()
         if len(_column_with_unique_requirement) == 0:
-            self._logger.debug("No data columns with unique value requirement")
+            self.logger.debug("No data columns with unique value requirement")
             return
         _data_table_unique_for_key_column = unique(
             self.data_table, keys=_column_with_unique_requirement
@@ -412,10 +455,10 @@ class DataValidator:
 
         for entry in self._data_description:
             if "input_processing" in entry and "remove_duplicates" in entry["input_processing"]:
-                self._logger.debug(f"Removing duplicates for column {entry['name']}")
+                self.logger.debug(f"Removing duplicates for column {entry['name']}")
                 _unique_required_column.append(entry["name"])
 
-        self._logger.debug(f"Unique required columns: {_unique_required_column}")
+        self.logger.debug(f"Unique required columns: {_unique_required_column}")
         return _unique_required_column
 
     def _get_reference_unit(self, column_name):
@@ -470,7 +513,7 @@ class DataValidator:
             dtype=dtype,
             allow_subtypes=(not self.check_exact_data_type),
         ):
-            self._logger.error(
+            self.logger.error(
                 f"Invalid data type in column '{column_name}'. "
                 f"Expected type '{reference_dtype}', found '{dtype}' "
                 f"(exact type: {self.check_exact_data_type})"
@@ -505,9 +548,9 @@ class DataValidator:
             data = np.array(data)
 
         if np.isnan(data).any():
-            self._logger.info(f"Column {col_name} contains NaN.")
+            self.logger.info(f"Column {col_name} contains NaN.")
         if np.isinf(data).any():
-            self._logger.info(f"Column {col_name} contains infinite value.")
+            self.logger.info(f"Column {col_name} contains infinite value.")
 
         entry = self._get_data_description(col_name)
         if "allow_nan" in entry.get("input_processing", {}):
@@ -593,7 +636,7 @@ class DataValidator:
             # ensure that the data type is preserved (e.g., integers)
             return (type(data)(u.Unit(column_unit).to(reference_unit) * data), reference_unit)
         except (u.core.UnitConversionError, ValueError) as exc:
-            self._logger.error(
+            self.logger.error(
                 f"Invalid unit in data column '{col_name}'. "
                 f"Expected type '{reference_unit}', found '{column_unit}'"
             )
@@ -696,9 +739,9 @@ class DataValidator:
         try:
             col_index = int(col_name)
             if col_index < max_logs:
-                self._logger.debug(message)
+                self.logger.debug(message)
         except (ValueError, TypeError):
-            self._logger.debug(message)
+            self.logger.debug(message)
 
     @staticmethod
     def _interval_check(data, axis_range, range_type):
@@ -817,7 +860,7 @@ class DataValidator:
         except IndexError as exc:
             if len(self._data_description) == 1:  # all columns are described by the same schema
                 return self._data_description[0]
-            self._logger.error(
+            self.logger.error(
                 f"Data column '{column_name}' not found in reference column definition"
             )
             raise exc
@@ -835,7 +878,7 @@ class DataValidator:
         try:
             return _entry[_index]
         except IndexError:
-            self._logger.error(
+            self.logger.error(
                 f"Data column '{column_name}' not found in reference column definition"
             )
             raise
@@ -868,42 +911,33 @@ class DataValidator:
         if isinstance(self.data_dict["unit"], list):
             self.data_dict["unit"] = gen.convert_list_to_string(self.data_dict["unit"])
 
-    def _check_version_string(self, version):
-        """
-        Check that version string follows semantic versioning.
-
-        Parameters
-        ----------
-        version: str
-            version string
-
-        Raises
-        ------
-        ValueError
-            if version string does not follow semantic versioning
-
-        """
-        if version is None:
-            return
-        semver_regex = r"^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$"
-        if not re.match(semver_regex, version):
-            raise ValueError(f"Invalid version string '{version}'")
-        self._logger.debug(f"Valid version string '{version}'")
-
     def _check_site_and_array_element_consistency(self, instrument, site):
         """
         Check that site and array element names are consistent.
 
         An example for an inconsistency is 'LSTN' at site 'South'
         """
-        if not all([instrument, site]) or "OBS" in instrument:
+        if not (instrument and site):
+            return
+
+        instruments = [instrument] if isinstance(instrument, str) else instrument
+        if any(inst.startswith("OBS") for inst in instruments):
             return
 
         def to_sorted_list(value):
             """Return value as sorted list."""
             return [value] if isinstance(value, str) else sorted(value)
 
-        instrument_site = to_sorted_list(names.get_site_from_array_element_name(instrument))
+        instrument_sites = []
+        for inst in instruments:
+            instrument_sites.append(names.get_site_from_array_element_name(inst))
+        # names.get_site_from_array_element_name might return a list
+        flat_sites = [
+            s
+            for sublist in instrument_sites
+            for s in (sublist if isinstance(sublist, list) else [sublist])
+        ]
+        instrument_site = to_sorted_list(set(flat_sites))
         site = to_sorted_list(site)
 
         if instrument_site != site: