gammasimtools 0.8.2__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/db/db_handler.py

@@ -2,16 +2,19 @@
 
 import logging
 import re
+from importlib.resources import files
 from pathlib import Path
 from threading import Lock
 
 import gridfs
+import jsonschema
 from bson.objectid import ObjectId
 from packaging.version import Version
 from pymongo import ASCENDING, MongoClient
 from pymongo.errors import BulkWriteError
 
-from simtools.db import db_array_elements, db_from_repo_handler
+from simtools.data_model import validate_data
+from simtools.db import db_array_elements
 from simtools.io_operations import io_handler
 from simtools.utils import names, value_conversion
 
@@ -24,6 +27,35 @@ logging.getLogger("pymongo").setLevel(logging.WARNING)
 # The above comment is because pylint does not know that DatabaseHandler.db_client is subscriptable
 
 
+jsonschema_db_dict = {
+    "$schema": "https://json-schema.org/draft/2020-12/schema#",
+    "type": "object",
+    "description": "MongoDB configuration",
+    "properties": {
+        "db_server": {"type": "string", "description": "DB server address"},
+        "db_api_port": {
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 65535,
+            "default": 27017,
+            "description": "Port to use",
+        },
+        "db_api_user": {"type": "string", "description": "API username"},
+        "db_api_pw": {"type": "string", "description": "Password for the API user"},
+        "db_api_authentication_database": {
+            "type": "string",
+            "default": "admin",
+            "description": "DB with user info (optional)",
+        },
+        "db_simulation_model": {
+            "type": "string",
+            "description": "Name of simulation model database",
+        },
+    },
+    "required": ["db_server", "db_api_port", "db_api_user", "db_api_pw", "db_simulation_model"],
+}
+
+
 class DatabaseHandler:
     """
     DatabaseHandler provides the interface to the DB.
@@ -31,13 +63,7 @@ class DatabaseHandler:
     Parameters
     ----------
     mongo_db_config: dict
-        Dictionary with the MongoDB configuration with the following entries:
-        "db_server" - DB server address
-        "db_api_port" - Port to use
-        "db_api_user" - API username
-        "db_api_pw" - Password for the API user
-        "db_api_authentication_database" - DB with user info (optional, default is "admin")
-        "db_simulation_model" - Name of simulation model database
+        Dictionary with the MongoDB configuration (see jsonschema_db_dict for details).
     """
 
     DB_CTA_SIMULATION_MODEL_DESCRIPTIONS = "CTA-Simulation-Model-Descriptions"
@@ -56,7 +82,7 @@ class DatabaseHandler:
         """Initialize the DatabaseHandler class."""
         self._logger = logging.getLogger(__name__)
 
-        self.mongo_db_config = mongo_db_config
+        self.mongo_db_config = self._validate_mongo_db_config(mongo_db_config)
         self.io_handler = io_handler.IOHandler()
         self.list_of_collections = {}
 
@@ -70,6 +96,16 @@ class DatabaseHandler:
             with lock:
                 DatabaseHandler.db_client = self._open_mongo_db()
 
+    def _validate_mongo_db_config(self, mongo_db_config):
+        """Validate the MongoDB configuration."""
+        if mongo_db_config is None or all(value is None for value in mongo_db_config.values()):
+            return None
+        try:
+            jsonschema.validate(instance=mongo_db_config, schema=jsonschema_db_dict)
+            return mongo_db_config
+        except jsonschema.exceptions.ValidationError as err:
+            raise ValueError("Invalid MongoDB configuration") from err
+
     def _open_mongo_db(self):
         """
         Open a connection to MongoDB and return the client to read/write to the DB with.
@@ -83,25 +119,21 @@ class DatabaseHandler:
         KeyError
             If the DB configuration is invalid
         """
-        try:
-            direct_connection = self.mongo_db_config["db_server"] in (
-                "localhost",
-                "simtools-mongodb",
-            )
-            return MongoClient(
-                self.mongo_db_config["db_server"],
-                port=self.mongo_db_config["db_api_port"],
-                username=self.mongo_db_config["db_api_user"],
-                password=self.mongo_db_config["db_api_pw"],
-                authSource=self.mongo_db_config.get("db_api_authentication_database", "admin"),
-                directConnection=direct_connection,
-                ssl=not direct_connection,
-                tlsallowinvalidhostnames=True,
-                tlsallowinvalidcertificates=True,
-            )
-        except KeyError:
-            self._logger.error("Invalid setting of DB configuration")
-            raise
+        direct_connection = self.mongo_db_config["db_server"] in (
+            "localhost",
+            "simtools-mongodb",
+        )
+        return MongoClient(
+            self.mongo_db_config["db_server"],
+            port=self.mongo_db_config["db_api_port"],
+            username=self.mongo_db_config["db_api_user"],
+            password=self.mongo_db_config["db_api_pw"],
+            authSource=self.mongo_db_config.get("db_api_authentication_database", "admin"),
+            directConnection=direct_connection,
+            ssl=not direct_connection,
+            tlsallowinvalidhostnames=True,
+            tlsallowinvalidcertificates=True,
+        )
 
     def _find_latest_simulation_model_db(self):
         """
@@ -207,15 +239,6 @@ class DatabaseHandler:
                         only_applicable=only_applicable,
                     )
                 )
-                if self.mongo_db_config.get("db_simulation_model_url", None) is not None:
-                    pars = db_from_repo_handler.update_model_parameters_from_repo(
-                        parameters=pars,
-                        site=_site,
-                        parameter_collection=collection,
-                        array_element_name=array_element,
-                        model_version=_model_version,
-                        db_simulation_model_url=self.mongo_db_config.get("db_simulation_model_url"),
-                    )
             DatabaseHandler.model_parameters_cached[_array_elements_cache_key] = pars
 
         return pars
@@ -278,7 +301,7 @@ class DatabaseHandler:
         Parameters
         ----------
         parameters: dict
-            Dict of model parameters
+            Dict of model parameters.
         dest: str or Path
             Location where to write the files to.
 
@@ -296,10 +319,6 @@ class DatabaseHandler:
                     continue
                 file = self._get_file_mongo_db(self._get_db_name(), info["value"])
                 self._write_file_from_mongo_to_disk(self._get_db_name(), dest, file)
-        if self.mongo_db_config.get("db_simulation_model_url", None) is not None:
-            self._logger.warning(
-                "Exporting model files from simulation model repository not yet implemented"
-            )
 
     @staticmethod
     def _is_file(value):
@@ -406,24 +425,14 @@ class DatabaseHandler:
         except KeyError:
             pass
 
-        _pars = self._get_site_parameters_mongo_db(
-            _db_name,
-            _site,
-            _model_version,
-            only_applicable,
-        )
-        # update simulation model using repository
-        if self.mongo_db_config.get("db_simulation_model_url", None) is not None:
-            _pars = db_from_repo_handler.update_model_parameters_from_repo(
-                parameters=_pars,
-                site=_site,
-                array_element_name=None,
-                parameter_collection="site",
-                model_version=_model_version,
-                db_simulation_model_url=self.mongo_db_config.get("db_simulation_model_url", None),
+        DatabaseHandler.site_parameters_cached[_site_cache_key] = (
+            self._get_site_parameters_mongo_db(
+                _db_name,
+                _site,
+                _model_version,
+                only_applicable,
             )
-
-        DatabaseHandler.site_parameters_cached[_site_cache_key] = _pars
+        )
         return DatabaseHandler.site_parameters_cached[_site_cache_key]
 
     def _get_site_parameters_mongo_db(self, db_name, site, model_version, only_applicable=False):
@@ -653,7 +662,7 @@ class DatabaseHandler:
         Copy a full array element configuration to a new array element name.
 
         Only a specific version is copied.
-        (This function should be rarely used, probably only during "construction".)
+        This function should be rarely used and is intended to simplify unit tests.
 
         Parameters
         ----------
@@ -710,298 +719,66 @@ class DatabaseHandler:
         except BulkWriteError as exc:
             raise BulkWriteError(str(exc.details)) from exc
 
-    def copy_documents(self, db_name, collection, query, db_to_copy_to, collection_to_copy_to=None):
-        """
-        Copy the documents matching to "query" to the DB "db_to_copy_to".
-
-        The documents are copied to the same collection as in "db_name".
-        (This function should be rarely used, probably only during "construction".)
-
-        Parameters
-        ----------
-        db_name: str
-            the name of the DB to copy from
-        collection: str
-            the name of the collection to copy from
-        query: dict
-            A dictionary with a query to search for documents to copy.
-            For example, the query below would copy all entries of version 6.0.0
-            from telescope LSTN-01 to "db_to_copy_to".
-
-            .. code-block:: python
-
-                query = {
-                    "instrument": "LSTN-01",
-                    "version": "6.0.0",
-                }
-        db_to_copy_to: str
-            The name of the DB to copy to.
-
-        Raises
-        ------
-        BulkWriteError
-
-        """
-        db_name = self._get_db_name(db_name)
-
-        _collection = self.get_collection(db_name, collection)
-        if collection_to_copy_to is None:
-            collection_to_copy_to = collection
-        db_entries = []
-
-        for post in _collection.find(query):
-            post.pop("_id", None)
-            db_entries.append(post)
-
-        self._logger.info(
-            f"Copying documents matching the following query {query}\nto {db_to_copy_to}"
-        )
-        _collection = self.get_collection(db_to_copy_to, collection_to_copy_to)
-        try:
-            _collection.insert_many(db_entries)
-        except BulkWriteError as exc:
-            raise BulkWriteError(str(exc.details)) from exc
-
-    def delete_query(self, db_name, collection, query):
-        """
-        Delete all entries from the DB which correspond to the provided query.
-
-        (This function should be rarely used, if at all.)
-
-        Parameters
-        ----------
-        db_name: str
-            the name of the DB
-        collection: str
-            the name of the collection to copy from
-        query: dict
-            A dictionary listing the fields/values to delete.
-            For example, the query below would delete the entire version 6.0.0
-            from telescope LSTN-01.
-
-            .. code-block:: python
-
-                query = {
-                    "instrument": "LSTN-01",
-                    "version": "6.0.0",
-                }
-
-        """
-        _collection = self.get_collection(db_name, collection)
-
-        if "version" in query:
-            query["version"] = self.model_version(query["version"])
-
-        self._logger.info(f"Deleting {_collection.count_documents(query)} entries from {db_name}")
-
-        _collection.delete_many(query)
-
-    def update_parameter_field(
-        self,
-        db_name,
-        model_version,
-        parameter,
-        field,
-        new_value,
-        array_element_name=None,
-        site=None,
-        collection_name="telescopes",
-    ):
-        """
-        Update a parameter field value for a specific array element/version.
-
-        This function only modifies the value of one of the following
-        DB entries: Applicable, units, Type, items, minimum, maximum.
-        These type of changes should be very rare. However they can
-        be done without changing the Object ID of the entry since
-        they are generally "harmless".
-
-        Parameters
-        ----------
-        db_name: str
-            the name of the DB
-        model_version: str
-            Which model version to update
-        parameter: str
-            Which parameter to update
-        field: str
-            Field to update (only options are Applicable, units, Type, items, minimum, maximum).
-            If the field does not exist, it will be added.
-        new_value: type identical to the original field type
-            The new value to set to the field given in "field".
-        array_element_name: str
-            Which array element to update, if None then update a site parameter
-        site: str, North or South
-            Update a site parameter (the array_element_name argument must be None)
-        collection_name: str
-            The name of the collection in which to update the parameter.
-
-        Raises
-        ------
-        ValueError
-            if field not in allowed fields
-
-        """
-        db_name = self._get_db_name(db_name)
-        allowed_fields = ["applicable", "unit", "type", "items", "minimum", "maximum"]
-        if field not in allowed_fields:
-            raise ValueError(f"The field {field} must be one of {', '.join(allowed_fields)}")
-
-        collection = self.get_collection(db_name, collection_name)
-        _model_version = self.model_version(model_version, db_name)
-
-        query = {
-            "version": _model_version,
-            "parameter": parameter,
-        }
-        if array_element_name is not None:
-            query["instrument"] = array_element_name
-            logger_info = f"instrument {array_element_name}"
-        elif site is not None and site in names.site_names():
-            query["site"] = site
-            logger_info = f"site {site}"
-        else:
-            raise ValueError("You need to specify an array element or a site.")
-
-        par_entry = collection.find_one(query)
-        if par_entry is None:
-            self._logger.warning(
-                f"The query {query} did not return any results. I will not make any changes."
-            )
-            return
-
-        if field in par_entry:
-            old_field_value = par_entry[field]
-
-            if old_field_value == new_value:
-                self._logger.warning(
-                    f"The value of the field {field} is already {new_value}. No changes necessary"
-                )
-                return
-
-            self._logger.info(
-                f"For {logger_info}, version {_model_version}, parameter {parameter}, "
-                f"replacing field {field} value from {old_field_value} to {new_value}"
-            )
-        else:
-            self._logger.info(
-                f"For {logger_info}, version {_model_version}, parameter {parameter}, "
-                f"the field {field} does not exist, adding it"
-            )
-
-        query_update = {"$set": {field: new_value}}
-        collection.update_one(query, query_update)
-
-        self._reset_parameter_cache(
-            site=(
-                site
-                if site is not None
-                else names.get_site_from_array_element_name(array_element_name)
-            ),
-            array_element_name=array_element_name,
-            model_version=_model_version,
-        )
-
     def add_new_parameter(
         self,
         db_name,
-        version,
-        parameter,
-        value,
-        array_element_name=None,
-        site=None,
+        par_dict,
         collection_name="telescopes",
         file_prefix=None,
-        **kwargs,
     ):
         """
-        Add a parameter value for a specific array element.
+        Add a parameter dictionary for a specific array element to the DB.
 
-        A new document will be added to the DB,
-        with all fields taken from the input parameters.
+        A new document will be added to the DB, with all fields taken from the input parameters.
+        Parameter dictionaries are validated before submission using the corresponding schema.
 
         Parameters
         ----------
         db_name: str
             the name of the DB
-        version: str
-            The version of the new parameter value
-        parameter: str
-            Which parameter to add
-        value: can be any type, preferably given in kwargs
-            The value to set for the new parameter
-        array_element_name: str
-            The name of the array element to add a parameter to
-            (only used if collection_name is not "sites").
-        site: str
-            Site name; ignored if collection_name is "telescopes".
+        par_dict: dict
+            dictionary with parameter data
         collection_name: str
             The name of the collection to add a parameter to.
         file_prefix: str or Path
             where to find files to upload to the DB
-        kwargs: dict
-            Any additional fields to add to the parameter
-
-        Raises
-        ------
-        ValueError
-            If key to collection_name is not valid.
-
         """
+        data_validator = validate_data.DataValidator(
+            schema_file=files("simtools")
+            / f"schemas/model_parameters/{par_dict['parameter']}.schema.yml",
+            data_dict=par_dict,
+            check_exact_data_type=False,
+        )
+        par_dict = data_validator.validate_and_transform(is_model_parameter=True)
+
         db_name = self._get_db_name(db_name)
         collection = self.get_collection(db_name, collection_name)
 
-        db_entry = {}
-        if any(
-            key in collection_name
-            for key in ["telescopes", "calibration_devices", "configuration_sim_telarray"]
-        ):
-            db_entry["instrument"] = names.validate_array_element_name(array_element_name)
-        elif "sites" in collection_name:
-            db_entry["instrument"] = names.validate_site_name(site)
-        elif "configuration_corsika" in collection_name:
-            db_entry["instrument"] = None
-        else:
-            raise ValueError(f"Cannot add parameter to collection {collection_name}")
-
-        db_entry["version"] = version
-        db_entry["parameter"] = parameter
-        if site is not None:
-            db_entry["site"] = names.validate_site_name(site)
-
-        _base_value, _base_unit, _base_type = value_conversion.get_value_unit_type(
-            value=value, unit_str=kwargs.get("unit", None)
+        par_dict["value"], _base_unit, _ = value_conversion.get_value_unit_type(
+            value=par_dict["value"], unit_str=par_dict.get("unit", None)
         )
-        db_entry["value"] = _base_value
-        if _base_unit is not None:
-            db_entry["unit"] = _base_unit
-        db_entry["type"] = kwargs["type"] if "type" in kwargs else _base_type
+        par_dict["unit"] = _base_unit if _base_unit else None
 
         files_to_add_to_db = set()
-        db_entry["file"] = False
-        if self._is_file(value):
-            db_entry["file"] = True
+        if par_dict["file"] and par_dict["value"]:
             if file_prefix is None:
                 raise FileNotFoundError(
                     "The location of the file to upload, "
-                    f"corresponding to the {parameter} parameter, must be provided."
+                    f"corresponding to the {par_dict['parameter']} parameter, must be provided."
                 )
-            file_path = Path(file_prefix).joinpath(value)
+            file_path = Path(file_prefix).joinpath(par_dict["value"])
             files_to_add_to_db.add(f"{file_path}")
 
-        kwargs.pop("type", None)
-        db_entry.update(kwargs)
-
         self._logger.info(
-            f"Will add the following entry to DB {db_name} and collection {db_name}:\n{db_entry}"
+            f"Adding a new entry to DB {db_name} and collection {db_name}:\n{par_dict}"
         )
+        collection.insert_one(par_dict)
 
-        collection.insert_one(db_entry)
         for file_to_insert_now in files_to_add_to_db:
             self._logger.info(f"Will also add the file {file_to_insert_now} to the DB")
             self.insert_file_to_db(file_to_insert_now, db_name)
 
-        self._reset_parameter_cache(site, array_element_name, version)
+        self._reset_parameter_cache(par_dict["site"], par_dict["instrument"], par_dict["version"])
 
     def _get_db_name(self, db_name=None):
         """

simtools/io_operations/legacy_data_handler.py

@@ -0,0 +1,61 @@
+#!/usr/bin/python3
+"""Reading of legacy data files (expect that this will be obsolete in future)."""
+
+import logging
+
+from astropy.table import Table
+
+__all__ = [
+    "read_legacy_data_as_table",
+    "read_legacy_lst_single_pe",
+]
+
+logger = logging.getLogger(__name__)
+
+
+def read_legacy_data_as_table(file_path, file_type):
+    """
+    Read legacy data file.
+
+    Parameters
+    ----------
+    file_path: Path
+        Path to the legacy data file.
+    file_type: str
+        Type of legacy data file.
+
+    Returns
+    -------
+    Table
+        Astropy table.
+
+    Raises
+    ------
+    ValueError
+        If unsupported legacy data file type.
+    """
+    logger.debug(f"Reading legacy data file of type {file_type} from {file_path}")
+
+    try:
+        return globals()[f"read_{file_type}"](file_path)
+    except KeyError as exc:
+        raise ValueError(f"Unsupported legacy data file type: {file_type}") from exc
+
+
+def read_legacy_lst_single_pe(file_path):
+    """
+    Read LST single pe file (in legacy data format).
+
+    File contains two columns: amplitude (in units of single p.e) and response.
+
+    Parameters
+    ----------
+    file_path: Path
+        Path to the legacy data file.
+
+    Returns
+    -------
+    Table
+        Astropy table.
+    """
+    return Table.read(file_path, format="ascii.csv", names=("amplitude", "response"))