gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/db/db_handler.py

@@ -2,7 +2,6 @@
 
 import logging
 import re
-from importlib.resources import files
 from pathlib import Path
 from threading import Lock
 
@@ -10,11 +9,9 @@ import gridfs
 import jsonschema
 from bson.objectid import ObjectId
 from packaging.version import Version
-from pymongo import ASCENDING, MongoClient
-from pymongo.errors import BulkWriteError
+from pymongo import MongoClient
 
 from simtools.data_model import validate_data
-from simtools.db import db_array_elements
 from simtools.io_operations import io_handler
 from simtools.utils import names, value_conversion
 
@@ -66,17 +63,11 @@ class DatabaseHandler:
         Dictionary with the MongoDB configuration (see jsonschema_db_dict for details).
     """
 
-    DB_CTA_SIMULATION_MODEL_DESCRIPTIONS = "CTA-Simulation-Model-Descriptions"
-    # DB collection with updates field names
-    DB_DERIVED_VALUES = "Staging-CTA-Simulation-Model-Derived-Values"
-
     ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"]
 
     db_client = None
-    site_parameters_cached = {}
+    production_table_cached = {}
     model_parameters_cached = {}
-    model_versions_cached = {}
-    corsika_configuration_parameters_cached = {}
 
     def __init__(self, mongo_db_config=None):
         """Initialize the DatabaseHandler class."""
@@ -150,12 +141,13 @@ class DatabaseHandler:
 
         """
         try:
-            if not self.mongo_db_config["db_simulation_model"].endswith("LATEST"):
+            db_simulation_model = self.mongo_db_config["db_simulation_model"]
+            if not db_simulation_model.endswith("LATEST"):
                 return
-        except TypeError:
+        except TypeError:  # db_simulation_model is None
             return
 
-        prefix = self.mongo_db_config["db_simulation_model"].replace("LATEST", "")
+        prefix = db_simulation_model.replace("LATEST", "")
         list_of_db_names = self.db_client.list_database_names()
         filtered_list_of_db_names = [s for s in list_of_db_names if s.startswith(prefix)]
         versioned_strings = []
@@ -180,69 +172,116 @@ class DatabaseHandler:
         else:
             raise ValueError("Found LATEST in the DB name but no matching versions found in DB.")
 
+    def get_model_parameter(
+        self,
+        parameter,
+        parameter_version,
+        site,
+        array_element_name,
+    ):
+        """
+        Get a model parameter using the parameter version.
+
+        Parameters
+        ----------
+        parameter: str
+            Name of the parameter.
+        parameter_version: str
+            Version of the parameter.
+        site: str
+            Site name.
+        array_element_name: str
+            Name of the array element model (e.g. MSTN, SSTS).
+
+        Returns
+        -------
+        dict containing the parameter
+
+        """
+        query = {
+            "parameter_version": parameter_version,
+            "parameter": parameter,
+        }
+        if array_element_name is not None:
+            query["instrument"] = array_element_name
+        if site is not None:
+            query["site"] = site
+        return self._read_mongo_db(
+            query=query, collection_name=names.get_collection_name_from_parameter_name(parameter)
+        )
+
     def get_model_parameters(
         self,
         site,
         array_element_name,
-        model_version,
-        collection="telescope",
-        only_applicable=False,
+        collection,
+        model_version=None,
     ):
         """
-        Get parameters from MongoDB or simulation model repository for an array element.
+        Get model parameters using the model version.
 
-        An array element can be e.g., a telescope or a calibration device.
-        Read parameters for design and for the specified array element (if necessary). This allows
-        to overwrite design parameters with specific parameters without having to copy
-        all model parameters when changing only a few.
+        Queries parameters for design and for the specified array element (if necessary).
 
         Parameters
         ----------
         site: str
             Site name.
         array_element_name: str
-            Name of the array element model (e.g. LSTN-01, MSTS-design)
-        model_version: str
-            Version of the model.
+            Name of the array element model (e.g. LSTN-01, MSTx-FlashCam, ILLN-01).
+        model_version: str, list
+            Version(s) of the model.
         collection: str
-            collection of array element (e.g. telescopes, calibration_devices)
-        only_applicable: bool
-            If True, only applicable parameters will be read.
+            Collection of array element (e.g. telescopes, calibration_devices).
 
         Returns
         -------
         dict containing the parameters
-
         """
-        _site, _array_element_name, _model_version = self._validate_model_input(
-            site, array_element_name, model_version
-        )
-        array_element_list = db_array_elements.get_array_element_list_for_db_query(
-            _array_element_name, self, _model_version, collection
+        model_versions = (
+            self.get_model_versions(collection) if model_version is None else [model_version]
         )
+
         pars = {}
-        for array_element in array_element_list:
-            _array_elements_cache_key = self._parameter_cache_key(
-                site, array_element, model_version, collection
+        for _model_version in model_versions:
+            production_table = self._read_production_table_from_mongo_db(collection, _model_version)
+            array_element_list = self._get_array_element_list(
+                array_element_name, site, production_table, collection
             )
-            try:
-                pars.update(DatabaseHandler.model_parameters_cached[_array_elements_cache_key])
-            except KeyError:
+            for array_element in array_element_list:
                 pars.update(
-                    self.read_mongo_db(
-                        self.mongo_db_config.get("db_simulation_model", None),
-                        array_element_name=array_element,
-                        model_version=_model_version,
-                        collection_name=collection,
-                        run_location=None,
-                        write_files=False,
-                        only_applicable=only_applicable,
+                    self._get_parameter_for_model_version(
+                        array_element, _model_version, site, collection, production_table
                     )
                 )
-            DatabaseHandler.model_parameters_cached[_array_elements_cache_key] = pars
-
         return pars
 
+    def _get_parameter_for_model_version(
+        self, array_element, model_version, site, collection, production_table
+    ):
+        cache_key, cache_dict = self._read_cache(
+            DatabaseHandler.model_parameters_cached,
+            names.validate_site_name(site) if site else None,
+            array_element,
+            model_version,
+            collection,
+        )
+        if cache_dict:
+            self._logger.debug(f"Found {array_element} in cache (key: {cache_key})")
+            return cache_dict
+        self._logger.debug(f"Did not find {array_element} in cache (key: {cache_key})")
+
+        try:
+            parameter_version_table = production_table["parameters"][array_element]
+        except KeyError:  # allow missing array elements (parameter dict is checked later)
+            return {}
+        DatabaseHandler.model_parameters_cached[cache_key] = self._read_mongo_db(
+            query=self._get_query_from_parameter_version_table(
+                parameter_version_table, array_element, site
+            ),
+            collection_name=collection,
+        )
+        return DatabaseHandler.model_parameters_cached[cache_key]
+
     def get_collection(self, db_name, collection_name):
         """
         Get a collection from the DB.
@@ -263,99 +302,102 @@ class DatabaseHandler:
         db_name = self._get_db_name(db_name)
         return DatabaseHandler.db_client[db_name][collection_name]
 
-    def export_file_db(self, db_name, dest, file_name):
+    def get_collections(self, db_name=None, model_collections_only=False):
         """
-        Get file from the DB and write to disk.
+        List of collections in the DB.
 
         Parameters
         ----------
         db_name: str
-            Name of the DB to search in.
-        dest: str or Path
-            Location where to write the file to.
-        file_name: str
-            Name of the file to get.
+            Database name.
+        model_collections_only: bool
+            If True, only return model collections (i.e. exclude fs.files, fs.chunks)
 
         Returns
         -------
-        file_id: GridOut._id
-            the database ID the file was assigned when it was inserted to the DB.
-
-        Raises
-        ------
-        FileNotFoundError
-            If the desired file is not found.
+        list
+            List of collection names
 
         """
-        db_name = self._get_db_name(db_name)
-
-        self._logger.debug(f"Getting {file_name} from {db_name} and writing it to {dest}")
-        file_path_instance = self._get_file_mongo_db(db_name, file_name)
-        self._write_file_from_mongo_to_disk(db_name, dest, file_path_instance)
-        return file_path_instance._id  # pylint: disable=protected-access;
+        db_name = db_name or self._get_db_name()
+        if db_name not in self.list_of_collections:
+            self.list_of_collections[db_name] = DatabaseHandler.db_client[
+                db_name
+            ].list_collection_names()
+        collections = self.list_of_collections[db_name]
+        if model_collections_only:
+            return [collection for collection in collections if not collection.startswith("fs.")]
+        return collections
 
-    def export_model_files(self, parameters, dest):
+    def export_model_files(self, parameters=None, file_names=None, dest=None, db_name=None):
         """
-        Export all the files in a model from the DB and write them to disk.
+        Export files from the DB to the model directory.
+
+        The files to be exported can be specified by file_name or retrieved from the database
+        using the parameters dictionary.
 
         Parameters
         ----------
         parameters: dict
-            Dict of model parameters.
+            Dict of model parameters
+        file_names: list, str
+            List (or string) of file names to export
         dest: str or Path
             Location where to write the files to.
 
-        Raises
-        ------
-        FileNotFoundError
-            if a file in parameters.values is not found
-
+        Returns
+        -------
+        file_id: dict of GridOut._id
+            Dict of database IDs of files.
         """
-        if self.mongo_db_config:
-            for info in parameters.values():
-                if not info or not info.get("file") or info["value"] is None:
-                    continue
-                if Path(dest).joinpath(info["value"]).exists():
-                    continue
-                file = self._get_file_mongo_db(self._get_db_name(), info["value"])
-                self._write_file_from_mongo_to_disk(self._get_db_name(), dest, file)
+        db_name = self._get_db_name(db_name)
 
-    @staticmethod
-    def _is_file(value):
-        """Verify if a parameter value is a file name."""
-        return any(ext in str(value) for ext in DatabaseHandler.ALLOWED_FILE_EXTENSIONS)
+        if file_names:
+            file_names = [file_names] if not isinstance(file_names, list) else file_names
+        elif parameters:
+            file_names = [
+                info["value"]
+                for info in parameters.values()
+                if info and info.get("file") and info["value"] is not None
+            ]
 
-    def read_mongo_db(
-        self,
-        db_name,
-        array_element_name,
-        model_version,
-        run_location,
-        collection_name,
-        write_files=True,
-        only_applicable=False,
+        instance_ids = {}
+        for file_name in file_names:
+            if Path(dest).joinpath(file_name).exists():
+                instance_ids[file_name] = "file exists"
+            else:
+                file_path_instance = self._get_file_mongo_db(self._get_db_name(), file_name)
+                self._write_file_from_mongo_to_disk(self._get_db_name(), dest, file_path_instance)
+                instance_ids[file_name] = file_path_instance._id  # pylint: disable=protected-access
+        return instance_ids
+
+    def _get_query_from_parameter_version_table(
+        self, parameter_version_table, array_element_name, site
     ):
-        """
-        Build and execute query to Read the MongoDB for a specific array element.
+        """Return query based on parameter version table."""
+        query_dict = {
+            "$or": [
+                {"parameter": param, "parameter_version": version}
+                for param, version in parameter_version_table.items()
+            ],
+        }
+        # 'xSTX-design' is a placeholder to ignore 'instrument' field in query.
+        if array_element_name and array_element_name != "xSTx-design":
+            query_dict["instrument"] = array_element_name
+        if site:
+            query_dict["site"] = site
+        return query_dict
 
-        Also writes the files listed in the parameter values into the sim_telarray run location
+    def _read_mongo_db(self, query, collection_name):
+        """
+        Query MongoDB.
 
         Parameters
         ----------
-        db_name: str
-            the name of the DB
-        array_element_name: str
-            Name of the array element model (e.g. MSTN-design ...)
-        model_version: str
-            Version of the model.
-        run_location: Path or str
-            The sim_telarray run location to write the tabulated data files into.
+        query: dict
+            Query to execute.
         collection_name: str
-            The name of the collection to read from.
-        write_files: bool
-            If true, write the files to the run_location.
-        only_applicable: bool
-            If True, only applicable parameters will be read.
+            Collection name.
 
         Returns
         -------
@@ -364,154 +406,153 @@ class DatabaseHandler:
         Raises
         ------
         ValueError
-            if query returned zero results.
-
+            if query returned no results.
         """
+        db_name = self._get_db_name()
         collection = self.get_collection(db_name, collection_name)
-        _parameters = {}
-
-        query = {
-            "instrument": array_element_name,
-            "version": self.model_version(model_version, db_name),
-        }
-
-        if only_applicable:
-            query["applicable"] = True
-        if collection.count_documents(query) < 1:
+        posts = list(collection.find(query))
+        if not posts:
             raise ValueError(
-                "The following query returned zero results! Check the input data and rerun.\n",
-                query,
+                f"The following query for {collection_name} returned zero results: {query} "
             )
-        for post in collection.find(query).sort("parameter", ASCENDING):
+        parameters = {}
+        for post in posts:
             par_now = post["parameter"]
-            _parameters[par_now] = post
-            _parameters[par_now].pop("parameter", None)
-            _parameters[par_now].pop("instrument", None)
-            _parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time
-            if _parameters[par_now]["file"] and write_files:
-                file = self._get_file_mongo_db(db_name, _parameters[par_now]["value"])
-                self._write_file_from_mongo_to_disk(db_name, run_location, file)
-
-        return _parameters
+            parameters[par_now] = post
+            parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time
+        return {k: parameters[k] for k in sorted(parameters)}
 
-    def get_site_parameters(
-        self,
-        site,
-        model_version,
-        only_applicable=False,
-    ):
+    def _read_production_table_from_mongo_db(self, collection_name, model_version):
         """
-        Get parameters from either MongoDB or simulation model repository for a specific site.
+        Read production table from MongoDB.
 
         Parameters
         ----------
-        site: str
-            Site name.
+        collection_name: str
+            Name of the collection.
         model_version: str
             Version of the model.
-        only_applicable: bool
-            If True, only applicable parameters will be read.
-
-        Returns
-        -------
-        dict containing the parameters
 
+        Raises
+        ------
+        ValueError
+            if query returned no results.
         """
-        _site, _, _model_version = self._validate_model_input(site, None, model_version)
-        _db_name = self._get_db_name()
-        _site_cache_key = self._parameter_cache_key(site, None, model_version)
         try:
-            return DatabaseHandler.site_parameters_cached[_site_cache_key]
+            return DatabaseHandler.production_table_cached[
+                self._cache_key(None, None, model_version, collection_name)
+            ]
         except KeyError:
             pass
 
-        DatabaseHandler.site_parameters_cached[_site_cache_key] = (
-            self._get_site_parameters_mongo_db(
-                _db_name,
-                _site,
-                _model_version,
-                only_applicable,
-            )
-        )
-        return DatabaseHandler.site_parameters_cached[_site_cache_key]
+        query = {"model_version": model_version, "collection": collection_name}
+        collection = self.get_collection(self._get_db_name(), "production_tables")
+        post = collection.find_one(query)
+        if not post:
+            raise ValueError(f"The following query returned zero results: {query}")
+
+        return {
+            "collection": post["collection"],
+            "model_version": post["model_version"],
+            "parameters": post["parameters"],
+            "design_model": post.get("design_model", {}),
+            "entry_date": ObjectId(post["_id"]).generation_time,
+        }
 
-    def _get_site_parameters_mongo_db(self, db_name, site, model_version, only_applicable=False):
+    def get_model_versions(self, collection_name="telescopes"):
         """
-        Get parameters from MongoDB for a specific site.
+        Get list of model versions from the DB.
 
         Parameters
         ----------
-        db_name: str
-            The name of the DB.
-        site: str
-            Site name.
-        model_version: str
-            Version of the model.
-        only_applicable: bool
-            If True, only applicable parameters will be read.
+        collection_name: str
+            Name of the collection.
 
         Returns
         -------
-        dict containing the parameters
-
-        Raises
-        ------
-        ValueError
-            if query returned zero results.
+        list
+            List of model versions
+        """
+        collection = self.get_collection(self._get_db_name(), "production_tables")
+        return sorted(
+            {post["model_version"] for post in collection.find({"collection": collection_name})}
+        )
 
+    def get_array_elements(self, model_version, collection="telescopes"):
         """
-        collection = self.get_collection(db_name, "sites")
-        _parameters = {}
+        Get list array elements for a given model version and collection from the DB.
 
-        query = {
-            "site": site,
-            "version": model_version,
-        }
-        if only_applicable:
-            query["applicable"] = True
-        if collection.count_documents(query) < 1:
-            raise ValueError(
-                "The following query returned zero results! Check the input data and rerun.\n",
-                query,
-            )
-        for post in collection.find(query).sort("parameter", ASCENDING):
-            par_now = post["parameter"]
-            _parameters[par_now] = post
-            _parameters[par_now].pop("parameter", None)
-            _parameters[par_now].pop("site", None)
-            _parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time
+        Parameters
+        ----------
+        model_version: str
+            Version of the model.
+        collection: str
+            Which collection to get the array elements from:
+            i.e. telescopes, calibration_devices.
 
-        return _parameters
+        Returns
+        -------
+        list
+            Sorted list of all array elements found in collection
+        """
+        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        return sorted([entry for entry in production_table["parameters"] if "-design" not in entry])
 
-    def get_derived_values(self, site, array_element_name, model_version):
+    def get_design_model(self, model_version, array_element_name, collection="telescopes"):
         """
-        Get all derived values from the DB for a specific array element.
+        Get the design model used for a given array element and a given model version.
 
         Parameters
         ----------
-        site: str
-            Site name.
-        array_element_name: str
-            Name of the array element model (e.g. MSTN, SSTS).
         model_version: str
             Version of the model.
+        array_element_name: str
+            Name of the array element model (e.g. MSTN, SSTS).
+        collection: str
+            Which collection to get the array elements from:
+            i.e. telescopes, calibration_devices.
 
         Returns
         -------
-        dict containing the parameters
+        str
+            Design model for a given array element.
+        """
+        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        try:
+            return production_table["design_model"][array_element_name]
+        except KeyError:
+            # for eg. array_element_name == 'LSTN-design' returns 'LSTN-design'
+            return array_element_name
 
+    def get_array_elements_of_type(self, array_element_type, model_version, collection):
         """
-        _, _array_element_name, _model_version = self._validate_model_input(
-            site, array_element_name, model_version
-        )
+        Get array elements of a certain type (e.g. 'LSTN') for a DB collection.
+
+        Does not return 'design' models.
 
-        return self.read_mongo_db(
-            DatabaseHandler.DB_DERIVED_VALUES,
-            _array_element_name,
-            _model_version,
-            run_location=None,
-            collection_name="derived_values",
-            write_files=False,
+        Parameters
+        ----------
+        array_element_type: str
+            Type of the array element (e.g. LSTN, MSTS).
+        model_version: str
+            Version of the model.
+        collection: str
+            Which collection to get the array elements from:
+            i.e. telescopes, calibration_devices.
+
+        Returns
+        -------
+        list
+            Sorted list of all array element names found in collection
+        """
+        production_table = self._read_production_table_from_mongo_db(collection, model_version)
+        all_array_elements = production_table["parameters"]
+        return sorted(
+            [
+                entry
+                for entry in all_array_elements
+                if entry.startswith(array_element_type) and "-design" not in entry
+            ]
         )
 
     def get_simulation_configuration_parameters(
@@ -541,63 +582,24 @@ class DatabaseHandler:
             if simulation_software is not valid.
         """
         if simulation_software == "corsika":
-            return self.get_corsika_configuration_parameters(model_version)
+            return self.get_model_parameters(
+                None,
+                None,
+                model_version=model_version,
+                collection="configuration_corsika",
+            )
         if simulation_software == "simtel":
-            if site and array_element_name:
-                return self.get_model_parameters(
-                    site, array_element_name, model_version, collection="configuration_sim_telarray"
+            return (
+                self.get_model_parameters(
+                    site,
+                    array_element_name,
+                    model_version=model_version,
+                    collection="configuration_sim_telarray",
                 )
-            return {}
-        raise ValueError(f"Unknown simulation software: {simulation_software}")
-
-    def get_corsika_configuration_parameters(self, model_version):
-        """
-        Get CORSIKA configuration parameters from the DB.
-
-        Parameters
-        ----------
-        model_version : str
-            Version of the model.
-
-        Returns
-        -------
-        dict
-            Configuration parameters for CORSIKA
-        """
-        _corsika_cache_key = self._parameter_cache_key(None, None, model_version)
-        try:
-            return DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key]
-        except KeyError:
-            pass
-        DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] = (
-            self.read_mongo_db(
-                db_name=self._get_db_name(),
-                array_element_name=None,
-                model_version=model_version,
-                run_location=None,
-                collection_name="configuration_corsika",
-                write_files=False,
+                if site and array_element_name
+                else {}
             )
-        )
-        return DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key]
-
-    def _validate_model_input(self, site, array_element_name, model_version):
-        """
-        Validate input for model parameter queries.
-
-        site: str
-            Site name.
-        array_element_name: str
-            Name of the array element model (e.g. LSTN-01, MSTS-design)
-        model_version: str
-            Version of the model.
-
-        """
-        return (
-            names.validate_site_name(site),
-            names.validate_array_element_name(array_element_name) if array_element_name else None,
-            self.model_version(model_version),
-        )
+        raise ValueError(f"Unknown simulation software: {simulation_software}")
 
     @staticmethod
     def _get_file_mongo_db(db_name, file_name):
@@ -648,76 +650,22 @@ class DatabaseHandler:
         with open(Path(path).joinpath(file.filename), "wb") as output_file:
             fs_output.download_to_stream_by_name(file.filename, output_file)
 
-    def copy_array_element(
-        self,
-        db_name,
-        element_to_copy,
-        version_to_copy,
-        new_array_element_name,
-        collection_name="telescopes",
-        db_to_copy_to=None,
-        collection_to_copy_to=None,
-    ):
+    def add_production_table(self, db_name, production_table):
         """
-        Copy a full array element configuration to a new array element name.
-
-        Only a specific version is copied.
-        This function should be rarely used and is intended to simplify unit tests.
+        Add a production table to the DB.
 
         Parameters
         ----------
         db_name: str
-            the name of the DB to copy from
-        element_to_copy: str
-            The array element to copy
-        version_to_copy: str
-            The version of the configuration to copy
-        new_array_element_name: str
-            The name of the new array element
-        collection_name: str
-            The name of the collection to copy from.
-        db_to_copy_to: str
-            The name of the DB to copy to.
-        collection_to_copy_to: str
-            The name of the collection to copy to.
-
-        Raises
-        ------
-        BulkWriteError
-
+            the name of the DB.
+        production_table: dict
+            The production table to add to the DB.
         """
         db_name = self._get_db_name(db_name)
-        if db_to_copy_to is None:
-            db_to_copy_to = db_name
-
-        if collection_to_copy_to is None:
-            collection_to_copy_to = collection_name
-
-        self._logger.info(
-            f"Copying version {version_to_copy} of {element_to_copy} "
-            f"to the new array element {new_array_element_name} in the {db_to_copy_to} DB"
-        )
-
-        collection = self.get_collection(db_name, collection_name)
-        db_entries = []
-
-        _version_to_copy = self.model_version(version_to_copy)
-
-        query = {
-            "instrument": element_to_copy,
-            "version": _version_to_copy,
-        }
-        for post in collection.find(query):
-            post["instrument"] = new_array_element_name
-            post.pop("_id", None)
-            db_entries.append(post)
-
-        self._logger.info(f"Creating new array element {new_array_element_name}")
-        collection = self.get_collection(db_to_copy_to, collection_to_copy_to)
-        try:
-            collection.insert_many(db_entries)
-        except BulkWriteError as exc:
-            raise BulkWriteError(str(exc.details)) from exc
+        collection = self.get_collection(db_name, "production_tables")
+        self._logger.info(f"Adding production for {production_table.get('collection')} to to DB")
+        collection.insert_one(production_table)
+        DatabaseHandler.production_table_cached.clear()
 
     def add_new_parameter(
         self,
@@ -727,7 +675,7 @@ class DatabaseHandler:
         file_prefix=None,
     ):
         """
-        Add a parameter dictionary for a specific array element to the DB.
+        Add a new parameter dictionary to the DB.
 
         A new document will be added to the DB, with all fields taken from the input parameters.
         Parameter dictionaries are validated before submission using the corresponding schema.
@@ -743,13 +691,7 @@ class DatabaseHandler:
         file_prefix: str or Path
             where to find files to upload to the DB
         """
-        data_validator = validate_data.DataValidator(
-            schema_file=files("simtools")
-            / f"schemas/model_parameters/{par_dict['parameter']}.schema.yml",
-            data_dict=par_dict,
-            check_exact_data_type=False,
-        )
-        par_dict = data_validator.validate_and_transform(is_model_parameter=True)
+        par_dict = validate_data.DataValidator.validate_model_parameter(par_dict)
 
         db_name = self._get_db_name(db_name)
         collection = self.get_collection(db_name, collection_name)
@@ -778,7 +720,7 @@ class DatabaseHandler:
             self._logger.info(f"Will also add the file {file_to_insert_now} to the DB")
             self.insert_file_to_db(file_to_insert_now, db_name)
 
-        self._reset_parameter_cache(par_dict["site"], par_dict["instrument"], par_dict["version"])
+        self._reset_parameter_cache()
 
     def _get_db_name(self, db_name=None):
         """
@@ -796,42 +738,6 @@ class DatabaseHandler:
         """
         return self.mongo_db_config["db_simulation_model"] if db_name is None else db_name
 
-    def model_version(self, version, db_name=None):
-        """
-        Return model version and check that it is valid.
-
-        Queries the database for all available model versions and check if the
-        requested version is valid.
-
-        Parameters
-        ----------
-        version : str
-            Model version.
-        db_name : str
-            Database name.
-
-        Returns
-        -------
-        str
-            Model version.
-
-        Raises
-        ------
-        ValueError
-            if version not valid.
-
-        """
-        _all_versions = self.get_all_versions(db_name=db_name)
-        if version in _all_versions:
-            return version
-        if len(_all_versions) == 0:
-            return None
-
-        raise ValueError(
-            f"Invalid model version {version} in DB {self._get_db_name(db_name)} "
-            f"(allowed are {_all_versions})"
-        )
-
     def insert_file_to_db(self, file_name, db_name=None, **kwargs):
         """
         Insert a file to the DB.
@@ -855,15 +761,11 @@ class DatabaseHandler:
 
         """
         db_name = self._get_db_name(db_name)
-
         db = DatabaseHandler.db_client[db_name]
         file_system = gridfs.GridFS(db)
 
-        if "content_type" not in kwargs:
-            kwargs["content_type"] = "ascii/dat"
-
-        if "filename" not in kwargs:
-            kwargs["filename"] = Path(file_name).name
+        kwargs.setdefault("content_type", "ascii/dat")
+        kwargs.setdefault("filename", Path(file_name).name)
 
         if file_system.exists({"filename": kwargs["filename"]}):
             self._logger.warning(
@@ -872,78 +774,11 @@ class DatabaseHandler:
             return file_system.find_one(  # pylint: disable=protected-access
                 {"filename": kwargs["filename"]}
             )._id
+        self._logger.debug(f"Writing file to DB: {file_name}")
         with open(file_name, "rb") as data_file:
             return file_system.put(data_file, **kwargs)
 
-    def get_all_versions(
-        self,
-        parameter=None,
-        array_element_name=None,
-        site=None,
-        db_name=None,
-        collection=None,
-    ):
-        """
-        Get all version entries in the DB of collection and/or a specific parameter.
-
-        Parameters
-        ----------
-        parameter: str
-            Which parameter to get the versions of
-        array_element_name: str
-            Which array element to get the versions of (in case "collection_name" is not "sites")
-        site: str
-            Site name.
-        db_name: str
-            Database name.
-        collection_name: str
-            The name of the collection in which to update the parameter.
-
-        Returns
-        -------
-        all_versions: list
-            List of all versions found
-
-        Raises
-        ------
-        ValueError
-            If key to collection_name is not valid.
-
-        """
-        db_name = self._get_db_name() if db_name is None else db_name
-        if not db_name:
-            self._logger.warning("No database name defined to determine list of model versions")
-            return []
-        _cache_key = f"model_versions_{db_name}-{collection}"
-
-        query = {}
-        if parameter is not None:
-            query["parameter"] = parameter
-            _cache_key = f"{_cache_key}-{parameter}"
-        if collection in ["telescopes", "calibration_devices"] and array_element_name is not None:
-            query["instrument"] = names.validate_array_element_name(array_element_name)
-            _cache_key = f"{_cache_key}-{query['instrument']}"
-        elif collection == "sites" and site is not None:
-            query["site"] = names.validate_site_name(site)
-            _cache_key = f"{_cache_key}-{query['site']}"
-
-        if _cache_key not in DatabaseHandler.model_versions_cached:
-            all_versions = set()
-            collections_to_query = (
-                [collection] if collection else self.get_collections(db_name, True)
-            )
-            for collection_name in collections_to_query:
-                db_collection = self.get_collection(db_name, collection_name)
-                sorted_posts = db_collection.find(query).sort("version", ASCENDING)
-                all_versions.update(post["version"] for post in sorted_posts)
-            DatabaseHandler.model_versions_cached[_cache_key] = list(all_versions)
-
-        if len(DatabaseHandler.model_versions_cached[_cache_key]) == 0:
-            self._logger.warning(f"The query {query} did not return any results. No versions found")
-
-        return DatabaseHandler.model_versions_cached[_cache_key]
-
-    def _parameter_cache_key(self, site, array_element_name, model_version, collection=None):
+    def _cache_key(self, site=None, array_element_name=None, model_version=None, collection=None):
         """
         Create a cache key for the parameter cache dictionaries.
 
@@ -963,61 +798,84 @@ class DatabaseHandler:
         str
             Cache key.
         """
-        parts = []
-        if site:
-            parts.append(site)
-        if array_element_name:
-            parts.append(array_element_name)
-        parts.append(model_version)
-        if collection:
-            parts.append(collection)
-        return "-".join(parts)
+        return "-".join(
+            part for part in [model_version, collection, site, array_element_name] if part
+        )
 
-    def _reset_parameter_cache(self, site, array_element_name, model_version):
+    def _read_cache(
+        self, cache_dict, site=None, array_element_name=None, model_version=None, collection=None
+    ):
         """
-        Reset the cache for the parameters.
+        Read parameters from cache.
 
         Parameters
         ----------
+        cache_dict: dict
+            Cache dictionary.
         site: str
             Site name.
         array_element_name: str
             Array element name.
         model_version: str
             Model version.
+        collection: str
+            DB collection name.
+
+        Returns
+        -------
+        str
+            Cache key.
         """
-        self._logger.debug(f"Resetting cache for {site} {array_element_name} {model_version}")
-        _cache_key = self._parameter_cache_key(site, array_element_name, model_version)
-        DatabaseHandler.site_parameters_cached.pop(_cache_key, None)
-        DatabaseHandler.model_parameters_cached.pop(_cache_key, None)
-        db_array_elements.get_array_elements.cache_clear()
+        cache_key = self._cache_key(site, array_element_name, model_version, collection)
+        try:
+            return cache_key, cache_dict[cache_key]
+        except KeyError:
+            return cache_key, None
 
-    def get_collections(self, db_name=None, model_collections_only=False):
+    def _reset_parameter_cache(self):
+        """Reset the cache for the parameters."""
+        DatabaseHandler.model_parameters_cached.clear()
+
+    def _get_array_element_list(self, array_element_name, site, production_table, collection):
         """
-        List of collections in the DB.
+        Return list of array elements for DB queries (add design model if needed).
+
+        Design model is added if found in the production table.
 
         Parameters
         ----------
-        db_name: str
-            Database name.
+        array_element_name: str
+            Name of the array element.
+        site: str
+            Site name.
+        production_table: dict
+            Production table.
+        collection: str
+            collection of array element (e.g. telescopes, calibration_devices).
 
         Returns
         -------
         list
-            List of collection names
-        model_collections_only: bool
-            If True, only return model collections (i.e. exclude fs.files, fs.chunks, metadata)
-
-        """
-        db_name = self._get_db_name() if db_name is None else db_name
-        if db_name not in self.list_of_collections:
-            self.list_of_collections[db_name] = DatabaseHandler.db_client[
-                db_name
-            ].list_collection_names()
-        if model_collections_only:
+            List of array elements
+        """
+        if collection == "configuration_corsika":
+            return ["xSTx-design"]  # placeholder to ignore 'instrument' field in query.
+        if collection == "sites":
+            return [f"OBS-{site}"]
+        if names.is_design_type(array_element_name):
+            return [array_element_name]
+        if collection == "configuration_sim_telarray":
+            # get design model from 'telescope' or 'calibration_device' production tables
+            production_table = self._read_production_table_from_mongo_db(
+                names.get_collection_name_from_array_element_name(array_element_name),
+                production_table["model_version"],
+            )
+        try:
             return [
-                collection
-                for collection in self.list_of_collections[db_name]
-                if not collection.startswith("fs.") and collection != "metadata"
+                production_table["design_model"][array_element_name],
+                array_element_name,
             ]
-        return self.list_of_collections[db_name]
+        except KeyError as exc:
+            raise KeyError(
+                f"Failed generated array element list for db query for {array_element_name}"
+            ) from exc