gammasimtools 0.20.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

simtools/db/db_handler.py

@@ -11,13 +11,13 @@ import gridfs
 import jsonschema
 from astropy.table import Table
 from bson.objectid import ObjectId
-from packaging.version import Version
 from pymongo import MongoClient
 
 from simtools.data_model import validate_data
 from simtools.io import ascii_handler, io_handler
 from simtools.simtel import simtel_table_reader
 from simtools.utils import names, value_conversion
+from simtools.version import resolve_version_to_latest_patch
 
 __all__ = ["DatabaseHandler"]
 
@@ -72,6 +72,11 @@ class DatabaseHandler:
     """
     DatabaseHandler provides the interface to the DB.
 
+    Note the two types of version variables used in this class:
+
+    - db_simulation_model_version (from mongo_db_config): version of the simulation model database
+    - model_version (from production_tables): version of the model contained in the database
+
     Parameters
     ----------
     mongo_db_config: dict
@@ -83,6 +88,7 @@ class DatabaseHandler:
     db_client = None
     production_table_cached = {}
     model_parameters_cached = {}
+    model_versions_cached = {}
 
     def __init__(self, mongo_db_config=None):
         """Initialize the DatabaseHandler class."""
@@ -93,10 +99,9 @@ class DatabaseHandler:
         self.list_of_collections = {}
 
         self._set_up_connection()
-        self._find_latest_simulation_model_db()
         self.db_name = (
             self.get_db_name(
-                model_version=self.mongo_db_config.get("db_simulation_model_version"),
+                db_simulation_model_version=self.mongo_db_config.get("db_simulation_model_version"),
                 model_name=self.mongo_db_config.get("db_simulation_model"),
             )
             if self.mongo_db_config
@@ -110,15 +115,43 @@ class DatabaseHandler:
             with lock:
                 DatabaseHandler.db_client = self._open_mongo_db()
 
-    def get_db_name(self, db_name=None, model_version=None, model_name=None):
+    def get_db_name(self, db_name=None, db_simulation_model_version=None, model_name=None):
         """Build DB name from configuration."""
         if db_name:
             return db_name
-        if model_version and model_name:
-            return f"{model_name}-{model_version.replace('.', '-')}"
-        if model_version or model_name:
+        if db_simulation_model_version and model_name:
+            return f"{model_name}-{db_simulation_model_version.replace('.', '-')}"
+        if db_simulation_model_version or model_name:
             return None
-        return None if (model_version or model_name) else self.db_name
+        return None if (db_simulation_model_version or model_name) else self.db_name
+
+    def print_connection_info(self):
+        """Print the connection information."""
+        if self.mongo_db_config:
+            self._logger.info(
+                f"Connected to MongoDB at {self.mongo_db_config['db_server']}:"
+                f"{self.mongo_db_config['db_api_port']} "
+                f"using database: {self.db_name}"
+            )
+        else:
+            self._logger.info("No MongoDB configuration provided.")
+
+    def is_remote_database(self):
+        """
+        Check if the database is remote.
+
+        Check for domain pattern like "cta-simpipe-protodb.zeuthen.desy.de"
+
+        Returns
+        -------
+        bool
+            True if the database is remote, False otherwise.
+        """
+        if self.mongo_db_config:
+            db_server = self.mongo_db_config["db_server"]
+            domain_pattern = r"^([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$"
+            return bool(re.match(domain_pattern, db_server))
+        return False
 
     def _validate_mongo_db_config(self, mongo_db_config):
         """Validate the MongoDB configuration."""
@@ -164,50 +197,45 @@ class DatabaseHandler:
             tlsallowinvalidcertificates=True,
         )
 
-    def _find_latest_simulation_model_db(self):
+    def generate_compound_indexes_for_databases(
+        self, db_name, db_simulation_model, db_simulation_model_version
+    ):
         """
-        Find the latest released version of the simulation model and update the DB config.
+        Generate compound indexes for several databases.
 
-        This is indicated by "LATEST" to the simulation model data base version.
-        Only released versions are considered, pre-releases are ignored.
+        Parameters
+        ----------
+        db_name: str
+            Name of the database.
+        db_simulation_model: str
+            Name of the simulation model.
+        db_simulation_model_version: str
+            Version of the simulation model.
 
         Raises
         ------
         ValueError
-            If the "LATEST" version is requested but no versions are found in the DB.
+            If the requested database is not found.
 
         """
-        try:
-            db_simulation_model_version = self.mongo_db_config["db_simulation_model_version"]
-            db_simulation_model = self.mongo_db_config["db_simulation_model"]
-            if db_simulation_model_version != "LATEST":
-                return
-        except TypeError:  # db_simulation_model_version is None
-            return
-
-        list_of_db_names = self.db_client.list_database_names()
-        filtered_list_of_db_names = [
-            s for s in list_of_db_names if s.startswith(db_simulation_model)
+        databases = [
+            d for d in self.db_client.list_database_names() if d not in ("config", "admin", "local")
         ]
-        pattern = re.compile(rf"{re.escape(db_simulation_model)}-v(\d+)-(\d+)-(\d+)(?:-(.+))?$")
-
-        versioned_strings = []
-        for s in filtered_list_of_db_names:
-            m = pattern.match(s)
-            if m:
-                # skip pre-releases (have suffix)
-                if m.group(4) is None:
-                    version_str = f"{m.group(1)}.{m.group(2)}.{m.group(3)}"
-                    versioned_strings.append((s, Version(version_str)))
-
-        if versioned_strings:
-            latest_string, _ = max(versioned_strings, key=lambda x: x[1])
-            self.mongo_db_config["db_simulation_model"] = latest_string
-            self._logger.info(
-                f"Updated the DB simulation model to the latest version {latest_string}"
+        requested = self.get_db_name(
+            db_name=db_name,
+            db_simulation_model_version=db_simulation_model_version,
+            model_name=db_simulation_model,
+        )
+        if requested != "all" and requested not in databases:
+            raise ValueError(
+                f"Requested database '{requested}' not found. "
+                f"Following databases are available: {', '.join(databases)}"
             )
-        else:
-            raise ValueError("LATEST requested but no released versions found in DB.")
+
+        databases = databases if requested == "all" else [requested]
+        for dbs in databases:
+            self._logger.info(f"Generating compound indexes for database: {dbs}")
+            self.generate_compound_indexes(db_name=dbs)
 
     def generate_compound_indexes(self, db_name=None):
         """
@@ -269,6 +297,9 @@ class DatabaseHandler:
                 raise ValueError(
                     "Only one model version can be passed to get_model_parameter, not a list."
                 )
+            model_version = resolve_version_to_latest_patch(
+                model_version, self.get_model_versions(collection_name)
+            )
             production_table = self.read_production_table_from_mongo_db(
                 collection_name, model_version
             )
@@ -315,6 +346,9 @@ class DatabaseHandler:
         dict containing the parameters
         """
         pars = {}
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         array_element_list = self._get_array_element_list(
             array_element_name, site, production_table, collection
@@ -361,6 +395,11 @@ class DatabaseHandler:
     def _get_parameter_for_model_version(
         self, array_element, model_version, site, collection, production_table
     ):
+        """
+        Get parameters for a specific model version and array element.
+
+        Uses caching wherever possible.
+        """
         cache_key, cache_dict = self._read_cache(
             DatabaseHandler.model_parameters_cached,
             names.validate_site_name(site) if site else None,
@@ -589,6 +628,9 @@ class DatabaseHandler:
         ValueError
             if query returned no results.
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection_name)
+        )
         try:
             return DatabaseHandler.production_table_cached[
                 self._cache_key(None, None, model_version, collection_name)
@@ -612,7 +654,7 @@ class DatabaseHandler:
 
     def get_model_versions(self, collection_name="telescopes"):
         """
-        Get list of model versions from the DB.
+        Get list of model versions from the DB with caching.
 
         Parameters
         ----------
@@ -624,10 +666,12 @@ class DatabaseHandler:
         list
             List of model versions
         """
-        collection = self.get_collection("production_tables", db_name=self.db_name)
-        return sorted(
-            {post["model_version"] for post in collection.find({"collection": collection_name})}
-        )
+        if collection_name not in DatabaseHandler.model_versions_cached:
+            collection = self.get_collection("production_tables", db_name=self.db_name)
+            DatabaseHandler.model_versions_cached[collection_name] = sorted(
+                {post["model_version"] for post in collection.find({"collection": collection_name})}
+            )
+        return DatabaseHandler.model_versions_cached[collection_name]
 
     def get_array_elements(self, model_version, collection="telescopes"):
         """
@@ -646,6 +690,9 @@ class DatabaseHandler:
         list
             Sorted list of all array elements found in collection
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         return sorted([entry for entry in production_table["parameters"] if "-design" not in entry])
 
@@ -668,6 +715,9 @@ class DatabaseHandler:
         str
             Design model for a given array element.
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         try:
             return production_table["design_model"][array_element_name]
@@ -696,6 +746,9 @@ class DatabaseHandler:
         list
             Sorted list of all array element names found in collection
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         all_array_elements = production_table["parameters"]
         return sorted(
@@ -843,9 +896,10 @@ class DatabaseHandler:
         """
         db_name = db_name or self.db_name
         collection = self.get_collection("production_tables", db_name=db_name or self.db_name)
-        self._logger.debug(f"Adding production for {production_table.get('collection')} to to DB")
+        self._logger.debug(f"Adding production for {production_table.get('collection')} to the DB")
         collection.insert_one(production_table)
         DatabaseHandler.production_table_cached.clear()
+        DatabaseHandler.model_versions_cached.clear()
 
     def add_new_parameter(
         self,
@@ -1001,6 +1055,7 @@ class DatabaseHandler:
     def _reset_parameter_cache(self):
         """Reset the cache for the parameters."""
         DatabaseHandler.model_parameters_cached.clear()
+        DatabaseHandler.model_versions_cached.clear()
 
     def _get_array_element_list(self, array_element_name, site, production_table, collection):
         """

simtools/db/db_model_upload.py

@@ -1,14 +1,85 @@
 """Upload a simulation model (parameters and production tables) to the database."""
 
 import logging
+import shutil
 from pathlib import Path
 
+from packaging.version import Version
+
 from simtools.io import ascii_handler
+from simtools.job_execution.job_manager import retry_command
 from simtools.utils import names
 
 logger = logging.getLogger(__name__)
 
 
+def add_complete_model(
+    tmp_dir,
+    db,
+    db_simulation_model,
+    db_simulation_model_version,
+    repository_url,
+    repository_branch=None,
+):
+    """
+    Upload a complete model including model parameters and production tables to the database.
+
+    Generate compound indexes for the new database.
+
+    Parameters
+    ----------
+    tmp_dir : Path or str
+        Temporary directory to use for cloning the repository.
+    db : DatabaseHandler
+        Database handler object.
+    db_simulation_model : str
+        Name of the simulation model in the database.
+    db_simulation_model_version : str
+        Version of the simulation model in the database.
+    repository_url : str
+        URL of the simulation model repository to clone.
+    repository_branch : str, optional
+        Branch of the repository to use. If None, the default branch is used.
+
+    Returns
+    -------
+    None
+        This function does not return a value.
+    """
+    if not _confirm_remote_database_upload(db):
+        return
+
+    repository_dir = None
+    try:
+        repository_dir = clone_simulation_model_repository(
+            tmp_dir,
+            repository_url,
+            db_simulation_model_version=db_simulation_model_version,
+            repository_branch=repository_branch,
+        )
+
+        add_model_parameters_to_db(
+            input_path=repository_dir / "simulation-models" / "model_parameters", db=db
+        )
+
+        add_production_tables_to_db(
+            input_path=repository_dir / "simulation-models" / "productions", db=db
+        )
+
+        db.generate_compound_indexes_for_databases(
+            db_name=None,
+            db_simulation_model=db_simulation_model,
+            db_simulation_model_version=db_simulation_model_version,
+        )
+    except Exception as exc:
+        raise RuntimeError(f"Upload of simulation model failed: {exc}") from exc
+    finally:
+        if repository_dir is not None and repository_dir.exists():
+            shutil.rmtree(repository_dir)
+
+    logger.info("Upload of simulation model completed successfully")
+
+
 def add_values_from_json_to_db(file, collection, db, file_prefix):
     """
     Upload new model parameter from json files to db.
@@ -51,8 +122,7 @@ def add_model_parameters_to_db(input_path, db):
     """
     input_path = Path(input_path)
     logger.info(f"Reading model parameters from repository path {input_path}")
-    array_elements = [d for d in input_path.iterdir() if d.is_dir()]
-    for element in array_elements:
+    for element in filter(Path.is_dir, input_path.iterdir()):
         collection = names.get_collection_name_from_array_element_name(element.name, False)
         if collection == "Files":
             logger.info("Files (tables) are uploaded with the corresponding model parameters")
@@ -85,20 +155,51 @@ def add_production_tables_to_db(input_path, db):
     input_path = Path(input_path)
     logger.info(f"Reading production tables from repository path {input_path}")
 
-    for model in filter(Path.is_dir, input_path.iterdir()):
+    for model in sorted(filter(Path.is_dir, input_path.iterdir())):
         logger.info(f"Reading production tables for model version {model.name}")
-        model_dict = {}
-        for file in sorted(model.rglob("*json")):
-            _read_production_table(model_dict, file, model.name)
+        model_dict = _read_production_tables(model)
 
         for collection, data in model_dict.items():
-            if not data["parameters"]:
+            if data["parameters"]:
+                logger.info(
+                    f"Adding production table for {collection} "
+                    f"(model version {model.name}) to the database"
+                )
+                db.add_production_table(production_table=data)
+            else:
                 logger.info(f"No production table for {collection} in model version {model.name}")
-                continue
-            logger.info(f"Adding production table for {collection} to the database")
-            db.add_production_table(
-                production_table=data,
-            )
+
+
+def _read_production_tables(model_path):
+    """
+    Read production tables from a directory.
+
+    Take into account that some productions include patch updates only. Read in this cases
+    all models from the model version history, starting with the earliest one.
+
+    Parameters
+    ----------
+    model_path : Path
+        Path to the directory containing the production tables for a specific model version.
+    """
+    model_dict = {}
+    models = [model_path.name]
+    if (model_path / "info.yml").exists():
+        info = ascii_handler.collect_data_from_file(file_name=model_path / "info.yml")
+        models.extend(info.get("model_version_history", []))
+    # sort oldest --> newest
+    models = sorted(set(models), key=Version, reverse=False)
+    for model in models:
+        for file in sorted((model_path.parent / model).rglob("*json")):
+            _read_production_table(model_dict, file, model)
+
+    # ensure that the for patch updates the model version is set correctly
+    for table in model_dict.values():
+        table["model_version"] = model_path.name
+
+    _remove_deprecated_model_parameters(model_dict)
+
+    return model_dict
 
 
 def _read_production_table(model_dict, file, model_name):
@@ -112,23 +213,153 @@ def _read_production_table(model_dict, file, model_name):
             "model_version": model_name,
             "parameters": {},
             "design_model": {},
+            "deprecated_parameters": [],
         },
     )
     parameter_dict = ascii_handler.collect_data_from_file(file_name=file)
-    logger.debug(f"Reading production table for {array_element} (collection {collection})")
+    logger.debug(
+        f"Reading production table for {array_element} "
+        f"(model_version {model_name}, collection {collection})"
+    )
     try:
         if array_element in ("configuration_corsika", "configuration_sim_telarray"):
             model_dict[collection]["parameters"] = parameter_dict["parameters"]
         else:
-            model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][
-                array_element
-            ]
+            model_dict[collection]["parameters"].setdefault(array_element, {}).update(
+                parameter_dict["parameters"][array_element]
+            )
     except KeyError as exc:
         logger.error(f"KeyError: {exc}")
         raise
+
     try:
         model_dict[collection]["design_model"][array_element] = parameter_dict["design_model"][
             array_element
         ]
     except KeyError:
         pass
+
+    try:
+        model_dict[collection]["deprecated_parameters"] = parameter_dict["deprecated_parameters"]
+    except KeyError:
+        pass
+
+    model_dict[collection]["model_version"] = model_name
+
+
+def _remove_deprecated_model_parameters(model_dict):
+    """
+    Remove deprecated parameters from all tables in a model dictionary.
+
+    Parameters
+    ----------
+    model_dict : dict
+        Production tables for a specific model version.
+    """
+    for table in model_dict.values():
+        for params in table.get("parameters", {}).values():
+            for param in table.get("deprecated_parameters", []):
+                if param in params:
+                    logger.info(
+                        f"Deprecated parameter {param} in production table {table['collection']}"
+                    )
+                    params.pop(param)
+
+
+def _confirm_remote_database_upload(db):
+    """
+    Confirm with the user that they want to upload to a remote database.
+
+    Returns True if the user confirms, False otherwise. Returns also True
+    if the database is not remote.
+
+    Parameters
+    ----------
+    db : DatabaseHandler
+        Database handler object.
+
+    Returns
+    -------
+    bool
+        True if user confirms upload, False otherwise.
+    """
+    abort_message = "Operation aborted."
+
+    if not db.is_remote_database():
+        return True
+
+    db_config = db.mongo_db_config
+    db_server = db_config.get("db_server", "unknown server") if db_config else "unknown server"
+
+    try:
+        # First confirmation
+        user_input = input(
+            f"Do you really want to upload to remote DB {db_server}? Type 'yes' to confirm: "
+        )
+        if user_input != "yes":
+            logger.info(abort_message)
+            return False
+
+        # Second confirmation
+        user_input = input(
+            f"Let be sure: do you really want to upload to remote DB {db_server}? "
+            "Type 'yes' to confirm: "
+        )
+        if user_input != "yes":
+            logger.info(abort_message)
+            return False
+
+        return True
+
+    except (EOFError, KeyboardInterrupt):
+        logger.info(abort_message)
+        return False
+
+
+def clone_simulation_model_repository(
+    target_dir, repository_url, db_simulation_model_version, repository_branch
+):
+    """
+    Clone a git repository containing simulation model parameters and production tables.
+
+    Parameters
+    ----------
+    target_dir : Path or str
+        Target directory to clone the repository into.
+    repository_url : str
+        URL of the git repository to clone.
+    db_simulation_model_version : str
+        Version tag of the simulation model to checkout.
+    repository_branch : str, optional
+        Branch of the repository to use. If None, the version tag is used.
+
+    Returns
+    -------
+    Path
+        Path to the cloned repository.
+    """
+    if repository_branch:
+        logger.info(f"Using branch: {repository_branch}")
+    else:
+        logger.info(f"Using version tag: {db_simulation_model_version}")
+
+    target_dir = Path(target_dir)
+    target_dir = target_dir if target_dir.is_absolute() else Path.cwd() / target_dir
+
+    if target_dir.exists():
+        shutil.rmtree(target_dir)
+
+    logger.info(f"Cloning model parameters from {repository_url}")
+
+    if repository_branch:
+        command = f'git clone --depth=1 -b "{repository_branch}" "{repository_url}" "{target_dir}"'
+    else:
+        # Use version tag with detached head (generates warning but that's fine)
+        command = (
+            f'git clone --branch "{db_simulation_model_version}" '
+            f'--depth 1 "{repository_url}" "{target_dir}"'
+        )
+
+    retry_command(command)
+
+    return target_dir