gammasimtools 0.19.0__py3-none-any.whl → 0.21.0__py3-none-any.whl

simtools/db/db_handler.py

@@ -1,5 +1,6 @@
 """Module to handle interaction with DB."""
 
+import io
 import logging
 import re
 from collections import defaultdict
@@ -8,6 +9,7 @@ from threading import Lock
 
 import gridfs
 import jsonschema
+from astropy.table import Table
 from bson.objectid import ObjectId
 from packaging.version import Version
 from pymongo import MongoClient
@@ -16,6 +18,7 @@ from simtools.data_model import validate_data
 from simtools.io import ascii_handler, io_handler
 from simtools.simtel import simtel_table_reader
 from simtools.utils import names, value_conversion
+from simtools.version import resolve_version_to_latest_patch
 
 __all__ = ["DatabaseHandler"]
 
@@ -50,8 +53,19 @@ jsonschema_db_dict = {
             "type": "string",
             "description": "Name of simulation model database",
         },
+        "db_simulation_model_version": {
+            "type": "string",
+            "description": "Version of simulation model database",
+        },
     },
-    "required": ["db_server", "db_api_port", "db_api_user", "db_api_pw", "db_simulation_model"],
+    "required": [
+        "db_server",
+        "db_api_port",
+        "db_api_user",
+        "db_api_pw",
+        "db_simulation_model",
+        "db_simulation_model_version",
+    ],
 }
 
 
@@ -59,6 +73,11 @@ class DatabaseHandler:
     """
     DatabaseHandler provides the interface to the DB.
 
+    Note the two types of version variables used in this class:
+
+    - db_simulation_model_version (from mongo_db_config): version of the simulation model database
+    - model_version (from production_tables): version of the model contained in the database
+
     Parameters
     ----------
     mongo_db_config: dict
@@ -70,6 +89,7 @@ class DatabaseHandler:
     db_client = None
     production_table_cached = {}
     model_parameters_cached = {}
+    model_versions_cached = {}
 
     def __init__(self, mongo_db_config=None):
         """Initialize the DatabaseHandler class."""
@@ -82,7 +102,12 @@ class DatabaseHandler:
         self._set_up_connection()
         self._find_latest_simulation_model_db()
         self.db_name = (
-            self.mongo_db_config.get("db_simulation_model", None) if self.mongo_db_config else None
+            self.get_db_name(
+                db_simulation_model_version=self.mongo_db_config.get("db_simulation_model_version"),
+                model_name=self.mongo_db_config.get("db_simulation_model"),
+            )
+            if self.mongo_db_config
+            else None
         )
 
     def _set_up_connection(self):
@@ -92,6 +117,16 @@ class DatabaseHandler:
             with lock:
                 DatabaseHandler.db_client = self._open_mongo_db()
 
+    def get_db_name(self, db_name=None, db_simulation_model_version=None, model_name=None):
+        """Build DB name from configuration."""
+        if db_name:
+            return db_name
+        if db_simulation_model_version and model_name:
+            return f"{model_name}-{db_simulation_model_version.replace('.', '-')}"
+        if db_simulation_model_version or model_name:
+            return None
+        return None if (db_simulation_model_version or model_name) else self.db_name
+
     def _validate_mongo_db_config(self, mongo_db_config):
         """Validate the MongoDB configuration."""
         if mongo_db_config is None or all(value is None for value in mongo_db_config.values()):
@@ -140,8 +175,7 @@ class DatabaseHandler:
         """
         Find the latest released version of the simulation model and update the DB config.
 
-        This is indicated by adding "LATEST" to the name of the simulation model database
-        (field "db_simulation_model" in the database configuration dictionary).
+        This is indicated by "LATEST" to the simulation model data base version.
         Only released versions are considered, pre-releases are ignored.
 
         Raises
@@ -151,27 +185,27 @@ class DatabaseHandler:
 
         """
         try:
+            db_simulation_model_version = self.mongo_db_config["db_simulation_model_version"]
             db_simulation_model = self.mongo_db_config["db_simulation_model"]
-            if not db_simulation_model.endswith("LATEST"):
+            if db_simulation_model_version != "LATEST":
                 return
-        except TypeError:  # db_simulation_model is None
+        except TypeError:  # db_simulation_model_version is None
             return
 
-        prefix = db_simulation_model.replace("LATEST", "")
         list_of_db_names = self.db_client.list_database_names()
-        filtered_list_of_db_names = [s for s in list_of_db_names if s.startswith(prefix)]
-        versioned_strings = []
-        version_pattern = re.compile(
-            rf"{re.escape(prefix)}v?(\d+)-(\d+)-(\d+)(?:-([a-zA-Z0-9_.]+))?"
-        )
+        filtered_list_of_db_names = [
+            s for s in list_of_db_names if s.startswith(db_simulation_model)
+        ]
+        pattern = re.compile(rf"{re.escape(db_simulation_model)}-v(\d+)-(\d+)-(\d+)(?:-(.+))?$")
 
+        versioned_strings = []
         for s in filtered_list_of_db_names:
-            match = version_pattern.search(s)
-            # A version is considered a pre-release if it contains a '-' character (re group 4)
-            if match and match.group(4) is None:
-                version_str = match.group(1) + "." + match.group(2) + "." + match.group(3)
-                version = Version(version_str)
-                versioned_strings.append((s, version))
+            m = pattern.match(s)
+            if m:
+                # skip pre-releases (have suffix)
+                if m.group(4) is None:
+                    version_str = f"{m.group(1)}.{m.group(2)}.{m.group(3)}"
+                    versioned_strings.append((s, Version(version_str)))
 
         if versioned_strings:
             latest_string, _ = max(versioned_strings, key=lambda x: x[1])
@@ -180,7 +214,7 @@ class DatabaseHandler:
                 f"Updated the DB simulation model to the latest version {latest_string}"
             )
         else:
-            raise ValueError("Found LATEST in the DB name but no matching versions found in DB.")
+            raise ValueError("LATEST requested but no released versions found in DB.")
 
     def generate_compound_indexes(self, db_name=None):
         """
@@ -242,6 +276,9 @@ class DatabaseHandler:
                 raise ValueError(
                     "Only one model version can be passed to get_model_parameter, not a list."
                 )
+            model_version = resolve_version_to_latest_patch(
+                model_version, self.get_model_versions(collection_name)
+            )
             production_table = self.read_production_table_from_mongo_db(
                 collection_name, model_version
             )
@@ -288,6 +325,9 @@ class DatabaseHandler:
         dict containing the parameters
         """
         pars = {}
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         array_element_list = self._get_array_element_list(
             array_element_name, site, production_table, collection
@@ -334,6 +374,11 @@ class DatabaseHandler:
     def _get_parameter_for_model_version(
         self, array_element, model_version, site, collection, production_table
     ):
+        """
+        Get parameters for a specific model version and array element.
+
+        Uses caching wherever possible.
+        """
         cache_key, cache_dict = self._read_cache(
             DatabaseHandler.model_parameters_cached,
             names.validate_site_name(site) if site else None,
@@ -562,6 +607,9 @@ class DatabaseHandler:
         ValueError
             if query returned no results.
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection_name)
+        )
         try:
             return DatabaseHandler.production_table_cached[
                 self._cache_key(None, None, model_version, collection_name)
@@ -585,7 +633,7 @@ class DatabaseHandler:
 
     def get_model_versions(self, collection_name="telescopes"):
         """
-        Get list of model versions from the DB.
+        Get list of model versions from the DB with caching.
 
         Parameters
         ----------
@@ -597,10 +645,12 @@ class DatabaseHandler:
         list
             List of model versions
         """
-        collection = self.get_collection("production_tables", db_name=self.db_name)
-        return sorted(
-            {post["model_version"] for post in collection.find({"collection": collection_name})}
-        )
+        if collection_name not in DatabaseHandler.model_versions_cached:
+            collection = self.get_collection("production_tables", db_name=self.db_name)
+            DatabaseHandler.model_versions_cached[collection_name] = sorted(
+                {post["model_version"] for post in collection.find({"collection": collection_name})}
+            )
+        return DatabaseHandler.model_versions_cached[collection_name]
 
     def get_array_elements(self, model_version, collection="telescopes"):
         """
@@ -619,6 +669,9 @@ class DatabaseHandler:
         list
             Sorted list of all array elements found in collection
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         return sorted([entry for entry in production_table["parameters"] if "-design" not in entry])
 
@@ -641,6 +694,9 @@ class DatabaseHandler:
         str
             Design model for a given array element.
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         try:
             return production_table["design_model"][array_element_name]
@@ -669,6 +725,9 @@ class DatabaseHandler:
         list
             Sorted list of all array element names found in collection
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         all_array_elements = production_table["parameters"]
         return sorted(
@@ -774,26 +833,57 @@ class DatabaseHandler:
         with open(Path(path).joinpath(file.filename), "wb") as output_file:
             fs_output.download_to_stream_by_name(file.filename, output_file)
 
-    def add_production_table(self, db_name, production_table):
+    def get_ecsv_file_as_astropy_table(self, file_name, db_name=None):
         """
-        Add a production table to the DB.
+        Read contents of an ECSV file from the database and return it as an Astropy Table.
+
+        Files are not written to disk.
 
         Parameters
         ----------
+        file_name: str
+            The name of the ECSV file.
         db_name: str
-            the name of the DB.
+            The name of the database.
+
+        Returns
+        -------
+        astropy.table.Table
+            The contents of the ECSV file as an Astropy Table.
+        """
+        db = DatabaseHandler.db_client[db_name or self.db_name]
+        fs = gridfs.GridFSBucket(db)
+
+        buf = io.BytesIO()
+        try:
+            fs.download_to_stream_by_name(file_name, buf)
+        except gridfs.errors.NoFile as exc:
+            raise FileNotFoundError(f"ECSV file '{file_name}' not found in DB.") from exc
+        buf.seek(0)
+        return Table.read(buf.getvalue().decode("utf-8"), format="ascii.ecsv")
+
+    def add_production_table(self, production_table, db_name=None):
+        """
+        Add a production table to the DB.
+
+        Parameters
+        ----------
         production_table: dict
             The production table to add to the DB.
+        db_name: str
+            the name of the DB.
         """
+        db_name = db_name or self.db_name
         collection = self.get_collection("production_tables", db_name=db_name or self.db_name)
         self._logger.debug(f"Adding production for {production_table.get('collection')} to to DB")
         collection.insert_one(production_table)
         DatabaseHandler.production_table_cached.clear()
+        DatabaseHandler.model_versions_cached.clear()
 
     def add_new_parameter(
         self,
-        db_name,
         par_dict,
+        db_name=None,
         collection_name="telescopes",
         file_prefix=None,
     ):
@@ -805,10 +895,10 @@ class DatabaseHandler:
 
         Parameters
         ----------
-        db_name: str
-            the name of the DB
         par_dict: dict
             dictionary with parameter data
+        db_name: str
+            the name of the DB
         collection_name: str
             The name of the collection to add a parameter to.
         file_prefix: str or Path
@@ -859,7 +949,7 @@ class DatabaseHandler:
             the name of the DB
         **kwargs (optional): keyword arguments for file creation.
             The full list of arguments can be found in, \
-            https://docs.mongodb.com/manual/core/gridfs/#the-files-collection
+            https://www.mongodb.com/docs/manual/core/gridfs/
             mostly these are unnecessary though.
 
         Returns
@@ -944,6 +1034,7 @@ class DatabaseHandler:
     def _reset_parameter_cache(self):
         """Reset the cache for the parameters."""
         DatabaseHandler.model_parameters_cached.clear()
+        DatabaseHandler.model_versions_cached.clear()
 
     def _get_array_element_list(self, array_element_name, site, production_table, collection):
         """

simtools/db/db_model_upload.py

@@ -3,26 +3,26 @@
 import logging
 from pathlib import Path
 
+from packaging.version import Version
+
 from simtools.io import ascii_handler
 from simtools.utils import names
 
 logger = logging.getLogger(__name__)
 
 
-def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
+def add_values_from_json_to_db(file, collection, db, file_prefix):
     """
     Upload new model parameter from json files to db.
 
     Parameters
     ----------
     file : list
-        Json file to be uploaded to the DB.
+        JSON file to be uploaded to the DB.
     collection : str
         The DB collection to which to add the file.
     db : DatabaseHandler
         Database handler object.
-    db_name : str
-        Name of the database to be created.
     file_prefix : str
         Path to location of all additional files to be uploaded.
     """
@@ -30,32 +30,30 @@ def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
     logger.debug(
         f"Adding the following parameter to the DB: {par_dict['parameter']} "
         f"version {par_dict['parameter_version']} "
-        f"(collection {collection} in database {db_name})"
+        f"(collection {collection} in database {db.get_db_name()})"
     )
 
     db.add_new_parameter(
-        db_name=db_name,
         par_dict=par_dict,
         collection_name=collection,
         file_prefix=file_prefix,
     )
 
 
-def add_model_parameters_to_db(args_dict, db):
+def add_model_parameters_to_db(input_path, db):
     """
     Read model parameters from a directory and upload them to the database.
 
     Parameters
     ----------
-    args_dict : dict
-        Command line arguments.
+    input_path : Path, str
+        Path to the directory containing the model parameters.
     db : DatabaseHandler
         Database handler object.
     """
-    input_path = Path(args_dict["input_path"])
+    input_path = Path(input_path)
     logger.info(f"Reading model parameters from repository path {input_path}")
-    array_elements = [d for d in input_path.iterdir() if d.is_dir()]
-    for element in array_elements:
+    for element in filter(Path.is_dir, input_path.iterdir()):
         collection = names.get_collection_name_from_array_element_name(element.name, False)
         if collection == "Files":
             logger.info("Files (tables) are uploaded with the corresponding model parameters")
@@ -67,12 +65,11 @@ def add_model_parameters_to_db(args_dict, db):
                 file=file,
                 collection=collection,
                 db=db,
-                db_name=args_dict["db_name"],
                 file_prefix=input_path / "Files",
             )
 
 
-def add_production_tables_to_db(args_dict, db):
+def add_production_tables_to_db(input_path, db):
     """
     Read production tables from a directory and upload them to the database.
 
@@ -81,29 +78,50 @@ def add_production_tables_to_db(args_dict, db):
 
     Parameters
     ----------
-    args_dict : dict
-        Command line arguments.
+    input_path : Path, str
+        Path to the directory containing the production tables.
     db : DatabaseHandler
         Database handler object.
     """
-    input_path = Path(args_dict["input_path"])
+    input_path = Path(input_path)
     logger.info(f"Reading production tables from repository path {input_path}")
 
     for model in filter(Path.is_dir, input_path.iterdir()):
         logger.info(f"Reading production tables for model version {model.name}")
-        model_dict = {}
-        for file in sorted(model.rglob("*json")):
-            _read_production_table(model_dict, file, model.name)
+        model_dict = _read_production_tables(model)
 
         for collection, data in model_dict.items():
-            if not data["parameters"]:
+            if data["parameters"]:
+                logger.info(f"Adding production table for {collection} to the database")
+                db.add_production_table(production_table=data)
+            else:
                 logger.info(f"No production table for {collection} in model version {model.name}")
-                continue
-            logger.info(f"Adding production table for {collection} to the database")
-            db.add_production_table(
-                db_name=args_dict["db_name"],
-                production_table=data,
-            )
+
+
+def _read_production_tables(model_path):
+    """
+    Read production tables from a directory.
+
+    Take into account that some productions include patch updates only. Read in this cases
+    the base models first.
+
+    Parameters
+    ----------
+    model_path : Path
+        Path to the directory containing the production tables for a specific model version.
+    """
+    model_dict = {}
+    models = [model_path.name]
+    if (model_path / "info.yml").exists():
+        info = ascii_handler.collect_data_from_file(file_name=model_path / "info.yml")
+        models.extend(info.get("model_version_history", []))
+    # sort oldest --> newest
+    models = sorted(set(models), key=Version, reverse=False)
+    for model in models:
+        for file in sorted((model_path.parent / model).rglob("*json")):
+            _read_production_table(model_dict, file, model)
+
+    return model_dict
 
 
 def _read_production_table(model_dict, file, model_name):
@@ -125,9 +143,10 @@ def _read_production_table(model_dict, file, model_name):
         if array_element in ("configuration_corsika", "configuration_sim_telarray"):
             model_dict[collection]["parameters"] = parameter_dict["parameters"]
         else:
-            model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][
-                array_element
-            ]
+            model_dict[collection]["parameters"].setdefault(array_element, {}).update(
+                parameter_dict["parameters"][array_element]
+            )
+
     except KeyError as exc:
         logger.error(f"KeyError: {exc}")
         raise
@@ -137,3 +156,5 @@ def _read_production_table(model_dict, file, model_name):
         ]
     except KeyError:
         pass
+
+    model_dict[collection]["model_version"] = model_name

simtools/dependencies.py

@@ -40,7 +40,8 @@ def get_version_string(db_config=None, run_time=None):
 
     """
     return (
-        f"Database version: {get_database_version(db_config)}\n"
+        f"Database name: {get_database_version_or_name(db_config, version=False)}\n"
+        f"Database version: {get_database_version_or_name(db_config, version=True)}\n"
         f"sim_telarray version: {get_sim_telarray_version(run_time)}\n"
         f"CORSIKA version: {get_corsika_version(run_time)}\n"
         f"Build options: {get_build_options(run_time)}\n"
@@ -48,25 +49,29 @@ def get_version_string(db_config=None, run_time=None):
     )
 
 
-def get_database_version(db_config):
+def get_database_version_or_name(db_config, version=True):
     """
-    Get the version of the simulation model data base used.
+    Get the version or name of the simulation model data base used.
 
     Parameters
     ----------
     db_config : dict
         Dictionary containing the database configuration.
+    version : bool
+        If True, return the version of the database. If False, return the name.
 
     Returns
     -------
     str
-        Version of the simulation model data base used.
+        Version or name of the simulation model data base used.
 
     """
     if db_config is None:
         return None
     db = DatabaseHandler(db_config)
-    return db.mongo_db_config.get("db_simulation_model")
+    return db.mongo_db_config.get(
+        "db_simulation_model_version" if version else "db_simulation_model"
+    )
 
 
 def get_sim_telarray_version(run_time):

simtools/layout/array_layout_utils.py

@@ -282,11 +282,7 @@ def get_array_layouts_from_db(
     """
     layout_names = []
     if layout_name:
-        layout_names.append(
-            layout_name[0]
-            if isinstance(layout_name, list) and len(layout_name) == 1
-            else layout_name
-        )
+        layout_names = gen.ensure_iterable(layout_name)
     else:
         site_model = SiteModel(site=site, model_version=model_version, mongo_db_config=db_config)
         layout_names = site_model.get_list_of_array_layouts()
@@ -394,3 +390,39 @@ def _get_array_layout_dict(
             coordinate_system=coordinate_system
         ),
     }
+
+
+def get_array_elements_from_db_for_layouts(layouts, site, model_version, db_config):
+    """
+    Get list of array elements from the database for given list of layout names.
+
+    Structure of the returned dictionary::
+
+        {
+            "layout_name_1": [telescope_id_1, telescope_id_2, ...],
+            "layout_name_2": [telescope_id_3, telescope_id_4, ...],
+            ...
+        }
+
+    Parameters
+    ----------
+    layouts : list[str]
+        List of layout names to read. If "all", read all available layouts.
+    site : str
+        Site name for the array layouts.
+    model_version : str
+        Model version for the array layouts.
+    db_config : dict
+        Database configuration dictionary.
+
+    Returns
+    -------
+    dict
+        Dictionary mapping layout names to telescope IDs.
+    """
+    site_model = SiteModel(site=site, model_version=model_version, mongo_db_config=db_config)
+    layout_names = site_model.get_list_of_array_layouts() if layouts == ["all"] else layouts
+    layout_dict = {}
+    for layout_name in layout_names:
+        layout_dict[layout_name] = site_model.get_array_elements_for_layout(layout_name)
+    return layout_dict

simtools/model/array_model.py

@@ -272,7 +272,7 @@ class ArrayModel:
             config_file_path=self.config_file_path,
             telescope_model=self.telescope_model,
             site_model=self.site_model,
-            sim_telarray_seeds=self.sim_telarray_seeds,
+            additional_metadata=self._get_additional_simtel_metadata(),
         )
         self._array_model_file_exported = True
 
@@ -484,3 +484,20 @@ class ArrayModel:
 
         table.sort("telescope_name")
         return table
+
+    def _get_additional_simtel_metadata(self):
+        """
+        Collect additional metadata to be included in sim_telarray output.
+
+        Returns
+        -------
+        dict
+            Dictionary with additional metadata.
+        """
+        metadata = {}
+        if self.sim_telarray_seeds is not None:
+            metadata.update(self.sim_telarray_seeds)
+
+        metadata["nsb_integrated_flux"] = self.site_model.get_nsb_integrated_flux()
+
+        return metadata