gammasimtools 0.21.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

simtools/data_model/validate_data.py

@@ -15,8 +15,6 @@ from simtools.data_model import schema
 from simtools.io import ascii_handler
 from simtools.utils import names, value_conversion
 
-__all__ = ["DataValidator"]
-
 
 class DataValidator:
     """
@@ -232,7 +230,7 @@ class DataValidator:
         else:
             self._check_data_type(np.array(value).dtype, index, value)
 
-        if self.data_dict.get("type") not in ("string", "dict", "file"):
+        if self._get_data_description(index).get("type", None) not in ("string", "dict", "file"):
             self._check_for_not_a_number(value, index)
             value, unit = self._check_and_convert_units(value, unit, index)
             for range_type in ("allowed_range", "required_range"):

simtools/db/db_handler.py

@@ -11,7 +11,6 @@ import gridfs
 import jsonschema
 from astropy.table import Table
 from bson.objectid import ObjectId
-from packaging.version import Version
 from pymongo import MongoClient
 
 from simtools.data_model import validate_data
@@ -20,8 +19,6 @@ from simtools.simtel import simtel_table_reader
 from simtools.utils import names, value_conversion
 from simtools.version import resolve_version_to_latest_patch
 
-__all__ = ["DatabaseHandler"]
-
 logging.getLogger("pymongo").setLevel(logging.WARNING)
 
 
@@ -100,7 +97,6 @@ class DatabaseHandler:
         self.list_of_collections = {}
 
         self._set_up_connection()
-        self._find_latest_simulation_model_db()
         self.db_name = (
             self.get_db_name(
                 db_simulation_model_version=self.mongo_db_config.get("db_simulation_model_version"),
@@ -127,6 +123,34 @@ class DatabaseHandler:
             return None
         return None if (db_simulation_model_version or model_name) else self.db_name
 
+    def print_connection_info(self):
+        """Print the connection information."""
+        if self.mongo_db_config:
+            self._logger.info(
+                f"Connected to MongoDB at {self.mongo_db_config['db_server']}:"
+                f"{self.mongo_db_config['db_api_port']} "
+                f"using database: {self.db_name}"
+            )
+        else:
+            self._logger.info("No MongoDB configuration provided.")
+
+    def is_remote_database(self):
+        """
+        Check if the database is remote.
+
+        Check for domain pattern like "cta-simpipe-protodb.zeuthen.desy.de"
+
+        Returns
+        -------
+        bool
+            True if the database is remote, False otherwise.
+        """
+        if self.mongo_db_config:
+            db_server = self.mongo_db_config["db_server"]
+            domain_pattern = r"^([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$"
+            return bool(re.match(domain_pattern, db_server))
+        return False
+
     def _validate_mongo_db_config(self, mongo_db_config):
         """Validate the MongoDB configuration."""
         if mongo_db_config is None or all(value is None for value in mongo_db_config.values()):
@@ -171,50 +195,45 @@ class DatabaseHandler:
             tlsallowinvalidcertificates=True,
         )
 
-    def _find_latest_simulation_model_db(self):
+    def generate_compound_indexes_for_databases(
+        self, db_name, db_simulation_model, db_simulation_model_version
+    ):
         """
-        Find the latest released version of the simulation model and update the DB config.
+        Generate compound indexes for several databases.
 
-        This is indicated by "LATEST" to the simulation model data base version.
-        Only released versions are considered, pre-releases are ignored.
+        Parameters
+        ----------
+        db_name: str
+            Name of the database.
+        db_simulation_model: str
+            Name of the simulation model.
+        db_simulation_model_version: str
+            Version of the simulation model.
 
         Raises
         ------
         ValueError
-            If the "LATEST" version is requested but no versions are found in the DB.
+            If the requested database is not found.
 
         """
-        try:
-            db_simulation_model_version = self.mongo_db_config["db_simulation_model_version"]
-            db_simulation_model = self.mongo_db_config["db_simulation_model"]
-            if db_simulation_model_version != "LATEST":
-                return
-        except TypeError:  # db_simulation_model_version is None
-            return
-
-        list_of_db_names = self.db_client.list_database_names()
-        filtered_list_of_db_names = [
-            s for s in list_of_db_names if s.startswith(db_simulation_model)
+        databases = [
+            d for d in self.db_client.list_database_names() if d not in ("config", "admin", "local")
         ]
-        pattern = re.compile(rf"{re.escape(db_simulation_model)}-v(\d+)-(\d+)-(\d+)(?:-(.+))?$")
-
-        versioned_strings = []
-        for s in filtered_list_of_db_names:
-            m = pattern.match(s)
-            if m:
-                # skip pre-releases (have suffix)
-                if m.group(4) is None:
-                    version_str = f"{m.group(1)}.{m.group(2)}.{m.group(3)}"
-                    versioned_strings.append((s, Version(version_str)))
-
-        if versioned_strings:
-            latest_string, _ = max(versioned_strings, key=lambda x: x[1])
-            self.mongo_db_config["db_simulation_model"] = latest_string
-            self._logger.info(
-                f"Updated the DB simulation model to the latest version {latest_string}"
+        requested = self.get_db_name(
+            db_name=db_name,
+            db_simulation_model_version=db_simulation_model_version,
+            model_name=db_simulation_model,
+        )
+        if requested != "all" and requested not in databases:
+            raise ValueError(
+                f"Requested database '{requested}' not found. "
+                f"Following databases are available: {', '.join(databases)}"
             )
-        else:
-            raise ValueError("LATEST requested but no released versions found in DB.")
+
+        databases = databases if requested == "all" else [requested]
+        for dbs in databases:
+            self._logger.info(f"Generating compound indexes for database: {dbs}")
+            self.generate_compound_indexes(db_name=dbs)
 
     def generate_compound_indexes(self, db_name=None):
         """
@@ -875,7 +894,7 @@ class DatabaseHandler:
         """
         db_name = db_name or self.db_name
         collection = self.get_collection("production_tables", db_name=db_name or self.db_name)
-        self._logger.debug(f"Adding production for {production_table.get('collection')} to to DB")
+        self._logger.debug(f"Adding production for {production_table.get('collection')} to the DB")
         collection.insert_one(production_table)
         DatabaseHandler.production_table_cached.clear()
         DatabaseHandler.model_versions_cached.clear()

simtools/db/db_model_upload.py

@@ -1,16 +1,85 @@
 """Upload a simulation model (parameters and production tables) to the database."""
 
 import logging
+import shutil
 from pathlib import Path
 
 from packaging.version import Version
 
 from simtools.io import ascii_handler
+from simtools.job_execution.job_manager import retry_command
 from simtools.utils import names
 
 logger = logging.getLogger(__name__)
 
 
+def add_complete_model(
+    tmp_dir,
+    db,
+    db_simulation_model,
+    db_simulation_model_version,
+    repository_url,
+    repository_branch=None,
+):
+    """
+    Upload a complete model including model parameters and production tables to the database.
+
+    Generate compound indexes for the new database.
+
+    Parameters
+    ----------
+    tmp_dir : Path or str
+        Temporary directory to use for cloning the repository.
+    db : DatabaseHandler
+        Database handler object.
+    db_simulation_model : str
+        Name of the simulation model in the database.
+    db_simulation_model_version : str
+        Version of the simulation model in the database.
+    repository_url : str
+        URL of the simulation model repository to clone.
+    repository_branch : str, optional
+        Branch of the repository to use. If None, the default branch is used.
+
+    Returns
+    -------
+    None
+        This function does not return a value.
+    """
+    if not _confirm_remote_database_upload(db):
+        return
+
+    repository_dir = None
+    try:
+        repository_dir = clone_simulation_model_repository(
+            tmp_dir,
+            repository_url,
+            db_simulation_model_version=db_simulation_model_version,
+            repository_branch=repository_branch,
+        )
+
+        add_model_parameters_to_db(
+            input_path=repository_dir / "simulation-models" / "model_parameters", db=db
+        )
+
+        add_production_tables_to_db(
+            input_path=repository_dir / "simulation-models" / "productions", db=db
+        )
+
+        db.generate_compound_indexes_for_databases(
+            db_name=None,
+            db_simulation_model=db_simulation_model,
+            db_simulation_model_version=db_simulation_model_version,
+        )
+    except Exception as exc:
+        raise RuntimeError(f"Upload of simulation model failed: {exc}") from exc
+    finally:
+        if repository_dir is not None and repository_dir.exists():
+            shutil.rmtree(repository_dir)
+
+    logger.info("Upload of simulation model completed successfully")
+
+
 def add_values_from_json_to_db(file, collection, db, file_prefix):
     """
     Upload new model parameter from json files to db.
@@ -86,13 +155,16 @@ def add_production_tables_to_db(input_path, db):
     input_path = Path(input_path)
     logger.info(f"Reading production tables from repository path {input_path}")
 
-    for model in filter(Path.is_dir, input_path.iterdir()):
+    for model in sorted(filter(Path.is_dir, input_path.iterdir())):
         logger.info(f"Reading production tables for model version {model.name}")
         model_dict = _read_production_tables(model)
 
         for collection, data in model_dict.items():
             if data["parameters"]:
-                logger.info(f"Adding production table for {collection} to the database")
+                logger.info(
+                    f"Adding production table for {collection} "
+                    f"(model version {model.name}) to the database"
+                )
                 db.add_production_table(production_table=data)
             else:
                 logger.info(f"No production table for {collection} in model version {model.name}")
@@ -103,7 +175,7 @@ def _read_production_tables(model_path):
     Read production tables from a directory.
 
     Take into account that some productions include patch updates only. Read in this cases
-    the base models first.
+    all models from the model version history, starting with the earliest one.
 
     Parameters
     ----------
@@ -121,6 +193,12 @@ def _read_production_tables(model_path):
         for file in sorted((model_path.parent / model).rglob("*json")):
             _read_production_table(model_dict, file, model)
 
+    # ensure that the for patch updates the model version is set correctly
+    for table in model_dict.values():
+        table["model_version"] = model_path.name
+
+    _remove_deprecated_model_parameters(model_dict)
+
     return model_dict
 
 
@@ -135,10 +213,14 @@ def _read_production_table(model_dict, file, model_name):
             "model_version": model_name,
             "parameters": {},
             "design_model": {},
+            "deprecated_parameters": [],
         },
     )
     parameter_dict = ascii_handler.collect_data_from_file(file_name=file)
-    logger.debug(f"Reading production table for {array_element} (collection {collection})")
+    logger.debug(
+        f"Reading production table for {array_element} "
+        f"(model_version {model_name}, collection {collection})"
+    )
     try:
         if array_element in ("configuration_corsika", "configuration_sim_telarray"):
             model_dict[collection]["parameters"] = parameter_dict["parameters"]
@@ -146,10 +228,10 @@ def _read_production_table(model_dict, file, model_name):
             model_dict[collection]["parameters"].setdefault(array_element, {}).update(
                 parameter_dict["parameters"][array_element]
             )
-
     except KeyError as exc:
         logger.error(f"KeyError: {exc}")
         raise
+
     try:
         model_dict[collection]["design_model"][array_element] = parameter_dict["design_model"][
             array_element
@@ -157,4 +239,127 @@ def _read_production_table(model_dict, file, model_name):
     except KeyError:
         pass
 
+    try:
+        model_dict[collection]["deprecated_parameters"] = parameter_dict["deprecated_parameters"]
+    except KeyError:
+        pass
+
     model_dict[collection]["model_version"] = model_name
+
+
+def _remove_deprecated_model_parameters(model_dict):
+    """
+    Remove deprecated parameters from all tables in a model dictionary.
+
+    Parameters
+    ----------
+    model_dict : dict
+        Production tables for a specific model version.
+    """
+    for table in model_dict.values():
+        for params in table.get("parameters", {}).values():
+            for param in table.get("deprecated_parameters", []):
+                if param in params:
+                    logger.info(
+                        f"Deprecated parameter {param} in production table {table['collection']}"
+                    )
+                    params.pop(param)
+
+
+def _confirm_remote_database_upload(db):
+    """
+    Confirm with the user that they want to upload to a remote database.
+
+    Returns True if the user confirms, False otherwise. Returns also True
+    if the database is not remote.
+
+    Parameters
+    ----------
+    db : DatabaseHandler
+        Database handler object.
+
+    Returns
+    -------
+    bool
+        True if user confirms upload, False otherwise.
+    """
+    abort_message = "Operation aborted."
+
+    if not db.is_remote_database():
+        return True
+
+    db_config = db.mongo_db_config
+    db_server = db_config.get("db_server", "unknown server") if db_config else "unknown server"
+
+    try:
+        # First confirmation
+        user_input = input(
+            f"Do you really want to upload to remote DB {db_server}? Type 'yes' to confirm: "
+        )
+        if user_input != "yes":
+            logger.info(abort_message)
+            return False
+
+        # Second confirmation
+        user_input = input(
+            f"Let be sure: do you really want to upload to remote DB {db_server}? "
+            "Type 'yes' to confirm: "
+        )
+        if user_input != "yes":
+            logger.info(abort_message)
+            return False
+
+        return True
+
+    except (EOFError, KeyboardInterrupt):
+        logger.info(abort_message)
+        return False
+
+
+def clone_simulation_model_repository(
+    target_dir, repository_url, db_simulation_model_version, repository_branch
+):
+    """
+    Clone a git repository containing simulation model parameters and production tables.
+
+    Parameters
+    ----------
+    target_dir : Path or str
+        Target directory to clone the repository into.
+    repository_url : str
+        URL of the git repository to clone.
+    db_simulation_model_version : str
+        Version tag of the simulation model to checkout.
+    repository_branch : str, optional
+        Branch of the repository to use. If None, the version tag is used.
+
+    Returns
+    -------
+    Path
+        Path to the cloned repository.
+    """
+    if repository_branch:
+        logger.info(f"Using branch: {repository_branch}")
+    else:
+        logger.info(f"Using version tag: {db_simulation_model_version}")
+
+    target_dir = Path(target_dir)
+    target_dir = target_dir if target_dir.is_absolute() else Path.cwd() / target_dir
+
+    if target_dir.exists():
+        shutil.rmtree(target_dir)
+
+    logger.info(f"Cloning model parameters from {repository_url}")
+
+    if repository_branch:
+        command = f'git clone --depth=1 -b "{repository_branch}" "{repository_url}" "{target_dir}"'
+    else:
+        # Use version tag with detached head (generates warning but that's fine)
+        command = (
+            f'git clone --branch "{db_simulation_model_version}" '
+            f'--depth 1 "{repository_url}" "{target_dir}"'
+        )
+
+    retry_command(command)
+
+    return target_dir

simtools/io/hdf5_handler.py

@@ -10,11 +10,6 @@ from ctapipe.io import read_table
 
 from simtools.utils.names import sanitize_name
 
-__all__ = [
-    "fill_hdf5_table",
-    "read_hdf5",
-]
-
 _logger = logging.getLogger(__name__)
 
 

simtools/io/io_handler.py

@@ -1,15 +1,8 @@
-"""Handle input and output paths."""
+"""Handle input and output directories and file paths."""
 
-import datetime
 import logging
 from pathlib import Path
 
-__all__ = ["IOHandler", "IOHandlerSingleton"]
-
-
-class IncompleteIOHandlerInitError(Exception):
-    """Exception raised when IOHandler is not initialized."""
-
 
 class IOHandlerSingleton(type):
     """Singleton base class."""
@@ -17,32 +10,24 @@ class IOHandlerSingleton(type):
     _instances = {}
 
     def __call__(cls, *args, **kwargs):
-        """
-        Ensure a single instance of the IOHandlerSingleton class.
-
-        Creates a new instance if it doesn't exist, otherwise returns the existing instance.
-        """
+        """Create a new instance if it doesn't exist, otherwise returns the existing instance."""
         if cls not in cls._instances:
             cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]
 
 
 class IOHandler(metaclass=IOHandlerSingleton):
-    """Handle input and output paths."""
+    """Handle input and output directories and file paths."""
 
     def __init__(self):
         """Initialize IOHandler."""
-        self._logger = logging.getLogger(__name__)
-        self._logger.debug("Init IOHandler")
+        self.logger = logging.getLogger(__name__)
+        self.logger.debug("Init IOHandler")
 
         self.output_path = None
-        self.use_plain_output_path = False
-        self.data_path = None
         self.model_path = None
 
-    def set_paths(
-        self, output_path=None, data_path=None, model_path=None, use_plain_output_path=False
-    ):
+    def set_paths(self, output_path=None, model_path=None):
         """
         Set paths for input and output.
 
@@ -50,28 +35,19 @@ class IOHandler(metaclass=IOHandlerSingleton):
         ----------
         output_path: str or Path
             Path pointing to the output directory.
-        data_path: str or Path
-            Path pointing to the data files (e.g., CORSIKA or sim_telarray output).
         model_path: str or Path
             Path pointing to the model file directory.
-        use_plain_output_path: bool
-            Use plain output path without adding tool name and date
-
         """
         self.output_path = output_path
-        self.use_plain_output_path = use_plain_output_path
-        self.data_path = data_path
         self.model_path = model_path
 
-    def get_output_directory(self, label=None, sub_dir=None):
+    def get_output_directory(self, sub_dir=None):
         """
-        Return path to output directory.
+        Create and get path of an output directory.
 
         Parameters
         ----------
-        label: str
-            Instance label.
-        sub_dir: str
+        sub_dir: str or list of str, optional
             Name of the subdirectory (ray_tracing, model etc)
 
         Returns
@@ -81,34 +57,24 @@ class IOHandler(metaclass=IOHandlerSingleton):
         Raises
         ------
         FileNotFoundError
-            if error creating directory
-        TypeError
-            raised for errors while creating directory name
+            if the directory cannot be created
         """
-        path = Path(self.output_path)
-        if not self.use_plain_output_path:
-            path = (
-                path
-                if str(self.output_path).endswith("-output")
-                else path.joinpath("simtools-output")
-            )
-            label_dir = label if label is not None else "d-" + str(datetime.date.today())
-            path = (
-                path.joinpath(label_dir) if sub_dir is None else path.joinpath(label_dir, sub_dir)
-            )
-
-        return self._mkdir(path)
-
-    def _mkdir(self, path):
-        """Create a directory and return path."""
+        if sub_dir is None:
+            parts = []
+        elif isinstance(sub_dir, list | tuple):
+            parts = sub_dir
+        else:
+            parts = [sub_dir]
+        path = Path(self.output_path, *parts)
+
         try:
             path.mkdir(parents=True, exist_ok=True)
         except FileNotFoundError as exc:
             raise FileNotFoundError(f"Error creating directory {path!s}") from exc
 
-        return path.absolute()
+        return path.resolve()
 
-    def get_output_file(self, file_name, label=None, sub_dir=None):
+    def get_output_file(self, file_name, sub_dir=None):
         """
         Get path of an output file.
 
@@ -116,51 +82,31 @@ class IOHandler(metaclass=IOHandlerSingleton):
         ----------
         files_name: str
             File name.
-        label: str
-            Instance label.
-        sub_dir: str
+        sub_dir: sub_dir: str or list of str, optional
             Name of the subdirectory (ray_tracing, model etc)
 
         Returns
         -------
         Path
         """
-        return (
-            self.get_output_directory(label=label, sub_dir=sub_dir).joinpath(file_name).absolute()
-        )
+        return self.get_output_directory(sub_dir).joinpath(file_name).absolute()
 
-    def get_input_data_file(self, parent_dir=None, file_name=None, test=False):
+    def get_test_data_file(self, file_name=None):
         """
-        Get path of a data file, using data_path.
+        Get path of a data file in the test resources directory.
 
         Parameters
         ----------
-        parent_dir: str
-            Parent directory of the file.
         files_name: str
             File name.
-        test: bool
-            If true, return test resources location
 
         Returns
         -------
         Path
-
-        Raises
-        ------
-        IncompleteIOHandlerInitError
-            if data_path is not set
-
         """
-        if test:
-            file_prefix = Path("tests/resources/")
-        elif self.data_path is not None:
-            file_prefix = Path(self.data_path).joinpath(parent_dir)
-        else:
-            raise IncompleteIOHandlerInitError
-        return file_prefix.joinpath(file_name).absolute()
+        return Path("tests/resources", file_name).resolve()
 
-    def get_model_configuration_directory(self, label, model_version):
+    def get_model_configuration_directory(self, model_version, sub_dir=None):
         """
         Get path of the simulation model configuration directory.
 
@@ -168,13 +114,15 @@ class IOHandler(metaclass=IOHandlerSingleton):
 
         Parameters
         ----------
-        label: str
-            Instance label.
         model_version: str
             Model version.
+        sub_dir: str
+            subdirectory
 
         Returns
         -------
         Path
         """
-        return self._mkdir(self.get_output_directory(label=label).joinpath("model", model_version))
+        return self.get_output_directory(
+            sub_dir=[sub_dir, "model", model_version] if sub_dir else ["model", model_version]
+        )

simtools/io/legacy_data_handler.py

@@ -5,11 +5,6 @@ import logging
 
 from astropy.table import Table
 
-__all__ = [
-    "read_legacy_data_as_table",
-    "read_legacy_lst_single_pe",
-]
-
 logger = logging.getLogger(__name__)