gammasimtools 0.21.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

simtools/db/db_model_upload.py

@@ -1,16 +1,85 @@
 """Upload a simulation model (parameters and production tables) to the database."""
 
 import logging
+import shutil
 from pathlib import Path
 
 from packaging.version import Version
 
 from simtools.io import ascii_handler
+from simtools.job_execution.job_manager import retry_command
 from simtools.utils import names
 
 logger = logging.getLogger(__name__)
 
 
+def add_complete_model(
+    tmp_dir,
+    db,
+    db_simulation_model,
+    db_simulation_model_version,
+    repository_url,
+    repository_branch=None,
+):
+    """
+    Upload a complete model including model parameters and production tables to the database.
+
+    Generate compound indexes for the new database.
+
+    Parameters
+    ----------
+    tmp_dir : Path or str
+        Temporary directory to use for cloning the repository.
+    db : DatabaseHandler
+        Database handler object.
+    db_simulation_model : str
+        Name of the simulation model in the database.
+    db_simulation_model_version : str
+        Version of the simulation model in the database.
+    repository_url : str
+        URL of the simulation model repository to clone.
+    repository_branch : str, optional
+        Branch of the repository to use. If None, the default branch is used.
+
+    Returns
+    -------
+    None
+        This function does not return a value.
+    """
+    if not _confirm_remote_database_upload(db):
+        return
+
+    repository_dir = None
+    try:
+        repository_dir = clone_simulation_model_repository(
+            tmp_dir,
+            repository_url,
+            db_simulation_model_version=db_simulation_model_version,
+            repository_branch=repository_branch,
+        )
+
+        add_model_parameters_to_db(
+            input_path=repository_dir / "simulation-models" / "model_parameters", db=db
+        )
+
+        add_production_tables_to_db(
+            input_path=repository_dir / "simulation-models" / "productions", db=db
+        )
+
+        db.generate_compound_indexes_for_databases(
+            db_name=None,
+            db_simulation_model=db_simulation_model,
+            db_simulation_model_version=db_simulation_model_version,
+        )
+    except Exception as exc:
+        raise RuntimeError(f"Upload of simulation model failed: {exc}") from exc
+    finally:
+        if repository_dir is not None and repository_dir.exists():
+            shutil.rmtree(repository_dir)
+
+    logger.info("Upload of simulation model completed successfully")
+
+
 def add_values_from_json_to_db(file, collection, db, file_prefix):
     """
     Upload new model parameter from json files to db.
@@ -86,13 +155,16 @@ def add_production_tables_to_db(input_path, db):
     input_path = Path(input_path)
     logger.info(f"Reading production tables from repository path {input_path}")
 
-    for model in filter(Path.is_dir, input_path.iterdir()):
+    for model in sorted(filter(Path.is_dir, input_path.iterdir())):
         logger.info(f"Reading production tables for model version {model.name}")
         model_dict = _read_production_tables(model)
 
         for collection, data in model_dict.items():
             if data["parameters"]:
-                logger.info(f"Adding production table for {collection} to the database")
+                logger.info(
+                    f"Adding production table for {collection} "
+                    f"(model version {model.name}) to the database"
+                )
                 db.add_production_table(production_table=data)
             else:
                 logger.info(f"No production table for {collection} in model version {model.name}")
@@ -103,7 +175,7 @@ def _read_production_tables(model_path):
     Read production tables from a directory.
 
     Take into account that some productions include patch updates only. Read in this cases
-    the base models first.
+    all models from the model version history, starting with the earliest one.
 
     Parameters
     ----------
@@ -121,6 +193,12 @@ def _read_production_tables(model_path):
         for file in sorted((model_path.parent / model).rglob("*json")):
             _read_production_table(model_dict, file, model)
 
+    # ensure that the for patch updates the model version is set correctly
+    for table in model_dict.values():
+        table["model_version"] = model_path.name
+
+    _remove_deprecated_model_parameters(model_dict)
+
     return model_dict
 
 
@@ -135,10 +213,14 @@ def _read_production_table(model_dict, file, model_name):
             "model_version": model_name,
             "parameters": {},
             "design_model": {},
+            "deprecated_parameters": [],
         },
     )
     parameter_dict = ascii_handler.collect_data_from_file(file_name=file)
-    logger.debug(f"Reading production table for {array_element} (collection {collection})")
+    logger.debug(
+        f"Reading production table for {array_element} "
+        f"(model_version {model_name}, collection {collection})"
+    )
     try:
         if array_element in ("configuration_corsika", "configuration_sim_telarray"):
             model_dict[collection]["parameters"] = parameter_dict["parameters"]
@@ -146,10 +228,10 @@ def _read_production_table(model_dict, file, model_name):
             model_dict[collection]["parameters"].setdefault(array_element, {}).update(
                 parameter_dict["parameters"][array_element]
             )
-
     except KeyError as exc:
         logger.error(f"KeyError: {exc}")
         raise
+
     try:
         model_dict[collection]["design_model"][array_element] = parameter_dict["design_model"][
             array_element
@@ -157,4 +239,127 @@ def _read_production_table(model_dict, file, model_name):
     except KeyError:
         pass
 
+    try:
+        model_dict[collection]["deprecated_parameters"] = parameter_dict["deprecated_parameters"]
+    except KeyError:
+        pass
+
     model_dict[collection]["model_version"] = model_name
+
+
+def _remove_deprecated_model_parameters(model_dict):
+    """
+    Remove deprecated parameters from all tables in a model dictionary.
+
+    Parameters
+    ----------
+    model_dict : dict
+        Production tables for a specific model version.
+    """
+    for table in model_dict.values():
+        for params in table.get("parameters", {}).values():
+            for param in table.get("deprecated_parameters", []):
+                if param in params:
+                    logger.info(
+                        f"Deprecated parameter {param} in production table {table['collection']}"
+                    )
+                    params.pop(param)
+
+
+def _confirm_remote_database_upload(db):
+    """
+    Confirm with the user that they want to upload to a remote database.
+
+    Returns True if the user confirms, False otherwise. Returns also True
+    if the database is not remote.
+
+    Parameters
+    ----------
+    db : DatabaseHandler
+        Database handler object.
+
+    Returns
+    -------
+    bool
+        True if user confirms upload, False otherwise.
+    """
+    abort_message = "Operation aborted."
+
+    if not db.is_remote_database():
+        return True
+
+    db_config = db.mongo_db_config
+    db_server = db_config.get("db_server", "unknown server") if db_config else "unknown server"
+
+    try:
+        # First confirmation
+        user_input = input(
+            f"Do you really want to upload to remote DB {db_server}? Type 'yes' to confirm: "
+        )
+        if user_input != "yes":
+            logger.info(abort_message)
+            return False
+
+        # Second confirmation
+        user_input = input(
+            f"Let be sure: do you really want to upload to remote DB {db_server}? "
+            "Type 'yes' to confirm: "
+        )
+        if user_input != "yes":
+            logger.info(abort_message)
+            return False
+
+        return True
+
+    except (EOFError, KeyboardInterrupt):
+        logger.info(abort_message)
+        return False
+
+
+def clone_simulation_model_repository(
+    target_dir, repository_url, db_simulation_model_version, repository_branch
+):
+    """
+    Clone a git repository containing simulation model parameters and production tables.
+
+    Parameters
+    ----------
+    target_dir : Path or str
+        Target directory to clone the repository into.
+    repository_url : str
+        URL of the git repository to clone.
+    db_simulation_model_version : str
+        Version tag of the simulation model to checkout.
+    repository_branch : str, optional
+        Branch of the repository to use. If None, the version tag is used.
+
+    Returns
+    -------
+    Path
+        Path to the cloned repository.
+    """
+    if repository_branch:
+        logger.info(f"Using branch: {repository_branch}")
+    else:
+        logger.info(f"Using version tag: {db_simulation_model_version}")
+
+    target_dir = Path(target_dir)
+    target_dir = target_dir if target_dir.is_absolute() else Path.cwd() / target_dir
+
+    if target_dir.exists():
+        shutil.rmtree(target_dir)
+
+    logger.info(f"Cloning model parameters from {repository_url}")
+
+    if repository_branch:
+        command = f'git clone --depth=1 -b "{repository_branch}" "{repository_url}" "{target_dir}"'
+    else:
+        # Use version tag with detached head (generates warning but that's fine)
+        command = (
+            f'git clone --branch "{db_simulation_model_version}" '
+            f'--depth 1 "{repository_url}" "{target_dir}"'
+        )
+
+    retry_command(command)
+
+    return target_dir

simtools/io/io_handler.py

@@ -1,15 +1,8 @@
-"""Handle input and output paths."""
+"""Handle input and output directories and file paths."""
 
-import datetime
 import logging
 from pathlib import Path
 
-__all__ = ["IOHandler", "IOHandlerSingleton"]
-
-
-class IncompleteIOHandlerInitError(Exception):
-    """Exception raised when IOHandler is not initialized."""
-
 
 class IOHandlerSingleton(type):
     """Singleton base class."""
@@ -17,32 +10,24 @@ class IOHandlerSingleton(type):
     _instances = {}
 
     def __call__(cls, *args, **kwargs):
-        """
-        Ensure a single instance of the IOHandlerSingleton class.
-
-        Creates a new instance if it doesn't exist, otherwise returns the existing instance.
-        """
+        """Create a new instance if it doesn't exist, otherwise returns the existing instance."""
         if cls not in cls._instances:
             cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]
 
 
 class IOHandler(metaclass=IOHandlerSingleton):
-    """Handle input and output paths."""
+    """Handle input and output directories and file paths."""
 
     def __init__(self):
         """Initialize IOHandler."""
-        self._logger = logging.getLogger(__name__)
-        self._logger.debug("Init IOHandler")
+        self.logger = logging.getLogger(__name__)
+        self.logger.debug("Init IOHandler")
 
         self.output_path = None
-        self.use_plain_output_path = False
-        self.data_path = None
         self.model_path = None
 
-    def set_paths(
-        self, output_path=None, data_path=None, model_path=None, use_plain_output_path=False
-    ):
+    def set_paths(self, output_path=None, model_path=None):
         """
         Set paths for input and output.
 
@@ -50,28 +35,19 @@ class IOHandler(metaclass=IOHandlerSingleton):
         ----------
         output_path: str or Path
             Path pointing to the output directory.
-        data_path: str or Path
-            Path pointing to the data files (e.g., CORSIKA or sim_telarray output).
         model_path: str or Path
             Path pointing to the model file directory.
-        use_plain_output_path: bool
-            Use plain output path without adding tool name and date
-
         """
         self.output_path = output_path
-        self.use_plain_output_path = use_plain_output_path
-        self.data_path = data_path
         self.model_path = model_path
 
-    def get_output_directory(self, label=None, sub_dir=None):
+    def get_output_directory(self, sub_dir=None):
         """
-        Return path to output directory.
+        Create and get path of an output directory.
 
         Parameters
         ----------
-        label: str
-            Instance label.
-        sub_dir: str
+        sub_dir: str or list of str, optional
             Name of the subdirectory (ray_tracing, model etc)
 
         Returns
@@ -81,34 +57,24 @@ class IOHandler(metaclass=IOHandlerSingleton):
         Raises
         ------
         FileNotFoundError
-            if error creating directory
-        TypeError
-            raised for errors while creating directory name
+            if the directory cannot be created
         """
-        path = Path(self.output_path)
-        if not self.use_plain_output_path:
-            path = (
-                path
-                if str(self.output_path).endswith("-output")
-                else path.joinpath("simtools-output")
-            )
-            label_dir = label if label is not None else "d-" + str(datetime.date.today())
-            path = (
-                path.joinpath(label_dir) if sub_dir is None else path.joinpath(label_dir, sub_dir)
-            )
-
-        return self._mkdir(path)
-
-    def _mkdir(self, path):
-        """Create a directory and return path."""
+        if sub_dir is None:
+            parts = []
+        elif isinstance(sub_dir, list | tuple):
+            parts = sub_dir
+        else:
+            parts = [sub_dir]
+        path = Path(self.output_path, *parts)
+
         try:
             path.mkdir(parents=True, exist_ok=True)
         except FileNotFoundError as exc:
             raise FileNotFoundError(f"Error creating directory {path!s}") from exc
 
-        return path.absolute()
+        return path.resolve()
 
-    def get_output_file(self, file_name, label=None, sub_dir=None):
+    def get_output_file(self, file_name, sub_dir=None):
         """
         Get path of an output file.
 
@@ -116,51 +82,31 @@ class IOHandler(metaclass=IOHandlerSingleton):
         ----------
         files_name: str
             File name.
-        label: str
-            Instance label.
-        sub_dir: str
+        sub_dir: sub_dir: str or list of str, optional
             Name of the subdirectory (ray_tracing, model etc)
 
         Returns
         -------
         Path
         """
-        return (
-            self.get_output_directory(label=label, sub_dir=sub_dir).joinpath(file_name).absolute()
-        )
+        return self.get_output_directory(sub_dir).joinpath(file_name).absolute()
 
-    def get_input_data_file(self, parent_dir=None, file_name=None, test=False):
+    def get_test_data_file(self, file_name=None):
         """
-        Get path of a data file, using data_path.
+        Get path of a data file in the test resources directory.
 
         Parameters
         ----------
-        parent_dir: str
-            Parent directory of the file.
         files_name: str
             File name.
-        test: bool
-            If true, return test resources location
 
         Returns
         -------
         Path
-
-        Raises
-        ------
-        IncompleteIOHandlerInitError
-            if data_path is not set
-
         """
-        if test:
-            file_prefix = Path("tests/resources/")
-        elif self.data_path is not None:
-            file_prefix = Path(self.data_path).joinpath(parent_dir)
-        else:
-            raise IncompleteIOHandlerInitError
-        return file_prefix.joinpath(file_name).absolute()
+        return Path("tests/resources", file_name).resolve()
 
-    def get_model_configuration_directory(self, label, model_version):
+    def get_model_configuration_directory(self, model_version, sub_dir=None):
         """
         Get path of the simulation model configuration directory.
 
@@ -168,13 +114,15 @@ class IOHandler(metaclass=IOHandlerSingleton):
 
         Parameters
         ----------
-        label: str
-            Instance label.
         model_version: str
             Model version.
+        sub_dir: str
+            subdirectory
 
         Returns
         -------
         Path
         """
-        return self._mkdir(self.get_output_directory(label=label).joinpath("model", model_version))
+        return self.get_output_directory(
+            sub_dir=[sub_dir, "model", model_version] if sub_dir else ["model", model_version]
+        )

simtools/job_execution/job_manager.py

@@ -2,6 +2,7 @@
 
 import logging
 import subprocess
+import time
 from pathlib import Path
 
 import simtools.utils.general as gen
@@ -9,10 +10,54 @@ import simtools.utils.general as gen
 __all__ = ["JobExecutionError", "JobManager"]
 
 
+logger = logging.getLogger(__name__)
+
+
 class JobExecutionError(Exception):
     """Job execution error."""
 
 
+def retry_command(command, max_attempts=3, delay=10):
+    """
+    Execute a shell command with retry logic for network-related failures.
+
+    Parameters
+    ----------
+    command : str
+        Shell command to execute.
+    max_attempts : int
+        Maximum number of retry attempts (default: 3).
+    delay : int
+        Delay in seconds between attempts (default: 10).
+
+    Returns
+    -------
+    bool
+        True if command succeeded, False if all attempts failed.
+
+    Raises
+    ------
+    subprocess.CalledProcessError
+        If command fails after all retry attempts.
+    """
+    for attempt in range(1, max_attempts + 1):
+        logger.info(f"Attempt {attempt} of {max_attempts}: {command}")
+        try:
+            subprocess.run(command, shell=True, check=True, text=True)
+            logger.info(f"Command succeeded on attempt {attempt}")
+            return True
+        except subprocess.CalledProcessError as exc:
+            logger.warning(f"Command failed on attempt {attempt}")
+            if attempt < max_attempts:
+                logger.info(f"Waiting {delay}s before retry...")
+                time.sleep(delay)
+            else:
+                logger.error(f"Command failed after {max_attempts} attempts")
+                raise exc from None
+
+    return False
+
+
 class JobManager:
     """
     Job manager for submitting jobs to a compute node.

simtools/layout/array_layout_utils.py

@@ -154,10 +154,7 @@ def write_array_layouts(array_layouts, args_dict, db_config):
     _logger.info(f"Writing updated array layouts to the database for site {site}.")
 
     io_handler_instance = io_handler.IOHandler()
-    io_handler_instance.set_paths(
-        output_path=args_dict["output_path"],
-        use_plain_output_path=args_dict["use_plain_output_path"],
-    )
+    io_handler_instance.set_paths(output_path=args_dict["output_path"])
     output_file = io_handler_instance.get_output_file(
         f"array-layouts-{args_dict['updated_parameter_version']}.json"
     )
@@ -168,7 +165,6 @@ def write_array_layouts(array_layouts, args_dict, db_config):
         instrument=site,
         parameter_version=args_dict.get("updated_parameter_version"),
         output_file=output_file,
-        use_plain_output_path=args_dict["use_plain_output_path"],
         db_config=db_config,
     )
     MetadataCollector.dump(