gammasimtools 0.21.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

simtools/applications/db_add_value_from_json_to_db.py

@@ -25,18 +25,21 @@ r"""
 
 """
 
-import logging
 import uuid
 from pathlib import Path
 
 import simtools.utils.general as gen
+from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
 from simtools.db import db_handler
 from simtools.io import ascii_handler
 
 
 def _parse():
-    config = configurator.Configurator(description="Add a new parameter to the DB.")
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        label=get_application_label(__file__), description="Add a new parameter to the DB."
+    )
     group = config.parser.add_mutually_exclusive_group(required=True)
     group.add_argument("--file_name", help="file to be added", type=str)
     group.add_argument(
@@ -55,22 +58,22 @@ def _parse():
     return config.initialize(db_config=True)
 
 
-def main():  # noqa: D103
-    args_dict, db_config = _parse()
-
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+def main():
+    """Add value from JSON to database."""
+    app_context = startup_application(_parse)
 
-    if args_dict.get("test_db", False):
-        db_config["db_simulation_model_version"] = str(uuid.uuid4())
-        logger.info(f"Using test database version {db_config['db_simulation_model_version']}")
-    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    if app_context.args.get("test_db", False):
+        app_context.db_config["db_simulation_model_version"] = str(uuid.uuid4())
+        app_context.logger.info(
+            f"Using test database version {app_context.db_config['db_simulation_model_version']}"
+        )
+    db = db_handler.DatabaseHandler(mongo_db_config=app_context.db_config)
 
     files_to_insert = []
-    if args_dict.get("file_name", None) is not None:
-        files_to_insert.append(args_dict["file_name"])
+    if app_context.args.get("file_name", None) is not None:
+        files_to_insert.append(app_context.args["file_name"])
     else:
-        files_to_insert.extend(Path(args_dict["input_path"]).glob("*json"))
+        files_to_insert.extend(Path(app_context.args["input_path"]).glob("*json"))
 
     if len(files_to_insert) < 1:
         raise ValueError("No files were provided to upload")
@@ -78,32 +81,34 @@ def main():  # noqa: D103
 
     print(
         f"Should the following parameter{plural} be inserted to the "
-        f"{args_dict['db_collection']} DB collection?:\n"
+        f"{app_context.args['db_collection']} DB collection?:\n"
     )
     print(*files_to_insert, sep="\n")
     print()
 
-    logger.info(f"DB {db.get_db_name()} selected.")
+    app_context.logger.info(f"DB {db.get_db_name()} selected.")
 
     if gen.user_confirm():
         for file_to_insert_now in files_to_insert:
             par_dict = ascii_handler.collect_data_from_file(file_name=file_to_insert_now)
-            logger.info(f"Adding the following parameter to the DB: {par_dict['parameter']}")
+            app_context.logger.info(
+                f"Adding the following parameter to the DB: {par_dict['parameter']}"
+            )
             db.add_new_parameter(
                 par_dict=par_dict,
-                collection_name=args_dict["db_collection"],
+                collection_name=app_context.args["db_collection"],
                 file_prefix="./",
             )
-            logger.info(
+            app_context.logger.info(
                 f"Value for {par_dict['parameter']} added to "
-                f"{args_dict['db_collection']} collection."
+                f"{app_context.args['db_collection']} collection."
             )
     else:
-        logger.info("Aborted, no change applied to the database")
+        app_context.logger.info("Aborted, no change applied to the database")
 
     # drop test database; be safe and required DB name is sandbox
-    if args_dict.get("test_db", False) and "sandbox" in db.get_db_name():
-        logger.info(f"Test database used. Dropping all data from {db.get_db_name()}")
+    if app_context.args.get("test_db", False) and "sandbox" in db.get_db_name():
+        app_context.logger.info(f"Test database used. Dropping all data from {db.get_db_name()}")
         db.db_client.drop_database(db.get_db_name())
 
 

simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py

@@ -33,35 +33,23 @@
 
 """
 
-import logging
 from pathlib import Path
 
 import astropy.table
 
-import simtools.utils.general as gen
+from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
 from simtools.data_model.model_data_writer import ModelDataWriter
 from simtools.io import ascii_handler
 from simtools.model.array_model import ArrayModel
 
 
-def _parse(label=None, description=None):
-    """
-    Parse command line configuration.
-
-    Parameters
-    ----------
-    label : str
-        Label describing application.
-    description : str
-        Description of application.
-
-    Returns
-    -------
-    CommandLineParser
-        Command line parser object.
-    """
-    config = configurator.Configurator(label=label, description=description)
+def _parse():
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        label=get_application_label(__file__),
+        description="Add array element positions to model parameter repository",
+    )
     config.parser.add_argument(
         "--input",
         help="File containing a table of array element positions.",
@@ -96,8 +84,8 @@ def write_utm_array_elements_to_repository(args_dict, logger):
     ----------
     args_dict : dict
         Command line arguments.
-    logger : Logger
-        Logger object.
+    app_context.logger : app_context.logger
+        app_context.logger object.
 
     """
     array_elements = astropy.table.Table.read(args_dict["input"])
@@ -131,8 +119,8 @@ def write_ground_array_elements_to_repository(args_dict, db_config, logger):
         Command line arguments.
     db_config : dict
         Database configuration.
-    logger : Logger
-        Logger object.
+    logger : logger
+        logger object.
 
     """
     array_model = ArrayModel(
@@ -155,19 +143,16 @@ def write_ground_array_elements_to_repository(args_dict, db_config, logger):
 
 def main():
     """Application main."""
-    label = Path(__file__).stem
-    args_dict, db_config = _parse(
-        label, description="Add array element positions to model parameter repository"
-    )
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+    app_context = startup_application(_parse)
 
-    if args_dict["coordinate_system"] == "utm":
-        write_utm_array_elements_to_repository(args_dict, logger)
-    elif args_dict["coordinate_system"] == "ground":
-        write_ground_array_elements_to_repository(args_dict, db_config, logger)
+    if app_context.args["coordinate_system"] == "utm":
+        write_utm_array_elements_to_repository(app_context.args, app_context.logger)
+    elif app_context.args["coordinate_system"] == "ground":
+        write_ground_array_elements_to_repository(
+            app_context.args, app_context.db_config, app_context.logger
+        )
     else:
-        logger.error("Invalid coordinate system. Allowed are 'utm' and 'ground'.")
+        app_context.logger.error("Invalid coordinate system. Allowed are 'utm' and 'ground'.")
         raise ValueError
 
 

simtools/applications/db_generate_compound_indexes.py

@@ -13,16 +13,16 @@ db_name (str, optional)
     Database name (use "all" for all databases)
 """
 
-import logging
-
-import simtools.utils.general as gen
+from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
 from simtools.db import db_handler
 
 
 def _parse():
+    """Parse command line configuration."""
     config = configurator.Configurator(
-        description="Generate compound indexes for a specific database"
+        description="Generate compound indexes for a specific database",
+        label=get_application_label(__file__),
     )
     config.parser.add_argument(
         "--db_name",
@@ -33,32 +33,17 @@ def _parse():
     return config.initialize(db_config=True)
 
 
-def main():  # noqa: D103
-    args_dict, db_config = _parse()
+def main():
+    """Generate compound indexes for the specified database."""
+    app_context = startup_application(_parse, setup_io_handler=False)
 
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+    db = db_handler.DatabaseHandler(mongo_db_config=app_context.db_config)
 
-    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
-    # databases without internal databases we don't have rights to modify
-    databases = [
-        d for d in db.db_client.list_database_names() if d not in ("config", "admin", "local")
-    ]
-    requested = db.get_db_name(
-        db_name=args_dict["db_name"],
-        db_simulation_model_version=args_dict.get("db_simulation_model_version"),
-        model_name=args_dict.get("db_simulation_model"),
+    db.generate_compound_indexes_for_databases(
+        db_name=app_context.args["db_name"],
+        db_simulation_model=app_context.args.get("db_simulation_model"),
+        db_simulation_model_version=app_context.args.get("db_simulation_model_version"),
     )
-    if requested != "all" and requested not in databases:
-        raise ValueError(
-            f"Requested database '{requested}' not found. "
-            f"Following databases are available: {', '.join(databases)}"
-        )
-
-    databases = databases if requested == "all" else [requested]
-    for db_name in databases:
-        logger.info(f"Generating compound indexes for database: {db_name}")
-        db.generate_compound_indexes(db_name=db_name)
 
 
 if __name__ == "__main__":

simtools/applications/db_get_array_layouts_from_db.py

@@ -51,33 +51,19 @@ Retrieve telescope positions from database (utm coordinate system) and write to
       --output_file telescope_positions-test_layout.ecsv
 """
 
-import logging
-from pathlib import Path
-
 import simtools.data_model.model_data_writer as writer
-import simtools.utils.general as gen
+from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
 from simtools.model.array_model import ArrayModel
 from simtools.model.site_model import SiteModel
 
 
-def _parse(label, description):
-    """
-    Parse command line configuration.
-
-    Parameters
-    ----------
-    label : str
-        Label describing the application.
-    description : str
-        Description of the application.
-
-    Returns
-    -------
-    CommandLineParser
-        Command line parser object.
-    """
-    config = configurator.Configurator(label=label, description=description)
+def _parse():
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        label=get_application_label(__file__),
+        description="Get list of array elements as defined in the db (array layout).",
+    )
 
     input_group = config.parser.add_mutually_exclusive_group()
     input_group.add_argument(
@@ -128,33 +114,27 @@ def _layout_from_db(args_dict, db_config):
 
 
 def main():
-    """Get list of array elements as defined in the db (array layout)."""
-    label = Path(__file__).stem
-    args_dict, db_config = _parse(
-        label,
-        "Get list of array elements as defined in the db (array layout).",
-    )
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+    """Get list of array layouts or list of elements for a given layout as defined in the db."""
+    app_context = startup_application(_parse)
 
-    if args_dict.get("list_available_layouts", False):
-        if args_dict.get("site", None) is None:
+    if app_context.args.get("list_available_layouts", False):
+        if app_context.args.get("site", None) is None:
             raise ValueError("Site must be provided to list available layouts.")
         site_model = SiteModel(
-            mongo_db_config=db_config,
-            model_version=args_dict["model_version"],
-            site=args_dict["site"],
+            mongo_db_config=app_context.db_config,
+            model_version=app_context.args["model_version"],
+            site=app_context.args["site"],
         )
         print(site_model.get_list_of_array_layouts())
     else:
-        logger.info("Array layout: %s", args_dict["array_layout_name"])
-        layout = _layout_from_db(args_dict, db_config)
+        app_context.logger.info("Array layout: %s", app_context.args["array_layout_name"])
+        layout = _layout_from_db(app_context.args, app_context.db_config)
         layout.pprint()
 
-        if not args_dict.get("output_file_from_default", False):
+        if not app_context.args.get("output_file_from_default", False):
             writer.ModelDataWriter.dump(
-                args_dict=args_dict,
-                output_file=args_dict["output_file"],
+                args_dict=app_context.args,
+                output_file=app_context.args["output_file"],
                 metadata=None,
                 product_data=layout,
             )

simtools/applications/db_get_file_from_db.py

@@ -33,16 +33,15 @@
 
 """
 
-import logging
-
-import simtools.utils.general as gen
+from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
 from simtools.db import db_handler
-from simtools.io import io_handler
 
 
 def _parse():
+    """Parse command line configuration."""
     config = configurator.Configurator(
+        label=get_application_label(__file__),
         description="Get file(s) from the DB.",
         usage="simtools-get-file-from-db --file_name mirror_CTA-S-LST_v2020-04-07.dat",
     )
@@ -57,24 +56,23 @@ def _parse():
     return config.initialize(db_config=True, output=True)
 
 
-def main():  # noqa: D103
-    args_dict, db_config = _parse()
-
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
-    _io_handler = io_handler.IOHandler()
+def main():
+    """Get file from database."""
+    app_context = startup_application(_parse)
 
-    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    db = db_handler.DatabaseHandler(mongo_db_config=app_context.db_config)
     file_id = db.export_model_files(
-        dest=_io_handler.get_output_directory(),
-        file_names=args_dict["file_name"],
+        dest=app_context.io_handler.get_output_directory(),
+        file_names=app_context.args["file_name"],
     )
     if file_id is None:
-        logger.error(f"The file {args_dict['file_name']} was not found in {db.db_name}.")
+        app_context.logger.error(
+            f"The file {app_context.args['file_name']} was not found in {db.db_name}."
+        )
         raise FileNotFoundError
-    logger.info(
-        f"Got file {args_dict['file_name']} from DB {db.db_name} "
-        f"and saved into {_io_handler.get_output_directory()}"
+    app_context.logger.info(
+        f"Got file {app_context.args['file_name']} from DB {db.db_name} "
+        f"and saved into {app_context.io_handler.get_output_directory()}"
     )
 
 

simtools/applications/db_get_parameter_from_db.py

@@ -60,23 +60,23 @@ r"""
 
 """
 
-import logging
-from pathlib import Path
 from pprint import pprint
 
-import simtools.utils.general as gen
+from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
 from simtools.db import db_handler
-from simtools.io import ascii_handler, io_handler
+from simtools.io import ascii_handler
 
 
 def _parse():
+    """Parse command line configuration."""
     config = configurator.Configurator(
+        label=get_application_label(__file__),
         description=(
             "Get a parameter entry from DB for a specific telescope or a site. "
             "The application receives a parameter name, a site, a telescope (if applicable), "
             "and a version. It then prints out the parameter entry. "
-        )
+        ),
     )
 
     config.parser.add_argument("--parameter", help="Parameter name", type=str, required=True)
@@ -104,48 +104,46 @@ def _parse():
     )
 
 
-def main():  # noqa: D103
-    args_dict, db_config = _parse()
-
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+def main():
+    """Get a parameter entry from DB for a specific telescope or a site."""
+    app_context = startup_application(_parse)
 
-    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    db = db_handler.DatabaseHandler(mongo_db_config=app_context.db_config)
 
     pars = db.get_model_parameter(
-        parameter=args_dict["parameter"],
-        site=args_dict["site"],
-        array_element_name=args_dict.get("telescope"),
-        parameter_version=args_dict.get("parameter_version"),
-        model_version=args_dict.get("model_version"),
+        parameter=app_context.args["parameter"],
+        site=app_context.args["site"],
+        array_element_name=app_context.args.get("telescope"),
+        parameter_version=app_context.args.get("parameter_version"),
+        model_version=app_context.args.get("model_version"),
     )
-    if args_dict["export_model_file"] or args_dict["export_model_file_as_table"]:
+    if app_context.args["export_model_file"] or app_context.args["export_model_file_as_table"]:
         table = db.export_model_file(
-            parameter=args_dict["parameter"],
-            site=args_dict["site"],
-            array_element_name=args_dict["telescope"],
-            parameter_version=args_dict.get("parameter_version"),
-            model_version=args_dict.get("model_version"),
-            export_file_as_table=args_dict["export_model_file_as_table"],
+            parameter=app_context.args["parameter"],
+            site=app_context.args["site"],
+            array_element_name=app_context.args["telescope"],
+            parameter_version=app_context.args.get("parameter_version"),
+            model_version=app_context.args.get("model_version"),
+            export_file_as_table=app_context.args["export_model_file_as_table"],
         )
-        param_value = pars[args_dict["parameter"]]["value"]
-        table_file = Path(io_handler.IOHandler().get_output_directory()) / f"{param_value}"
-        logger.info(f"Exported model file {param_value} to {table_file}")
+        param_value = pars[app_context.args["parameter"]]["value"]
+        table_file = app_context.io_handler.get_output_file(param_value)
+        app_context.logger.info(f"Exported model file {param_value} to {table_file}")
         if table and table_file.suffix != ".ecsv":
             table.write(table_file.with_suffix(".ecsv"), format="ascii.ecsv", overwrite=True)
-            logger.info(f"Exported model file {param_value} to {table_file.with_suffix('.ecsv')}")
+            app_context.logger.info(
+                f"Exported model file {param_value} to {table_file.with_suffix('.ecsv')}"
+            )
 
-    if args_dict["output_file"] is not None:
-        _output_file = (
-            Path(io_handler.IOHandler().get_output_directory()) / args_dict["output_file"]
-        )
-        pars[args_dict["parameter"]].pop("_id")
-        pars[args_dict["parameter"]].pop("entry_date")
+    if app_context.args["output_file"] is not None:
+        pars[app_context.args["parameter"]].pop("_id")
+        pars[app_context.args["parameter"]].pop("entry_date")
         ascii_handler.write_data_to_file(
-            data=pars[args_dict["parameter"]], output_file=_output_file
+            data=pars[app_context.args["parameter"]],
+            output_file=app_context.io_handler.get_output_file(app_context.args["output_file"]),
         )
     else:
-        pprint(pars[args_dict["parameter"]])
+        pprint(pars[app_context.args["parameter"]])
 
 
 if __name__ == "__main__":

simtools/applications/db_inspect_databases.py

@@ -9,15 +9,16 @@ db_name (str, optional)
     Database name (use "all" for all databases)
 """
 
-import logging
-
-import simtools.utils.general as gen
+from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
 from simtools.db import db_handler
 
 
 def _parse():
-    config = configurator.Configurator(description="Inspect databases")
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        label=get_application_label(__file__), description="Inspect databases"
+    )
     config.parser.add_argument(
         "--db_name",
         help="Database name",
@@ -27,18 +28,16 @@ def _parse():
     return config.initialize(db_config=True)
 
 
-def main():  # noqa: D103
-    args_dict, db_config = _parse()
-
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+def main():
+    """Inspect databases."""
+    app_context = startup_application(_parse, setup_io_handler=False)
 
-    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    db = db_handler.DatabaseHandler(mongo_db_config=app_context.db_config)
     # databases without internal databases we don't have rights to modify
     databases = [
         d for d in db.db_client.list_database_names() if d not in ("config", "admin", "local")
     ]
-    requested = args_dict["db_name"]
+    requested = app_context.args["db_name"]
     if requested != "all" and requested not in databases:
         raise ValueError(
             f"Requested database '{requested}' not found. "

simtools/applications/db_upload_model_repository.py

@@ -0,0 +1,104 @@
+#!/usr/bin/python3
+r"""
+Upload model parameters from simulation model repository to a local or remote database.
+
+This script clones the CTAO simulation model repository and uploads model parameters
+and production tables to a MongoDB database. It includes retry functionality for
+network operations and confirmation prompts for remote database uploads.
+
+Command line arguments
+----------------------
+db_simulation_model (str, required)
+    Name of the database simulation model.
+db_simulation_model_version (str, required)
+    Version of the database simulation model.
+branch (str, optional)
+    Repository branch to clone (if not provided, uses the version tag).
+tmp_dir (str, optional)
+    Temporary directory for cloning the repository (default: ./tmp_model_parameters).
+
+Examples
+--------
+Upload model repository to database using a version tag
+(see simulation model repository for available tags):
+
+.. code-block:: console
+
+    simtools-db-upload-model-repository \\
+        --db_simulation_model CTAO-Simulation-Model \\
+        --db_simulation_model_version v10.10.10
+
+Upload model repository using specific branch (the version tag is
+used to name the database, but no tag checkout is done):
+
+.. code-block:: console
+
+    simtools-db-upload-model-repository \\
+        --db_simulation_model CTAO-Simulation-Model \\
+        --db_simulation_model_version v10.10.10 \\
+        --branch main
+
+"""
+
+from simtools.application_control import get_application_label, startup_application
+from simtools.configuration import configurator
+from simtools.db import db_handler, db_model_upload
+
+DEFAULT_REPOSITORY_URL = (
+    "https://gitlab.cta-observatory.org/cta-science/simulations/"
+    "simulation-model/simulation-models.git"
+)
+
+
+def _parse():
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        label=get_application_label(__file__),
+        description="Upload model parameters from repository to database",
+    )
+    config.parser.add_argument(
+        "--branch",
+        help="Repository branch to clone (optional, defaults to using version tag).",
+        type=str,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--tmp_dir",
+        help="Temporary directory for cloning the repository (default: ./tmp_model_parameters).",
+        type=str,
+        default="tmp_model_parameters",
+        required=False,
+    )
+
+    args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True)
+
+    if args_dict.get("db_simulation_model_version"):
+        db_config["db_simulation_model"] = args_dict.get(
+            "db_simulation_model", "CTAO-Simulation-Model"
+        )
+        db_config["db_simulation_model_version"] = args_dict["db_simulation_model_version"]
+    else:
+        raise ValueError("Setting of db_simulation_model_version is required.")
+
+    return args_dict, db_config
+
+
+def main():
+    """Application main."""
+    app_context = startup_application(_parse)
+
+    db = db_handler.DatabaseHandler(mongo_db_config=app_context.db_config)
+    db.print_connection_info()
+
+    db_model_upload.add_complete_model(
+        tmp_dir=app_context.args.get("tmp_dir"),
+        db=db,
+        db_simulation_model=app_context.args.get("db_simulation_model"),
+        db_simulation_model_version=app_context.args.get("db_simulation_model_version"),
+        repository_url=DEFAULT_REPOSITORY_URL,
+        repository_branch=app_context.args.get("branch"),
+    )
+
+
+if __name__ == "__main__":
+    main()