gammasimtools 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/applications/convert_all_model_parameters_from_simtel.py

@@ -6,6 +6,8 @@ r"""
     ready to be submitted to the model database. Prints out parameters which are not found
     in simtel configuration file and parameters which are not found in simtools schema files.
 
+    Note that all parameters are assigned the same parameter version.
+
     Command line arguments
     ----------------------
     simtel_cfg_file (str)
@@ -30,7 +32,7 @@ r"""
           --simtel_cfg_file all_telescope_config_la_palma.cfg\\
           --simtel_telescope_name CT1\\
           --telescope LSTN-01\\
-          --model_version "2024-03-06"
+          --parameter_version "1.0.0"
 
     The export of the model parameters from sim_telarray for 6.0.0 can be done e.g., as follows:
 
@@ -59,7 +61,7 @@ import numpy as np
 import simtools.data_model.model_data_writer as writer
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
+from simtools.data_model import schema
 from simtools.io_operations.io_handler import IOHandler
 from simtools.simtel.simtel_config_reader import SimtelConfigReader
 
@@ -103,27 +105,7 @@ def _parse(label=None, description=None):
         type=str,
         required=True,
     )
-    return config.initialize(simulation_model="telescope")
-
-
-def get_list_of_parameters_and_schema_files(schema_directory):
-    """
-    Return list of parameters and schema files located in schema file directory.
-
-    Returns
-    -------
-    list
-        List of parameters found in schema file directory.
-    list
-        List of schema files found in schema file directory.
-
-    """
-    schema_files = sorted(Path(schema_directory).rglob("*.schema.yml"))
-    parameters = []
-    for schema_file in schema_files:
-        schema_dict = gen.collect_data_from_file(file_name=schema_file)
-        parameters.append(schema_dict.get("name"))
-    return parameters, schema_files
+    return config.initialize(simulation_model=["telescope", "parameter_version"])
 
 
 def get_list_of_simtel_parameters(simtel_config_file, logger):
@@ -203,7 +185,7 @@ def get_number_of_camera_pixel(args_dict, logger):
     """
     try:
         simtel_config_reader = SimtelConfigReader(
-            schema_file=MODEL_PARAMETER_SCHEMA_PATH / "camera_pixels.schema.yml",
+            schema_file=schema.get_model_parameter_schema_file("camera_pixels"),
             simtel_config_file=args_dict["simtel_cfg_file"],
             simtel_telescope_name=args_dict["simtel_telescope_name"],
         )
@@ -219,7 +201,6 @@ def read_and_export_parameters(args_dict, logger):
     """
     Read and export parameters from simtel configuration file to json files.
 
-    Only applicable parameters are exported to json.
     Provide extensive logging information on the parameters found in the simtel
     configuration file.
 
@@ -238,8 +219,8 @@ def read_and_export_parameters(args_dict, logger):
         List of simtools parameter not found in simtel configuration file.
 
     """
-    _parameters, _schema_files = get_list_of_parameters_and_schema_files(
-        args_dict.get("schema_directory", MODEL_PARAMETER_SCHEMA_PATH)
+    _parameters, _schema_files = schema.get_get_model_parameter_schema_files(
+        args_dict.get("schema_directory")
     )
     _simtel_parameters = get_list_of_simtel_parameters(args_dict["simtel_cfg_file"], logger)
 
@@ -266,7 +247,7 @@ def read_and_export_parameters(args_dict, logger):
             parameter_name=_parameter,
             value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
             instrument=args_dict["telescope"],
-            model_version=args_dict["model_version"],
+            parameter_version=args_dict["parameter_version"],
             output_file=io_handler.get_output_file(f"{_parameter}.json"),
         )
 

simtools/applications/convert_geo_coordinates_of_array_elements.py

@@ -1,56 +1,56 @@
 #!/usr/bin/python3
 """Convert and print a list of array element positions in different coordinate systems.
 
-    Description
-    -----------
-
-    Convert array element positions in different CTAO coordinate systems.
-    Available coordinate systems are:
-
-    1. UTM system
-    2. ground system (similar to sim_telarray system with x-axis pointing toward geographic north
-       and y-axis pointing towards the west); altitude relative to the CORSIKA observation level.
-       Altitude is the height of the elevation rotation axis (plus some possible mirror offset).
-    3. Mercator system
-
-    Command line arguments
-    ----------------------
-    input (str)
-        File name with list of array element positions.
-        Input can be given as astropy table file (ecsv) or a single array element in
-        a json file.
-    print (str)
-        Print in requested coordinate system; possible are ground, utm, mercator
-    export (str)
-        Export array element list to file in requested coordinate system;
-          possible are ground, utm, mercator
-    select_assets (str)
-        Select a subset of array elements / telescopes (e.g., MSTN, LSTN)
-
-    Example
-    -------
-    Convert a list of array elements using a list of telescope positions in UTM coordinates.
+Description
+-----------
+
+Convert array element positions in different CTAO coordinate systems.
+Available coordinate systems are:
+
+1. UTM system
+2. ground system (similar to sim_telarray system with x-axis pointing toward geographic north
+   and y-axis pointing towards the west); altitude relative to the CORSIKA observation level.
+   Altitude is the height of the elevation rotation axis (plus some possible mirror offset).
+3. Mercator system
+
+Command line arguments
+----------------------
+input (str)
+    File name with list of array element positions.
+    Input can be given as astropy table file (ecsv) or a single array element in
+    a json file.
+print (str)
+    Print in requested coordinate system; possible are ground, utm, mercator
+export (str)
+    Export array element list to file in requested coordinate system;
+      possible are ground, utm, mercator
+select_assets (str)
+    Select a subset of array elements / telescopes (e.g., MSTN, LSTN)
+
+Example
+-------
+Convert a list of array elements using a list of telescope positions in UTM coordinates.
 
-    .. code-block:: console
+.. code-block:: console
 
-        simtools-convert-geo-coordinates-of-array-elements
-            --input tests/resources/telescope_positions-North-utm.ecsv
-            --print ground
+    simtools-convert-geo-coordinates-of-array-elements
+        --input tests/resources/telescope_positions-North-utm.ecsv
+        --print ground
 
-    The converted list of telescope positions in ground coordinates is printed to the screen.
+The converted list of telescope positions in ground coordinates is printed to the screen.
 
-    The following example converts a list of telescope positions in UTM coordinates
-    and writes the output to a file in ground (sim_telarray) coordinates. Also selects
-    only a subset of the array elements (telescopes; ignore calibration devices):
+The following example converts a list of telescope positions in UTM coordinates
+and writes the output to a file in ground (sim_telarray) coordinates. Also selects
+only a subset of the array elements (telescopes; ignore calibration devices):
 
-    .. code-block:: console
+.. code-block:: console
 
-        simtools-convert-geo-coordinates-of-array-elements
-            --input tests/resources/telescope_positions-North-utm.ecsv
-            --export ground
-            --select_assets LSTN
+    simtools-convert-geo-coordinates-of-array-elements
+        --input tests/resources/telescope_positions-North-utm.ecsv
+        --export ground
+        --select_assets LSTN
 
-    Expected output is a ecsv file in the directory printed to the screen.
+Expected output is a ecsv file in the directory printed to the screen.
 
 """
 
@@ -130,7 +130,10 @@ def _parse(label=None, description=None):
         action="store_true",
     )
     return config.initialize(
-        output=True, require_command_line=True, db_config=True, simulation_model=["version", "site"]
+        output=True,
+        require_command_line=True,
+        db_config=True,
+        simulation_model=["model_version", "parameter_version", "site"],
     )
 
 
@@ -146,16 +149,12 @@ def main():
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    json_type = args_dict.get("input", "").endswith(".json")
-    # simplified metadata treatment for model parameter json files
-    if json_type:
+    if args_dict.get("input", "").endswith(".json"):
         site = args_dict.get("site", None)
-        top_level_meta = None
-        validate_schema_file = None
+        metadata, validate_schema_file = None, None
     else:
         metadata = MetadataCollector(args_dict=args_dict, data_model_name=data_model_name)
         site = metadata.get_site(from_input_meta=True)
-        top_level_meta = metadata.top_level_meta
         validate_schema_file = metadata.get_data_model_schema_file_name()
 
     layout = array_layout.ArrayLayout(
@@ -171,13 +170,15 @@ def main():
 
     if args_dict["export"] is not None:
         product_data = (
-            layout.export_one_telescope_as_json(crs_name=args_dict["export"])
-            if json_type
+            layout.export_one_telescope_as_json(
+                crs_name=args_dict["export"], parameter_version=args_dict.get("parameter_version")
+            )
+            if args_dict.get("input", "").endswith(".json")
             else layout.export_telescope_list_table(crs_name=args_dict["export"])
         )
         writer.ModelDataWriter.dump(
             args_dict=args_dict,
-            metadata=top_level_meta,
+            metadata=metadata.get_top_level_metadata() if metadata else None,
             product_data=product_data,
             validate_schema_file=validate_schema_file,
         )

simtools/applications/convert_model_parameter_from_simtel.py

@@ -78,7 +78,7 @@ def _parse(label=None, description=None):
         type=str,
         required=True,
     )
-    return config.initialize(simulation_model="telescope", output=True)
+    return config.initialize(simulation_model=["telescope", "parameter_version"], output=True)
 
 
 def main():  # noqa: D103
@@ -107,7 +107,7 @@ def main():  # noqa: D103
         parameter_name=simtel_config_reader.parameter_name,
         value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
         instrument=args_dict["telescope"],
-        model_version=args_dict["model_version"],
+        parameter_version=args_dict["parameter_version"],
         output_file=args_dict["output_file"],
         output_path=args_dict.get("output_path"),
         use_plain_output_path=args_dict.get("use_plain_output_path"),

simtools/applications/db_add_file_to_db.py

@@ -18,8 +18,7 @@
         A directory with files to upload to the DB. \
         All files in the directory with a predefined list of extensions will be uploaded.
     db (str)
-        The DB to insert the files to. \
-        The choices are either the default CTA simulation DB or a sandbox for testing.
+        The DB to insert the files to.
 
     Example
     -------

simtools/applications/db_add_simulation_model_from_repository_to_db.py

@@ -0,0 +1,110 @@
+#!/usr/bin/python3
+r"""
+    Add parameters and production tables from a simulation model repository to a new database.
+
+    Generates a new database with all required collections.
+    Follows the structure of the CTAO gitlab simulation model repository.
+
+    This is an application for experts and should not be used by the general user.
+
+    Command line arguments
+
+    input_path (str, required)
+        Path of local copy of model parameter repository.
+    db_name (str, required)
+        Name of new DB to be created.
+    type (str, optional)
+        Type of data to be uploaded to the DB. Options are: model_parameters, production_tables.
+
+    Examples
+    --------
+    Upload model data repository to the DB:
+
+    .. code-block:: console
+
+        simtools-db-simulation-model-from-repository-to-db \
+            --input_path /path/to/repository \
+            --db_name new_db_name \
+            --type model_parameters
+
+    Upload production tables to the DB:
+
+    .. code-block:: console
+
+        simtools-db-simulation-model-from-repository-to-db \
+            --input_path /path/to/repository \
+            --db_name new_db_name \
+            --type production_tables
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler, db_model_upload
+
+
+def _parse(label=None, description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing application.
+    description : str
+        Description of application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description)
+    config.parser.add_argument(
+        "--input_path",
+        help="Path to simulation model repository.",
+        type=Path,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--db_name",
+        help="Name of the new simulation model database to be created.",
+        type=str.strip,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--type",
+        help="Type of data to be uploaded to the database.",
+        type=str,
+        required=False,
+        default="model_parameters",
+        choices=["model_parameters", "production_tables"],
+    )
+
+    args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True)
+    db_config["db_simulation_model"] = args_dict["db_name"]  # overwrite explicitly DB configuration
+    return args_dict, db_config
+
+
+def main():
+    """Application main."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(
+        label, description="Add or update a model parameter database to the DB"
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    if args_dict.get("type") == "model_parameters":
+        db_model_upload.add_model_parameters_to_db(args_dict, db)
+    elif args_dict.get("type") == "production_tables":
+        db_model_upload.add_production_tables_to_db(args_dict, db)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/db_add_value_from_json_to_db.py

@@ -10,8 +10,7 @@ r"""
     db_collection (str, required)
         The DB collection to which to add the file.
     db (str)
-        The DB to insert the files to. \
-        The choices are either the default CTA simulation DB or a sandbox for testing.
+        The DB to insert the files to.
 
     Example
     -------
@@ -81,16 +80,8 @@ def main():  # noqa: D103
             logger.info(f"Adding the following parameter to the DB: {par_dict['parameter']}")
             db.add_new_parameter(
                 db_name=db_config["db_simulation_model"],
-                array_element_name=par_dict["instrument"],
-                parameter=par_dict["parameter"],
-                version=par_dict["version"],
-                value=par_dict["value"],
-                site=par_dict["site"],
-                type=par_dict["type"],
+                par_dict=par_dict,
                 collection_name=args_dict["db_collection"],
-                applicable=par_dict["applicable"],
-                file=par_dict["file"],
-                unit=par_dict.get("unit", None),
                 file_prefix="./",
             )
             logger.info(

simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py

@@ -11,8 +11,8 @@
         File containing a table of array element positions.
     repository_path : str
         Path of local copy of model parameter repository.
-    model_version : str
-        Model version.
+    parameter_version : str
+        Parameter version.
     site : str
         Observatory site.
     coordinate_system : str
@@ -27,7 +27,7 @@
         simtools-write-array-element-positions-to-repository \
             --input /path/to/positions.txt \
             --repository_path /path/to/repository \
-            --model_version 1.0.0 \
+            --parameter_version 0.1.0 \
             --coordinate_system ground \
             --site North
 
@@ -82,7 +82,7 @@ def _parse(label=None, description=None):
         choices=["ground", "utm"],
     )
 
-    return config.initialize(db_config=True, simulation_model="site")
+    return config.initialize(db_config=True, simulation_model=["site", "parameter_version"])
 
 
 def write_utm_array_elements_to_repository(args_dict, logger):
@@ -115,7 +115,7 @@ def write_utm_array_elements_to_repository(args_dict, logger):
             parameter_name="array_element_position_utm",
             instrument=instrument,
             value=f"{row['utm_east']} {row['utm_north']} {row['altitude']}",
-            model_version=args_dict["model_version"],
+            parameter_version=args_dict["parameter_version"],
             output_path=output_path,
             output_file="array_element_position_utm.json",
         )
@@ -137,7 +137,7 @@ def write_ground_array_elements_to_repository(args_dict, db_config, logger):
     """
     array_model = ArrayModel(
         mongo_db_config=db_config,
-        model_version=args_dict["model_version"],
+        model_version=None,
         site=args_dict["site"],
         array_elements=args_dict["input"],
     )

simtools/applications/db_get_array_layouts_from_db.py

@@ -94,7 +94,9 @@ def _parse(label, description):
         default="ground",
         choices=["ground", "utm"],
     )
-    return config.initialize(db_config=True, simulation_model=["site", "layout"], output=True)
+    return config.initialize(
+        db_config=True, simulation_model=["site", "layout", "model_version"], output=True
+    )
 
 
 def _layout_from_db(args_dict, db_config):

simtools/applications/db_get_file_from_db.py

@@ -49,8 +49,9 @@ def _parse():
 
     config.parser.add_argument(
         "--file_name",
-        help="The name of the file to be downloaded.",
+        help="The name of the file(s) to be downloaded (single file or space-separated list).",
         type=str,
+        nargs="+",
         required=True,
     )
     return config.initialize(db_config=True, output=True)
@@ -66,15 +67,14 @@ def main():  # noqa: D103
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
     available_dbs = [
         db_config["db_simulation_model"],
-        db.DB_CTA_SIMULATION_MODEL_DESCRIPTIONS,
-        db.DB_DERIVED_VALUES,
-        "sandbox",
     ]
-    file_id = None
+    file_id = {}
     for db_name in available_dbs:
         try:
-            file_id = db.export_file_db(
-                db_name, _io_handler.get_output_directory(), args_dict["file_name"]
+            file_id = db.export_model_files(
+                db_name=db_name,
+                dest=_io_handler.get_output_directory(),
+                file_names=args_dict["file_name"],
             )
             logger.info(
                 f"Got file {args_dict['file_name']} from DB {db_name} "
@@ -84,11 +84,10 @@ def main():  # noqa: D103
         except FileNotFoundError:
             continue
 
-    if file_id is None:
-        logger.error(
-            f"The file {args_dict['file_name']} was not found in any of the available DBs."
-        )
-        raise FileNotFoundError
+    for key, value in file_id.items():
+        if value is None:
+            logger.error(f"The file {key} was not found in any of the available DBs.")
+            raise FileNotFoundError
 
 
 if __name__ == "__main__":

simtools/applications/db_get_parameter_from_db.py

@@ -26,7 +26,7 @@ r"""
 
     Example
     -------
-    Get the mirror_list parameter from the DB.
+    Get the mirror_list parameter used for a given model_version from the DB.
 
     .. code-block:: console
 
@@ -34,18 +34,13 @@ r"""
                 --site North --telescope LSTN-01 \\
                 --model_version 5.0.0
 
-    Expected final print-out message:
+    Get the mirror_list parameter using the parameter_version from the DB.
 
     .. code-block:: console
 
-        {'Applicable': True,
-         'File': True,
-         'Type': 'str',
-         'Value': 'mirror_CTA-N-LST1_v2019-03-31.dat',
-         'Version': '5.0.0',
-         '_id': ObjectId('608834f257df2db2531b8e78'),
-         'entry_date': datetime.datetime(2021, 4, 27, 15, 59, 46, tzinfo=<bson.tz_util.FixedOffset \
-          object at 0x7f601dd51d80>)}
+        simtools-db-get-parameter-from-db --parameter mirror_list \\
+                --site North --telescope LSTN-01 \\
+                --parameter_version 1.0.0
 
 """
 
@@ -73,7 +68,6 @@ def _parse():
     config.parser.add_argument(
         "--db_collection",
         help="DB collection to which to add the file",
-        default="telescopes",
         required=False,
     )
     config.parser.add_argument(
@@ -83,7 +77,9 @@ def _parse():
         required=False,
     )
 
-    return config.initialize(db_config=True, simulation_model="telescope")
+    return config.initialize(
+        db_config=True, simulation_model=["telescope", "parameter_version", "model_version"]
+    )
 
 
 def main():  # noqa: D103
@@ -94,37 +90,53 @@ def main():  # noqa: D103
 
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
 
-    if args_dict["db_collection"] == "configuration_sim_telarray":
-        pars = db.get_model_parameters(
-            args_dict["site"],
-            args_dict["telescope"],
-            args_dict["model_version"],
-            collection="configuration_sim_telarray",
-        )
-    elif args_dict["db_collection"] == "configuration_corsika":
-        pars = db.get_corsika_configuration_parameters(args_dict["model_version"])
-    elif args_dict["telescope"] is not None:
-        pars = db.get_model_parameters(
-            args_dict["site"],
-            args_dict["telescope"],
-            args_dict["model_version"],
-            collection="telescopes",
+    # get parameter using 'parameter_version'
+    if args_dict["parameter_version"] is not None:
+        pars = db.get_model_parameter(
+            parameter=args_dict["parameter"],
+            parameter_version=args_dict["parameter_version"],
+            site=args_dict["site"],
+            array_element_name=args_dict["telescope"],
+            collection=(
+                args_dict["db_collection"] if args_dict.get("db_collection") else "telescopes"
+            ),
         )
+    # get parameter using 'model_version'
+    elif args_dict["model_version"] is not None:
+        if args_dict["telescope"]:
+            pars = db.get_model_parameters(
+                site=args_dict["site"],
+                array_element_name=args_dict["telescope"],
+                model_version=args_dict["model_version"],
+                collection=(
+                    "configuration_sim_telarray"
+                    if args_dict.get("db_collection") == "configuration_sim_telarray"
+                    else "telescopes"
+                ),
+            )
+        else:
+            pars = db.get_model_parameters(
+                site=args_dict.get("site"),
+                model_version=args_dict["model_version"],
+                collection=(
+                    args_dict["db_collection"] if args_dict.get("db_collection") else "sites"
+                ),
+                array_element_name=None,
+            )
     else:
-        pars = db.get_site_parameters(args_dict["site"], args_dict["model_version"])
+        raise ValueError("Either 'parameter_version' or 'model_version' must be provided.")
     if args_dict["parameter"] not in pars:
         raise KeyError(f"The requested parameter, {args_dict['parameter']}, does not exist.")
     if args_dict["output_file"] is not None:
-        _io_handler = io_handler.IOHandler()
+        _output_file = (
+            Path(io_handler.IOHandler().get_output_directory()) / args_dict["output_file"]
+        )
         pars[args_dict["parameter"]].pop("_id")
         pars[args_dict["parameter"]].pop("entry_date")
-        _output_file = Path(_io_handler.get_output_directory()) / args_dict["output_file"]
         with open(_output_file, "w", encoding="utf-8") as json_file:
             json.dump(pars[args_dict["parameter"]], json_file, indent=4)
     else:
-        print()
         pprint(pars[args_dict["parameter"]])
-        print()
 
 
 if __name__ == "__main__":

simtools/applications/derive_mirror_rnda.py

@@ -178,6 +178,13 @@ def _parse(label):
         type=str,
         required=False,
     )
+    config.parser.add_argument(
+        "--rtol_psf_containment",
+        help="Relative tolerance for the containment diameter (default is 0.1).",
+        type=float,
+        required=False,
+        default=0.1,
+    )
     config.parser.add_argument(
         "--use_random_focal_length",
         help=("Use random focal lengths."),
@@ -200,7 +207,9 @@ def _parse(label):
         action="store_true",
         required=False,
     )
-    return config.initialize(db_config=True, output=True, simulation_model="telescope")
+    return config.initialize(
+        db_config=True, output=True, simulation_model=["telescope", "model_version"]
+    )
 
 
 def main():  # noqa: D103