gammasimtools 0.9.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

simtools/applications/calculate_trigger_rate.py

@@ -73,7 +73,7 @@ def _parse(label, description):
 
     config.parser.add_argument(
         "--simtel_file_names",
-        help="Name of the simtel_array output files to be calculate the trigger rate from  or the "
+        help="Name of the simtel_array output files to be calculate the trigger rate from or the "
         "text file containing the list of simtel_array output files.",
         nargs="+",
         required=True,
@@ -82,19 +82,19 @@ def _parse(label, description):
 
     config.parser.add_argument(
         "--save_tables",
-        help="If true, saves the trigger rates per energy bin into ECSV files.",
+        help="Save trigger rates per energy bin into ECSV files.",
         action="store_true",
     )
 
     config.parser.add_argument(
         "--area_from_distribution",
-        help="If true, calculates the trigger rates using the event distribution.",
+        help="Calculate trigger rates using the event distribution.",
         action="store_true",
     )
 
     config.parser.add_argument(
         "--stack_files",
-        help="If true, stacks all the histograms.",
+        help="Stacks all histograms.",
         action="store_true",
     )
 
@@ -124,22 +124,13 @@ def _get_simulation_parameters(config_parser):
         The view cone used in the simulation.
 
     """
-    if config_parser["energy_range"] is not None:
-        energy_range = [
-            config_parser["energy_range"][0].to("TeV").value,
-            config_parser["energy_range"][1].to("TeV").value,
-        ]
-    else:
-        energy_range = None
-    if config_parser["view_cone"] is not None:
-        view_cone = [
-            config_parser["view_cone"][0].to("deg").value,
-            config_parser["view_cone"][1].to("deg").value,
-        ]
-    else:
-        view_cone = None
-
-    return energy_range, view_cone
+
+    def convert(param, unit):
+        return [param[0].to(unit).value, param[1].to(unit).value] if param else None
+
+    return convert(config_parser.get("energy_range"), "TeV"), convert(
+        config_parser.get("view_cone"), "deg"
+    )
 
 
 def main():  # noqa: D103
@@ -152,23 +143,9 @@ def main():  # noqa: D103
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(config_parser["log_level"]))
 
-    # Building list of simtel_array files from the input files
-    simtel_array_files = []
-    for one_file in config_parser["simtel_file_names"]:
-        try:
-            if Path(one_file).suffix in [".zst", ".simtel", ".hdata"]:
-                simtel_array_files.append(one_file)
-            else:
-                # Collecting hist files
-                with open(one_file, encoding="utf-8") as file:
-                    for line in file:
-                        # Removing '\n' from filename, in case it is left there.
-                        simtel_array_files.append(line.replace("\n", ""))
-        except FileNotFoundError as exc:
-            msg = f"{one_file} is not a file."
-            logger.error(msg)
-            raise FileNotFoundError from exc
-
+    simtel_array_files = gen.get_list_of_files_from_command_line(
+        config_parser["simtel_file_names"], [".zst", ".simtel", ".hdata"]
+    )
     energy_range, view_cone = _get_simulation_parameters(config_parser)
 
     histograms = SimtelIOHistograms(
@@ -192,7 +169,7 @@ def main():  # noqa: D103
     for i_hist, _ in enumerate(sim_event_rates):
         print(f"\nFile {histograms.histogram_files[i_hist]}\n")
         print(
-            f"System trigger rate (Hz): {triggered_event_rates[i_hist].value:.4e} \u00B1 "
+            f"System trigger rate (Hz): {triggered_event_rates[i_hist].value:.4e} \u00b1 "
             f"{triggered_event_rate_uncertainties[i_hist].value:.4e} Hz"
         )
     if config_parser["save_tables"]:

simtools/applications/convert_all_model_parameters_from_simtel.py

@@ -6,6 +6,8 @@ r"""
     ready to be submitted to the model database. Prints out parameters which are not found
     in simtel configuration file and parameters which are not found in simtools schema files.
 
+    Note that all parameters are assigned the same parameter version.
+
     Command line arguments
     ----------------------
     simtel_cfg_file (str)
@@ -30,7 +32,7 @@ r"""
           --simtel_cfg_file all_telescope_config_la_palma.cfg\\
           --simtel_telescope_name CT1\\
           --telescope LSTN-01\\
-          --model_version "2024-03-06"
+          --parameter_version "1.0.0"
 
     The export of the model parameters from sim_telarray for 6.0.0 can be done e.g., as follows:
 
@@ -59,7 +61,7 @@ import numpy as np
 import simtools.data_model.model_data_writer as writer
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
+from simtools.data_model import schema
 from simtools.io_operations.io_handler import IOHandler
 from simtools.simtel.simtel_config_reader import SimtelConfigReader
 
@@ -103,27 +105,7 @@ def _parse(label=None, description=None):
         type=str,
         required=True,
     )
-    return config.initialize(simulation_model=["telescope", "model_version"])
-
-
-def get_list_of_parameters_and_schema_files(schema_directory):
-    """
-    Return list of parameters and schema files located in schema file directory.
-
-    Returns
-    -------
-    list
-        List of parameters found in schema file directory.
-    list
-        List of schema files found in schema file directory.
-
-    """
-    schema_files = sorted(Path(schema_directory).rglob("*.schema.yml"))
-    parameters = []
-    for schema_file in schema_files:
-        schema_dict = gen.collect_data_from_file(file_name=schema_file)
-        parameters.append(schema_dict.get("name"))
-    return parameters, schema_files
+    return config.initialize(simulation_model=["telescope", "parameter_version"])
 
 
 def get_list_of_simtel_parameters(simtel_config_file, logger):
@@ -203,7 +185,7 @@ def get_number_of_camera_pixel(args_dict, logger):
     """
     try:
         simtel_config_reader = SimtelConfigReader(
-            schema_file=MODEL_PARAMETER_SCHEMA_PATH / "camera_pixels.schema.yml",
+            schema_file=schema.get_model_parameter_schema_file("camera_pixels"),
             simtel_config_file=args_dict["simtel_cfg_file"],
             simtel_telescope_name=args_dict["simtel_telescope_name"],
         )
@@ -219,7 +201,6 @@ def read_and_export_parameters(args_dict, logger):
     """
     Read and export parameters from simtel configuration file to json files.
 
-    Only applicable parameters are exported to json.
     Provide extensive logging information on the parameters found in the simtel
     configuration file.
 
@@ -238,8 +219,8 @@ def read_and_export_parameters(args_dict, logger):
         List of simtools parameter not found in simtel configuration file.
 
     """
-    _parameters, _schema_files = get_list_of_parameters_and_schema_files(
-        args_dict.get("schema_directory", MODEL_PARAMETER_SCHEMA_PATH)
+    _parameters, _schema_files = schema.get_get_model_parameter_schema_files(
+        args_dict.get("schema_directory")
     )
     _simtel_parameters = get_list_of_simtel_parameters(args_dict["simtel_cfg_file"], logger)
 
@@ -251,7 +232,6 @@ def read_and_export_parameters(args_dict, logger):
     _parameters_not_in_simtel = []
 
     for _parameter, _schema_file in zip(_parameters, _schema_files):
-
         logger.info(f"Parameter: {_parameter} Schema file: {_schema_file}")
         simtel_config_reader = read_simtel_config_file(
             args_dict, logger, _schema_file, _camera_pixel
@@ -266,7 +246,7 @@ def read_and_export_parameters(args_dict, logger):
             parameter_name=_parameter,
             value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
             instrument=args_dict["telescope"],
-            model_version=args_dict["model_version"],
+            parameter_version=args_dict["parameter_version"],
             output_file=io_handler.get_output_file(f"{_parameter}.json"),
         )
 

simtools/applications/convert_geo_coordinates_of_array_elements.py

@@ -1,56 +1,56 @@
 #!/usr/bin/python3
 """Convert and print a list of array element positions in different coordinate systems.
 
-    Description
-    -----------
-
-    Convert array element positions in different CTAO coordinate systems.
-    Available coordinate systems are:
-
-    1. UTM system
-    2. ground system (similar to sim_telarray system with x-axis pointing toward geographic north
-       and y-axis pointing towards the west); altitude relative to the CORSIKA observation level.
-       Altitude is the height of the elevation rotation axis (plus some possible mirror offset).
-    3. Mercator system
-
-    Command line arguments
-    ----------------------
-    input (str)
-        File name with list of array element positions.
-        Input can be given as astropy table file (ecsv) or a single array element in
-        a json file.
-    print (str)
-        Print in requested coordinate system; possible are ground, utm, mercator
-    export (str)
-        Export array element list to file in requested coordinate system;
-          possible are ground, utm, mercator
-    select_assets (str)
-        Select a subset of array elements / telescopes (e.g., MSTN, LSTN)
-
-    Example
-    -------
-    Convert a list of array elements using a list of telescope positions in UTM coordinates.
+Description
+-----------
+
+Convert array element positions in different CTAO coordinate systems.
+Available coordinate systems are:
+
+1. UTM system
+2. ground system (similar to sim_telarray system with x-axis pointing toward geographic north
+   and y-axis pointing towards the west); altitude relative to the CORSIKA observation level.
+   Altitude is the height of the elevation rotation axis (plus some possible mirror offset).
+3. Mercator system
+
+Command line arguments
+----------------------
+input (str)
+    File name with list of array element positions.
+    Input can be given as astropy table file (ecsv) or a single array element in
+    a json file.
+print (str)
+    Print in requested coordinate system; possible are ground, utm, mercator
+export (str)
+    Export array element list to file in requested coordinate system;
+      possible are ground, utm, mercator
+select_assets (str)
+    Select a subset of array elements / telescopes (e.g., MSTN, LSTN)
+
+Example
+-------
+Convert a list of array elements using a list of telescope positions in UTM coordinates.
 
-    .. code-block:: console
+.. code-block:: console
 
-        simtools-convert-geo-coordinates-of-array-elements
-            --input tests/resources/telescope_positions-North-utm.ecsv
-            --print ground
+    simtools-convert-geo-coordinates-of-array-elements
+        --input tests/resources/telescope_positions-North-utm.ecsv
+        --print ground
 
-    The converted list of telescope positions in ground coordinates is printed to the screen.
+The converted list of telescope positions in ground coordinates is printed to the screen.
 
-    The following example converts a list of telescope positions in UTM coordinates
-    and writes the output to a file in ground (sim_telarray) coordinates. Also selects
-    only a subset of the array elements (telescopes; ignore calibration devices):
+The following example converts a list of telescope positions in UTM coordinates
+and writes the output to a file in ground (sim_telarray) coordinates. Also selects
+only a subset of the array elements (telescopes; ignore calibration devices):
 
-    .. code-block:: console
+.. code-block:: console
 
-        simtools-convert-geo-coordinates-of-array-elements
-            --input tests/resources/telescope_positions-North-utm.ecsv
-            --export ground
-            --select_assets LSTN
+    simtools-convert-geo-coordinates-of-array-elements
+        --input tests/resources/telescope_positions-North-utm.ecsv
+        --export ground
+        --select_assets LSTN
 
-    Expected output is a ecsv file in the directory printed to the screen.
+Expected output is a ecsv file in the directory printed to the screen.
 
 """
 
@@ -133,7 +133,7 @@ def _parse(label=None, description=None):
         output=True,
         require_command_line=True,
         db_config=True,
-        simulation_model=["model_version", "site"],
+        simulation_model=["model_version", "parameter_version", "site"],
     )
 
 
@@ -170,7 +170,9 @@ def main():
 
     if args_dict["export"] is not None:
         product_data = (
-            layout.export_one_telescope_as_json(crs_name=args_dict["export"])
+            layout.export_one_telescope_as_json(
+                crs_name=args_dict["export"], parameter_version=args_dict.get("parameter_version")
+            )
             if args_dict.get("input", "").endswith(".json")
             else layout.export_telescope_list_table(crs_name=args_dict["export"])
         )

simtools/applications/convert_model_parameter_from_simtel.py

@@ -78,11 +78,10 @@ def _parse(label=None, description=None):
         type=str,
         required=True,
     )
-    return config.initialize(simulation_model=["telescope", "model_version"], output=True)
+    return config.initialize(simulation_model=["telescope", "parameter_version"], output=True)
 
 
 def main():  # noqa: D103
-
     args_dict, _ = _parse(
         label=Path(__file__).stem,
         description="Convert simulation model parameter from sim_telarray to simtools format.",
@@ -107,7 +106,7 @@ def main():  # noqa: D103
         parameter_name=simtel_config_reader.parameter_name,
         value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
         instrument=args_dict["telescope"],
-        model_version=args_dict["model_version"],
+        parameter_version=args_dict["parameter_version"],
         output_file=args_dict["output_file"],
         output_path=args_dict.get("output_path"),
         use_plain_output_path=args_dict.get("use_plain_output_path"),

simtools/applications/db_add_file_to_db.py

@@ -18,8 +18,7 @@
         A directory with files to upload to the DB. \
         All files in the directory with a predefined list of extensions will be uploaded.
     db (str)
-        The DB to insert the files to. \
-        The choices are either the default CTA simulation DB or a sandbox for testing.
+        The DB to insert the files to.
 
     Example
     -------
@@ -47,7 +46,6 @@ from simtools.db import db_handler
 
 
 def _parse():
-
     config = configurator.Configurator(
         description="Add file to the DB.",
         usage="simtools-add-file-to-db --file_name test_application.dat --db test-data",

simtools/applications/db_add_simulation_model_from_repository_to_db.py

@@ -0,0 +1,110 @@
+#!/usr/bin/python3
+r"""
+    Add parameters and production tables from a simulation model repository to a new database.
+
+    Generates a new database with all required collections.
+    Follows the structure of the CTAO gitlab simulation model repository.
+
+    This is an application for experts and should not be used by the general user.
+
+    Command line arguments
+
+    input_path (str, required)
+        Path of local copy of model parameter repository.
+    db_name (str, required)
+        Name of new DB to be created.
+    type (str, optional)
+        Type of data to be uploaded to the DB. Options are: model_parameters, production_tables.
+
+    Examples
+    --------
+    Upload model data repository to the DB:
+
+    .. code-block:: console
+
+        simtools-db-simulation-model-from-repository-to-db \
+            --input_path /path/to/repository \
+            --db_name new_db_name \
+            --type model_parameters
+
+    Upload production tables to the DB:
+
+    .. code-block:: console
+
+        simtools-db-simulation-model-from-repository-to-db \
+            --input_path /path/to/repository \
+            --db_name new_db_name \
+            --type production_tables
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler, db_model_upload
+
+
+def _parse(label=None, description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing application.
+    description : str
+        Description of application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description)
+    config.parser.add_argument(
+        "--input_path",
+        help="Path to simulation model repository.",
+        type=Path,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--db_name",
+        help="Name of the new simulation model database to be created.",
+        type=str.strip,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--type",
+        help="Type of data to be uploaded to the database.",
+        type=str,
+        required=False,
+        default="model_parameters",
+        choices=["model_parameters", "production_tables"],
+    )
+
+    args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True)
+    db_config["db_simulation_model"] = args_dict["db_name"]  # overwrite explicitly DB configuration
+    return args_dict, db_config
+
+
+def main():
+    """Application main."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(
+        label, description="Add or update a model parameter database to the DB"
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    if args_dict.get("type") == "model_parameters":
+        db_model_upload.add_model_parameters_to_db(args_dict, db)
+    elif args_dict.get("type") == "production_tables":
+        db_model_upload.add_production_tables_to_db(args_dict, db)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/db_add_value_from_json_to_db.py

@@ -10,8 +10,7 @@ r"""
     db_collection (str, required)
         The DB collection to which to add the file.
     db (str)
-        The DB to insert the files to. \
-        The choices are either the default CTA simulation DB or a sandbox for testing.
+        The DB to insert the files to.
 
     Example
     -------

simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py

@@ -11,8 +11,8 @@
         File containing a table of array element positions.
     repository_path : str
         Path of local copy of model parameter repository.
-    model_version : str
-        Model version.
+    parameter_version : str
+        Parameter version.
     site : str
         Observatory site.
     coordinate_system : str
@@ -27,7 +27,7 @@
         simtools-write-array-element-positions-to-repository \
             --input /path/to/positions.txt \
             --repository_path /path/to/repository \
-            --model_version 1.0.0 \
+            --parameter_version 0.1.0 \
             --coordinate_system ground \
             --site North
 
@@ -82,7 +82,7 @@ def _parse(label=None, description=None):
         choices=["ground", "utm"],
     )
 
-    return config.initialize(db_config=True, simulation_model="site")
+    return config.initialize(db_config=True, simulation_model=["site", "parameter_version"])
 
 
 def write_utm_array_elements_to_repository(args_dict, logger):
@@ -115,7 +115,7 @@ def write_utm_array_elements_to_repository(args_dict, logger):
             parameter_name="array_element_position_utm",
             instrument=instrument,
             value=f"{row['utm_east']} {row['utm_north']} {row['altitude']}",
-            model_version=args_dict["model_version"],
+            parameter_version=args_dict["parameter_version"],
             output_path=output_path,
             output_file="array_element_position_utm.json",
         )
@@ -137,7 +137,7 @@ def write_ground_array_elements_to_repository(args_dict, db_config, logger):
     """
     array_model = ArrayModel(
         mongo_db_config=db_config,
-        model_version=args_dict["model_version"],
+        model_version=None,
         site=args_dict["site"],
         array_elements=args_dict["input"],
     )

simtools/applications/db_get_file_from_db.py

@@ -49,8 +49,9 @@ def _parse():
 
     config.parser.add_argument(
         "--file_name",
-        help="The name of the file to be downloaded.",
+        help="The name of the file(s) to be downloaded (single file or space-separated list).",
         type=str,
+        nargs="+",
         required=True,
     )
     return config.initialize(db_config=True, output=True)
@@ -66,15 +67,14 @@ def main():  # noqa: D103
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
     available_dbs = [
         db_config["db_simulation_model"],
-        db.DB_CTA_SIMULATION_MODEL_DESCRIPTIONS,
-        db.DB_DERIVED_VALUES,
-        "sandbox",
     ]
-    file_id = None
+    file_id = {}
     for db_name in available_dbs:
         try:
-            file_id = db.export_file_db(
-                db_name, _io_handler.get_output_directory(), args_dict["file_name"]
+            file_id = db.export_model_files(
+                db_name=db_name,
+                dest=_io_handler.get_output_directory(),
+                file_names=args_dict["file_name"],
             )
             logger.info(
                 f"Got file {args_dict['file_name']} from DB {db_name} "
@@ -84,11 +84,10 @@ def main():  # noqa: D103
         except FileNotFoundError:
             continue
 
-    if file_id is None:
-        logger.error(
-            f"The file {args_dict['file_name']} was not found in any of the available DBs."
-        )
-        raise FileNotFoundError
+    for key, value in file_id.items():
+        if value is None:
+            logger.error(f"The file {key} was not found in any of the available DBs.")
+            raise FileNotFoundError
 
 
 if __name__ == "__main__":