gammasimtools 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/applications/convert_geo_coordinates_of_array_elements.py

@@ -1,56 +1,56 @@
 #!/usr/bin/python3
 """Convert and print a list of array element positions in different coordinate systems.
 
-    Description
-    -----------
-
-    Convert array element positions in different CTAO coordinate systems.
-    Available coordinate systems are:
-
-    1. UTM system
-    2. ground system (similar to sim_telarray system with x-axis pointing toward geographic north
-       and y-axis pointing towards the west); altitude relative to the CORSIKA observation level.
-       Altitude is the height of the elevation rotation axis (plus some possible mirror offset).
-    3. Mercator system
-
-    Command line arguments
-    ----------------------
-    input (str)
-        File name with list of array element positions.
-        Input can be given as astropy table file (ecsv) or a single array element in
-        a json file.
-    print (str)
-        Print in requested coordinate system; possible are ground, utm, mercator
-    export (str)
-        Export array element list to file in requested coordinate system;
-          possible are ground, utm, mercator
-    select_assets (str)
-        Select a subset of array elements / telescopes (e.g., MSTN, LSTN)
-
-    Example
-    -------
-    Convert a list of array elements using a list of telescope positions in UTM coordinates.
+Description
+-----------
+
+Convert array element positions in different CTAO coordinate systems.
+Available coordinate systems are:
+
+1. UTM system
+2. ground system (similar to sim_telarray system with x-axis pointing toward geographic north
+   and y-axis pointing towards the west); altitude relative to the CORSIKA observation level.
+   Altitude is the height of the elevation rotation axis (plus some possible mirror offset).
+3. Mercator system
+
+Command line arguments
+----------------------
+input (str)
+    File name with list of array element positions.
+    Input can be given as astropy table file (ecsv) or a single array element in
+    a json file.
+print (str)
+    Print in requested coordinate system; possible are ground, utm, mercator
+export (str)
+    Export array element list to file in requested coordinate system;
+      possible are ground, utm, mercator
+select_assets (str)
+    Select a subset of array elements / telescopes (e.g., MSTN, LSTN)
+
+Example
+-------
+Convert a list of array elements using a list of telescope positions in UTM coordinates.
 
-    .. code-block:: console
+.. code-block:: console
 
-        simtools-convert-geo-coordinates-of-array-elements
-            --input tests/resources/telescope_positions-North-utm.ecsv
-            --print ground
+    simtools-convert-geo-coordinates-of-array-elements
+        --input tests/resources/telescope_positions-North-utm.ecsv
+        --print ground
 
-    The converted list of telescope positions in ground coordinates is printed to the screen.
+The converted list of telescope positions in ground coordinates is printed to the screen.
 
-    The following example converts a list of telescope positions in UTM coordinates
-    and writes the output to a file in ground (sim_telarray) coordinates. Also selects
-    only a subset of the array elements (telescopes; ignore calibration devices):
+The following example converts a list of telescope positions in UTM coordinates
+and writes the output to a file in ground (sim_telarray) coordinates. Also selects
+only a subset of the array elements (telescopes; ignore calibration devices):
 
-    .. code-block:: console
+.. code-block:: console
 
-        simtools-convert-geo-coordinates-of-array-elements
-            --input tests/resources/telescope_positions-North-utm.ecsv
-            --export ground
-            --select_assets LSTN
+    simtools-convert-geo-coordinates-of-array-elements
+        --input tests/resources/telescope_positions-North-utm.ecsv
+        --export ground
+        --select_assets LSTN
 
-    Expected output is a ecsv file in the directory printed to the screen.
+Expected output is a ecsv file in the directory printed to the screen.
 
 """
 
@@ -133,7 +133,7 @@ def _parse(label=None, description=None):
         output=True,
         require_command_line=True,
         db_config=True,
-        simulation_model=["model_version", "site"],
+        simulation_model=["model_version", "parameter_version", "site"],
     )
 
 
@@ -170,7 +170,9 @@ def main():
 
     if args_dict["export"] is not None:
         product_data = (
-            layout.export_one_telescope_as_json(crs_name=args_dict["export"])
+            layout.export_one_telescope_as_json(
+                crs_name=args_dict["export"], parameter_version=args_dict.get("parameter_version")
+            )
             if args_dict.get("input", "").endswith(".json")
             else layout.export_telescope_list_table(crs_name=args_dict["export"])
         )

simtools/applications/convert_model_parameter_from_simtel.py

@@ -78,7 +78,7 @@ def _parse(label=None, description=None):
         type=str,
         required=True,
     )
-    return config.initialize(simulation_model=["telescope", "model_version"], output=True)
+    return config.initialize(simulation_model=["telescope", "parameter_version"], output=True)
 
 
 def main():  # noqa: D103
@@ -107,7 +107,7 @@ def main():  # noqa: D103
         parameter_name=simtel_config_reader.parameter_name,
         value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
         instrument=args_dict["telescope"],
-        model_version=args_dict["model_version"],
+        parameter_version=args_dict["parameter_version"],
         output_file=args_dict["output_file"],
         output_path=args_dict.get("output_path"),
         use_plain_output_path=args_dict.get("use_plain_output_path"),

simtools/applications/db_add_file_to_db.py

@@ -18,8 +18,7 @@
         A directory with files to upload to the DB. \
         All files in the directory with a predefined list of extensions will be uploaded.
     db (str)
-        The DB to insert the files to. \
-        The choices are either the default CTA simulation DB or a sandbox for testing.
+        The DB to insert the files to.
 
     Example
     -------

simtools/applications/db_add_simulation_model_from_repository_to_db.py

@@ -0,0 +1,110 @@
+#!/usr/bin/python3
+r"""
+    Add parameters and production tables from a simulation model repository to a new database.
+
+    Generates a new database with all required collections.
+    Follows the structure of the CTAO gitlab simulation model repository.
+
+    This is an application for experts and should not be used by the general user.
+
+    Command line arguments
+
+    input_path (str, required)
+        Path of local copy of model parameter repository.
+    db_name (str, required)
+        Name of new DB to be created.
+    type (str, optional)
+        Type of data to be uploaded to the DB. Options are: model_parameters, production_tables.
+
+    Examples
+    --------
+    Upload model data repository to the DB:
+
+    .. code-block:: console
+
+        simtools-db-simulation-model-from-repository-to-db \
+            --input_path /path/to/repository \
+            --db_name new_db_name \
+            --type model_parameters
+
+    Upload production tables to the DB:
+
+    .. code-block:: console
+
+        simtools-db-simulation-model-from-repository-to-db \
+            --input_path /path/to/repository \
+            --db_name new_db_name \
+            --type production_tables
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler, db_model_upload
+
+
+def _parse(label=None, description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing application.
+    description : str
+        Description of application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description)
+    config.parser.add_argument(
+        "--input_path",
+        help="Path to simulation model repository.",
+        type=Path,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--db_name",
+        help="Name of the new simulation model database to be created.",
+        type=str.strip,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--type",
+        help="Type of data to be uploaded to the database.",
+        type=str,
+        required=False,
+        default="model_parameters",
+        choices=["model_parameters", "production_tables"],
+    )
+
+    args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True)
+    db_config["db_simulation_model"] = args_dict["db_name"]  # overwrite explicitly DB configuration
+    return args_dict, db_config
+
+
+def main():
+    """Application main."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(
+        label, description="Add or update a model parameter database to the DB"
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    if args_dict.get("type") == "model_parameters":
+        db_model_upload.add_model_parameters_to_db(args_dict, db)
+    elif args_dict.get("type") == "production_tables":
+        db_model_upload.add_production_tables_to_db(args_dict, db)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/db_add_value_from_json_to_db.py

@@ -10,8 +10,7 @@ r"""
     db_collection (str, required)
         The DB collection to which to add the file.
     db (str)
-        The DB to insert the files to. \
-        The choices are either the default CTA simulation DB or a sandbox for testing.
+        The DB to insert the files to.
 
     Example
     -------

simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py

@@ -11,8 +11,8 @@
         File containing a table of array element positions.
     repository_path : str
         Path of local copy of model parameter repository.
-    model_version : str
-        Model version.
+    parameter_version : str
+        Parameter version.
     site : str
         Observatory site.
     coordinate_system : str
@@ -27,7 +27,7 @@
         simtools-write-array-element-positions-to-repository \
             --input /path/to/positions.txt \
             --repository_path /path/to/repository \
-            --model_version 1.0.0 \
+            --parameter_version 0.1.0 \
             --coordinate_system ground \
             --site North
 
@@ -82,7 +82,7 @@ def _parse(label=None, description=None):
         choices=["ground", "utm"],
     )
 
-    return config.initialize(db_config=True, simulation_model="site")
+    return config.initialize(db_config=True, simulation_model=["site", "parameter_version"])
 
 
 def write_utm_array_elements_to_repository(args_dict, logger):
@@ -115,7 +115,7 @@ def write_utm_array_elements_to_repository(args_dict, logger):
             parameter_name="array_element_position_utm",
             instrument=instrument,
             value=f"{row['utm_east']} {row['utm_north']} {row['altitude']}",
-            model_version=args_dict["model_version"],
+            parameter_version=args_dict["parameter_version"],
             output_path=output_path,
             output_file="array_element_position_utm.json",
         )
@@ -137,7 +137,7 @@ def write_ground_array_elements_to_repository(args_dict, db_config, logger):
     """
     array_model = ArrayModel(
         mongo_db_config=db_config,
-        model_version=args_dict["model_version"],
+        model_version=None,
         site=args_dict["site"],
         array_elements=args_dict["input"],
     )

simtools/applications/db_get_file_from_db.py

@@ -49,8 +49,9 @@ def _parse():
 
     config.parser.add_argument(
         "--file_name",
-        help="The name of the file to be downloaded.",
+        help="The name of the file(s) to be downloaded (single file or space-separated list).",
         type=str,
+        nargs="+",
         required=True,
     )
     return config.initialize(db_config=True, output=True)
@@ -66,15 +67,14 @@ def main():  # noqa: D103
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
     available_dbs = [
         db_config["db_simulation_model"],
-        db.DB_CTA_SIMULATION_MODEL_DESCRIPTIONS,
-        db.DB_DERIVED_VALUES,
-        "sandbox",
     ]
-    file_id = None
+    file_id = {}
     for db_name in available_dbs:
         try:
-            file_id = db.export_file_db(
-                db_name, _io_handler.get_output_directory(), args_dict["file_name"]
+            file_id = db.export_model_files(
+                db_name=db_name,
+                dest=_io_handler.get_output_directory(),
+                file_names=args_dict["file_name"],
             )
             logger.info(
                 f"Got file {args_dict['file_name']} from DB {db_name} "
@@ -84,11 +84,10 @@ def main():  # noqa: D103
         except FileNotFoundError:
             continue
 
-    if file_id is None:
-        logger.error(
-            f"The file {args_dict['file_name']} was not found in any of the available DBs."
-        )
-        raise FileNotFoundError
+    for key, value in file_id.items():
+        if value is None:
+            logger.error(f"The file {key} was not found in any of the available DBs.")
+            raise FileNotFoundError
 
 
 if __name__ == "__main__":

simtools/applications/db_get_parameter_from_db.py

@@ -26,7 +26,7 @@ r"""
 
     Example
     -------
-    Get the mirror_list parameter from the DB.
+    Get the mirror_list parameter used for a given model_version from the DB.
 
     .. code-block:: console
 
@@ -34,18 +34,13 @@ r"""
                 --site North --telescope LSTN-01 \\
                 --model_version 5.0.0
 
-    Expected final print-out message:
+    Get the mirror_list parameter using the parameter_version from the DB.
 
     .. code-block:: console
 
-        {'Applicable': True,
-         'File': True,
-         'Type': 'str',
-         'Value': 'mirror_CTA-N-LST1_v2019-03-31.dat',
-         'Version': '5.0.0',
-         '_id': ObjectId('608834f257df2db2531b8e78'),
-         'entry_date': datetime.datetime(2021, 4, 27, 15, 59, 46, tzinfo=<bson.tz_util.FixedOffset \
-          object at 0x7f601dd51d80>)}
+        simtools-db-get-parameter-from-db --parameter mirror_list \\
+                --site North --telescope LSTN-01 \\
+                --parameter_version 1.0.0
 
 """
 
@@ -73,7 +68,6 @@ def _parse():
     config.parser.add_argument(
         "--db_collection",
         help="DB collection to which to add the file",
-        default="telescopes",
         required=False,
     )
     config.parser.add_argument(
@@ -83,7 +77,9 @@ def _parse():
         required=False,
     )
 
-    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
+    return config.initialize(
+        db_config=True, simulation_model=["telescope", "parameter_version", "model_version"]
+    )
 
 
 def main():  # noqa: D103
@@ -94,37 +90,53 @@ def main():  # noqa: D103
 
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
 
-    if args_dict["db_collection"] == "configuration_sim_telarray":
-        pars = db.get_model_parameters(
-            args_dict["site"],
-            args_dict["telescope"],
-            args_dict["model_version"],
-            collection="configuration_sim_telarray",
-        )
-    elif args_dict["db_collection"] == "configuration_corsika":
-        pars = db.get_corsika_configuration_parameters(args_dict["model_version"])
-    elif args_dict["telescope"] is not None:
-        pars = db.get_model_parameters(
-            args_dict["site"],
-            args_dict["telescope"],
-            args_dict["model_version"],
-            collection="telescopes",
+    # get parameter using 'parameter_version'
+    if args_dict["parameter_version"] is not None:
+        pars = db.get_model_parameter(
+            parameter=args_dict["parameter"],
+            parameter_version=args_dict["parameter_version"],
+            site=args_dict["site"],
+            array_element_name=args_dict["telescope"],
+            collection=(
+                args_dict["db_collection"] if args_dict.get("db_collection") else "telescopes"
+            ),
         )
+    # get parameter using 'model_version'
+    elif args_dict["model_version"] is not None:
+        if args_dict["telescope"]:
+            pars = db.get_model_parameters(
+                site=args_dict["site"],
+                array_element_name=args_dict["telescope"],
+                model_version=args_dict["model_version"],
+                collection=(
+                    "configuration_sim_telarray"
+                    if args_dict.get("db_collection") == "configuration_sim_telarray"
+                    else "telescopes"
+                ),
+            )
+        else:
+            pars = db.get_model_parameters(
+                site=args_dict.get("site"),
+                model_version=args_dict["model_version"],
+                collection=(
+                    args_dict["db_collection"] if args_dict.get("db_collection") else "sites"
+                ),
+                array_element_name=None,
+            )
     else:
-        pars = db.get_site_parameters(args_dict["site"], args_dict["model_version"])
+        raise ValueError("Either 'parameter_version' or 'model_version' must be provided.")
     if args_dict["parameter"] not in pars:
         raise KeyError(f"The requested parameter, {args_dict['parameter']}, does not exist.")
     if args_dict["output_file"] is not None:
-        _io_handler = io_handler.IOHandler()
+        _output_file = (
+            Path(io_handler.IOHandler().get_output_directory()) / args_dict["output_file"]
+        )
         pars[args_dict["parameter"]].pop("_id")
         pars[args_dict["parameter"]].pop("entry_date")
-        _output_file = Path(_io_handler.get_output_directory()) / args_dict["output_file"]
         with open(_output_file, "w", encoding="utf-8") as json_file:
             json.dump(pars[args_dict["parameter"]], json_file, indent=4)
     else:
-        print()
         pprint(pars[args_dict["parameter"]])
-        print()
 
 
 if __name__ == "__main__":

simtools/applications/derive_photon_electron_spectrum.py

@@ -0,0 +1,99 @@
+#!/usr/bin/python3
+
+r"""
+    Derive single photon electron spectrum from a given amplitude spectrum.
+
+    Normalizes singe-p.e. amplitude distribution to mean amplitude of 1.0,
+    as required by sim_telarray. Allows to fold in afterpulse distribution
+    to prompt a spectrum. Uses the sim_telarray tool 'norm_spe' to normalize
+    the spectra.
+
+    Input files can be in ecsv format (preferred) or in the sim_telarray legacy format.
+
+    Two output files with identical data are written to the output directory:
+
+    - 'output_file'.ecsv: Single photon electron spectrum in ecsv format (data and metadata).
+    - 'output_file'.dat: Single photon electron spectrum in sim_telarray format.
+
+    Example
+    -------
+
+    .. code-block:: console
+
+        simtools-derive-photon-electron-spectrum \\
+            --input_spectrum spectrum_photon_electron.ecsv \\
+            --afterpulse_spectrum spectrum_afterpulse.ecsv \\
+            --step_size 0.02 \\
+            --max_amplitude 42.0 \\
+            --use_norm_spe \\
+            --output_path ./tests/output \\
+            --output_file spectrum_photon_electron_afterpulse.ecsv
+
+    For an example of how to plot the single photon electron spectrum, see the
+    integration test 'tests/integration_tests/config/plot_tabular_data_for_single_pe_data.yml'.
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.camera.single_photon_electron_spectrum import SinglePhotonElectronSpectrum
+from simtools.configuration import configurator
+
+
+def _parse(label):
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        label=label,
+        description="Derive single photon electron spectrum from a given amplitude spectrum.",
+    )
+    config.parser.add_argument(
+        "--input_spectrum",
+        help="File with amplitude spectrum.",
+        type=Path,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--afterpulse_spectrum",
+        help="File with afterpulse spectrum.",
+        type=Path,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--step_size",
+        help="Step size in amplitude spectrum",
+        type=float,
+        default=0.02,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--max_amplitude",
+        help="Maximum amplitude in amplitude spectrum",
+        type=float,
+        default=42.0,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--use_norm_spe",
+        help="Use sim_telarray tool 'norm_spe' to normalize the spectrum.",
+        action="store_true",
+        required=False,
+    )
+
+    return config.initialize(db_config=False, output=True)
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse(Path(__file__).stem)
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    single_pe = SinglePhotonElectronSpectrum(args_dict)
+    single_pe.derive_single_pe_spectrum()
+    single_pe.write_single_pe_spectrum()
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/generate_array_config.py

@@ -1,28 +1,28 @@
 #!/usr/bin/python3
 """
-    Generate sim_telarray configuration files for a given array.
+Generate sim_telarray configuration files for a given array.
 
-    The applications generates the sim_telarray configuration files for a given array, site,
-    and model_version using the model parameters stored in the database.
+The applications generates the sim_telarray configuration files for a given array, site,
+and model_version using the model parameters stored in the database.
 
-    Command line arguments
-    ----------------------
-    site : str
-        Site name (e.g., North, South).
-    array_layout_name : str
-        Name of the layout array (e.g., test_layout, alpha, 4mst, etc.).
-    array_element_list : list
-        List of array elements (e.g., telescopes) to plot (e.g., ``LSTN-01 LSTN-02 MSTN``).
+Command line arguments
+----------------------
+site : str
+    Site name (e.g., North, South).
+array_layout_name : str
+    Name of the layout array (e.g., test_layout, alpha, 4mst, etc.).
+array_element_list : list
+    List of array elements (e.g., telescopes) to plot (e.g., ``LSTN-01 LSTN-02 MSTN``).
 
-    Example
-    -------
-    North - 5.0.0:
+Example
+-------
+North - 5.0.0:
 
-    .. code-block:: console
+.. code-block:: console
 
-        simtools-generate-array-config --site North --array_layout_name alpha --model_version 5.0.0
+    simtools-generate-array-config --site North --array_layout_name alpha --model_version 5.0.0
 
-    The output is saved in simtools-output/test/model.
+The output is saved in simtools-output/test/model.
 """
 
 import logging