gammasimtools 0.6.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

simtools/applications/{get_file_from_db.py → db_get_file_from_db.py}

@@ -1,8 +1,6 @@
 #!/usr/bin/python3
 
 """
-    Summary
-    -------
     Get a file from the DB.
 
     The name of the file is required.
@@ -17,8 +15,6 @@
     output_path (str)
         Name of the local output directory where to save the files.
         Default it $CWD.
-    verbosity (str, optional)
-        Log level to print.
 
     Example
     -------
@@ -26,13 +22,13 @@
 
     .. code-block:: console
 
-        simtools-get-file-from-db --file_name mirror_CTA-N-LST1_v2019-03-31.dat
+        simtools-db-get-file-from-db --file_name mirror_CTA-N-LST1_v2019-03-31.dat
 
     Expected final print-out message:
 
     .. code-block:: console
 
-        INFO::get_file_from_db(l82)::main::Got file mirror_CTA-N-LST1_v2019-03-31.dat from DB \
+        INFO::db_get_file_from_db(l82)::main::Got file mirror_CTA-N-LST1_v2019-03-31.dat from DB \
         CTA-Simulation-Model and saved into .
 
 """
@@ -40,11 +36,12 @@
 import logging
 
 import simtools.utils.general as gen
-from simtools import db_handler
 from simtools.configuration import configurator
+from simtools.db import db_handler
+from simtools.io_operations import io_handler
 
 
-def main():
+def _parse():
     config = configurator.Configurator(
         description="Get file(s) from the DB.",
         usage="simtools-get-file-from-db --file_name mirror_CTA-S-LST_v2020-04-07.dat",
@@ -52,47 +49,46 @@ def main():
 
     config.parser.add_argument(
         "--file_name",
-        help=("The name of the file to be downloaded."),
+        help="The name of the file to be downloaded.",
         type=str,
         required=True,
     )
-    args_dict, db_config = config.initialize(db_config=True)
+    return config.initialize(db_config=True, output=True)
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse()
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+    _io_handler = io_handler.IOHandler()
 
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
     available_dbs = [
-        db.DB_TABULATED_DATA,
-        db.DB_CTA_SIMULATION_MODEL,
+        db_config["db_simulation_model"],
         db.DB_CTA_SIMULATION_MODEL_DESCRIPTIONS,
-        db.DB_REFERENCE_DATA,
         db.DB_DERIVED_VALUES,
         "sandbox",
-        "test-data",
     ]
     file_id = None
-    if args_dict["output_path"].exists():
-        for db_name in available_dbs:
-            try:
-                file_id = db.export_file_db(
-                    db_name, args_dict["output_path"], args_dict["file_name"]
-                )
-                logger.info(
-                    f"Got file {args_dict['file_name']} from DB {db_name} "
-                    f"and saved into {args_dict['output_path']}"
-                )
-                break
-            except FileNotFoundError:
-                continue
-
-        if file_id is None:
-            logger.error(
-                f"The file {args_dict['file_name']} was not found in any of the available DBs."
+    for db_name in available_dbs:
+        try:
+            file_id = db.export_file_db(
+                db_name, _io_handler.get_output_directory(), args_dict["file_name"]
+            )
+            logger.info(
+                f"Got file {args_dict['file_name']} from DB {db_name} "
+                f"and saved into {_io_handler.get_output_directory()}"
             )
-            raise FileNotFoundError
-    else:
-        logger.error(f"Aborted, directory {args_dict['output_path']} does not exist")
+            break
+        except FileNotFoundError:
+            continue
+
+    if file_id is None:
+        logger.error(
+            f"The file {args_dict['file_name']} was not found in any of the available DBs."
+        )
+        raise FileNotFoundError
 
 
 if __name__ == "__main__":

simtools/applications/db_get_parameter_from_db.py

@@ -0,0 +1,131 @@
+#!/usr/bin/python3
+
+r"""
+    Get a parameter entry from DB for a specific telescope or a site.
+
+    The application receives a parameter name, a site, a telescope (if applicable) and \
+    a version. It then prints out the parameter entry.
+
+    Command line arguments
+    ----------------------
+    parameter (str, required)
+        Parameter name
+
+    site (str, required)
+        South or North.
+
+    telescope (str, optional)
+        Telescope model name (e.g. LST-1, SST-D, ...)
+
+    log_level (str, optional)
+        Log level to print.
+
+    Raises
+    ------
+    KeyError in case the parameter requested does not exist in the model parameters.
+
+    Example
+    -------
+    Get the mirror_list parameter from the DB.
+
+    .. code-block:: console
+
+        simtools-db-get-parameter-from-db --parameter mirror_list \\
+                --site North --telescope LSTN-01 \\
+                --model_version 5.0.0
+
+    Expected final print-out message:
+
+    .. code-block:: console
+
+        {'Applicable': True,
+         'File': True,
+         'Type': 'str',
+         'Value': 'mirror_CTA-N-LST1_v2019-03-31.dat',
+         'Version': '5.0.0',
+         '_id': ObjectId('608834f257df2db2531b8e78'),
+         'entry_date': datetime.datetime(2021, 4, 27, 15, 59, 46, tzinfo=<bson.tz_util.FixedOffset \
+          object at 0x7f601dd51d80>)}
+
+"""
+
+import json
+import logging
+from pathlib import Path
+from pprint import pprint
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler
+from simtools.io_operations import io_handler
+
+
+def _parse():
+    config = configurator.Configurator(
+        description=(
+            "Get a parameter entry from DB for a specific telescope or a site. "
+            "The application receives a parameter name, a site, a telescope (if applicable), "
+            "and a version. It then prints out the parameter entry. "
+        )
+    )
+
+    config.parser.add_argument("--parameter", help="Parameter name", type=str, required=True)
+    config.parser.add_argument(
+        "--db_collection",
+        help="DB collection to which to add the file",
+        default="telescopes",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--output_file",
+        help="output file name (if not given: print to stdout)",
+        type=str,
+        required=False,
+    )
+
+    return config.initialize(db_config=True, simulation_model="telescope")
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    if args_dict["db_collection"] == "configuration_sim_telarray":
+        pars = db.get_model_parameters(
+            args_dict["site"],
+            args_dict["telescope"],
+            args_dict["model_version"],
+            collection="configuration_sim_telarray",
+        )
+    elif args_dict["db_collection"] == "configuration_corsika":
+        pars = db.get_corsika_configuration_parameters(args_dict["model_version"])
+    elif args_dict["telescope"] is not None:
+        pars = db.get_model_parameters(
+            args_dict["site"],
+            args_dict["telescope"],
+            args_dict["model_version"],
+            collection="telescopes",
+        )
+    else:
+        pars = db.get_site_parameters(args_dict["site"], args_dict["model_version"])
+    if args_dict["parameter"] not in pars:
+        raise KeyError(f"The requested parameter, {args_dict['parameter']}, does not exist.")
+    if args_dict["output_file"] is not None:
+        _io_handler = io_handler.IOHandler()
+        pars[args_dict["parameter"]].pop("_id")
+        pars[args_dict["parameter"]].pop("entry_date")
+        _output_file = Path(_io_handler.get_output_directory()) / args_dict["output_file"]
+        with open(_output_file, "w", encoding="utf-8") as json_file:
+            json.dump(pars[args_dict["parameter"]], json_file, indent=4)
+    else:
+        print()
+        pprint(pars[args_dict["parameter"]])
+        print()
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/db_inspect_databases.py

@@ -0,0 +1,52 @@
+#!/usr/bin/python3
+
+"""
+Inspect databases and print (available database names and collections).
+
+Command line arguments
+----------------------
+db_name (str, optional)
+    Inspect a specific database.
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler
+
+
+def _parse():
+    config = configurator.Configurator(description="Inspect databases")
+    config.parser.add_argument(
+        "--db_name",
+        help="Inspect a specific database (use all to print all databases)",
+        default="all",
+        required=True,
+    )
+    return config.initialize(db_config=True, simulation_model="telescope")
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    databases = db.db_client.list_database_names()
+
+    for db_name in databases:
+        if args_dict["db_name"] != "all" and db_name != args_dict["db_name"]:
+            continue
+        # missing admin rights; skip config and admin
+        if db_name in ("config", "admin", "local"):
+            continue
+        print("Database:", db_name)
+        collections = db.get_collections(db_name=db_name)
+        print("   Collections:", collections)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/derive_mirror_rnda.py

@@ -1,11 +1,11 @@
 #!/usr/bin/python3
 
-"""
-    Summary
-    -------
-    Derive the simulation model parameter **mirror_reflection_random_angle**
-    (sometimes called mirror roughness) to match the measured containment diameter
-    of the optical point-spread function (PSF) of individual mirror panels.
+r"""
+    Derive the simulation model parameter mirror_reflection_random_angle.
+
+    This parameter, often referred to as "mirror roughness," is used to align the simulation
+    with the measured containment diameter of the optical point-spread function (PSF) for
+    individual mirror panels.
 
     Description
     -----------
@@ -30,11 +30,11 @@
         * file (table) with measured focal lengths per mirror panel
           (provided through ``--mirror_list``)
         * randomly generated focal lengths using an expected spread (value given through
-          ``--random_flen``) around the mean focal length (provided through the
-          :ref:`Model Parameters DB`). This option is switched with ``--use_random_flen``.
+          ``--random_focal_length``) around the mean focal length (provided through the
+          Model Parameters DB). This option is switched with ``--use_random_focal_length``.
 
     The tuning algorithm requires a starting value for the random reflection angle. This is either
-    taken from the :ref:`Model Parameters DB` (default) or can be set using the argument ``--rnda``.
+    taken from the Model Parameters DB (default) or can be set using the argument ``--rnda``.
 
     Ray-tracing simulations are performed for single mirror configurations for each
     mirror given in the mirror_list. The mean simulated containment diameter for all the mirrors
@@ -83,76 +83,59 @@
         default model is read from the simulation model database.
     mirror_list (file, optional)
         Table with mirror ID and panel radius.
-    use_random_flen (activation mode, optional)
-        Use random focal lengths, instead of the measured ones. The argument random_flen can be
-        used to replace the default random_focal_length from the model.
-    random_flen (float, optional)
+    use_random_focal_length (activation mode, optional)
+        Use random focal lengths, instead of the measured ones. The argument random_focal_length
+        can be used to replace the default random_focal_length from the model.
+    random_focal_length (float, optional)
         Value of the random focal lengths to replace the default random_focal_length. Only used if
-         use_random_flen is activated.
+        'use_random_focal_length' is activated.
     no_tuning (activation mode, optional)
         Turn off the tuning - A single case will be simulated and plotted.
     test (activation mode, optional)
         If activated, application will be faster by simulating only few mirrors.
-    verbosity (str, optional)
-        Log level to print.
 
     Example
     -------
     Derive mirror random reflection angle for a mid-sized telescope (MST),
-    simulation production Prod5.
+    simulation production 5.0.0
 
     .. code-block:: console
 
         simtools-derive-mirror-rnda \\
-            --site North \\
-            --telescope MST-FlashCam-D \\
+            --site South \\
+            --telescope LSTS-design \\
+            --model_version 6.0.0 \\
             --containment_fraction 0.8 \\
-            --mirror_list MLTdata-preproduction.ecsv \\
-            --psf_measurement MLTdata-preproduction.ecsv \\
-            --rnda 0.0063 \\
-            --test
-
-    Runtime about 4 min.
-
-    The output is saved in `simtools-output/derive_mirror_rnda`.
-    Use the parameter ``--output_path`` to change the output directory.
+            --mirror_list ./tests/resources/mirror_list_CTA-N-LST1_v2019-03-31_rotated.ecsv
+            --rnda 0.003 \\
+            --psf_measurement_containment_mean 1.4 \\
 
     Expected final print-out message:
 
     .. code-block:: console
 
         Measured D80:
-        Mean = 1.403 cm, StdDev = 0.163 cm
+        Mean = 1.400 cm
 
         Simulated D80:
-        Mean = 1.404 cm, StdDev = 0.608 cm
+        Mean = 1.406 cm, StdDev = 0.005 cm
 
         mirror_random_reflection_angle
-        Previous value = 0.006300
-        New value = 0.004975
+        Previous value = 0.003000
+        New value = 0.003824
 
 """
 
 import logging
 from pathlib import Path
 
-import astropy.units as u
-import numpy as np
-from astropy.table import QTable, Table
-
-import simtools.data_model.model_data_writer as writer
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.data_model.metadata_collector import MetadataCollector
-from simtools.model.telescope_model import TelescopeModel
-from simtools.ray_tracing import RayTracing
+from simtools.ray_tracing.mirror_panel_psf import MirrorPanelPSF
 
 
 def _parse(label):
-    """
-    Parse command line configuration
-    """
-
+    """Parse command line configuration."""
     config = configurator.Configurator(
         description="Derive mirror random reflection angle.", label=label
     )
@@ -196,14 +179,17 @@ def _parse(label):
         required=False,
     )
     config.parser.add_argument(
-        "--use_random_flen",
+        "--use_random_focal_length",
         help=("Use random focal lengths."),
         action="store_true",
         required=False,
     )
     config.parser.add_argument(
-        "--random_flen",
-        help="Value of the random focal length. Only used if use_random_flen is activated.",
+        "--random_focal_length",
+        help=(
+            "Value of the random focal length. "
+            "Only used if 'use_random_focal_length' is activated."
+        ),
         default=None,
         type=float,
         required=False,
@@ -214,131 +200,10 @@ def _parse(label):
         action="store_true",
         required=False,
     )
-    return config.initialize(db_config=True, output=True, telescope_model=True)
-
+    return config.initialize(db_config=True, output=True, simulation_model="telescope")
 
-def _define_telescope_model(label, args_dict, db_config):
-    """
-    Define telescope model and update configuration
-    with mirror list and/or random focal length given
-    as input
-
-    Attributes
-    ----------
-    label: str
-        Application label.
-    args_dict: dict
-        Dictionary with configuration parameters.
-    db_config:
-        Dictionary with database configuration.
-
-    Returns
-    -------
-    tel TelescopeModel
-        telescope model
 
-    """
-
-    tel = TelescopeModel(
-        site=args_dict["site"],
-        telescope_model_name=args_dict["telescope"],
-        model_version=args_dict["model_version"],
-        mongo_db_config=db_config,
-        label=label,
-    )
-    if args_dict["mirror_list"] is not None:
-        mirror_list_file = gen.find_file(name=args_dict["mirror_list"], loc=args_dict["model_path"])
-        tel.change_parameter("mirror_list", args_dict["mirror_list"])
-        tel.add_parameter_file("mirror_list", mirror_list_file)
-    if args_dict["random_flen"] is not None:
-        tel.change_parameter("random_focal_length", str(args_dict["random_flen"]))
-
-    return tel
-
-
-def _print_and_write_results(
-    args_dict, rnda_start, rnda_opt, mean_d80, sig_d80, results_rnda, results_mean, results_sig
-):
-    """
-    Print results to screen and write metadata and data files
-    in the requested format
-
-    """
-
-    containment_fraction_percent = int(args_dict["containment_fraction"] * 100)
-
-    # Printing results to stdout
-    print(f"\nMeasured D{containment_fraction_percent}:")
-    if args_dict["psf_measurement_containment_sigma"] is not None:
-        print(
-            f"Mean = {args_dict['psf_measurement_containment_mean']:.3f} cm, "
-            f"StdDev = {args_dict['psf_measurement_containment_sigma']:.3f} cm"
-        )
-    else:
-        print(f"Mean = {args_dict['psf_measurement_containment_mean']:.3f} cm")
-    print(f"\nSimulated D{containment_fraction_percent}:")
-    print(f"Mean = {mean_d80:.3f} cm, StdDev = {sig_d80:.3f} cm")
-    print("\nmirror_random_reflection_angle")
-    print(f"Previous value = {rnda_start:.6f}")
-    print(f"New value = {rnda_opt:.6f}\n")
-
-    # Result table written to ecsv file using file_writer
-    # First entry is always the best fit result
-    result_table = QTable(
-        [
-            [True] + [False] * len(results_rnda),
-            ([rnda_opt] + results_rnda) * u.deg,
-            ([0.0] * (len(results_rnda) + 1)),
-            ([0.0] * (len(results_rnda) + 1)) * u.deg,
-            ([mean_d80] + results_mean) * u.cm,
-            ([sig_d80] + results_sig) * u.cm,
-        ],
-        names=(
-            "best_fit",
-            "mirror_reflection_random_angle_sigma1",
-            "mirror_reflection_random_angle_fraction2",
-            "mirror_reflection_random_angle_sigma2",
-            f"containment_radius_D{containment_fraction_percent}",
-            f"containment_radius_sigma_D{containment_fraction_percent}",
-        ),
-    )
-    writer.ModelDataWriter.dump(
-        args_dict=args_dict,
-        metadata=MetadataCollector(args_dict=args_dict).top_level_meta,
-        product_data=result_table,
-    )
-
-
-def _get_psf_containment(logger, args_dict):
-    """
-    Read measured single-mirror point-spread function (containment)
-    from file and return mean and sigma
-
-    """
-
-    # If this is a test, read just the first few lines since we only simulate those mirrors
-    data_end = args_dict["number_of_mirrors_to_test"] + 1 if args_dict["test"] else None
-    _psf_list = Table.read(args_dict["psf_measurement"], format="ascii.ecsv", data_end=data_end)
-    try:
-        args_dict["psf_measurement_containment_mean"] = np.nanmean(
-            np.array(_psf_list["psf_opt"].to("cm").value)
-        )
-        args_dict["psf_measurement_containment_sigma"] = np.nanstd(
-            np.array(_psf_list["psf_opt"].to("cm").value)
-        )
-    except KeyError:
-        logger.debug(
-            f"Missing column for psf measurement (psf_opt) in {args_dict['psf_measurement']}"
-        )
-        raise
-
-    logger.info(
-        f"Determined PSF containment to {args_dict['psf_measurement_containment_mean']:.4} "
-        f"+- {args_dict['psf_measurement_containment_sigma']:.4} cm"
-    )
-
-
-def main():
+def main():  # noqa: D103
     label = Path(__file__).stem
 
     args_dict, db_config = _parse(label)
@@ -346,91 +211,10 @@ def main():
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    tel = _define_telescope_model(label, args_dict, db_config)
-
-    if args_dict["test"]:
-        args_dict["number_of_mirrors_to_test"] = 4
-
-    if args_dict["psf_measurement"]:
-        _get_psf_containment(logger, args_dict)
-    if not args_dict["psf_measurement_containment_mean"]:
-        logger.error("Missing PSF measurement")
-        raise ValueError
-
-    def run(rnda):
-        """Runs the simulations for one given value of rnda"""
-        tel.change_parameter("mirror_reflection_random_angle", str(rnda))
-        ray = RayTracing.from_kwargs(
-            telescope_model=tel,
-            single_mirror_mode=True,
-            mirror_numbers=list(range(1, args_dict["number_of_mirrors_to_test"] + 1))
-            if args_dict["test"]
-            else "all",
-            simtel_source_path=args_dict.get("simtel_path", None),
-            use_random_focal_length=args_dict["use_random_flen"],
-        )
-        ray.simulate(test=False, force=True)  # force has to be True, always
-        ray.analyze(force=True)
-
-        return (
-            ray.get_mean("d80_cm").to(u.cm).value,
-            ray.get_std_dev("d80_cm").to(u.cm).value,
-        )
-
-    # First - rnda from previous model or from command line
-    if args_dict["rnda"] != 0:
-        rnda_start = args_dict["rnda"]
-    else:
-        rnda_start = tel.get_parameter("mirror_reflection_random_angle")["Value"]
-        if isinstance(rnda_start, str):
-            rnda_start = float(rnda_start.split()[0])
-
-    logger.info(f"Start value for mirror_reflection_random_angle: {rnda_start} deg")
-
-    results_rnda = []
-    results_mean = []
-    results_sig = []
-    if args_dict["no_tuning"]:
-        rnda_opt = rnda_start
-    else:
-
-        def collect_results(rnda, mean, sig):
-            results_rnda.append(rnda)
-            results_mean.append(mean)
-            results_sig.append(sig)
-
-        stop = False
-        mean_d80, sig_d80 = run(rnda_start)
-        rnda = rnda_start
-        sign_delta = np.sign(mean_d80 - args_dict["psf_measurement_containment_mean"])
-        collect_results(rnda, mean_d80, sig_d80)
-        while not stop:
-            rnda = rnda - (0.1 * rnda_start * sign_delta)
-            if rnda < 0:
-                rnda = 0
-                collect_results(rnda, mean_d80, sig_d80)
-                break
-            mean_d80, sig_d80 = run(rnda)
-            new_sign_delta = np.sign(mean_d80 - args_dict["psf_measurement_containment_mean"])
-            stop = new_sign_delta != sign_delta
-            sign_delta = new_sign_delta
-            collect_results(rnda, mean_d80, sig_d80)
-
-        # Linear interpolation using two last rnda values
-        results_rnda, results_mean, results_sig = gen.sort_arrays(
-            results_rnda, results_mean, results_sig
-        )
-        rnda_opt = np.interp(
-            x=args_dict["psf_measurement_containment_mean"],
-            xp=results_mean,
-            fp=results_rnda,
-        )
-
-    mean_d80, sig_d80 = run(rnda_opt)
-
-    _print_and_write_results(
-        args_dict, rnda_start, rnda_opt, mean_d80, sig_d80, results_rnda, results_mean, results_sig
-    )
+    panel_psf = MirrorPanelPSF(label, args_dict, db_config)
+    panel_psf.derive_random_reflection_angle(save_figures=True)
+    panel_psf.print_results()
+    panel_psf.write_optimization_data()
 
 
 if __name__ == "__main__":