gammasimtools 0.19.0__py3-none-any.whl → 0.21.0__py3-none-any.whl

simtools/applications/db_get_file_from_db.py

@@ -65,29 +65,17 @@ def main():  # noqa: D103
     _io_handler = io_handler.IOHandler()
 
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
-    available_dbs = [
-        db_config["db_simulation_model"],
-    ]
-    file_id = {}
-    for db_name in available_dbs:
-        try:
-            file_id = db.export_model_files(
-                db_name=db_name,
-                dest=_io_handler.get_output_directory(),
-                file_names=args_dict["file_name"],
-            )
-            logger.info(
-                f"Got file {args_dict['file_name']} from DB {db_name} "
-                f"and saved into {_io_handler.get_output_directory()}"
-            )
-            break
-        except FileNotFoundError:
-            continue
-
-    for key, value in file_id.items():
-        if value is None:
-            logger.error(f"The file {key} was not found in any of the available DBs.")
-            raise FileNotFoundError
+    file_id = db.export_model_files(
+        dest=_io_handler.get_output_directory(),
+        file_names=args_dict["file_name"],
+    )
+    if file_id is None:
+        logger.error(f"The file {args_dict['file_name']} was not found in {db.db_name}.")
+        raise FileNotFoundError
+    logger.info(
+        f"Got file {args_dict['file_name']} from DB {db.db_name} "
+        f"and saved into {_io_handler.get_output_directory()}"
+    )
 
 
 if __name__ == "__main__":

simtools/applications/derive_psf_parameters.py

@@ -6,25 +6,22 @@ r"""
     This includes parameters mirror_reflection_random_angle, \
     mirror_align_random_horizontal and mirror_align_random_vertical.
 
-    The telescope zenith angle and the source distance can be set by command line arguments.
-
     The measured cumulative PSF should be provided by using the command line argument data. \
     A file name is expected, in which the file should contain 3 columns: radial distance in mm, \
     differential value of photon intensity and its integral value.
 
-    The derivation is performed through a random search. A number of random combination of the \
-    parameters are tested and the best ones are selected based on the minimum value of \
-    the Root Mean Squared Deviation between data and simulations. The range in which the \
-    parameter are drawn uniformly are defined based on the previous value on the telescope model.
+    The derivation is performed through gradient descent optimization that minimizes either the \
+    Root Mean Squared Deviation (RMSD) between measured and simulated PSF curves (default) or the \
+    Kolmogorov-Smirnov (KS) statistic when the --ks_statistic flag is used.
 
     The optimization workflow includes:
 
     * Loading and preprocessing PSF data from measurement files
-    * Generating random parameter combinations for optimization
-    * Running ray-tracing simulations for each parameter set
-    * Calculating RMSD between measured and simulated PSF curves
-    * Identifying the best-fit parameters with minimum RMSD
-    * Creating comprehensive plots and D80 vs off-axis angle analysis
+    * Running gradient descent optimization to minimize RMSD
+    * Generating cumulative PSF plots for each iteration showing optimization progression
+    * Logging parameter evolution through gradient descent steps
+    * Creating convergence plots showing RMSD and D80 evolution
+    * Automatically generating D80 vs off-axis angle analysis for best parameters
     * Optionally exporting optimized parameters as simulation model files
 
     The assumption are:
@@ -36,7 +33,7 @@ r"""
     One example of the plot generated by this applications are shown below.
 
     .. _derive_psf_parameters_plot:
-    .. image::  images/derive_psf_parameters.png
+    .. image::  images/gradient_descent.png
       :width: 49 %
 
     Command line arguments
@@ -63,47 +60,53 @@ r"""
         If activated, application will be faster by simulating fewer photons.
     write_psf_parameters (activation mode, optional)
         Write the optimized PSF parameters as simulation model parameter files.
-    random_seed (int, optional)
-        Random seed for parameter generation.
-    n_runs (int, optional)
-        Number of parameter combinations to test.
+    rmsd_threshold (float, optional)
+        RMSD threshold for gradient descent convergence (default: 0.007).
+    learning_rate (float, optional)
+        Learning rate for gradient descent optimization (default: 0.01).
+    monte_carlo_analysis (activation mode, optional)
+        Run Monte Carlo analysis to find statistical uncertainties.
 
     Example
     -------
-    LSTN-01 5.0.0
-
-    Runtime < 3 min.
-
-    Get PSF data from the DB:
+    --telescope LSTN-01 --model_version 6.0.0
 
-    .. code-block:: console
 
-        simtools-db-get-file-from-db --file_name PSFcurve_data_v2.txt
 
     Run the application:
 
     .. code-block:: console
 
         simtools-derive-psf-parameters --site North --telescope LSTN-01 \\
-            --model_version 6.0.0 --data tests/resources/PSFcurve_data_v2.txt --plot_all --test
+            --model_version 6.0.0 --data tests/resources/PSFcurve_data_v2.ecsv --plot_all --test
 
     Run with parameter export:
 
     .. code-block:: console
 
-        simtools-derive-psf-parameters --site North --telescope LSTN-01 \\
-            --model_version 6.0.0 \\
-            --data tests/resources/PSFcurve_data_v2.txt --write_psf_parameters
+        simtools-derive-psf-parameters --site North --telescope LSTN-01 --model_version 6.0.0 \\
+            --plot_all --test --rmsd_threshold 0.01 --learning_rate 0.001 \\
+            --data tests/resources/PSFcurve_data_v2.ecsv \\
+            --write_psf_parameters
 
-    The output is saved in simtools-output/tune_psf.
+    Run monte carlo analysis:
+
+    .. code-block:: console
+
+        simtools-derive-psf-parameters --site North --telescope LSTN-01 --model_version 6.0.0 \\
+            --plot_all --test --monte_carlo_analysis \\
+            --data tests/resources/PSFcurve_data_v2.ecsv \\
+            --write_psf_parameters
+
+    The output is saved in simtools-output/derive_psf_parameters.
 
     Output files include:
 
-    * Parameter optimization results in tested_psf_parameters.txt
-    * PSF comparison plots in tune_psf_[telescope].pdf
+    * Gradient descent progression log in psf_gradient_descent_[telescope].log
+    * Gradient descent convergence plots in gradient_descent_convergence_[telescope].png
+    * PSF progression plots showing evolution through iterations (if --plot_all is specified)
     * D80 vs off-axis angle plots (d80_vs_offaxis_cm.png, d80_vs_offaxis_deg.png)
     * Optimized simulation model parameter files (if --write_psf_parameters is specified)
-    * Cumulative PSF plots for all tested combinations (if --plot_all is specified)
 
 """
 
@@ -154,16 +157,32 @@ def _parse():
         required=False,
     )
     config.parser.add_argument(
-        "--random_seed",
-        help="Random seed for parameter generation.",
-        type=int,
-        default=None,
+        "--rmsd_threshold",
+        help=(
+            "RMSD threshold for gradient descent convergence "
+            "(not used with --monte_carlo_analysis)."
+        ),
+        type=float,
+        default=0.01,
+    )
+    config.parser.add_argument(
+        "--learning_rate",
+        help=(
+            "Learning rate for gradient descent optimization "
+            "(not used with --monte_carlo_analysis)."
+        ),
+        type=float,
+        default=0.01,
     )
     config.parser.add_argument(
-        "--n_runs",
-        help="Number of parameter combinations to test.",
-        type=int,
-        default=5,
+        "--monte_carlo_analysis",
+        help="Run analysis to find monte carlo uncertainties.",
+        action="store_true",
+    )
+    config.parser.add_argument(
+        "--ks_statistic",
+        help="Use KS statistic for monte carlo uncertainty analysis.",
+        action="store_true",
     )
     return config.initialize(
         db_config=True,

simtools/applications/derive_trigger_rates.py

@@ -0,0 +1,91 @@
+r"""
+Derive cosmic-ray trigger rates for a single telescope or an array of telescopes.
+
+Uses simulated background events (e.g. from proton primaries) to calculate the trigger rates.
+Input is reduced event data generated from simulations for the given configuration.
+
+
+Command line arguments
+----------------------
+event_data_file (str, required)
+    Event data file containing reduced event data.
+array_layout_name (list, optional)
+    Name of the array layout to use for the simulation.
+telescope_ids (str, optional)
+    Path to a file containing telescope configurations.
+plot_histograms (bool, optional)
+    Plot histograms of the event data.
+model_version (str, optional)
+    Version of the simulation model to use.
+site (str, optional)
+    Name of the site where the simulation is being run.
+
+
+Example
+-------
+
+Derive trigger rates for the South Alpha layout:
+
+.. code-block:: console
+
+    simtools-derive-trigger-rates \\
+        --site South \\
+        --model_version 6.0.0 \\
+        --event_data_file /path/to/event_data_file.h5 \\
+        --array_layout_name alpha\\
+        --plot_histograms
+
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.telescope_trigger_rates import telescope_trigger_rates
+
+
+def _parse():
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        description="Derive trigger rates for a single telescope or an array of telescopes.",
+    )
+    config.parser.add_argument(
+        "--event_data_file",
+        type=str,
+        required=True,
+        help="Event data file containing reduced event data.",
+    )
+    config.parser.add_argument(
+        "--telescope_ids",
+        type=str,
+        required=False,
+        help="Path to a file containing telescope configurations.",
+    )
+    config.parser.add_argument(
+        "--plot_histograms",
+        help="Plot histograms of the event data.",
+        action="store_true",
+        default=False,
+    )
+    return config.initialize(
+        db_config=True,
+        output=True,
+        simulation_model=[
+            "site",
+            "model_version",
+            "layout",
+        ],
+    )
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "info")))
+
+    telescope_trigger_rates(args_dict, db_config)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/generate_corsika_histograms.py

@@ -165,15 +165,11 @@ r"""
 """
 
 import logging
-import re
 import time
 from pathlib import Path
 
-import numpy as np
-
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.corsika import corsika_histograms_visualize
 from simtools.corsika.corsika_histograms import CorsikaHistograms
 from simtools.io import io_handler
 
@@ -284,134 +280,6 @@ def _parse(label, description):
     return config_parser, _
 
 
-def _plot_figures(corsika_histograms_instance, test=False):
-    """
-    Auxiliary function to centralize the plotting functions.
-
-    Parameters
-    ----------
-    corsika_histograms_instance: CorsikaHistograms instance.
-        The CorsikaHistograms instance created in main.
-    test: bool
-        If true plots the figures for the first two functions only.
-    """
-    plot_function_names = [
-        plotting_method
-        for plotting_method in dir(corsika_histograms_visualize)
-        if plotting_method.startswith("plot_")
-        and "event_header_distribution" not in plotting_method
-    ]
-    if test:
-        plot_function_names = plot_function_names[:2]
-
-    figure_list = []
-    for function_name in plot_function_names:
-        plot_function = getattr(corsika_histograms_visualize, function_name)
-        figures = plot_function(corsika_histograms_instance)
-        for fig in figures:
-            figure_list.append(fig)
-
-    figure_list = np.array(figure_list).flatten()
-    core_name = re.sub(r"\.hdf5$", "", corsika_histograms_instance.hdf5_file_name)
-    output_file_name = Path(corsika_histograms_instance.output_path).joinpath(f"{core_name}.pdf")
-    corsika_histograms_visualize.save_figs_to_pdf(figure_list, output_file_name)
-
-
-def _derive_event_1d_histograms(
-    corsika_histograms_instance, event_1d_header_keys, pdf, hdf5, overwrite=False
-):
-    """
-    Auxiliary function to derive the histograms for the arguments given by event_1d_histograms.
-
-    Parameters
-    ----------
-    corsika_histograms_instance: CorsikaHistograms instance.
-        The CorsikaHistograms instance created in main.
-    event_1d_header_keys: str
-        Generate 1D histograms for elements given in event_1d_header_keys from the CORSIKA event
-        header and save into hdf5/pdf files.
-    pdf: bool
-        If true, histograms are saved into a pdf file.
-    hdf5: bool
-        If true, histograms are saved into hdf5 files.
-    overwrite: bool
-        If true, overwrites the current output hdf5 file.
-    """
-    figure_list = []
-    for event_header_element in event_1d_header_keys:
-        if pdf:
-            figure = corsika_histograms_visualize.plot_1d_event_header_distribution(
-                corsika_histograms_instance, event_header_element
-            )
-            figure_list.append(figure)
-        if hdf5:
-            corsika_histograms_instance.export_event_header_1d_histogram(
-                event_header_element, bins=50, hist_range=None, overwrite=overwrite
-            )
-    if pdf:
-        figures_list = np.array(figure_list).flatten()
-        output_file_name = Path(corsika_histograms_instance.output_path).joinpath(
-            f"{corsika_histograms_instance.hdf5_file_name}_event_1d_histograms.pdf"
-        )
-        corsika_histograms_visualize.save_figs_to_pdf(figures_list, output_file_name)
-
-
-def _derive_event_2d_histograms(
-    corsika_histograms_instance, event_2d_header_keys, pdf, hdf5, overwrite=False
-):
-    """
-    Auxiliary function to derive the histograms for the arguments given by event_1d_histograms.
-
-    If an odd number of event header keys are given, the last one is discarded.
-
-    Parameters
-    ----------
-    corsika_histograms_instance: CorsikaHistograms instance.
-        The CorsikaHistograms instance created in main.
-    event_2d_header_keys: str
-        Generate 1D histograms for elements given in event_1d_header_keys from the CORSIKA event
-        header and save into hdf5/pdf files.
-    pdf: bool
-        If true, histograms are saved into a pdf file.
-    hdf5: bool
-        If true, histograms are saved into hdf5 files.
-    overwrite: bool
-        If true, overwrites the current output hdf5 file.
-    """
-    figure_list = []
-    for i_event_header_element, _ in enumerate(event_2d_header_keys[::2]):
-        # [::2] to discard the last one in case an odd number of keys are passed
-
-        if len(event_2d_header_keys) % 2 == 1:  # if odd number of keys
-            msg = (
-                "An odd number of keys was passed to generate 2D histograms."
-                "The last key is being ignored."
-            )
-            logger.warning(msg)
-
-        if pdf:
-            figure = corsika_histograms_visualize.plot_2d_event_header_distribution(
-                corsika_histograms_instance,
-                event_2d_header_keys[i_event_header_element],
-                event_2d_header_keys[i_event_header_element + 1],
-            )
-            figure_list.append(figure)
-        if hdf5:
-            corsika_histograms_instance.export_event_header_2d_histogram(
-                event_2d_header_keys[i_event_header_element],
-                event_2d_header_keys[i_event_header_element + 1],
-                bins=50,
-                hist_range=None,
-                overwrite=overwrite,
-            )
-    if pdf:
-        figures_list = np.array(figure_list).flatten()
-        output_file_name = Path(corsika_histograms_instance.output_path).joinpath(
-            f"{corsika_histograms_instance.hdf5_file_name}_event_2d_histograms.pdf"
-        )
-        corsika_histograms_visualize.save_figs_to_pdf(figures_list, output_file_name)
-
-
 def main():  # noqa: D103
     label = Path(__file__).stem
     description = "Generate histograms for the Cherenkov photons saved in the CORSIKA IACT file."
@@ -427,62 +295,17 @@ def main():  # noqa: D103
     corsika_histograms_instance = CorsikaHistograms(
         args_dict["iact_file"], output_path=output_path, hdf5_file_name=args_dict["hdf5_file_name"]
     )
-    if args_dict["telescope_indices"] is not None:
-        try:
-            indices = np.array(args_dict["telescope_indices"]).astype(int)
-        except ValueError:
-            msg = (
-                f"{args_dict['telescope_indices']} not a valid input. "
-                f"Please use integer numbers for telescope_indices"
-            )
-            logger.error(msg)
-            raise
-    else:
-        indices = None
-    # If the hdf5 output file already exists, the results are appended to it.
-    if (Path(corsika_histograms_instance.hdf5_file_name).exists()) and (
-        args_dict["hdf5"] or args_dict["event_1d_histograms"] or args_dict["event_2d_histograms"]
-    ):
-        msg = (
-            f"Output hdf5 file {corsika_histograms_instance.hdf5_file_name} already exists. "
-            f"Overwriting it."
-        )
-        logger.warning(msg)
-        overwrite = True
-    else:
-        overwrite = False
-    corsika_histograms_instance.set_histograms(
-        telescope_indices=indices,
+    corsika_histograms_instance.run_export_pipeline(
         individual_telescopes=args_dict["individual_telescopes"],
         hist_config=args_dict["hist_config"],
+        indices_arg=args_dict["telescope_indices"],
+        write_pdf=args_dict["pdf"],
+        write_hdf5=args_dict["hdf5"],
+        event1d=args_dict["event_1d_histograms"],
+        event2d=args_dict["event_2d_histograms"],
+        test=args_dict["test"],
     )
 
-    # Cherenkov photons
-    if args_dict["pdf"]:
-        _plot_figures(
-            corsika_histograms_instance=corsika_histograms_instance, test=args_dict["test"]
-        )
-    if args_dict["hdf5"]:
-        corsika_histograms_instance.export_histograms(overwrite=overwrite)
-
-    # Event information
-    if args_dict["event_1d_histograms"] is not None:
-        _derive_event_1d_histograms(
-            corsika_histograms_instance,
-            args_dict["event_1d_histograms"],
-            args_dict["pdf"],
-            args_dict["hdf5"],
-            overwrite=not args_dict["hdf5"],
-        )
-    if args_dict["event_2d_histograms"] is not None:
-        _derive_event_2d_histograms(
-            corsika_histograms_instance,
-            args_dict["event_2d_histograms"],
-            args_dict["pdf"],
-            args_dict["hdf5"],
-            overwrite=not (args_dict["hdf5"] or args_dict["event_1d_histograms"]),
-        )
-
     final_time = time.time()
     logger.info(
         f"Finalizing the application. Total time needed: {round(final_time - initial_time)}s."

simtools/applications/maintain_simulation_model_add_production.py

@@ -0,0 +1,105 @@
+r"""
+Generate a new simulation model production and update tables and model parameters.
+
+This script is used to maintain the simulation model repository. It allows to create
+new production tables by copying an existing base version and applies modifications
+to production tables and model parameters as provided in a YAML file (see the example file below).
+
+Two main use cases are covered by this script:
+
+1. full_update: Create a complete new set of production tables (e.g. for new major or minor
+   versions of the simulation models). This will copy all production tables from the source
+   directory and apply the modifications to the tables that are listed in the modifications file.
+
+2. patch_update: Create a set of new production tables including the changes defined in the
+   modifications file. No unmodified tables are copied. For new production tables with patch
+   modifications, the key-value pair 'base_model_version: <base_model version>' is added.
+
+Both use cases will also apply the modifications to the model parameters as defined in the
+modifications file.
+
+Example
+-------
+
+The following example applies a patch update with changes defined in a YAML file.
+
+.. code-block:: console
+
+    simtools-maintain-simulation-model-add-new-production \\
+        --simulation_models_path ../simulation-models-dev/simulation-models/ \\
+        --base_model_version 6.0.0 \\
+        --modifications tests/resources/production_tables_changes_for_threshold_study_6.2.0.yml \\
+        --patch_update
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.model import model_repository
+
+
+def _parse(label, description):
+    """
+    Parse command line arguments.
+
+    Returns
+    -------
+    dict
+        Parsed command-line arguments.
+    """
+    config = configurator.Configurator(label=label, description=description)
+    config.parser.add_argument(
+        "--simulation_models_path",
+        type=str,
+        required=True,
+        help="Path to the simulation models repository.",
+    )
+    config.parser.add_argument(
+        "--base_model_version",
+        type=str,
+        required=True,
+        help="Base model version (which is the source production table subdirectory to copy from).",
+    )
+    config.parser.add_argument(
+        "--modifications",
+        type=str,
+        required=True,
+        help="File containing the list of changes to apply.",
+    )
+    update_group = config.parser.add_mutually_exclusive_group(required=True)
+    update_group.add_argument(
+        "--full_update",
+        action="store_true",
+        default=False,
+        help=(
+            "Create a full new set of production tables by copying all tables from the "
+            "base version and applying the modifications to the relevant tables."
+        ),
+    )
+    update_group.add_argument(
+        "--patch_update",
+        action="store_true",
+        default=False,
+        help=(
+            "Create a new set of production tables including only the changes defined in the "
+            "modifications file. No unmodified tables are copied."
+        ),
+    )
+
+    return config.initialize(db_config=False, output=False)
+
+
+def main():  # noqa: D103
+    label = Path(__file__).stem
+    args_dict, _ = _parse(label=label, description="Generate a new simulation model production")
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    model_repository.generate_new_production(args_dict)
+
+
+if __name__ == "__main__":
+    main()