gammasimtools 0.18.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/applications/derive_psf_parameters.py

@@ -17,6 +17,16 @@ r"""
     the Root Mean Squared Deviation between data and simulations. The range in which the \
     parameter are drawn uniformly are defined based on the previous value on the telescope model.
 
+    The optimization workflow includes:
+
+    * Loading and preprocessing PSF data from measurement files
+    * Generating random parameter combinations for optimization
+    * Running ray-tracing simulations for each parameter set
+    * Calculating RMSD between measured and simulated PSF curves
+    * Identifying the best-fit parameters with minimum RMSD
+    * Creating comprehensive plots and D80 vs off-axis angle analysis
+    * Optionally exporting optimized parameters as simulation model files
+
     The assumption are:
 
     a) mirror_align_random_horizontal and mirror_align_random_vertical are the same.
@@ -37,6 +47,8 @@ r"""
         Telescope model name (e.g. LST-1, SST-D, ...).
     model_version (str, optional)
         Model version.
+    parameter_version (str, optional)
+        Parameter version for model parameter file export.
     src_distance (float, optional)
         Source distance in km.
     zenith (float, optional)
@@ -49,6 +61,12 @@ r"""
         Keep the first entry of mirror_reflection_random_angle fixed.
     test (activation mode, optional)
         If activated, application will be faster by simulating fewer photons.
+    write_psf_parameters (activation mode, optional)
+        Write the optimized PSF parameters as simulation model parameter files.
+    random_seed (int, optional)
+        Random seed for parameter generation.
+    n_runs (int, optional)
+        Number of parameter combinations to test.
 
     Example
     -------
@@ -67,58 +85,36 @@ r"""
     .. code-block:: console
 
         simtools-derive-psf-parameters --site North --telescope LSTN-01 \\
-            --model_version 6.0.0 --data PSFcurve_data_v2.txt --plot_all --test
+            --model_version 6.0.0 --data tests/resources/PSFcurve_data_v2.txt --plot_all --test
 
-    The output is saved in simtools-output/derive_psf_parameters.
-
-    Expected final print-out message:
+    Run with parameter export:
 
     .. code-block:: console
 
-        Best parameters:
-        mirror_reflection_random_angle = [0.006, 0.133, 0.005]
-        mirror_align_random_horizontal = [0.005, 28.0, 0.0, 0.0]
-        mirror_align_random_vertical = [0.005, 28.0, 0.0, 0.0]
+        simtools-derive-psf-parameters --site North --telescope LSTN-01 \\
+            --model_version 6.0.0 \\
+            --data tests/resources/PSFcurve_data_v2.txt --write_psf_parameters
+
+    The output is saved in simtools-output/tune_psf.
+
+    Output files include:
+
+    * Parameter optimization results in tested_psf_parameters.txt
+    * PSF comparison plots in tune_psf_[telescope].pdf
+    * D80 vs off-axis angle plots (d80_vs_offaxis_cm.png, d80_vs_offaxis_deg.png)
+    * Optimized simulation model parameter files (if --write_psf_parameters is specified)
+    * Cumulative PSF plots for all tested combinations (if --plot_all is specified)
 
 """
 
 import logging
-from collections import OrderedDict
-
-import astropy.units as u
-import matplotlib.pyplot as plt
-import numpy as np
-from matplotlib.backends.backend_pdf import PdfPages
+from pathlib import Path
 
-import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.model.model_utils import initialize_simulation_models
-from simtools.ray_tracing.ray_tracing import RayTracing
-from simtools.visualization import visualize
-
-
-def load_data(data_file):
-    """
-    Load data from a text file containing cumulative PSF measurements.
-
-    Parameters
-    ----------
-    data_file : str
-        Name of the data file with the measured cumulative PSF.
-
-    Returns
-    -------
-    numpy.ndarray
-        Loaded and processed data from the file.
-    """
-    radius_cm = "Radius [cm]"
-    cumulative_psf = "Cumulative PSF"
-    d_type = {"names": (radius_cm, cumulative_psf), "formats": ("f8", "f8")}
-    data = np.loadtxt(data_file, dtype=d_type, usecols=(0, 2))
-    data[radius_cm] *= 0.1
-    data[cumulative_psf] /= np.max(np.abs(data[cumulative_psf]))
-    return data
+from simtools.ray_tracing import psf_parameter_optimisation as psf_opt
+from simtools.utils.general import get_log_level_from_user
 
 
 def _parse():
@@ -151,256 +147,37 @@ def _parse():
         help=("Keep the first entry of mirror_reflection_random_angle fixed."),
         action="store_true",
     )
-    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
-
-
-def add_parameters(
-    all_parameters,
-    mirror_reflection,
-    mirror_align,
-    mirror_reflection_fraction=0.15,
-    mirror_reflection_2=0.035,
-):
-    """
-    Transform and add parameters to the all_parameters list.
-
-    Parameters
-    ----------
-    mirror_reflection : float
-        The random angle of mirror reflection.
-
-    mirror_align : float
-        The random angle for mirror alignment (both horizontal and vertical).
-
-    mirror_reflection_fraction : float, optional
-        The fraction of the mirror reflection. Default is 0.15.
-
-    mirror_reflection_2 : float, optional
-        A secondary random angle for mirror reflection. Default is 0.035.
-
-    Returns
-    -------
-    None
-        Updates the all_parameters list in place.
-    """
-    # If we want to start from values different than the ones currently in the model:
-    # align = 0.0046
-    # pars_to_change = {
-    #     'mirror_reflection_random_angle': '0.0075 0.125 0.0037',
-    #     'mirror_align_random_horizontal': f'{align} 28 0 0',
-    #     'mirror_align_random_vertical': f'{align} 28 0 0',
-    # }
-    # tel_model.change_multiple_parameters(**pars_to_change)
-    pars = {
-        "mirror_reflection_random_angle": [
-            mirror_reflection,
-            mirror_reflection_fraction,
-            mirror_reflection_2,
-        ],
-        "mirror_align_random_horizontal": [mirror_align, 28.0, 0.0, 0.0],
-        "mirror_align_random_vertical": [mirror_align, 28.0, 0.0, 0.0],
-    }
-    all_parameters.append(pars)
-
-
-def get_previous_values(tel_model, logger):
-    """
-    Retrieve previous parameter values from the telescope model.
-
-    Parameters
-    ----------
-    tel_model : TelescopeModel
-        Telescope model object.
-    logger : logging.Logger
-        Logger object for logging messages.
-
-    Returns
-    -------
-    tuple
-        Tuple containing the previous values of mirror_reflection_random_angle (first entry),
-        mirror_reflection_fraction, second entry), mirror_reflection_random_angle (third entry),
-        and mirror_align_random_horizontal/vertical.
-    """
-    split_par = tel_model.get_parameter_value("mirror_reflection_random_angle")
-    mrra_0, mfr_0, mrra2_0 = split_par[0], split_par[1], split_par[2]
-    mar_0 = tel_model.get_parameter_value("mirror_align_random_horizontal")[0]
-
-    logger.debug(
-        "Previous parameter values:\n"
-        f"MRRA = {mrra_0!s}\n"
-        f"MRF = {mfr_0!s}\n"
-        f"MRRA2 = {mrra2_0!s}\n"
-        f"MAR = {mar_0!s}\n"
+    config.parser.add_argument(
+        "--write_psf_parameters",
+        help=("Write the optimized PSF parameters as simulation model parameter files"),
+        action="store_true",
+        required=False,
     )
-
-    return mrra_0, mfr_0, mrra2_0, mar_0
-
-
-def generate_random_parameters(
-    all_parameters, n_runs, args_dict, mrra_0, mfr_0, mrra2_0, mar_0, logger
-):
-    """
-    Generate random parameters for tuning.
-
-    Parameters
-    ----------
-    all_parameters : list
-        List to store all parameter sets.
-    n_runs : int
-        Number of random parameter combinations to test.
-    args_dict : dict
-        Dictionary containing parsed command-line arguments.
-    mrra_0 : float
-        Initial value of mirror_reflection_random_angle.
-    mfr_0 : float
-        Initial value of mirror_reflection_fraction.
-    mrra2_0 : float
-        Initial value of the second mirror_reflection_random_angle.
-    mar_0 : float
-        Initial value of mirror_align_random_horizontal/vertical.
-    logger : logging.Logger
-        Logger object for logging messages.
-    """
-    # Range around the previous values are hardcoded
-    # Number of runs is hardcoded
-    if args_dict["fixed"]:
-        logger.debug("fixed=True - First entry of mirror_reflection_random_angle is kept fixed.")
-
-    for _ in range(n_runs):
-        mrra_range = 0.004 if not args_dict["fixed"] else 0
-        mrf_range = 0.1
-        mrra2_range = 0.03
-        mar_range = 0.005
-        rng = np.random.default_rng()
-        mrra = rng.uniform(max(mrra_0 - mrra_range, 0), mrra_0 + mrra_range)
-        mrf = rng.uniform(max(mfr_0 - mrf_range, 0), mfr_0 + mrf_range)
-        mrra2 = rng.uniform(max(mrra2_0 - mrra2_range, 0), mrra2_0 + mrra2_range)
-        mar = rng.uniform(max(mar_0 - mar_range, 0), mar_0 + mar_range)
-        add_parameters(all_parameters, mrra, mar, mrf, mrra2)
-
-
-def load_and_process_data(args_dict):
-    """
-    Load and process data if specified in the command-line arguments.
-
-    Returns
-    -------
-    - data_to_plot: OrderedDict containing loaded and processed data.
-    - radius: Radius data from loaded data (if available).
-    """
-    data_to_plot = OrderedDict()
-    radius = None
-
-    if args_dict["data"] is not None:
-        data_file = gen.find_file(args_dict["data"], args_dict["model_path"])
-        data_to_plot["measured"] = load_data(data_file)
-        radius = data_to_plot["measured"]["Radius [cm]"]
-
-    return data_to_plot, radius
-
-
-def calculate_rmsd(data, sim):
-    """Calculate Root Mean Squared Deviation to be used as metric to find the best parameters."""
-    return np.sqrt(np.mean((data - sim) ** 2))
-
-
-def run_pars(tel_model, site_model, args_dict, pars, data_to_plot, radius, pdf_pages):
-    """
-    Run the tuning for one set of parameters, add a plot to the pdfPages and return RMSD and D80.
-
-    Plotting is optional (if plot=True).
-    """
-    cumulative_psf = "Cumulative PSF"
-
-    if pars is not None:
-        tel_model.change_multiple_parameters(**pars)
-    else:
-        raise ValueError("No best parameters found")
-
-    ray = RayTracing(
-        telescope_model=tel_model,
-        site_model=site_model,
-        simtel_path=args_dict["simtel_path"],
-        zenith_angle=args_dict["zenith"] * u.deg,
-        source_distance=args_dict["src_distance"] * u.km,
-        off_axis_angle=[0.0] * u.deg,
+    config.parser.add_argument(
+        "--random_seed",
+        help="Random seed for parameter generation.",
+        type=int,
+        default=None,
     )
-
-    ray.simulate(test=args_dict["test"], force=True)
-    ray.analyze(force=True, use_rx=False)
-
-    # Plotting cumulative PSF
-    im = ray.images()[0]
-    d80 = im.get_psf()
-
-    if radius is not None:
-        # Simulated cumulative PSF
-        data_to_plot["simulated"] = im.get_cumulative_data(radius * u.cm)
-    else:
-        raise ValueError("Radius data is not available.")
-
-    rmsd = calculate_rmsd(
-        data_to_plot["measured"][cumulative_psf], data_to_plot["simulated"][cumulative_psf]
+    config.parser.add_argument(
+        "--n_runs",
+        help="Number of parameter combinations to test.",
+        type=int,
+        default=5,
+    )
+    return config.initialize(
+        db_config=True,
+        simulation_model=["telescope", "model_version", "parameter_version"],
     )
-
-    if args_dict["plot_all"]:
-        fig = visualize.plot_1d(
-            data_to_plot,
-            plot_difference=True,
-            no_markers=True,
-        )
-        ax = fig.get_axes()[0]
-        ax.set_ylim(0, 1.05)
-        ax.set_title(
-            f"refl_rnd={pars['mirror_reflection_random_angle']}, "
-            f"align_rnd={pars['mirror_align_random_vertical']}"
-        )
-
-        ax.text(
-            0.8,
-            0.3,
-            f"D80 = {d80:.3f} cm\nRMSD = {rmsd:.4f}",
-            verticalalignment="center",
-            horizontalalignment="center",
-            transform=ax.transAxes,
-        )
-        plt.tight_layout()
-        pdf_pages.savefig(fig)
-        plt.clf()
-
-    return d80, rmsd
-
-
-def find_best_parameters(
-    all_parameters, tel_model, site_model, args_dict, data_to_plot, radius, pdf_pages
-):
-    """
-    Find the best parameters from all parameter sets.
-
-    Returns
-    -------
-    - Tuple of best parameters and their D80 value.
-    """
-    min_rmsd = 100
-    best_pars = None
-
-    for pars in all_parameters:
-        _, rmsd = run_pars(tel_model, site_model, args_dict, pars, data_to_plot, radius, pdf_pages)
-        if rmsd < min_rmsd:
-            min_rmsd = rmsd
-            best_pars = pars
-
-    return best_pars, min_rmsd
 
 
 def main():  # noqa: D103
     args_dict, db_config = _parse()
 
-    label = "tune_psf"
+    label = label = Path(__file__).stem
     logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+    logger.setLevel(get_log_level_from_user(args_dict["log_level"]))
 
-    # Output directory to save files related directly to this app
     _io_handler = io_handler.IOHandler()
     output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
     tel_model, site_model = initialize_simulation_models(
@@ -411,34 +188,7 @@ def main():  # noqa: D103
         model_version=args_dict["model_version"],
     )
 
-    all_parameters = []
-    mrra_0, mfr_0, mrra2_0, mar_0 = get_previous_values(tel_model, logger)
-
-    n_runs = 5 if args_dict["test"] else 50
-    generate_random_parameters(
-        all_parameters, n_runs, args_dict, mrra_0, mfr_0, mrra2_0, mar_0, logger
-    )
-
-    data_to_plot, radius = load_and_process_data(args_dict)
-
-    # Preparing figure name
-    plot_file_name = "_".join((label, tel_model.name + ".pdf"))
-    plot_file = output_dir.joinpath(plot_file_name)
-    pdf_pages = PdfPages(plot_file)
-
-    best_pars, _ = find_best_parameters(
-        all_parameters, tel_model, site_model, args_dict, data_to_plot, radius, pdf_pages
-    )
-
-    # Rerunning and plotting the best pars
-    run_pars(tel_model, site_model, args_dict, best_pars, data_to_plot, radius, pdf_pages)
-    plt.close()
-    pdf_pages.close()
-
-    # Printing the results
-    print("Best parameters:")
-    for par, value in best_pars.items():
-        print(f"{par} = {value}")
+    psf_opt.run_psf_optimization_workflow(tel_model, site_model, args_dict, output_dir)
 
 
 if __name__ == "__main__":

simtools/applications/docs_produce_array_element_report.py

@@ -11,7 +11,7 @@ import logging
 from pathlib import Path
 
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.reporting.docs_auto_report_generator import ReportGenerator
 from simtools.reporting.docs_read_parameters import ReadParameters
 from simtools.utils import general as gen

simtools/applications/docs_produce_calibration_reports.py

@@ -5,7 +5,7 @@ r"""Produces a markdown file for calibration reports."""
 import logging
 
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.reporting.docs_read_parameters import ReadParameters
 from simtools.utils import general as gen
 

simtools/applications/docs_produce_model_parameter_reports.py

@@ -11,7 +11,7 @@ Currently only implemented for telescopes.
 import logging
 
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.reporting.docs_auto_report_generator import ReportGenerator
 from simtools.reporting.docs_read_parameters import ReadParameters
 from simtools.utils import general as gen

simtools/applications/docs_produce_simulation_configuration_report.py

@@ -5,7 +5,7 @@ r"""Produces a markdown file for a given simulation configuration."""
 import logging
 
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.reporting.docs_read_parameters import ReadParameters
 from simtools.utils import general as gen
 

simtools/applications/generate_corsika_histograms.py

@@ -175,7 +175,7 @@ import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.corsika import corsika_histograms_visualize
 from simtools.corsika.corsika_histograms import CorsikaHistograms
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 
 logger = logging.getLogger()
 

simtools/applications/generate_default_metadata.py

@@ -21,16 +21,13 @@ r"""
 
     """
 
-import json
 import logging
 from pathlib import Path
 
-import yaml
-
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.data_model import metadata_model
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 
 
 def _parse(label, description):
@@ -74,8 +71,8 @@ def main():  # noqa: D103
         label, description="Generate a default simtools metadata file from a json schema."
     )
 
-    _logger = logging.getLogger()
-    _logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     default_values = metadata_model.get_default_metadata_dict(args_dict["schema"])
 
@@ -83,24 +80,11 @@ def main():  # noqa: D103
         print(default_values)
     else:
         _io_handler = io_handler.IOHandler()
-        _out_file = _io_handler.get_output_file(args_dict["output_file"])
-        _logger.info(f"Writing default values to {_out_file}")
-        if args_dict["output_file"].endswith((".yml", ".yaml")):
-            with open(_out_file, "w", encoding="utf-8") as file:
-                yaml.dump(
-                    default_values,
-                    file,
-                    default_flow_style=False,
-                    sort_keys=False,
-                )
-        if args_dict["output_file"].endswith(".json"):
-            with open(_out_file, "w", encoding="utf-8") as file:
-                json.dump(
-                    default_values,
-                    file,
-                    indent=4,
-                    sort_keys=False,
-                )
+        output_file = _io_handler.get_output_file(args_dict["output_file"])
+        logger.info(f"Writing default values to {output_file}")
+        ascii_handler.write_data_to_file(
+            data=default_values, output_file=output_file, sort_keys=False
+        )
 
 
 if __name__ == "__main__":

simtools/applications/generate_sim_telarray_histograms.py

@@ -53,7 +53,7 @@ from matplotlib.backends.backend_pdf import PdfPages
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.simtel.simtel_io_histograms import SimtelIOHistograms
 
 

simtools/applications/generate_simtel_event_data.py

@@ -24,19 +24,19 @@ The output consists of an HDF5 or FITS file containing the following tables:
 +-------------------+---------+-----------------------------------------------+
 | energy_max        | float32 | Maximum simulated energy (TeV)                |
 +-------------------+---------+-----------------------------------------------+
-| viewcone_min      | float32 | Min viewcone angle (rad)                      |
+| viewcone_min      | float32 | Min viewcone angle (deg)                      |
 +-------------------+---------+-----------------------------------------------+
-| viewcone_max      | float32 | Max viewcone angle (rad)                      |
+| viewcone_max      | float32 | Max viewcone angle (deg)                      |
 +-------------------+---------+-----------------------------------------------+
 | core_scatter_min  | float32 | Min core scatter radius (m)                   |
 +-------------------+---------+-----------------------------------------------+
 | core_scatter_max  | float32 | Max core scatter radius (m)                   |
 +-------------------+---------+-----------------------------------------------+
-| zenith            | float32 | Zenith angle (rad)                            |
+| zenith            | float32 | Zenith angle (deg)                            |
 +-------------------+---------+-----------------------------------------------+
-| azimuth           | float32 | Azimuth angle (rad)                           |
+| azimuth           | float32 | Azimuth angle (deg)                           |
 +-------------------+---------+-----------------------------------------------+
-| nsb_level         | float64 | Night sky background level (photons/deg^2/ns) |
+| nsb_level         | float64 | Night sky background level (factor to dark)   |
 +-------------------+---------+-----------------------------------------------+
 
 **SHOWERS**
@@ -56,9 +56,9 @@ The output consists of an HDF5 or FITS file containing the following tables:
 +------------------+---------+-----------------------------------------------+
 | y_core           | float64 | Shower core Y position on ground (m)          |
 +------------------+---------+-----------------------------------------------+
-| shower_azimuth   | float64 | Direction of shower azimuth (rad)             |
+| shower_azimuth   | float64 | Direction of shower azimuth (deg)             |
 +------------------+---------+-----------------------------------------------+
-| shower_altitude  | float64 | Direction of shower altitude (rad)            |
+| shower_altitude  | float64 | Direction of shower altitude (deg)            |
 +------------------+---------+-----------------------------------------------+
 | area_weight      | float64 | Weighting factor for sampling area            |
 +------------------+---------+-----------------------------------------------+
@@ -74,9 +74,9 @@ The output consists of an HDF5 or FITS file containing the following tables:
 +-----------------+---------+-----------------------------------------------+
 | file_id         | int64   | Internal unique identifier for the file       |
 +-----------------+---------+-----------------------------------------------+
-| array_altitude  | float64 | Altitude of array pointing direction (rad)    |
+| array_altitude  | float64 | Altitude of array pointing direction (deg)    |
 +-----------------+---------+-----------------------------------------------+
-| array_azimuth   | float64 | Azimuth of array pointing direction (rad)     |
+| array_azimuth   | float64 | Azimuth of array pointing direction (deg)     |
 +-----------------+---------+-----------------------------------------------+
 | telescope_list  | string  | Comma-separated list of triggered telescopes  |
 +-----------------+---------+-----------------------------------------------+
@@ -130,7 +130,7 @@ from pathlib import Path
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.data_model.metadata_collector import MetadataCollector
-from simtools.io_operations import io_handler, io_table_handler
+from simtools.io import io_handler, table_handler
 from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
 
 
@@ -187,7 +187,7 @@ def main():  # noqa: D103
     output_filepath = io_handler.IOHandler().get_output_file(args_dict["output_file"])
     generator = SimtelIOEventDataWriter(files, args_dict["max_files"])
     tables = generator.process_files()
-    io_table_handler.write_tables(tables, output_filepath, overwrite_existing=True)
+    table_handler.write_tables(tables, output_filepath, overwrite_existing=True)
     MetadataCollector.dump(args_dict=args_dict, output_file=output_filepath.with_suffix(".yml"))
 
     if args_dict["print_dataset_information"] > 0: