gammasimtools 0.20.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

simtools/applications/generate_corsika_histograms.py

@@ -165,15 +165,11 @@ r"""
 """
 
 import logging
-import re
 import time
 from pathlib import Path
 
-import numpy as np
-
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.corsika import corsika_histograms_visualize
 from simtools.corsika.corsika_histograms import CorsikaHistograms
 from simtools.io import io_handler
 
@@ -284,141 +280,13 @@ def _parse(label, description):
     return config_parser, _
 
 
-def _plot_figures(corsika_histograms_instance, test=False):
-    """
-    Auxiliary function to centralize the plotting functions.
-
-    Parameters
-    ----------
-    corsika_histograms_instance: CorsikaHistograms instance.
-        The CorsikaHistograms instance created in main.
-    test: bool
-        If true plots the figures for the first two functions only.
-    """
-    plot_function_names = [
-        plotting_method
-        for plotting_method in dir(corsika_histograms_visualize)
-        if plotting_method.startswith("plot_")
-        and "event_header_distribution" not in plotting_method
-    ]
-    if test:
-        plot_function_names = plot_function_names[:2]
-
-    figure_list = []
-    for function_name in plot_function_names:
-        plot_function = getattr(corsika_histograms_visualize, function_name)
-        figures = plot_function(corsika_histograms_instance)
-        for fig in figures:
-            figure_list.append(fig)
-
-    figure_list = np.array(figure_list).flatten()
-    core_name = re.sub(r"\.hdf5$", "", corsika_histograms_instance.hdf5_file_name)
-    output_file_name = Path(corsika_histograms_instance.output_path).joinpath(f"{core_name}.pdf")
-    corsika_histograms_visualize.save_figs_to_pdf(figure_list, output_file_name)
-
-
-def _derive_event_1d_histograms(
-    corsika_histograms_instance, event_1d_header_keys, pdf, hdf5, overwrite=False
-):
-    """
-    Auxiliary function to derive the histograms for the arguments given by event_1d_histograms.
-
-    Parameters
-    ----------
-    corsika_histograms_instance: CorsikaHistograms instance.
-        The CorsikaHistograms instance created in main.
-    event_1d_header_keys: str
-        Generate 1D histograms for elements given in event_1d_header_keys from the CORSIKA event
-        header and save into hdf5/pdf files.
-    pdf: bool
-        If true, histograms are saved into a pdf file.
-    hdf5: bool
-        If true, histograms are saved into hdf5 files.
-    overwrite: bool
-        If true, overwrites the current output hdf5 file.
-    """
-    figure_list = []
-    for event_header_element in event_1d_header_keys:
-        if pdf:
-            figure = corsika_histograms_visualize.plot_1d_event_header_distribution(
-                corsika_histograms_instance, event_header_element
-            )
-            figure_list.append(figure)
-        if hdf5:
-            corsika_histograms_instance.export_event_header_1d_histogram(
-                event_header_element, bins=50, hist_range=None, overwrite=overwrite
-            )
-    if pdf:
-        figures_list = np.array(figure_list).flatten()
-        output_file_name = Path(corsika_histograms_instance.output_path).joinpath(
-            f"{corsika_histograms_instance.hdf5_file_name}_event_1d_histograms.pdf"
-        )
-        corsika_histograms_visualize.save_figs_to_pdf(figures_list, output_file_name)
-
-
-def _derive_event_2d_histograms(
-    corsika_histograms_instance, event_2d_header_keys, pdf, hdf5, overwrite=False
-):
-    """
-    Auxiliary function to derive the histograms for the arguments given by event_1d_histograms.
-
-    If an odd number of event header keys are given, the last one is discarded.
-
-    Parameters
-    ----------
-    corsika_histograms_instance: CorsikaHistograms instance.
-        The CorsikaHistograms instance created in main.
-    event_2d_header_keys: str
-        Generate 1D histograms for elements given in event_1d_header_keys from the CORSIKA event
-        header and save into hdf5/pdf files.
-    pdf: bool
-        If true, histograms are saved into a pdf file.
-    hdf5: bool
-        If true, histograms are saved into hdf5 files.
-    overwrite: bool
-        If true, overwrites the current output hdf5 file.
-    """
-    figure_list = []
-    for i_event_header_element, _ in enumerate(event_2d_header_keys[::2]):
-        # [::2] to discard the last one in case an odd number of keys are passed
-
-        if len(event_2d_header_keys) % 2 == 1:  # if odd number of keys
-            msg = (
-                "An odd number of keys was passed to generate 2D histograms."
-                "The last key is being ignored."
-            )
-            logger.warning(msg)
-
-        if pdf:
-            figure = corsika_histograms_visualize.plot_2d_event_header_distribution(
-                corsika_histograms_instance,
-                event_2d_header_keys[i_event_header_element],
-                event_2d_header_keys[i_event_header_element + 1],
-            )
-            figure_list.append(figure)
-        if hdf5:
-            corsika_histograms_instance.export_event_header_2d_histogram(
-                event_2d_header_keys[i_event_header_element],
-                event_2d_header_keys[i_event_header_element + 1],
-                bins=50,
-                hist_range=None,
-                overwrite=overwrite,
-            )
-    if pdf:
-        figures_list = np.array(figure_list).flatten()
-        output_file_name = Path(corsika_histograms_instance.output_path).joinpath(
-            f"{corsika_histograms_instance.hdf5_file_name}_event_2d_histograms.pdf"
-        )
-        corsika_histograms_visualize.save_figs_to_pdf(figures_list, output_file_name)
-
-
 def main():  # noqa: D103
     label = Path(__file__).stem
     description = "Generate histograms for the Cherenkov photons saved in the CORSIKA IACT file."
     io_handler_instance = io_handler.IOHandler()
     args_dict, _ = _parse(label, description)
 
-    output_path = io_handler_instance.get_output_directory(label, sub_dir="application-plots")
+    output_path = io_handler_instance.get_output_directory()
 
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
     initial_time = time.time()
@@ -427,62 +295,17 @@ def main():  # noqa: D103
     corsika_histograms_instance = CorsikaHistograms(
         args_dict["iact_file"], output_path=output_path, hdf5_file_name=args_dict["hdf5_file_name"]
     )
-    if args_dict["telescope_indices"] is not None:
-        try:
-            indices = np.array(args_dict["telescope_indices"]).astype(int)
-        except ValueError:
-            msg = (
-                f"{args_dict['telescope_indices']} not a valid input. "
-                f"Please use integer numbers for telescope_indices"
-            )
-            logger.error(msg)
-            raise
-    else:
-        indices = None
-    # If the hdf5 output file already exists, the results are appended to it.
-    if (Path(corsika_histograms_instance.hdf5_file_name).exists()) and (
-        args_dict["hdf5"] or args_dict["event_1d_histograms"] or args_dict["event_2d_histograms"]
-    ):
-        msg = (
-            f"Output hdf5 file {corsika_histograms_instance.hdf5_file_name} already exists. "
-            f"Overwriting it."
-        )
-        logger.warning(msg)
-        overwrite = True
-    else:
-        overwrite = False
-    corsika_histograms_instance.set_histograms(
-        telescope_indices=indices,
+    corsika_histograms_instance.run_export_pipeline(
         individual_telescopes=args_dict["individual_telescopes"],
         hist_config=args_dict["hist_config"],
+        indices_arg=args_dict["telescope_indices"],
+        write_pdf=args_dict["pdf"],
+        write_hdf5=args_dict["hdf5"],
+        event1d=args_dict["event_1d_histograms"],
+        event2d=args_dict["event_2d_histograms"],
+        test=args_dict["test"],
     )
 
-    # Cherenkov photons
-    if args_dict["pdf"]:
-        _plot_figures(
-            corsika_histograms_instance=corsika_histograms_instance, test=args_dict["test"]
-        )
-    if args_dict["hdf5"]:
-        corsika_histograms_instance.export_histograms(overwrite=overwrite)
-
-    # Event information
-    if args_dict["event_1d_histograms"] is not None:
-        _derive_event_1d_histograms(
-            corsika_histograms_instance,
-            args_dict["event_1d_histograms"],
-            args_dict["pdf"],
-            args_dict["hdf5"],
-            overwrite=not args_dict["hdf5"],
-        )
-    if args_dict["event_2d_histograms"] is not None:
-        _derive_event_2d_histograms(
-            corsika_histograms_instance,
-            args_dict["event_2d_histograms"],
-            args_dict["pdf"],
-            args_dict["hdf5"],
-            overwrite=not (args_dict["hdf5"] or args_dict["event_1d_histograms"]),
-        )
-
     final_time = time.time()
     logger.info(
         f"Finalizing the application. Total time needed: {round(final_time - initial_time)}s."

simtools/applications/maintain_simulation_model_add_production.py

@@ -0,0 +1,81 @@
+r"""
+Generate a new simulation model production and update tables and model parameters.
+
+This script is used to maintain the simulation model repository. It allows to create
+new production tables by copying an existing base version and applies modifications
+to production tables and model parameters as provided in a YAML file (see the example file below).
+
+Two main use cases are covered by this script:
+
+1. full update: Create a complete new set of production tables (e.g. for new major or minor
+   versions of the simulation models). This will copy all production tables from the source
+   directory and apply the modifications to the tables that are listed in the modifications file.
+
+2. patch update: Create a set of new production tables including the changes defined in the
+   modifications file. No unmodified tables are copied. For new production tables with patch
+   modifications, the key-value pair 'base_model_version: <base_model version>' is added.
+
+Both use cases will also apply the modifications to the model parameters as defined in the
+modifications file.
+
+Example
+-------
+
+The following example applies a patch update with changes defined in a YAML file.
+
+.. code-block:: console
+
+    simtools-maintain-simulation-model-add-new-production \\
+        --simulation_models_path ../simulation-models-dev/simulation-models/ \\
+        --modifications tests/resources/production_tables_changes_for_threshold_study_6.2.0.yml
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.model import model_repository
+
+
+def _parse(label, description):
+    """
+    Parse command line arguments.
+
+    Returns
+    -------
+    dict
+        Parsed command-line arguments.
+    """
+    config = configurator.Configurator(label=label, description=description)
+    config.parser.add_argument(
+        "--simulation_models_path",
+        type=str,
+        required=True,
+        help="Path to the simulation models repository.",
+    )
+    config.parser.add_argument(
+        "--modifications",
+        type=str,
+        required=True,
+        help="File containing the list of changes to apply.",
+    )
+
+    return config.initialize(db_config=False, output=False)
+
+
+def main():  # noqa: D103
+    label = Path(__file__).stem
+    args_dict, _ = _parse(label=label, description="Generate a new simulation model production")
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    model_repository.generate_new_production(
+        modifications=args_dict["modifications"],
+        simulation_models_path=Path(args_dict["simulation_models_path"]),
+    )
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/merge_tables.py

@@ -88,7 +88,7 @@ def main():  # noqa: D103
         args_dict["input_files"], [".hdf5", ".gz"]
     )
 
-    output_path = io_handler.IOHandler().get_output_directory(label)
+    output_path = io_handler.IOHandler().get_output_directory()
     output_filepath = Path(output_path).joinpath(f"{args_dict['output_file']}")
 
     table_handler.merge_tables(

simtools/applications/plot_array_layout.py

@@ -299,8 +299,7 @@ def main():
 
         visualize.save_figure(
             fig_out,
-            io_handler_instance.get_output_directory(label, sub_dir="application-plots")
-            / plot_file_name,
+            io_handler_instance.get_output_directory() / plot_file_name,
             dpi=400,
         )
         plt.close()

simtools/applications/plot_simtel_events.py

@@ -81,87 +81,8 @@ from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.corsika.corsika_histograms_visualize import save_figs_to_pdf
-from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io import io_handler
-from simtools.visualization.plot_simtel_events import (
-    plot_simtel_event_image,
-    plot_simtel_integrated_pedestal_image,
-    plot_simtel_integrated_signal_image,
-    plot_simtel_peak_timing,
-    plot_simtel_step_traces,
-    plot_simtel_time_traces,
-    plot_simtel_waveform_matrix,
-)
-
-PLOT_CHOICES = {
-    "event_image": "event_image",
-    "time_traces": "time_traces",
-    "waveform_matrix": "waveform_matrix",
-    "step_traces": "step_traces",
-    "integrated_signal_image": "integrated_signal_image",
-    "integrated_pedestal_image": "integrated_pedestal_image",
-    "peak_timing": "peak_timing",
-    "all": "all",
-}
-
-
-def _call_peak_timing(
-    filename,
-    *,
-    tel_id=None,
-    sum_threshold=10.0,
-    peak_width=8,
-    examples=3,
-    timing_bins=None,
-    event_index=None,
-):
-    """Call ``plot_simtel_peak_timing`` and support optional ``return_stats``.
-
-    Parameters
-    ----------
-    filename : pathlib.Path or str
-        Path to the input simtel file.
-    tel_id : int, optional
-        Telescope ID to visualize.
-    sum_threshold : float, default 10.0
-        Minimum pixel sum to consider a pixel.
-    peak_width : int, default 8
-        Expected peak width in samples.
-    examples : int, default 3
-        Number of example traces to draw.
-    timing_bins : int or None, optional
-        Number of bins for timing histogram (contiguous if not set).
-    event_index : int or None, optional
-        0-based index of the event to plot; default is the first event.
-
-    Returns
-    -------
-    object or None
-        The matplotlib Figure if available, otherwise ``None``.
-    """
-    try:
-        fig_stats = plot_simtel_peak_timing(
-            filename,
-            tel_id=tel_id,
-            sum_threshold=sum_threshold,
-            peak_width=peak_width,
-            examples=examples,
-            timing_bins=timing_bins,
-            return_stats=True,
-            event_index=event_index,
-        )
-        return fig_stats[0] if isinstance(fig_stats, tuple) else fig_stats
-    except TypeError:
-        return plot_simtel_peak_timing(
-            filename,
-            tel_id=tel_id,
-            sum_threshold=sum_threshold,
-            peak_width=peak_width,
-            examples=examples,
-            timing_bins=timing_bins,
-            event_index=event_index,
-        )
+from simtools.visualization.plot_simtel_events import PLOT_CHOICES, generate_and_save_plots
 
 
 def _parse(label: str):
@@ -186,7 +107,6 @@ def _parse(label: str):
         default=["event_image"],
         choices=sorted(PLOT_CHOICES),
     )
-    # common plotting options
     config.parser.add_argument("--tel_id", type=int, default=None, help="Telescope ID")
     config.parser.add_argument(
         "--n_pixels", type=int, default=3, help="For time_traces: number of pixel traces"
@@ -237,7 +157,6 @@ def _parse(label: str):
         default=None,
         help="0-based index of the event to plot; default is the first event",
     )
-    # outputs
     config.parser.add_argument(
         "--output_file",
         type=str,
@@ -257,123 +176,6 @@ def _parse(label: str):
     return config.initialize(db_config=False, require_command_line=True)
 
 
-def _save_png(fig, out_dir: Path, stem: str, suffix: str, dpi: int):
-    """Save ``fig`` as a PNG into ``out_dir`` using ``stem`` and ``suffix``.
-
-    Errors during saving are logged as warnings and otherwise ignored.
-    """
-    png_path = out_dir.joinpath(f"{stem}_{suffix}.png")
-    try:
-        fig.savefig(png_path, dpi=dpi, bbox_inches="tight")
-    except Exception as ex:  # pylint:disable=broad-except
-        logging.getLogger(__name__).warning("Failed to save PNG %s: %s", png_path, ex)
-
-
-def _make_output_paths(
-    ioh: io_handler.IOHandler, base: str | None, input_file: Path
-) -> tuple[Path, Path]:
-    """Return (out_dir, pdf_path) based on base and input_file."""
-    out_dir = ioh.get_output_directory(label=Path(__file__).stem)
-    if base:
-        pdf_path = ioh.get_output_file(f"{base}_{input_file.stem}")
-    else:
-        pdf_path = ioh.get_output_file(input_file.stem)
-    pdf_path = Path(f"{pdf_path}.pdf") if pdf_path.suffix != ".pdf" else Path(pdf_path)
-    return out_dir, pdf_path
-
-
-def _collect_figures_for_file(
-    filename: Path,
-    plots: list[str],
-    args: dict,
-    out_dir: Path,
-    base_stem: str,
-    save_pngs: bool,
-    dpi: int,
-):
-    """Generate the selected plots for a single sim_telarray file.
-
-    Returns a list of figures. If ``save_pngs`` is True, also writes PNGs to
-    ``out_dir`` using ``base_stem`` for filenames.
-    """
-    logger = logging.getLogger(__name__)
-    figures: list[object] = []
-
-    def add(fig, tag: str):
-        if fig is not None:
-            figures.append(fig)
-            if save_pngs:
-                _save_png(fig, out_dir, base_stem, tag, dpi)
-        else:
-            logger.warning("Plot '%s' returned no figure for %s", tag, filename)
-
-    plots_to_run = (
-        [
-            "event_image",
-            "time_traces",
-            "waveform_matrix",
-            "step_traces",
-            "integrated_signal_image",
-            "integrated_pedestal_image",
-            "peak_timing",
-        ]
-        if "all" in plots
-        else list(plots)
-    )
-
-    # function name -> (callable, defaults)
-    dispatch: dict[str, tuple[object, dict[str, object]]] = {
-        "event_image": (
-            plot_simtel_event_image,
-            {"distance": None, "event_index": None},
-        ),
-        "time_traces": (
-            plot_simtel_time_traces,
-            {"tel_id": None, "n_pixels": 3, "event_index": None},
-        ),
-        "waveform_matrix": (
-            plot_simtel_waveform_matrix,
-            {"tel_id": None, "vmax": None, "event_index": None},
-        ),
-        "step_traces": (
-            plot_simtel_step_traces,
-            {"tel_id": None, "pixel_step": None, "max_pixels": None, "event_index": None},
-        ),
-        "integrated_signal_image": (
-            plot_simtel_integrated_signal_image,
-            {"tel_id": None, "half_width": 8, "event_index": None},
-        ),
-        "integrated_pedestal_image": (
-            plot_simtel_integrated_pedestal_image,
-            {"tel_id": None, "half_width": 8, "offset": 16, "event_index": None},
-        ),
-        "peak_timing": (
-            _call_peak_timing,
-            {
-                "tel_id": None,
-                "sum_threshold": 10.0,
-                "peak_width": 8,
-                "examples": 3,
-                "timing_bins": None,
-                "event_index": None,
-            },
-        ),
-    }
-
-    for plot_name in plots_to_run:
-        entry = dispatch.get(plot_name)
-        if entry is None:
-            logger.warning("Unknown plot selection '%s'", plot_name)
-            continue
-        func, defaults = entry
-        # Build kwargs with user args overriding defaults
-        kwargs = {k: args.get(k, v) for k, v in defaults.items()}
-        fig = func(filename, **kwargs)  # type: ignore[misc]
-        add(fig, plot_name)
-
-    return figures
-
-
 def main():
     """Generate plots from sim_telarray files."""
     label = Path(__file__).stem
@@ -383,38 +185,10 @@ def main():
     logger.setLevel(gen.get_log_level_from_user(args.get("log_level", "INFO")))
 
     ioh = io_handler.IOHandler()
-
     simtel_files = [Path(p).expanduser() for p in gen.ensure_iterable(args["simtel_files"])]
     plots = list(gen.ensure_iterable(args.get("plots")))
 
-    for simtel in simtel_files:
-        out_dir, pdf_path = _make_output_paths(ioh, args.get("output_file"), simtel)
-        figures = _collect_figures_for_file(
-            filename=simtel,
-            plots=plots,
-            args=args,
-            out_dir=out_dir,
-            base_stem=simtel.stem,
-            save_pngs=bool(args.get("save_pngs", False)),
-            dpi=int(args.get("dpi", 300)),
-        )
-
-        if not figures:
-            logger.warning("No figures produced for %s", simtel)
-            continue
-
-        # Save a multipage PDF
-        try:
-            save_figs_to_pdf(figures, pdf_path)
-            logger.info("Saved PDF: %s", pdf_path)
-        except Exception as ex:  # pylint:disable=broad-except
-            logger.error("Failed to save PDF %s: %s", pdf_path, ex)
-
-        # Dump run metadata alongside PDF
-        try:
-            MetadataCollector.dump(args, pdf_path, add_activity_name=True)
-        except Exception as ex:  # pylint:disable=broad-except
-            logger.warning("Failed to write metadata for %s: %s", pdf_path, ex)
+    generate_and_save_plots(simtel_files=simtel_files, plots=plots, args=args, ioh=ioh)
 
 
 if __name__ == "__main__":

simtools/applications/print_version.py

@@ -39,7 +39,7 @@ def _parse(label, description, usage):
     """
     config = configurator.Configurator(label=label, description=description, usage=usage)
 
-    return config.initialize(db_config=True, output=True)
+    return config.initialize(db_config=True, output=True, require_command_line=False)
 
 
 def main():
@@ -68,12 +68,13 @@ def main():
         key, value = version_entry.split(": ", 1)
         version_dict[key] = value
 
-    ascii_handler.write_data_to_file(
-        data=version_dict,
-        output_file=io_handler_instance.get_output_file(
-            args_dict.get("output_file", "simtools_version.json"), label=label
-        ),
-    )
+    if not args_dict.get("output_file_from_default", False):
+        ascii_handler.write_data_to_file(
+            data=version_dict,
+            output_file=io_handler_instance.get_output_file(
+                args_dict.get("output_file", "simtools_version.json")
+            ),
+        )
 
 
 if __name__ == "__main__":

simtools/applications/production_derive_statistics.py

@@ -149,9 +149,8 @@ def main():
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     _io_handler = io_handler.IOHandler()
-    output_path = _io_handler.get_output_directory(label, sub_dir="")
 
-    manager = ProductionStatisticsHandler(args_dict, output_path=output_path)
+    manager = ProductionStatisticsHandler(args_dict, output_path=_io_handler.get_output_directory())
     manager.run()
 
 

simtools/applications/production_generate_grid.py

@@ -157,7 +157,7 @@ def main():
     _logger = logging.getLogger()
     _logger.setLevel(logging.INFO)
 
-    output_path = io_handler.IOHandler().get_output_directory(label)
+    output_path = io_handler.IOHandler().get_output_directory()
     output_filepath = Path(output_path).joinpath(f"{args_dict['output_file']}")
 
     axes = load_axes(args_dict["axes"])