gammasimtools 0.20.0__py3-none-any.whl → 0.21.0__py3-none-any.whl

simtools/applications/plot_simtel_events.py

@@ -81,87 +81,8 @@ from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.corsika.corsika_histograms_visualize import save_figs_to_pdf
-from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io import io_handler
-from simtools.visualization.plot_simtel_events import (
-    plot_simtel_event_image,
-    plot_simtel_integrated_pedestal_image,
-    plot_simtel_integrated_signal_image,
-    plot_simtel_peak_timing,
-    plot_simtel_step_traces,
-    plot_simtel_time_traces,
-    plot_simtel_waveform_matrix,
-)
-
-PLOT_CHOICES = {
-    "event_image": "event_image",
-    "time_traces": "time_traces",
-    "waveform_matrix": "waveform_matrix",
-    "step_traces": "step_traces",
-    "integrated_signal_image": "integrated_signal_image",
-    "integrated_pedestal_image": "integrated_pedestal_image",
-    "peak_timing": "peak_timing",
-    "all": "all",
-}
-
-
-def _call_peak_timing(
-    filename,
-    *,
-    tel_id=None,
-    sum_threshold=10.0,
-    peak_width=8,
-    examples=3,
-    timing_bins=None,
-    event_index=None,
-):
-    """Call ``plot_simtel_peak_timing`` and support optional ``return_stats``.
-
-    Parameters
-    ----------
-    filename : pathlib.Path or str
-        Path to the input simtel file.
-    tel_id : int, optional
-        Telescope ID to visualize.
-    sum_threshold : float, default 10.0
-        Minimum pixel sum to consider a pixel.
-    peak_width : int, default 8
-        Expected peak width in samples.
-    examples : int, default 3
-        Number of example traces to draw.
-    timing_bins : int or None, optional
-        Number of bins for timing histogram (contiguous if not set).
-    event_index : int or None, optional
-        0-based index of the event to plot; default is the first event.
-
-    Returns
-    -------
-    object or None
-        The matplotlib Figure if available, otherwise ``None``.
-    """
-    try:
-        fig_stats = plot_simtel_peak_timing(
-            filename,
-            tel_id=tel_id,
-            sum_threshold=sum_threshold,
-            peak_width=peak_width,
-            examples=examples,
-            timing_bins=timing_bins,
-            return_stats=True,
-            event_index=event_index,
-        )
-        return fig_stats[0] if isinstance(fig_stats, tuple) else fig_stats
-    except TypeError:
-        return plot_simtel_peak_timing(
-            filename,
-            tel_id=tel_id,
-            sum_threshold=sum_threshold,
-            peak_width=peak_width,
-            examples=examples,
-            timing_bins=timing_bins,
-            event_index=event_index,
-        )
+from simtools.visualization.plot_simtel_events import PLOT_CHOICES, generate_and_save_plots
 
 
 def _parse(label: str):
@@ -186,7 +107,6 @@ def _parse(label: str):
         default=["event_image"],
         choices=sorted(PLOT_CHOICES),
     )
-    # common plotting options
     config.parser.add_argument("--tel_id", type=int, default=None, help="Telescope ID")
     config.parser.add_argument(
         "--n_pixels", type=int, default=3, help="For time_traces: number of pixel traces"
@@ -237,7 +157,6 @@ def _parse(label: str):
         default=None,
         help="0-based index of the event to plot; default is the first event",
     )
-    # outputs
     config.parser.add_argument(
         "--output_file",
         type=str,
@@ -257,123 +176,6 @@ def _parse(label: str):
     return config.initialize(db_config=False, require_command_line=True)
 
 
-def _save_png(fig, out_dir: Path, stem: str, suffix: str, dpi: int):
-    """Save ``fig`` as a PNG into ``out_dir`` using ``stem`` and ``suffix``.
-
-    Errors during saving are logged as warnings and otherwise ignored.
-    """
-    png_path = out_dir.joinpath(f"{stem}_{suffix}.png")
-    try:
-        fig.savefig(png_path, dpi=dpi, bbox_inches="tight")
-    except Exception as ex:  # pylint:disable=broad-except
-        logging.getLogger(__name__).warning("Failed to save PNG %s: %s", png_path, ex)
-
-
-def _make_output_paths(
-    ioh: io_handler.IOHandler, base: str | None, input_file: Path
-) -> tuple[Path, Path]:
-    """Return (out_dir, pdf_path) based on base and input_file."""
-    out_dir = ioh.get_output_directory(label=Path(__file__).stem)
-    if base:
-        pdf_path = ioh.get_output_file(f"{base}_{input_file.stem}")
-    else:
-        pdf_path = ioh.get_output_file(input_file.stem)
-    pdf_path = Path(f"{pdf_path}.pdf") if pdf_path.suffix != ".pdf" else Path(pdf_path)
-    return out_dir, pdf_path
-
-
-def _collect_figures_for_file(
-    filename: Path,
-    plots: list[str],
-    args: dict,
-    out_dir: Path,
-    base_stem: str,
-    save_pngs: bool,
-    dpi: int,
-):
-    """Generate the selected plots for a single sim_telarray file.
-
-    Returns a list of figures. If ``save_pngs`` is True, also writes PNGs to
-    ``out_dir`` using ``base_stem`` for filenames.
-    """
-    logger = logging.getLogger(__name__)
-    figures: list[object] = []
-
-    def add(fig, tag: str):
-        if fig is not None:
-            figures.append(fig)
-            if save_pngs:
-                _save_png(fig, out_dir, base_stem, tag, dpi)
-        else:
-            logger.warning("Plot '%s' returned no figure for %s", tag, filename)
-
-    plots_to_run = (
-        [
-            "event_image",
-            "time_traces",
-            "waveform_matrix",
-            "step_traces",
-            "integrated_signal_image",
-            "integrated_pedestal_image",
-            "peak_timing",
-        ]
-        if "all" in plots
-        else list(plots)
-    )
-
-    # function name -> (callable, defaults)
-    dispatch: dict[str, tuple[object, dict[str, object]]] = {
-        "event_image": (
-            plot_simtel_event_image,
-            {"distance": None, "event_index": None},
-        ),
-        "time_traces": (
-            plot_simtel_time_traces,
-            {"tel_id": None, "n_pixels": 3, "event_index": None},
-        ),
-        "waveform_matrix": (
-            plot_simtel_waveform_matrix,
-            {"tel_id": None, "vmax": None, "event_index": None},
-        ),
-        "step_traces": (
-            plot_simtel_step_traces,
-            {"tel_id": None, "pixel_step": None, "max_pixels": None, "event_index": None},
-        ),
-        "integrated_signal_image": (
-            plot_simtel_integrated_signal_image,
-            {"tel_id": None, "half_width": 8, "event_index": None},
-        ),
-        "integrated_pedestal_image": (
-            plot_simtel_integrated_pedestal_image,
-            {"tel_id": None, "half_width": 8, "offset": 16, "event_index": None},
-        ),
-        "peak_timing": (
-            _call_peak_timing,
-            {
-                "tel_id": None,
-                "sum_threshold": 10.0,
-                "peak_width": 8,
-                "examples": 3,
-                "timing_bins": None,
-                "event_index": None,
-            },
-        ),
-    }
-
-    for plot_name in plots_to_run:
-        entry = dispatch.get(plot_name)
-        if entry is None:
-            logger.warning("Unknown plot selection '%s'", plot_name)
-            continue
-        func, defaults = entry
-        # Build kwargs with user args overriding defaults
-        kwargs = {k: args.get(k, v) for k, v in defaults.items()}
-        fig = func(filename, **kwargs)  # type: ignore[misc]
-        add(fig, plot_name)
-
-    return figures
-
-
 def main():
     """Generate plots from sim_telarray files."""
     label = Path(__file__).stem
@@ -383,38 +185,10 @@ def main():
     logger.setLevel(gen.get_log_level_from_user(args.get("log_level", "INFO")))
 
     ioh = io_handler.IOHandler()
-
     simtel_files = [Path(p).expanduser() for p in gen.ensure_iterable(args["simtel_files"])]
     plots = list(gen.ensure_iterable(args.get("plots")))
 
-    for simtel in simtel_files:
-        out_dir, pdf_path = _make_output_paths(ioh, args.get("output_file"), simtel)
-        figures = _collect_figures_for_file(
-            filename=simtel,
-            plots=plots,
-            args=args,
-            out_dir=out_dir,
-            base_stem=simtel.stem,
-            save_pngs=bool(args.get("save_pngs", False)),
-            dpi=int(args.get("dpi", 300)),
-        )
-
-        if not figures:
-            logger.warning("No figures produced for %s", simtel)
-            continue
-
-        # Save a multipage PDF
-        try:
-            save_figs_to_pdf(figures, pdf_path)
-            logger.info("Saved PDF: %s", pdf_path)
-        except Exception as ex:  # pylint:disable=broad-except
-            logger.error("Failed to save PDF %s: %s", pdf_path, ex)
-
-        # Dump run metadata alongside PDF
-        try:
-            MetadataCollector.dump(args, pdf_path, add_activity_name=True)
-        except Exception as ex:  # pylint:disable=broad-except
-            logger.warning("Failed to write metadata for %s: %s", pdf_path, ex)
+    generate_and_save_plots(simtel_files=simtel_files, plots=plots, args=args, ioh=ioh)
 
 
 if __name__ == "__main__":

simtools/applications/print_version.py

@@ -39,7 +39,7 @@ def _parse(label, description, usage):
     """
     config = configurator.Configurator(label=label, description=description, usage=usage)
 
-    return config.initialize(db_config=True, output=True)
+    return config.initialize(db_config=True, output=True, require_command_line=False)
 
 
 def main():
@@ -68,12 +68,13 @@ def main():
         key, value = version_entry.split(": ", 1)
         version_dict[key] = value
 
-    ascii_handler.write_data_to_file(
-        data=version_dict,
-        output_file=io_handler_instance.get_output_file(
-            args_dict.get("output_file", "simtools_version.json"), label=label
-        ),
-    )
+    if not args_dict.get("output_file_from_default", False):
+        ascii_handler.write_data_to_file(
+            data=version_dict,
+            output_file=io_handler_instance.get_output_file(
+                args_dict.get("output_file", "simtools_version.json"), label=label
+            ),
+        )
 
 
 if __name__ == "__main__":

simtools/corsika/corsika_histograms.py

@@ -22,6 +22,7 @@ from simtools.io.ascii_handler import collect_data_from_file
 from simtools.io.hdf5_handler import fill_hdf5_table
 from simtools.utils.geometry import convert_2d_to_radial_distr, rotate
 from simtools.utils.names import sanitize_name
+from simtools.visualization import plot_corsika_histograms as visualize
 
 X_AXIS_STRING = "x axis"
 Y_AXIS_STRING = "y axis"
@@ -111,6 +112,86 @@ class CorsikaHistograms:
         self.read_event_information()
         self._initialize_header()
 
+    def parse_telescope_indices(self, indices_arg):
+        """Return telescope indices as ndarray[int] or None.
+
+        Accepts None, a sequence of strings/ints. Raises ValueError on invalid input.
+        """
+        if indices_arg is None:
+            return None
+        try:
+            return np.array(indices_arg).astype(int)
+        except ValueError as exc:
+            msg = (
+                f"{indices_arg} not a valid input. Please use integer numbers for telescope_indices"
+            )
+            self._logger.error(msg)
+            raise ValueError(msg) from exc
+
+    def should_overwrite(
+        self, write_hdf5: bool, event1d: list | None, event2d: list | None
+    ) -> bool:
+        """Return True if output HDF5 exists and any writing flag is requested."""
+        exists = Path(self.hdf5_file_name).exists()
+        if exists and (write_hdf5 or bool(event1d) or bool(event2d)):
+            self._logger.warning(
+                f"Output hdf5 file {self.hdf5_file_name} already exists. Overwriting it."
+            )
+            return True
+        return False
+
+    def run_export_pipeline(
+        self,
+        *,
+        individual_telescopes: bool,
+        hist_config,
+        indices_arg,
+        write_pdf: bool,
+        write_hdf5: bool,
+        event1d: list | None,
+        event2d: list | None,
+        test: bool = False,
+    ) -> dict:
+        """Run the full histogram export pipeline and return output artifact paths.
+
+        Returns a dict with optional keys: pdf_photons, pdf_event1d, pdf_event2d.
+        """
+        outputs: dict[str, Path | None] = {
+            "pdf_photons": None,
+            "pdf_event_1d": None,
+            "pdf_event_2d": None,
+        }
+
+        indices = self.parse_telescope_indices(indices_arg)
+        overwrite = self.should_overwrite(write_hdf5, event1d, event2d)
+
+        self.set_histograms(
+            telescope_indices=indices,
+            individual_telescopes=individual_telescopes,
+            hist_config=hist_config,
+        )
+
+        if write_pdf:
+            pdf_path = visualize.export_all_photon_figures_pdf(self, test=test)
+            outputs["pdf_photons"] = pdf_path
+        if write_hdf5:
+            self.export_histograms(overwrite=overwrite)
+
+        if event1d is not None:
+            outputs["pdf_event_1d"] = visualize.derive_event_1d_histograms(
+                self, event1d, pdf=write_pdf, hdf5=write_hdf5, overwrite=not write_hdf5
+            )
+        if event2d is not None:
+            outputs["pdf_event_2d"] = visualize.derive_event_2d_histograms(
+                self,
+                event2d,
+                pdf=write_pdf,
+                hdf5=write_hdf5,
+                overwrite=not (write_hdf5 or bool(event1d)),
+            )
+
+        return outputs
+
     @property
     def hdf5_file_name(self):
         """

simtools/db/db_handler.py

@@ -18,6 +18,7 @@ from simtools.data_model import validate_data
 from simtools.io import ascii_handler, io_handler
 from simtools.simtel import simtel_table_reader
 from simtools.utils import names, value_conversion
+from simtools.version import resolve_version_to_latest_patch
 
 __all__ = ["DatabaseHandler"]
 
@@ -72,6 +73,11 @@ class DatabaseHandler:
     """
     DatabaseHandler provides the interface to the DB.
 
+    Note the two types of version variables used in this class:
+
+    - db_simulation_model_version (from mongo_db_config): version of the simulation model database
+    - model_version (from production_tables): version of the model contained in the database
+
     Parameters
     ----------
     mongo_db_config: dict
@@ -83,6 +89,7 @@ class DatabaseHandler:
     db_client = None
     production_table_cached = {}
     model_parameters_cached = {}
+    model_versions_cached = {}
 
     def __init__(self, mongo_db_config=None):
         """Initialize the DatabaseHandler class."""
@@ -96,7 +103,7 @@ class DatabaseHandler:
         self._find_latest_simulation_model_db()
         self.db_name = (
             self.get_db_name(
-                model_version=self.mongo_db_config.get("db_simulation_model_version"),
+                db_simulation_model_version=self.mongo_db_config.get("db_simulation_model_version"),
                 model_name=self.mongo_db_config.get("db_simulation_model"),
             )
             if self.mongo_db_config
@@ -110,15 +117,15 @@ class DatabaseHandler:
             with lock:
                 DatabaseHandler.db_client = self._open_mongo_db()
 
-    def get_db_name(self, db_name=None, model_version=None, model_name=None):
+    def get_db_name(self, db_name=None, db_simulation_model_version=None, model_name=None):
         """Build DB name from configuration."""
         if db_name:
             return db_name
-        if model_version and model_name:
-            return f"{model_name}-{model_version.replace('.', '-')}"
-        if model_version or model_name:
+        if db_simulation_model_version and model_name:
+            return f"{model_name}-{db_simulation_model_version.replace('.', '-')}"
+        if db_simulation_model_version or model_name:
             return None
-        return None if (model_version or model_name) else self.db_name
+        return None if (db_simulation_model_version or model_name) else self.db_name
 
     def _validate_mongo_db_config(self, mongo_db_config):
         """Validate the MongoDB configuration."""
@@ -269,6 +276,9 @@ class DatabaseHandler:
                 raise ValueError(
                     "Only one model version can be passed to get_model_parameter, not a list."
                 )
+            model_version = resolve_version_to_latest_patch(
+                model_version, self.get_model_versions(collection_name)
+            )
             production_table = self.read_production_table_from_mongo_db(
                 collection_name, model_version
             )
@@ -315,6 +325,9 @@ class DatabaseHandler:
         dict containing the parameters
         """
         pars = {}
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         array_element_list = self._get_array_element_list(
             array_element_name, site, production_table, collection
@@ -361,6 +374,11 @@ class DatabaseHandler:
     def _get_parameter_for_model_version(
         self, array_element, model_version, site, collection, production_table
     ):
+        """
+        Get parameters for a specific model version and array element.
+
+        Uses caching wherever possible.
+        """
         cache_key, cache_dict = self._read_cache(
             DatabaseHandler.model_parameters_cached,
             names.validate_site_name(site) if site else None,
@@ -589,6 +607,9 @@ class DatabaseHandler:
         ValueError
             if query returned no results.
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection_name)
+        )
         try:
             return DatabaseHandler.production_table_cached[
                 self._cache_key(None, None, model_version, collection_name)
@@ -612,7 +633,7 @@ class DatabaseHandler:
 
     def get_model_versions(self, collection_name="telescopes"):
         """
-        Get list of model versions from the DB.
+        Get list of model versions from the DB with caching.
 
         Parameters
         ----------
@@ -624,10 +645,12 @@ class DatabaseHandler:
         list
             List of model versions
         """
-        collection = self.get_collection("production_tables", db_name=self.db_name)
-        return sorted(
-            {post["model_version"] for post in collection.find({"collection": collection_name})}
-        )
+        if collection_name not in DatabaseHandler.model_versions_cached:
+            collection = self.get_collection("production_tables", db_name=self.db_name)
+            DatabaseHandler.model_versions_cached[collection_name] = sorted(
+                {post["model_version"] for post in collection.find({"collection": collection_name})}
+            )
+        return DatabaseHandler.model_versions_cached[collection_name]
 
     def get_array_elements(self, model_version, collection="telescopes"):
         """
@@ -646,6 +669,9 @@ class DatabaseHandler:
         list
             Sorted list of all array elements found in collection
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         return sorted([entry for entry in production_table["parameters"] if "-design" not in entry])
 
@@ -668,6 +694,9 @@ class DatabaseHandler:
         str
             Design model for a given array element.
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         try:
             return production_table["design_model"][array_element_name]
@@ -696,6 +725,9 @@ class DatabaseHandler:
         list
             Sorted list of all array element names found in collection
         """
+        model_version = resolve_version_to_latest_patch(
+            model_version, self.get_model_versions(collection)
+        )
         production_table = self.read_production_table_from_mongo_db(collection, model_version)
         all_array_elements = production_table["parameters"]
         return sorted(
@@ -846,6 +878,7 @@ class DatabaseHandler:
         self._logger.debug(f"Adding production for {production_table.get('collection')} to to DB")
         collection.insert_one(production_table)
         DatabaseHandler.production_table_cached.clear()
+        DatabaseHandler.model_versions_cached.clear()
 
     def add_new_parameter(
         self,
@@ -1001,6 +1034,7 @@ class DatabaseHandler:
     def _reset_parameter_cache(self):
         """Reset the cache for the parameters."""
         DatabaseHandler.model_parameters_cached.clear()
+        DatabaseHandler.model_versions_cached.clear()
 
     def _get_array_element_list(self, array_element_name, site, production_table, collection):
         """

simtools/db/db_model_upload.py

@@ -3,6 +3,8 @@
 import logging
 from pathlib import Path
 
+from packaging.version import Version
+
 from simtools.io import ascii_handler
 from simtools.utils import names
 
@@ -51,8 +53,7 @@ def add_model_parameters_to_db(input_path, db):
     """
     input_path = Path(input_path)
     logger.info(f"Reading model parameters from repository path {input_path}")
-    array_elements = [d for d in input_path.iterdir() if d.is_dir()]
-    for element in array_elements:
+    for element in filter(Path.is_dir, input_path.iterdir()):
         collection = names.get_collection_name_from_array_element_name(element.name, False)
         if collection == "Files":
             logger.info("Files (tables) are uploaded with the corresponding model parameters")
@@ -87,18 +88,40 @@ def add_production_tables_to_db(input_path, db):
 
     for model in filter(Path.is_dir, input_path.iterdir()):
         logger.info(f"Reading production tables for model version {model.name}")
-        model_dict = {}
-        for file in sorted(model.rglob("*json")):
-            _read_production_table(model_dict, file, model.name)
+        model_dict = _read_production_tables(model)
 
         for collection, data in model_dict.items():
-            if not data["parameters"]:
+            if data["parameters"]:
+                logger.info(f"Adding production table for {collection} to the database")
+                db.add_production_table(production_table=data)
+            else:
                 logger.info(f"No production table for {collection} in model version {model.name}")
-                continue
-            logger.info(f"Adding production table for {collection} to the database")
-            db.add_production_table(
-                production_table=data,
-            )
+
+
+def _read_production_tables(model_path):
+    """
+    Read production tables from a directory.
+
+    Take into account that some productions include patch updates only. Read in this cases
+    the base models first.
+
+    Parameters
+    ----------
+    model_path : Path
+        Path to the directory containing the production tables for a specific model version.
+    """
+    model_dict = {}
+    models = [model_path.name]
+    if (model_path / "info.yml").exists():
+        info = ascii_handler.collect_data_from_file(file_name=model_path / "info.yml")
+        models.extend(info.get("model_version_history", []))
+    # sort oldest --> newest
+    models = sorted(set(models), key=Version, reverse=False)
+    for model in models:
+        for file in sorted((model_path.parent / model).rglob("*json")):
+            _read_production_table(model_dict, file, model)
+
+    return model_dict
 
 
 def _read_production_table(model_dict, file, model_name):
@@ -120,9 +143,10 @@ def _read_production_table(model_dict, file, model_name):
         if array_element in ("configuration_corsika", "configuration_sim_telarray"):
             model_dict[collection]["parameters"] = parameter_dict["parameters"]
         else:
-            model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][
-                array_element
-            ]
+            model_dict[collection]["parameters"].setdefault(array_element, {}).update(
+                parameter_dict["parameters"][array_element]
+            )
+
     except KeyError as exc:
         logger.error(f"KeyError: {exc}")
         raise
@@ -132,3 +156,5 @@ def _read_production_table(model_dict, file, model_name):
         ]
     except KeyError:
         pass
+
+    model_dict[collection]["model_version"] = model_name