gammasimtools 0.19.0__py3-none-any.whl → 0.20.0__py3-none-any.whl

simtools/applications/derive_trigger_rates.py

@@ -0,0 +1,91 @@
+r"""
+Derive cosmic-ray trigger rates for a single telescope or an array of telescopes.
+
+Uses simulated background events (e.g. from proton primaries) to calculate the trigger rates.
+Input is reduced event data generated from simulations for the given configuration.
+
+
+Command line arguments
+----------------------
+event_data_file (str, required)
+    Event data file containing reduced event data.
+array_layout_name (list, optional)
+    Name of the array layout to use for the simulation.
+telescope_ids (str, optional)
+    Path to a file containing telescope configurations.
+plot_histograms (bool, optional)
+    Plot histograms of the event data.
+model_version (str, optional)
+    Version of the simulation model to use.
+site (str, optional)
+    Name of the site where the simulation is being run.
+
+
+Example
+-------
+
+Derive trigger rates for the South Alpha layout:
+
+.. code-block:: console
+
+    simtools-derive-trigger-rates \\
+        --site South \\
+        --model_version 6.0.0 \\
+        --event_data_file /path/to/event_data_file.h5 \\
+        --array_layout_name alpha\\
+        --plot_histograms
+
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.telescope_trigger_rates import telescope_trigger_rates
+
+
+def _parse():
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        description="Derive trigger rates for a single telescope or an array of telescopes.",
+    )
+    config.parser.add_argument(
+        "--event_data_file",
+        type=str,
+        required=True,
+        help="Event data file containing reduced event data.",
+    )
+    config.parser.add_argument(
+        "--telescope_ids",
+        type=str,
+        required=False,
+        help="Path to a file containing telescope configurations.",
+    )
+    config.parser.add_argument(
+        "--plot_histograms",
+        help="Plot histograms of the event data.",
+        action="store_true",
+        default=False,
+    )
+    return config.initialize(
+        db_config=True,
+        output=True,
+        simulation_model=[
+            "site",
+            "model_version",
+            "layout",
+        ],
+    )
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "info")))
+
+    telescope_trigger_rates(args_dict, db_config)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/plot_simtel_events.py

@@ -3,9 +3,9 @@
 r"""
 Plot simulated events.
 
-This application produces figures from one or more sim_telarray (.simtel.zst) files
-by calling functions in ``simtools.visualization.simtel_event_plots``. It is meant to
-run after simulations (e.g., simtools-simulate-flasher, simtools-simulate-illuminator).
+Produces figures from one or more sim_telarray (.simtel.zst) files
+It is meant to run after simulations (e.g., simtools-simulate-flasher,
+simtools-simulate-illuminator).
 
 What it does
 ------------
@@ -84,7 +84,7 @@ from simtools.configuration import configurator
 from simtools.corsika.corsika_histograms_visualize import save_figs_to_pdf
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io import io_handler
-from simtools.visualization.simtel_event_plots import (
+from simtools.visualization.plot_simtel_events import (
     plot_simtel_event_image,
     plot_simtel_integrated_pedestal_image,
     plot_simtel_integrated_signal_image,
@@ -106,6 +106,64 @@ PLOT_CHOICES = {
 }
 
 
+def _call_peak_timing(
+    filename,
+    *,
+    tel_id=None,
+    sum_threshold=10.0,
+    peak_width=8,
+    examples=3,
+    timing_bins=None,
+    event_index=None,
+):
+    """Call ``plot_simtel_peak_timing`` and support optional ``return_stats``.
+
+    Parameters
+    ----------
+    filename : pathlib.Path or str
+        Path to the input simtel file.
+    tel_id : int, optional
+        Telescope ID to visualize.
+    sum_threshold : float, default 10.0
+        Minimum pixel sum to consider a pixel.
+    peak_width : int, default 8
+        Expected peak width in samples.
+    examples : int, default 3
+        Number of example traces to draw.
+    timing_bins : int or None, optional
+        Number of bins for timing histogram (contiguous if not set).
+    event_index : int or None, optional
+        0-based index of the event to plot; default is the first event.
+
+    Returns
+    -------
+    object or None
+        The matplotlib Figure if available, otherwise ``None``.
+    """
+    try:
+        fig_stats = plot_simtel_peak_timing(
+            filename,
+            tel_id=tel_id,
+            sum_threshold=sum_threshold,
+            peak_width=peak_width,
+            examples=examples,
+            timing_bins=timing_bins,
+            return_stats=True,
+            event_index=event_index,
+        )
+        return fig_stats[0] if isinstance(fig_stats, tuple) else fig_stats
+    except TypeError:
+        return plot_simtel_peak_timing(
+            filename,
+            tel_id=tel_id,
+            sum_threshold=sum_threshold,
+            peak_width=peak_width,
+            examples=examples,
+            timing_bins=timing_bins,
+            event_index=event_index,
+        )
+
+
 def _parse(label: str):
     """Parse command line configuration."""
     config = configurator.Configurator(
@@ -263,30 +321,6 @@ def _collect_figures_for_file(
         else list(plots)
     )
 
-    def _call_peak_timing():
-        try:
-            fig_stats = plot_simtel_peak_timing(
-                filename,
-                tel_id=args.get("tel_id"),
-                sum_threshold=args.get("sum_threshold", 10.0),
-                peak_width=args.get("peak_width", 8),
-                examples=args.get("examples", 3),
-                timing_bins=args.get("timing_bins"),
-                return_stats=True,
-                event_index=args.get("event_index"),
-            )
-            return fig_stats[0] if isinstance(fig_stats, tuple) else fig_stats
-        except TypeError:
-            return plot_simtel_peak_timing(
-                filename,
-                tel_id=args.get("tel_id"),
-                sum_threshold=args.get("sum_threshold", 10.0),
-                peak_width=args.get("peak_width", 8),
-                examples=args.get("examples", 3),
-                timing_bins=args.get("timing_bins"),
-                event_index=args.get("event_index"),
-            )
-
     # function name -> (callable, defaults)
     dispatch: dict[str, tuple[object, dict[str, object]]] = {
         "event_image": (
@@ -313,12 +347,20 @@ def _collect_figures_for_file(
             plot_simtel_integrated_pedestal_image,
             {"tel_id": None, "half_width": 8, "offset": 16, "event_index": None},
         ),
+        "peak_timing": (
+            _call_peak_timing,
+            {
+                "tel_id": None,
+                "sum_threshold": 10.0,
+                "peak_width": 8,
+                "examples": 3,
+                "timing_bins": None,
+                "event_index": None,
+            },
+        ),
     }
 
     for plot_name in plots_to_run:
-        if plot_name == "peak_timing":
-            add(_call_peak_timing(), "peak_timing")
-            continue
         entry = dispatch.get(plot_name)
         if entry is None:
             logger.warning("Unknown plot selection '%s'", plot_name)

simtools/applications/validate_file_using_schema.py

@@ -145,10 +145,13 @@ def validate_dict_using_schema(args_dict, logger):
         except FileNotFoundError as exc:
             raise FileNotFoundError(f"Error reading schema file from {file_name}") from exc
         data = data if isinstance(data, list) else [data]
-        for data_dict in data:
-            schema.validate_dict_using_schema(
-                data_dict, _get_schema_file_name(args_dict, data_dict)
-            )
+        try:
+            for data_dict in data:
+                schema.validate_dict_using_schema(
+                    data_dict, _get_schema_file_name(args_dict, data_dict)
+                )
+        except Exception as exc:
+            raise ValueError(f"Validation of file {file_name} failed") from exc
         logger.info(f"Successful validation of file {file_name}")
 
 

simtools/configuration/commandline_parser.py

@@ -222,6 +222,13 @@ class CommandLineParser(argparse.ArgumentParser):
             required=False,
             default=None,
         )
+        _job_group.add_argument(
+            "--db_simulation_model_version",
+            help="version of simulation model database",
+            type=str.strip,
+            required=False,
+            default=None,
+        )
 
     def initialize_simulation_model_arguments(self, model_options):
         """
@@ -740,28 +747,27 @@ class CommandLineParser(argparse.ArgumentParser):
     @staticmethod
     def parse_quantity_pair(string):
         """
-        Parse a string representing a pair of astropy quantities separated by a space.
-
-        Args:
-            string: The input string (e.g., "0 deg 1.5 deg").
+        Parse a string representing a pair of astropy quantities.
 
         Returns
         -------
-            tuple: A tuple containing two astropy.units.Quantity objects.
+        tuple
+            A tuple of two astropy.units.Quantity objects.
 
         Raises
         ------
-            ValueError: If the string is not formatted correctly (e.g., missing space).
+        ValueError
+            If the string cannot be parsed into exactly two quantities.
         """
-        pattern = r"(\d+\.?\d*)\s*([a-zA-Z]+)"
+        pattern = r"(?>[\d\.eE+-]+)\s*(?>[A-Za-z]+)"
         matches = re.findall(pattern, string)
         if len(matches) != 2:
             raise ValueError("Input string does not contain exactly two quantities.")
 
-        return (
-            u.Quantity(float(matches[0][0]), matches[0][1]),
-            u.Quantity(float(matches[1][0]), matches[1][1]),
-        )
+        try:
+            return tuple(u.Quantity(m) for m in matches)
+        except Exception as exc:
+            raise ValueError(f"Could not parse quantities: {exc}") from exc
 
     @staticmethod
     def parse_integer_and_quantity(input_string):

simtools/data_model/validate_data.py

@@ -187,9 +187,14 @@ class DataValidator:
         value_as_list, unit_as_list = self._get_value_and_units_as_lists()
 
         for index, (value, unit) in enumerate(zip(value_as_list, unit_as_list)):
-            value_as_list[index], unit_as_list[index] = self._validate_value_and_unit(
-                value, unit, index
-            )
+            try:
+                value_as_list[index], unit_as_list[index] = self._validate_value_and_unit(
+                    value, unit, index
+                )
+            except TypeError as ex:
+                raise TypeError(
+                    f"Error validating dictionary using {self.schema_file_name}"
+                ) from ex
 
         if len(value_as_list) == 1:
             self.data_dict["value"], self.data_dict["unit"] = value_as_list[0], unit_as_list[0]

simtools/db/db_handler.py

@@ -1,5 +1,6 @@
 """Module to handle interaction with DB."""
 
+import io
 import logging
 import re
 from collections import defaultdict
@@ -8,6 +9,7 @@ from threading import Lock
 
 import gridfs
 import jsonschema
+from astropy.table import Table
 from bson.objectid import ObjectId
 from packaging.version import Version
 from pymongo import MongoClient
@@ -50,8 +52,19 @@ jsonschema_db_dict = {
             "type": "string",
             "description": "Name of simulation model database",
         },
+        "db_simulation_model_version": {
+            "type": "string",
+            "description": "Version of simulation model database",
+        },
     },
-    "required": ["db_server", "db_api_port", "db_api_user", "db_api_pw", "db_simulation_model"],
+    "required": [
+        "db_server",
+        "db_api_port",
+        "db_api_user",
+        "db_api_pw",
+        "db_simulation_model",
+        "db_simulation_model_version",
+    ],
 }
 
 
@@ -82,7 +95,12 @@ class DatabaseHandler:
         self._set_up_connection()
         self._find_latest_simulation_model_db()
         self.db_name = (
-            self.mongo_db_config.get("db_simulation_model", None) if self.mongo_db_config else None
+            self.get_db_name(
+                model_version=self.mongo_db_config.get("db_simulation_model_version"),
+                model_name=self.mongo_db_config.get("db_simulation_model"),
+            )
+            if self.mongo_db_config
+            else None
         )
 
     def _set_up_connection(self):
@@ -92,6 +110,16 @@ class DatabaseHandler:
             with lock:
                 DatabaseHandler.db_client = self._open_mongo_db()
 
+    def get_db_name(self, db_name=None, model_version=None, model_name=None):
+        """Build DB name from configuration."""
+        if db_name:
+            return db_name
+        if model_version and model_name:
+            return f"{model_name}-{model_version.replace('.', '-')}"
+        if model_version or model_name:
+            return None
+        return None if (model_version or model_name) else self.db_name
+
     def _validate_mongo_db_config(self, mongo_db_config):
         """Validate the MongoDB configuration."""
         if mongo_db_config is None or all(value is None for value in mongo_db_config.values()):
@@ -140,8 +168,7 @@ class DatabaseHandler:
         """
         Find the latest released version of the simulation model and update the DB config.
 
-        This is indicated by adding "LATEST" to the name of the simulation model database
-        (field "db_simulation_model" in the database configuration dictionary).
+        This is indicated by "LATEST" to the simulation model data base version.
         Only released versions are considered, pre-releases are ignored.
 
         Raises
@@ -151,27 +178,27 @@ class DatabaseHandler:
 
         """
         try:
+            db_simulation_model_version = self.mongo_db_config["db_simulation_model_version"]
             db_simulation_model = self.mongo_db_config["db_simulation_model"]
-            if not db_simulation_model.endswith("LATEST"):
+            if db_simulation_model_version != "LATEST":
                 return
-        except TypeError:  # db_simulation_model is None
+        except TypeError:  # db_simulation_model_version is None
             return
 
-        prefix = db_simulation_model.replace("LATEST", "")
         list_of_db_names = self.db_client.list_database_names()
-        filtered_list_of_db_names = [s for s in list_of_db_names if s.startswith(prefix)]
-        versioned_strings = []
-        version_pattern = re.compile(
-            rf"{re.escape(prefix)}v?(\d+)-(\d+)-(\d+)(?:-([a-zA-Z0-9_.]+))?"
-        )
+        filtered_list_of_db_names = [
+            s for s in list_of_db_names if s.startswith(db_simulation_model)
+        ]
+        pattern = re.compile(rf"{re.escape(db_simulation_model)}-v(\d+)-(\d+)-(\d+)(?:-(.+))?$")
 
+        versioned_strings = []
         for s in filtered_list_of_db_names:
-            match = version_pattern.search(s)
-            # A version is considered a pre-release if it contains a '-' character (re group 4)
-            if match and match.group(4) is None:
-                version_str = match.group(1) + "." + match.group(2) + "." + match.group(3)
-                version = Version(version_str)
-                versioned_strings.append((s, version))
+            m = pattern.match(s)
+            if m:
+                # skip pre-releases (have suffix)
+                if m.group(4) is None:
+                    version_str = f"{m.group(1)}.{m.group(2)}.{m.group(3)}"
+                    versioned_strings.append((s, Version(version_str)))
 
         if versioned_strings:
             latest_string, _ = max(versioned_strings, key=lambda x: x[1])
@@ -180,7 +207,7 @@ class DatabaseHandler:
                 f"Updated the DB simulation model to the latest version {latest_string}"
             )
         else:
-            raise ValueError("Found LATEST in the DB name but no matching versions found in DB.")
+            raise ValueError("LATEST requested but no released versions found in DB.")
 
     def generate_compound_indexes(self, db_name=None):
         """
@@ -774,17 +801,47 @@ class DatabaseHandler:
         with open(Path(path).joinpath(file.filename), "wb") as output_file:
             fs_output.download_to_stream_by_name(file.filename, output_file)
 
-    def add_production_table(self, db_name, production_table):
+    def get_ecsv_file_as_astropy_table(self, file_name, db_name=None):
         """
-        Add a production table to the DB.
+        Read contents of an ECSV file from the database and return it as an Astropy Table.
+
+        Files are not written to disk.
 
         Parameters
         ----------
+        file_name: str
+            The name of the ECSV file.
         db_name: str
-            the name of the DB.
+            The name of the database.
+
+        Returns
+        -------
+        astropy.table.Table
+            The contents of the ECSV file as an Astropy Table.
+        """
+        db = DatabaseHandler.db_client[db_name or self.db_name]
+        fs = gridfs.GridFSBucket(db)
+
+        buf = io.BytesIO()
+        try:
+            fs.download_to_stream_by_name(file_name, buf)
+        except gridfs.errors.NoFile as exc:
+            raise FileNotFoundError(f"ECSV file '{file_name}' not found in DB.") from exc
+        buf.seek(0)
+        return Table.read(buf.getvalue().decode("utf-8"), format="ascii.ecsv")
+
+    def add_production_table(self, production_table, db_name=None):
+        """
+        Add a production table to the DB.
+
+        Parameters
+        ----------
         production_table: dict
             The production table to add to the DB.
+        db_name: str
+            the name of the DB.
         """
+        db_name = db_name or self.db_name
         collection = self.get_collection("production_tables", db_name=db_name or self.db_name)
         self._logger.debug(f"Adding production for {production_table.get('collection')} to to DB")
         collection.insert_one(production_table)
@@ -792,8 +849,8 @@ class DatabaseHandler:
 
     def add_new_parameter(
         self,
-        db_name,
         par_dict,
+        db_name=None,
         collection_name="telescopes",
         file_prefix=None,
     ):
@@ -805,10 +862,10 @@ class DatabaseHandler:
 
         Parameters
         ----------
-        db_name: str
-            the name of the DB
         par_dict: dict
             dictionary with parameter data
+        db_name: str
+            the name of the DB
         collection_name: str
             The name of the collection to add a parameter to.
         file_prefix: str or Path
@@ -859,7 +916,7 @@ class DatabaseHandler:
             the name of the DB
         **kwargs (optional): keyword arguments for file creation.
             The full list of arguments can be found in, \
-            https://docs.mongodb.com/manual/core/gridfs/#the-files-collection
+            https://www.mongodb.com/docs/manual/core/gridfs/
             mostly these are unnecessary though.
 
         Returns

simtools/db/db_model_upload.py

@@ -9,20 +9,18 @@ from simtools.utils import names
 logger = logging.getLogger(__name__)
 
 
-def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
+def add_values_from_json_to_db(file, collection, db, file_prefix):
     """
     Upload new model parameter from json files to db.
 
     Parameters
     ----------
     file : list
-        Json file to be uploaded to the DB.
+        JSON file to be uploaded to the DB.
     collection : str
         The DB collection to which to add the file.
     db : DatabaseHandler
         Database handler object.
-    db_name : str
-        Name of the database to be created.
     file_prefix : str
         Path to location of all additional files to be uploaded.
     """
@@ -30,29 +28,28 @@ def add_values_from_json_to_db(file, collection, db, db_name, file_prefix):
     logger.debug(
         f"Adding the following parameter to the DB: {par_dict['parameter']} "
         f"version {par_dict['parameter_version']} "
-        f"(collection {collection} in database {db_name})"
+        f"(collection {collection} in database {db.get_db_name()})"
     )
 
     db.add_new_parameter(
-        db_name=db_name,
         par_dict=par_dict,
         collection_name=collection,
         file_prefix=file_prefix,
     )
 
 
-def add_model_parameters_to_db(args_dict, db):
+def add_model_parameters_to_db(input_path, db):
     """
     Read model parameters from a directory and upload them to the database.
 
     Parameters
     ----------
-    args_dict : dict
-        Command line arguments.
+    input_path : Path, str
+        Path to the directory containing the model parameters.
     db : DatabaseHandler
         Database handler object.
     """
-    input_path = Path(args_dict["input_path"])
+    input_path = Path(input_path)
     logger.info(f"Reading model parameters from repository path {input_path}")
     array_elements = [d for d in input_path.iterdir() if d.is_dir()]
     for element in array_elements:
@@ -67,12 +64,11 @@ def add_model_parameters_to_db(args_dict, db):
                 file=file,
                 collection=collection,
                 db=db,
-                db_name=args_dict["db_name"],
                 file_prefix=input_path / "Files",
             )
 
 
-def add_production_tables_to_db(args_dict, db):
+def add_production_tables_to_db(input_path, db):
     """
     Read production tables from a directory and upload them to the database.
 
@@ -81,12 +77,12 @@ def add_production_tables_to_db(args_dict, db):
 
     Parameters
     ----------
-    args_dict : dict
-        Command line arguments.
+    input_path : Path, str
+        Path to the directory containing the production tables.
     db : DatabaseHandler
         Database handler object.
     """
-    input_path = Path(args_dict["input_path"])
+    input_path = Path(input_path)
     logger.info(f"Reading production tables from repository path {input_path}")
 
     for model in filter(Path.is_dir, input_path.iterdir()):
@@ -101,7 +97,6 @@ def add_production_tables_to_db(args_dict, db):
                 continue
             logger.info(f"Adding production table for {collection} to the database")
             db.add_production_table(
-                db_name=args_dict["db_name"],
                 production_table=data,
             )
 

simtools/dependencies.py

@@ -40,7 +40,8 @@ def get_version_string(db_config=None, run_time=None):
 
     """
     return (
-        f"Database version: {get_database_version(db_config)}\n"
+        f"Database name: {get_database_version_or_name(db_config, version=False)}\n"
+        f"Database version: {get_database_version_or_name(db_config, version=True)}\n"
         f"sim_telarray version: {get_sim_telarray_version(run_time)}\n"
         f"CORSIKA version: {get_corsika_version(run_time)}\n"
         f"Build options: {get_build_options(run_time)}\n"
@@ -48,25 +49,29 @@ def get_version_string(db_config=None, run_time=None):
     )
 
 
-def get_database_version(db_config):
+def get_database_version_or_name(db_config, version=True):
     """
-    Get the version of the simulation model data base used.
+    Get the version or name of the simulation model data base used.
 
     Parameters
     ----------
     db_config : dict
         Dictionary containing the database configuration.
+    version : bool
+        If True, return the version of the database. If False, return the name.
 
     Returns
     -------
     str
-        Version of the simulation model data base used.
+        Version or name of the simulation model data base used.
 
     """
     if db_config is None:
         return None
     db = DatabaseHandler(db_config)
-    return db.mongo_db_config.get("db_simulation_model")
+    return db.mongo_db_config.get(
+        "db_simulation_model_version" if version else "db_simulation_model"
+    )
 
 
 def get_sim_telarray_version(run_time):