gammasimtools 0.23.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

simtools/dependencies.py

@@ -16,8 +16,8 @@ from pathlib import Path
 
 import yaml
 
-from simtools.db.db_handler import DatabaseHandler
 from simtools.io import ascii_handler
+from simtools.version import __version__
 
 _logger = logging.getLogger(__name__)
 
@@ -49,6 +49,30 @@ def get_version_string(db_config=None, run_time=None):
     )
 
 
+def get_software_version(software):
+    """
+    Return the version of the specified software package.
+
+    Parameters
+    ----------
+    software : str
+        Name of the software package.
+
+    Returns
+    -------
+    str
+        Version of the specified software package.
+    """
+    if software.lower() == "simtools":
+        return __version__
+
+    try:
+        version_call = f"get_{software.lower()}_version"
+        return globals()[version_call]()
+    except KeyError as exc:
+        raise ValueError(f"Unknown software: {software}") from exc
+
+
 def get_database_version_or_name(db_config, version=True):
     """
     Get the version or name of the simulation model data base used.
@@ -66,18 +90,17 @@ def get_database_version_or_name(db_config, version=True):
         Version or name of the simulation model data base used.
 
     """
-    if db_config is None:
-        return None
-    db = DatabaseHandler(db_config)
-    return db.mongo_db_config.get(
-        "db_simulation_model_version" if version else "db_simulation_model"
-    )
+    if version:
+        return db_config and db_config.get("db_simulation_model_version")
+    return db_config and db_config.get("db_simulation_model")
 
 
-def get_sim_telarray_version(run_time):
+def get_sim_telarray_version(run_time=None):
     """
     Get the version of the sim_telarray package using 'sim_telarray --version'.
 
+    Version strings for sim_telarray are of the form "2024.271.0" (year.day_of_year.patch).
+
     Parameters
     ----------
     run_time : list, optional
@@ -116,6 +139,8 @@ def get_corsika_version(run_time=None):
     """
     Get the version of the CORSIKA package.
 
+    Version strings for CORSIKA are of the form "7.7550" (major.minor with 4-digit minor).
+
     Parameters
     ----------
     run_time : list, optional
@@ -126,12 +151,11 @@ def get_corsika_version(run_time=None):
     str
         Version of the CORSIKA package.
     """
-    version = None
-    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if sim_telarray_path is None:
+    corsika_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
+    if corsika_path is None:
         _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
         return None
-    corsika_command = Path(sim_telarray_path) / "corsika-run" / "corsika"
+    corsika_command = Path(corsika_path) / "corsika-run" / "corsika"
 
     if run_time is None:
         command = [str(corsika_command)]
@@ -148,11 +172,8 @@ def get_corsika_version(run_time=None):
         text=True,
     )
 
-    # Capture output until it waits for input
-    while True:
-        line = process.stdout.readline()
-        if not line:
-            break
+    version = None
+    for line in process.stdout:
         # Extract the version from the line "NUMBER OF VERSION :  7.7550"
         if "NUMBER OF VERSION" in line:
             version = line.split(":")[1].strip()

simtools/io/eventio_handler.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+"""Read file info and run headers from eventio (CORSIKA IACT, sim_telarray) files."""
+
+import warnings
+
+from eventio import EventIOFile, iact
+from eventio.simtel import MCRunHeader, MCShower, RunHeader
+
+# Suppress all UserWarnings from corsikaio - no CORSIKA versions <7.7 are supported anyway
+warnings.filterwarnings("ignore", category=UserWarning, module=r"corsikaio\.subblocks\..*")
+
+
+def get_corsika_run_number(file):
+    """
+    Return the CORSIKA run number from an eventio (CORSIKA IACT or sim_telarray) file.
+
+    Parameters
+    ----------
+    file: str
+        Path to the eventio file.
+
+    Returns
+    -------
+    int, None
+        CORSIKA run number. Returns None if not found.
+    """
+    run_header = get_combined_corsika_run_header(file)
+    if run_header and "run" in run_header:
+        return run_header["run"]
+    run_header, _ = get_corsika_run_and_event_headers(file)
+    try:
+        return int(run_header["run_number"])
+    except (TypeError, KeyError, ValueError):
+        return None
+
+
+def get_combined_corsika_run_header(sim_telarray_file):
+    """
+    Return the CORSIKA run header information from an sim_telarray file.
+
+    Reads both RunHeader and MCRunHeader object from file and returns a merged dictionary.
+    Adds primary id from the first event.
+
+    Parameters
+    ----------
+    sim_telarray_file: str
+        Path to the sim_telarray file.
+
+    Returns
+    -------
+    dict, None
+        CORSIKA run header. Returns None if not found.
+    """
+    run_header = mc_run_header = None
+    primary_id = None
+
+    with EventIOFile(sim_telarray_file) as f:
+        for o in f:
+            if isinstance(o, RunHeader) and run_header is None:
+                run_header = o.parse()
+            elif isinstance(o, MCRunHeader) and mc_run_header is None:
+                mc_run_header = o.parse()
+            elif isinstance(o, MCShower):  # get primary_id from first MCShower
+                primary_id = o.parse().get("primary_id")
+            if run_header and mc_run_header and primary_id is not None:
+                break
+
+    run_header = run_header or {}
+    mc_run_header = mc_run_header or {}
+    if primary_id is not None:
+        mc_run_header["primary_id"] = primary_id
+    return run_header | mc_run_header or None
+
+
+def get_corsika_run_and_event_headers(corsika_iact_file):
+    """
+    Return the CORSIKA run and event headers from a CORSIKA IACT eventio file.
+
+    Parameters
+    ----------
+    corsika_iact_file: str, Path
+        Path to the CORSIKA IACT eventio file.
+
+    Returns
+    -------
+    tuple
+        CORSIKA run header and event header as dictionaries.
+    """
+    run_header = event_header = None
+
+    with EventIOFile(corsika_iact_file) as f:
+        for o in f:
+            if isinstance(o, iact.RunHeader) and run_header is None:
+                run_header = o.parse()
+            elif isinstance(o, iact.EventHeader) and event_header is None:
+                event_header = o.parse()
+            if run_header and event_header:
+                break
+
+    return run_header, event_header
+
+
+def get_simulated_events(event_io_file):
+    """
+    Return the number of shower and MC events from a simulation (eventio) file.
+
+    For a sim_telarray file, the number of simulated showers and MC events is
+    determined by counting the number of MCShower (type id 2020) and MCEvent
+    objects (type id 2021). For a CORSIKA IACT file, the number of simulated
+    showers is determined by counting the number of IACTShower (type id 1202).
+
+    Parameters
+    ----------
+    event_io_file: str, Path
+        Path to the eventio file.
+
+    Returns
+    -------
+    tuple
+        Number of showers and number of MC events (MC events for sim_telarray files only).
+    """
+    counts = {1202: 0, 2020: 0, 2021: 0}
+    with EventIOFile(event_io_file) as f:
+        for o in f:
+            t = o.header.type
+            if t in counts:
+                counts[t] += 1
+    return counts[2020] if counts[2020] else counts[1202], counts[2021]

simtools/job_execution/htcondor_script_generator.py

@@ -136,8 +136,6 @@ simtools-simulate-prod \\
     --view_cone {view_cone_string} \\
     --run_number $((process_id)) \\
     --run_number_offset {run_number_offset} \\
-    --number_of_runs 1 \\
-    --data_directory /tmp/simtools-data \\
     --output_path /tmp/simtools-output \\
     --log_level {args_dict["log_level"]} \\
     --pack_for_grid_register simtools-output

simtools/layout/array_layout.py

@@ -32,8 +32,8 @@ class ArrayLayout:
 
     Parameters
     ----------
-    mongo_db_config: dict
-        MongoDB configuration.
+    db_config: dict
+        Database configuration.
     site: str
         Site name or location (e.g., North/South or LaPalma/Paranal)
     model_version: str
@@ -52,7 +52,7 @@ class ArrayLayout:
 
     def __init__(
         self,
-        mongo_db_config,
+        db_config,
         site,
         model_version,
         label=None,
@@ -67,7 +67,7 @@ class ArrayLayout:
         self.model_version = model_version
         self.label = label
         self.name = name
-        self.mongo_db_config = mongo_db_config
+        self.db_config = db_config
         self.site = None if site is None else names.validate_site_name(site)
         self.site_model = None
         self.io_handler = io_handler.IOHandler()
@@ -95,13 +95,13 @@ class ArrayLayout:
     def _initialize_site_parameters_from_db(self):
         """Initialize site parameters required for transformations using the database."""
         self._logger.debug("Initialize parameters from DB")
-        if self.mongo_db_config is None:
+        if self.db_config is None:
             raise ValueError("No database configuration provided")
 
         self.site_model = SiteModel(
             site=self.site,
             model_version=self.model_version,
-            mongo_db_config=self.mongo_db_config,
+            db_config=self.db_config,
         )
         self._corsika_observation_level = self.site_model.get_corsika_site_parameters().get(
             "corsika_observation_level", None
@@ -419,7 +419,7 @@ class ArrayLayout:
             site=self.site,
             telescope_name=telescope_name,
             model_version=self.model_version,
-            mongo_db_config=self.mongo_db_config,
+            db_config=self.db_config,
             label=self.label,
         )
 

simtools/layout/array_layout_utils.py

@@ -162,7 +162,7 @@ def write_array_layouts(array_layouts, args_dict, db_config):
     ModelDataWriter.dump_model_parameter(
         parameter_name="array_layouts",
         value=array_layouts["value"],
-        instrument=site,
+        instrument=f"OBS-{site}",
         parameter_version=args_dict.get("updated_parameter_version"),
         output_file=output_file,
         db_config=db_config,
@@ -280,7 +280,7 @@ def get_array_layouts_from_db(
     if layout_name:
         layout_names = gen.ensure_iterable(layout_name)
     else:
-        site_model = SiteModel(site=site, model_version=model_version, mongo_db_config=db_config)
+        site_model = SiteModel(site=site, model_version=model_version, db_config=db_config)
         layout_names = site_model.get_list_of_array_layouts()
 
     layouts = []
@@ -373,7 +373,7 @@ def _get_array_layout_dict(
 ):
     """Return array layout dictionary for a given telescope list."""
     array_model = ArrayModel(
-        mongo_db_config=db_config,
+        db_config=db_config,
         model_version=model_version,
         site=site,
         array_elements=telescope_list,
@@ -416,7 +416,7 @@ def get_array_elements_from_db_for_layouts(layouts, site, model_version, db_conf
     dict
         Dictionary mapping layout names to telescope IDs.
     """
-    site_model = SiteModel(site=site, model_version=model_version, mongo_db_config=db_config)
+    site_model = SiteModel(site=site, model_version=model_version, db_config=db_config)
     layout_names = site_model.get_list_of_array_layouts() if layouts == ["all"] else layouts
     layout_dict = {}
     for layout_name in layout_names:

simtools/model/array_model.py

@@ -7,7 +7,6 @@ import astropy.units as u
 from astropy.table import QTable
 
 from simtools.data_model import data_reader, schema
-from simtools.db import db_handler
 from simtools.io import io_handler
 from simtools.model.calibration_model import CalibrationModel
 from simtools.model.site_model import SiteModel
@@ -22,8 +21,8 @@ class ArrayModel:
 
     Parameters
     ----------
-    mongo_db_config: dict
-        MongoDB configuration.
+    db_config: dict
+        Database configuration.
     model_version: str
         Model version.
     label: str, optional
@@ -35,30 +34,30 @@ class ArrayModel:
     array_elements: Union[str, Path, List[str]], optional
         Array element definitions (list of array element or path to file with
         the array element positions).
-    sim_telarray_seeds : dict, optional
-        Dictionary with configuration for sim_telarray random instrument setup.
-    simtel_path: str, Path, optional
-        Path to the sim_telarray installation directory.
     calibration_device_types: List[str], optional
         List of calibration device types (e.g., 'flat_fielding') attached to each telescope.
+    simtel_path: str, Path, optional
+        Path to the sim_telarray installation directory.
+    overwrite_model_parameters: str, optional
+        File name to overwrite model parameters from DB with provided values.
     """
 
     def __init__(
         self,
-        mongo_db_config,
+        db_config,
         model_version,
         label=None,
         site=None,
         layout_name=None,
         array_elements=None,
-        sim_telarray_seeds=None,
-        simtel_path=None,
         calibration_device_types=None,
+        simtel_path=None,
+        overwrite_model_parameters=None,
     ):
         """Initialize ArrayModel."""
         self._logger = logging.getLogger(__name__)
         self._logger.debug("Init ArrayModel")
-        self.mongo_db_config = mongo_db_config
+        self.db_config = db_config
         self.model_version = model_version
         self.label = label
         self.layout_name = (
@@ -69,7 +68,8 @@ class ArrayModel:
         self._config_file_path = None
         self._config_file_directory = None
         self.io_handler = io_handler.IOHandler()
-        self.db = db_handler.DatabaseHandler(mongo_db_config=mongo_db_config)
+
+        self.overwrite_model_parameters = overwrite_model_parameters
 
         self.array_elements, self.site_model, self.telescope_models, self.calibration_models = (
             self._initialize(site, array_elements, calibration_device_types)
@@ -77,7 +77,7 @@ class ArrayModel:
 
         self._telescope_model_files_exported = False
         self._array_model_file_exported = False
-        self.sim_telarray_seeds = sim_telarray_seeds
+        self._sim_telarray_seeds = None
         self.simtel_path = simtel_path
 
     def _initialize(self, site, array_elements_config, calibration_device_types):
@@ -105,9 +105,10 @@ class ArrayModel:
         self._logger.debug(f"Getting site parameters from DB ({site})")
         site_model = SiteModel(
             site=names.validate_site_name(site),
-            mongo_db_config=self.mongo_db_config,
+            db_config=self.db_config,
             model_version=self.model_version,
             label=self.label,
+            overwrite_model_parameters=self.overwrite_model_parameters,
         )
 
         # Case 1: array_elements is a file name
@@ -117,7 +118,7 @@ class ArrayModel:
             )
         # Case 2: array elements is a list of elements
         elif isinstance(array_elements_config, list):
-            array_elements = self._get_array_elements_from_list(array_elements_config)
+            array_elements = self._get_array_elements_from_list(array_elements_config, site_model)
         # Case 3: array elements defined in DB by array layout name
         elif self.layout_name is not None:
             array_elements = self._get_array_elements_from_list(
@@ -135,6 +136,41 @@ class ArrayModel:
 
         return array_elements, site_model, telescope_models, calibration_models
 
+    @property
+    def sim_telarray_seeds(self):
+        """
+        Return sim_telarray seeds.
+
+        Returns
+        -------
+        dict
+            Dictionary with sim_telarray seeds.
+        """
+        return self._sim_telarray_seeds
+
+    @sim_telarray_seeds.setter
+    def sim_telarray_seeds(self, value):
+        """
+        Set sim_telarray seeds.
+
+        Parameters
+        ----------
+        value: dict
+            Dictionary with sim_telarray seeds.
+        """
+        if isinstance(value, dict):
+            required_keys = {
+                "seed",
+                "random_instrument_instances",
+                "seed_file_name",
+            }
+            if not required_keys.issubset(value):
+                raise ValueError(
+                    "sim_telarray_seeds dictionary must contain the following keys: "
+                    f"{required_keys}"
+                )
+        self._sim_telarray_seeds = value
+
     @property
     def config_file_path(self):
         """
@@ -149,7 +185,6 @@ class ArrayModel:
             config_file_name = names.simtel_config_file_name(
                 array_name=self.layout_name,
                 site=self.site_model.site,
-                model_version=self.model_version,
                 label=self.label,
             )
             self._config_file_path = self.get_config_directory().joinpath(config_file_name)
@@ -168,7 +203,7 @@ class ArrayModel:
         return len(self.telescope_models)
 
     @property
-    def site(self) -> str:
+    def site(self):
         """
         Return site.
 
@@ -179,34 +214,6 @@ class ArrayModel:
         """
         return self.site_model.site
 
-    @property
-    def model_version(self):
-        """Model version."""
-        return self._model_version
-
-    @model_version.setter
-    def model_version(self, model_version):
-        """
-        Set model version.
-
-        Parameters
-        ----------
-        _model_version: str or list
-            Model version (e.g., "6.0.0").
-            If a list is passed, it must contain exactly one element,
-            and only that element will be used.
-
-        Raises
-        ------
-        ValueError
-            If more than one model version is passed.
-        """
-        if isinstance(model_version, list):
-            raise ValueError(
-                f"Only one model version can be passed to {self.__class__.__name__}, not a list."
-            )
-        self._model_version = model_version
-
     def _build_telescope_models(self, site_model, array_elements, calibration_device_types):
         """
         Build telescope models for all telescopes of this array.
@@ -243,8 +250,9 @@ class ArrayModel:
                 site=site_model.site,
                 telescope_name=element_name,
                 model_version=self.model_version,
-                mongo_db_config=self.mongo_db_config,
+                db_config=self.db_config,
                 label=self.label,
+                overwrite_model_parameters=self.overwrite_model_parameters,
             )
             calibration_models[element_name] = self._build_calibration_models(
                 telescope_models[element_name],
@@ -274,9 +282,10 @@ class ArrayModel:
             calibration_models[device_name] = CalibrationModel(
                 site=site_model.site,
                 calibration_device_model_name=device_name,
-                mongo_db_config=self.mongo_db_config,
+                db_config=self.db_config,
                 model_version=self.model_version,
                 label=self.label,
+                overwrite_model_parameters=self.overwrite_model_parameters,
             )
         return calibration_models
 
@@ -289,9 +298,7 @@ class ArrayModel:
         """Export sim_telarray configuration files for all telescopes into the model directory."""
         exported_models = []
         for tel_model in self.telescope_models.values():
-            name = tel_model.name + (
-                "_" + tel_model.extra_label if tel_model.extra_label != "" else ""
-            )
+            name = tel_model.name
             if name not in exported_models:
                 self._logger.debug(f"Exporting configuration file for telescope {name}")
                 tel_model.write_sim_telarray_config_file(
@@ -336,7 +343,7 @@ class ArrayModel:
         if not self._array_model_file_exported:
             self.export_sim_telarray_config_file()
 
-    def get_config_directory(self) -> Path:
+    def get_config_directory(self):
         """
         Get the path of the array config directory for sim_telarray.
 
@@ -370,9 +377,7 @@ class ArrayModel:
         self._logger.info(f"Packed model files into {archive_name}")
         return archive_name
 
-    def _load_array_element_positions_from_file(
-        self, array_elements_file: str | Path, site: str
-    ) -> dict:
+    def _load_array_element_positions_from_file(self, array_elements_file, site):
         """
         Load array element (e.g. telescope) positions from a file into a dict.
 
@@ -400,14 +405,8 @@ class ArrayModel:
         }
 
     def _get_telescope_position_parameter(
-        self,
-        telescope_name: str,
-        site: str,
-        x: u.Quantity,
-        y: u.Quantity,
-        z: u.Quantity,
-        parameter_version: str | None = None,
-    ) -> dict:
+        self, telescope_name, site, x, y, z, parameter_version=None
+    ):
         """
         Return dictionary with telescope position parameters (following DB model database format).
 
@@ -444,7 +443,7 @@ class ArrayModel:
             "model_parameter_schema_version": "0.1.0",
         }
 
-    def _get_array_elements_from_list(self, array_elements_list: list[str]) -> dict:
+    def _get_array_elements_from_list(self, array_elements_list, site_model=None):
         """
         Return dictionary with array elements from a list of telescope names.
 
@@ -456,6 +455,8 @@ class ArrayModel:
         ----------
         array_elements_list: list
             List of telescope names.
+        site_model: SiteModel
+            Site model.
 
         Returns
         -------
@@ -467,31 +468,30 @@ class ArrayModel:
             try:
                 array_elements_dict[names.validate_array_element_name(name)] = None
             except ValueError:
-                array_elements_dict.update(self._get_all_array_elements_of_type(name))
+                array_elements_dict.update(self._get_all_array_elements_of_type(name, site_model))
         return array_elements_dict
 
-    def _get_all_array_elements_of_type(self, array_element_type: str) -> dict:
+    def _get_all_array_elements_of_type(self, array_element_type, site_model):
         """
-        Return all array elements of a specific type using the database.
+        Return all array elements of a specific type.
 
         Parameters
         ----------
         array_element_type : str
             Type of the array element (e.g. LSTN, MSTS)
+        site_model: SiteModel
+            Site model.
 
         Returns
         -------
         dict
             Dict with array elements.
         """
-        all_elements = self.db.get_array_elements_of_type(
-            array_element_type=array_element_type,
-            model_version=self.model_version,
-            collection="telescopes",
+        return self._get_array_elements_from_list(
+            site_model.get_array_elements_of_type(array_element_type)
         )
-        return self._get_array_elements_from_list(all_elements)
 
-    def export_array_elements_as_table(self, coordinate_system: str = "ground") -> QTable:
+    def export_array_elements_as_table(self, coordinate_system="ground"):
         """
         Export array elements positions to astropy table.
 

simtools/model/calibration_model.py

@@ -17,31 +17,34 @@ class CalibrationModel(ModelParameter):
         Site name (e.g., South or North).
     calibration_device_model_name: str
         Calibration device model name (ex. ILLS-01, ILLN-01, ...).
-    mongo_db_config: dict
-        MongoDB configuration.
+    db_config: dict
+        Database configuration.
     model_version: str
         Model version.
     label: str, optional
         Instance label. Important for output file naming.
+    overwrite_model_parameters: str, optional
+        File name to overwrite model parameters from DB with provided values.
     """
 
     def __init__(
         self,
-        site: str,
-        calibration_device_model_name: str,
-        mongo_db_config: dict,
-        model_version: str,
-        label: str | None = None,
+        site,
+        calibration_device_model_name,
+        db_config,
+        model_version,
+        label=None,
+        overwrite_model_parameters=None,
     ):
         """Initialize CalibrationModel."""
         super().__init__(
             site=site,
             array_element_name=calibration_device_model_name,
             collection="calibration_devices",
-            mongo_db_config=mongo_db_config,
+            db_config=db_config,
             model_version=model_version,
-            db=None,
             label=label,
+            overwrite_model_parameters=overwrite_model_parameters,
         )
 
         self._logger = logging.getLogger(__name__)