gammasimtools 0.25.0__py3-none-any.whl → 0.26.0__py3-none-any.whl

simtools/corsika/primary_particle.py

@@ -185,7 +185,7 @@ class PrimaryParticle:
             "silicon": {"corsika7_id": 2814},
             "iron": {"corsika7_id": 5626},
         }
-        for _, ids in particles.items():
+        for ids in particles.values():
             ids["pdg_id"] = Corsika7ID(ids["corsika7_id"]).to_pdgid().numerator
             ids["pdg_name"] = Particle.findall(pdgid=ids["pdg_id"])[0].name
 

simtools/data_model/metadata_collector.py

@@ -360,6 +360,9 @@ class MetadataCollector:
                     "Metadata extraction from sim_telarray files is not supported yet."
                 )
                 continue
+            elif Path(metadata_file).name.endswith((".corsika.zst", ".corsika")):
+                self._logger.warning("Metadata extraction from CORSIKA files is not supported yet.")
+                continue
             else:
                 raise ValueError(f"Unknown metadata file format: {metadata_file}")
 
@@ -422,7 +425,7 @@ class MetadataCollector:
         product_dict["creation_time"] = gen.now_date_time_in_isoformat()
         product_dict["description"] = self.schema_dict.get("description", None)
 
-        # DATA:CATEGORY
+        # metadata DATA:CATEGORY
         product_dict["data"]["category"] = "SIM"
         product_dict["data"]["level"] = "R1"
         product_dict["data"]["type"] = "Service"
@@ -431,7 +434,7 @@ class MetadataCollector:
         except KeyError:
             pass
 
-        # DATA:MODEL
+        # metadata DATA:MODEL
         product_dict["data"]["model"]["name"] = (
             self.schema_dict.get("name")
             or self.args_dict.get("metadata_product_data_name")

simtools/data_model/metadata_model.py

@@ -8,13 +8,9 @@ Follows CTAO top-level data model definition.
 
 """
 
-import logging
-
 import simtools.data_model.schema
 import simtools.utils.general as gen
 
-_logger = logging.getLogger(__name__)
-
 
 def get_default_metadata_dict(
     schema_file=None, observatory="CTA", schema_version="latest", lower_case=True

simtools/data_model/model_data_writer.py

@@ -98,10 +98,10 @@ class ModelDataWriter:
         output_file,
         output_path=None,
         metadata_input_dict=None,
-        db_config=None,
         unit=None,
         meta_parameter=False,
         model_parameter_schema_version=None,
+        check_db_for_existing_parameter=True,
     ):
         """
         Generate DB-style model parameter dict and write it to json file.
@@ -122,14 +122,14 @@ class ModelDataWriter:
             Path to output file.
         metadata_input_dict: dict
             Input to metadata collector.
-        db_config: dict
-            Database configuration. If not None, check if parameter with the same version exists.
         unit: str
             Unit of the parameter value (if applicable and value is not of type astropy Quantity).
         meta_parameter: bool
             Setting for meta parameter flag.
         model_parameter_schema_version: str, None
             Version of the model parameter schema (if None, use schema version from schema dict).
+        check_db_for_existing_parameter: bool
+            If True, check if parameter with same version exists in DB before writing.
 
         Returns
         -------
@@ -142,10 +142,8 @@ class ModelDataWriter:
             args_dict=None,
             output_path=output_path,
         )
-        if db_config is not None:
-            writer.check_db_for_existing_parameter(
-                parameter_name, instrument, parameter_version, db_config
-            )
+        if check_db_for_existing_parameter:
+            writer.check_db_for_existing_parameter(parameter_name, instrument, parameter_version)
 
         unique_id = None
         if metadata_input_dict is not None:
@@ -170,9 +168,7 @@ class ModelDataWriter:
         writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         return _json_dict
 
-    def check_db_for_existing_parameter(
-        self, parameter_name, instrument, parameter_version, db_config
-    ):
+    def check_db_for_existing_parameter(self, parameter_name, instrument, parameter_version):
         """
         Check if a parameter with the same version exists in the simulation model database.
 
@@ -184,15 +180,15 @@ class ModelDataWriter:
             Name of the instrument.
         parameter_version: str
             Version of the parameter.
-        db_config: dict
-            Database configuration.
 
         Raises
         ------
         ValueError
             If parameter with the same version exists in the database.
         """
-        db = db_handler.DatabaseHandler(db_config=db_config)
+        db = db_handler.DatabaseHandler()
+        if not db.is_configured():
+            return
         try:
             db.get_model_parameter(
                 parameter=parameter_name,
@@ -373,8 +369,10 @@ class ModelDataWriter:
         """
         try:
             unit_list = []
-            for data in self.schema_dict["data"]:
-                unit_list.append(data["unit"] if data["unit"] != "dimensionless" else None)
+            unit_list = [
+                data["unit"] if data["unit"] != "dimensionless" else None
+                for data in self.schema_dict["data"]
+            ]
             return unit_list if len(unit_list) > 1 else unit_list[0]
         except (KeyError, IndexError):
             pass

simtools/data_model/validate_data.py

@@ -928,9 +928,7 @@ class DataValidator:
             """Return value as sorted list."""
             return [value] if isinstance(value, str) else sorted(value)
 
-        instrument_sites = []
-        for inst in instruments:
-            instrument_sites.append(names.get_site_from_array_element_name(inst))
+        instrument_sites = [names.get_site_from_array_element_name(inst) for inst in instruments]
         # names.get_site_from_array_element_name might return a list
         flat_sites = [
             s

simtools/db/db_handler.py

@@ -4,6 +4,7 @@ import logging
 from collections import defaultdict
 from pathlib import Path
 
+from simtools import settings
 from simtools.data_model import validate_data
 from simtools.db.mongo_db import MongoDBHandler
 from simtools.io import io_handler
@@ -20,11 +21,6 @@ class DatabaseHandler:
 
     - db_simulation_model_version (from db_config): version of the simulation model database
     - model_version (from production_tables): version of the model contained in the database
-
-    Parameters
-    ----------
-    db_config: dict
-        Dictionary with the DB configuration.
     """
 
     ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"]
@@ -33,13 +29,17 @@ class DatabaseHandler:
     model_parameters_cached = {}
     model_versions_cached = {}
 
-    def __init__(self, db_config=None):
+    def __init__(self):
         """Initialize the DatabaseHandler class."""
         self._logger = logging.getLogger(__name__)
 
-        self.db_config = MongoDBHandler.validate_db_config(db_config)
+        self.db_config = (
+            MongoDBHandler.validate_db_config(dict(settings.config.db_config))
+            if settings.config.db_config
+            else None
+        )
         self.io_handler = io_handler.IOHandler()
-        self.mongo_db_handler = MongoDBHandler(db_config) if self.db_config else None
+        self.mongo_db_handler = MongoDBHandler(self.db_config) if self.db_config else None
 
         self.db_name = (
             MongoDBHandler.get_db_name(
@@ -50,6 +50,17 @@ class DatabaseHandler:
             else None
         )
 
+    def is_configured(self):
+        """
+        Check if the DatabaseHandler is configured.
+
+        Returns
+        -------
+        bool
+            True if the DatabaseHandler is configured, False otherwise.
+        """
+        return self.mongo_db_handler is not None
+
     def get_db_name(self, db_name=None, db_simulation_model_version=None, model_name=None):
         """Build DB name from configuration."""
         if db_name:

simtools/dependencies.py

@@ -9,27 +9,26 @@ This modules provides two main functionalities:
 """
 
 import logging
-import os
 import re
 import subprocess
 from pathlib import Path
 
 import yaml
 
+from simtools import settings
 from simtools.io import ascii_handler
+from simtools.utils import general as gen
 from simtools.version import __version__
 
 _logger = logging.getLogger(__name__)
 
 
-def get_version_string(db_config=None, run_time=None):
+def get_version_string(run_time=None):
     """
     Print the versions of the dependencies.
 
     Parameters
     ----------
-    db_config : dict, optional
-        Database configuration dictionary.
     run_time : list, optional
         Runtime environment command (e.g., Docker).
 
@@ -40,10 +39,13 @@ def get_version_string(db_config=None, run_time=None):
 
     """
     return (
-        f"Database name: {get_database_version_or_name(db_config, version=False)}\n"
-        f"Database version: {get_database_version_or_name(db_config, version=True)}\n"
+        f"Database name: {get_database_version_or_name(version=False)}\n"
+        f"Database version: {get_database_version_or_name(version=True)}\n"
         f"sim_telarray version: {get_sim_telarray_version(run_time)}\n"
+        "sim_telarray exe: "
+        f"{settings.config.sim_telarray_exe if settings.config.sim_telarray_exe else 'None'}\n"
         f"CORSIKA version: {get_corsika_version(run_time)}\n"
+        f"CORSIKA exe: {settings.config.corsika_exe if settings.config.corsika_exe else 'None'}\n"
         f"Build options: {get_build_options(run_time)}\n"
         f"Runtime environment: {run_time if run_time else 'None'}\n"
     )
@@ -73,14 +75,12 @@ def get_software_version(software):
         raise ValueError(f"Unknown software: {software}") from exc
 
 
-def get_database_version_or_name(db_config, version=True):
+def get_database_version_or_name(version=True):
     """
     Get the version or name of the simulation model data base used.
 
     Parameters
     ----------
-    db_config : dict
-        Dictionary containing the database configuration.
     version : bool
         If True, return the version of the database. If False, return the name.
 
@@ -91,8 +91,10 @@ def get_database_version_or_name(db_config, version=True):
 
     """
     if version:
-        return db_config and db_config.get("db_simulation_model_version")
-    return db_config and db_config.get("db_simulation_model")
+        return settings.config.db_config and settings.config.db_config.get(
+            "db_simulation_model_version"
+        )
+    return settings.config.db_config and settings.config.db_config.get("db_simulation_model")
 
 
 def get_sim_telarray_version(run_time=None):
@@ -111,16 +113,13 @@ def get_sim_telarray_version(run_time=None):
     str
         Version of the sim_telarray package.
     """
-    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if sim_telarray_path is None:
-        _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
+    if settings.config.sim_telarray_exe is None:
+        _logger.warning("sim_telarray environment not configured.")
         return None
-    sim_telarray_path = Path(sim_telarray_path) / "sim_telarray" / "bin" / "sim_telarray"
-
     if run_time is None:
-        command = [str(sim_telarray_path), "--version"]
+        command = [str(settings.config.sim_telarray_exe), "--version"]
     else:
-        command = [*run_time, str(sim_telarray_path), "--version"]
+        command = [*run_time, str(settings.config.sim_telarray_exe), "--version"]
 
     _logger.debug(f"Running command: {command}")
     result = subprocess.run(command, capture_output=True, text=True, check=False)
@@ -151,19 +150,15 @@ def get_corsika_version(run_time=None):
     str
         Version of the CORSIKA package.
     """
-    corsika_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if corsika_path is None:
-        _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
+    if settings.config.corsika_exe is None:
+        _logger.warning("CORSIKA environment not configured.")
         return None
-    corsika_command = Path(corsika_path) / "corsika-run" / "corsika"
 
     if run_time is None:
-        command = [str(corsika_command)]
+        command = [str(settings.config.corsika_exe)]
     else:
-        command = [*run_time, str(corsika_command)]
+        command = [*run_time, str(settings.config.corsika_exe)]
 
-    # Below I do not use the standard context manager because
-    # it makes mocking in the tests significantly more difficult
     process = subprocess.Popen(  # pylint: disable=consider-using-with
         command,
         stdout=subprocess.PIPE,
@@ -197,9 +192,14 @@ def get_corsika_version(run_time=None):
 
 def get_build_options(run_time=None):
     """
-    Return CORSIKA / sim_telarray build options.
+    Return CORSIKA / sim_telarray config and build options.
+
+    For CORSIKA / sim_telarray build for simtools version >0.25.0:
+    expects build_opts.yml file in each CORSIKA and sim_telarray
+    directories.
 
-    Expects a build_opts.yml file in the sim_telarray directory.
+    For CORSIKA / sim_telarray build for simtools version <=0.25.0:
+    expects a build_opts.yml file in the sim_telarray directory.
 
     Parameters
     ----------
@@ -209,28 +209,71 @@ def get_build_options(run_time=None):
     Returns
     -------
     dict
-        Build options from build_opts.yml file.
+        CORSIKA / sim_telarray build options.
     """
-    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if sim_telarray_path is None:
-        raise ValueError("SIMTOOLS_SIMTEL_PATH not defined.")
-
-    build_opts_path = Path(sim_telarray_path) / "build_opts.yml"
-
+    build_opts = {}
+    for package in ["corsika", "sim_telarray"]:
+        path = _get_package_path(package)
+        if not path:
+            continue
+        try:
+            build_opts.update(_get_build_options_from_file(path / "build_opts.yml", run_time))
+        except (FileNotFoundError, TypeError, ValueError):
+            # legacy fallback only for sim_telarray
+            if package == "sim_telarray":
+                try:
+                    legacy_path = path.parent / "build_opts.yml"
+                    build_opts.update(_get_build_options_from_file(legacy_path, run_time))
+                except (FileNotFoundError, TypeError, ValueError):
+                    _logger.debug(f"No build options found for {package}.")
+    if not build_opts:
+        raise FileNotFoundError("No build option file found.")
+
+    return build_opts
+
+
+def _get_package_path(package):
+    """Get the package path from settings or environment variables."""
+    path = getattr(settings.config, f"{package}_path")
+    if path is None:
+        path = gen.load_environment_variables().get(f"{package}_path")
+    return Path(path) if path else None
+
+
+def _get_build_options_from_file(build_opts_path, run_time=None):
+    """Read build options from file."""
     if run_time is None:
         try:
             return ascii_handler.collect_data_from_file(build_opts_path)
         except FileNotFoundError as exc:
-            raise FileNotFoundError("No build_opts.yml file found.") from exc
+            raise FileNotFoundError("No build option file found.") from exc
 
     command = [*run_time, "cat", str(build_opts_path)]
-    _logger.debug(f"Reading build_opts.yml with command: {command}")
+    _logger.debug(f"Reading build option with command: {command}")
 
     result = subprocess.run(command, capture_output=True, text=True, check=False)
     if result.returncode:
-        raise FileNotFoundError(f"No build_opts.yml file found in container: {result.stderr}")
+        raise FileNotFoundError(f"No build option file found in container: {result.stderr}")
 
     try:
         return yaml.safe_load(result.stdout)
     except yaml.YAMLError as exc:
         raise ValueError(f"Error parsing build_opts.yml from container: {exc}") from exc
+
+
+def export_build_info(output_file, run_time=None):
+    """
+    Export build and version information to a file.
+
+    Parameters
+    ----------
+    output_file : str
+        Path to the output file.
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+    """
+    build_info = get_build_options(run_time)
+    build_info["simtools"] = __version__
+    build_info["database_name"] = get_database_version_or_name(version=False)
+    build_info["database_version"] = get_database_version_or_name(version=True)
+    ascii_handler.write_data_to_file(data=build_info, output_file=Path(output_file))

simtools/io/ascii_handler.py

@@ -208,9 +208,11 @@ def write_data_to_file(data, output_file, sort_keys=False, numpy_types=False):
     """
     output_file = Path(output_file)
     if output_file.suffix.lower() == ".json":
-        return _write_to_json(data, output_file, sort_keys, numpy_types)
+        _write_to_json(data, output_file, sort_keys, numpy_types)
+        return
     if output_file.suffix.lower() in [".yml", ".yaml"]:
-        return _write_to_yaml(data, output_file, sort_keys)
+        _write_to_yaml(data, output_file, sort_keys)
+        return
 
     raise ValueError(
         f"Unsupported file type {output_file.suffix}. Only .json, .yml, and .yaml are supported."

simtools/io/table_handler.py

@@ -295,7 +295,7 @@ def write_table_in_hdf5(table, output_file, table_name):
     None
     """
     for col in table.colnames:
-        if table[col].dtype.kind == "U":  # hd5 does not support unicode
+        if table[col].dtype.kind == "U":  # hdf5 does not support unicode
             table[col] = table[col].astype("S")
 
     with h5py.File(output_file, "a") as f:

simtools/layout/array_layout.py

@@ -32,8 +32,6 @@ class ArrayLayout:
 
     Parameters
     ----------
-    db_config: dict
-        Database configuration.
     site: str
         Site name or location (e.g., North/South or LaPalma/Paranal)
     model_version: str
@@ -52,7 +50,6 @@ class ArrayLayout:
 
     def __init__(
         self,
-        db_config,
         site,
         model_version,
         label=None,
@@ -67,7 +64,6 @@ class ArrayLayout:
         self.model_version = model_version
         self.label = label
         self.name = name
-        self.db_config = db_config
         self.site = None if site is None else names.validate_site_name(site)
         self.site_model = None
         self.io_handler = io_handler.IOHandler()
@@ -95,14 +91,11 @@ class ArrayLayout:
     def _initialize_site_parameters_from_db(self):
         """Initialize site parameters required for transformations using the database."""
         self._logger.debug("Initialize parameters from DB")
-        if self.db_config is None:
-            raise ValueError("No database configuration provided")
 
-        self.site_model = SiteModel(
-            site=self.site,
-            model_version=self.model_version,
-            db_config=self.db_config,
-        )
+        try:
+            self.site_model = SiteModel(site=self.site, model_version=self.model_version)
+        except RuntimeError as e:
+            raise ValueError("No database configuration provided") from e
         self._corsika_observation_level = self.site_model.get_corsika_site_parameters().get(
             "corsika_observation_level", None
         )
@@ -419,7 +412,6 @@ class ArrayLayout:
             site=self.site,
             telescope_name=telescope_name,
             model_version=self.model_version,
-            db_config=self.db_config,
             label=self.label,
         )