gammasimtools 0.24.0__py3-none-any.whl → 0.26.0__py3-none-any.whl

simtools/db/mongo_db.py

@@ -10,7 +10,7 @@ import gridfs
 import jsonschema
 from astropy.table import Table
 from bson.objectid import ObjectId
-from pymongo import MongoClient
+from pymongo import MongoClient, monitoring
 
 from simtools.io import ascii_handler
 
@@ -57,6 +57,61 @@ jsonschema_db_dict = {
 }
 
 
+class IdleConnectionMonitor(monitoring.ConnectionPoolListener):
+    """
+    A listener to track MongoDB connection pool activity.
+
+    Used to monitor idle connections and log connection events.
+    Switched on in debug mode.
+    """
+
+    def __init__(self):
+        self._logger = logging.getLogger("IdleConnectionMonitor")
+        self.open_connections = 0
+
+    def connection_created(self, event):
+        """Handle connection creation event."""
+        self.open_connections += 1
+        self._logger.debug(
+            f"MongoDB connection Created: {event.address}. Total in Pool: {self.open_connections}"
+        )
+
+    def connection_closed(self, event):
+        """Handle connection closure event."""
+        self.open_connections -= 1
+        self._logger.debug(
+            f"MongoDB connection Closed: {event.address}. Reason: {event.reason}. "
+            f"Total in Pool: {self.open_connections}"
+        )
+
+    def connection_check_out_started(self, event):
+        """Handle connection check out started event."""
+
+    def connection_check_out_failed(self, event):
+        """Handle connection check out failure event."""
+
+    def connection_checked_out(self, event):
+        """Handle connection checked out event."""
+
+    def connection_checked_in(self, event):
+        """Handle connection checked in event."""
+
+    def connection_ready(self, event):
+        """Handle connection ready event."""
+
+    def pool_created(self, event):
+        """Handle connection pool creation event."""
+
+    def pool_ready(self, event):
+        """Handle connection pool ready event."""
+
+    def pool_cleared(self, event):
+        """Handle connection pool cleared event."""
+
+    def pool_closed(self, event):
+        """Handle connection pool closure event."""
+
+
 class MongoDBHandler:  # pylint: disable=unsubscriptable-object
     """
     MongoDBHandler provides low-level interface to MongoDB operations.
@@ -70,19 +125,90 @@ class MongoDBHandler:  # pylint: disable=unsubscriptable-object
         Dictionary with the MongoDB configuration (see jsonschema_db_dict for details).
     """
 
-    db_client: MongoClient | None = None
+    db_client: MongoClient = None
     _lock = Lock()
+    _logger = logging.getLogger(__name__)
 
     def __init__(self, db_config=None):
         """Initialize the MongoDBHandler class."""
-        self._logger = logging.getLogger(__name__)
         self.db_config = MongoDBHandler.validate_db_config(db_config)
         self.list_of_collections = {}
 
-        if self.db_config and MongoDBHandler.db_client is None:
-            with MongoDBHandler._lock:
-                if MongoDBHandler.db_client is None:
-                    MongoDBHandler.db_client = self._open_db()
+        if self.db_config:
+            self._initialize_client(self.db_config)
+
+    @classmethod
+    def _initialize_client(cls, db_config):
+        """
+        Initialize the MongoDB client in a thread-safe manner.
+
+        Only initializes if it hasn't been done yet. Uses double-checked locking
+        to ensure thread safety.
+
+        Parameters
+        ----------
+        db_config: dict
+            Dictionary with the MongoDB configuration.
+        """
+        if cls.db_client is not None:
+            return
+        with cls._lock:
+            if cls.db_client is None:
+                try:
+                    uri = cls._build_uri(db_config)
+                    client_kwargs = {"maxIdleTimeMS": 10000}
+
+                    if cls._logger.isEnabledFor(logging.DEBUG):
+                        client_kwargs["event_listeners"] = [IdleConnectionMonitor()]
+
+                    cls.db_client = MongoClient(uri, **client_kwargs)
+                    cls._logger.debug("MongoDB client initialized successfully.")
+                except Exception as e:
+                    cls._logger.error(f"Failed to initialize MongoDB client: {e}")
+                    raise
+
+    @staticmethod
+    def _build_uri(db_config):
+        """
+        Build MongoDB URI from configuration.
+
+        Parameters
+        ----------
+        db_config: dict
+            Dictionary with the MongoDB configuration.
+
+        Returns
+        -------
+        str
+            MongoDB connection URI.
+        """
+        direct_connection = db_config["db_server"] in (
+            "localhost",
+            "simtools-mongodb",
+            "mongodb",
+        )
+        auth_source = (
+            db_config.get("db_api_authentication_database")
+            if db_config.get("db_api_authentication_database")
+            else "admin"
+        )
+
+        username = db_config["db_api_user"]
+        password = db_config["db_api_pw"]
+        server = db_config["db_server"]
+        port = db_config["db_api_port"]
+
+        uri_base = f"mongodb://{username}:{password}@{server}:{port}/"
+        params = [f"authSource={auth_source}"]
+
+        if direct_connection:
+            params.append("directConnection=true")
+        else:
+            params.append("ssl=true")
+            params.append("tlsAllowInvalidHostnames=true")
+            params.append("tlsAllowInvalidCertificates=true")
+
+        return f"{uri_base}?{'&'.join(params)}"
 
     @staticmethod
     def validate_db_config(db_config):
@@ -112,41 +238,6 @@ class MongoDBHandler:  # pylint: disable=unsubscriptable-object
         except jsonschema.exceptions.ValidationError as err:
             raise ValueError("Invalid MongoDB configuration") from err
 
-    def _open_db(self):
-        """
-        Open a connection to MongoDB and return the client.
-
-        Returns
-        -------
-        MongoClient
-            A PyMongo DB client
-
-        Raises
-        ------
-        KeyError
-            If the DB configuration is invalid
-        """
-        direct_connection = self.db_config["db_server"] in (
-            "localhost",
-            "simtools-mongodb",
-            "mongodb",
-        )
-        return MongoClient(
-            self.db_config["db_server"],
-            port=self.db_config["db_api_port"],
-            username=self.db_config["db_api_user"],
-            password=self.db_config["db_api_pw"],
-            authSource=(
-                self.db_config.get("db_api_authentication_database")
-                if self.db_config.get("db_api_authentication_database")
-                else "admin"
-            ),
-            directConnection=direct_connection,
-            ssl=not direct_connection,
-            tlsallowinvalidhostnames=True,
-            tlsallowinvalidcertificates=True,
-        )
-
     @staticmethod
     def get_db_name(db_name=None, db_simulation_model_version=None, model_name=None):
         """

simtools/dependencies.py

@@ -9,27 +9,26 @@ This modules provides two main functionalities:
 """
 
 import logging
-import os
 import re
 import subprocess
 from pathlib import Path
 
 import yaml
 
+from simtools import settings
 from simtools.io import ascii_handler
+from simtools.utils import general as gen
 from simtools.version import __version__
 
 _logger = logging.getLogger(__name__)
 
 
-def get_version_string(db_config=None, run_time=None):
+def get_version_string(run_time=None):
     """
     Print the versions of the dependencies.
 
     Parameters
     ----------
-    db_config : dict, optional
-        Database configuration dictionary.
     run_time : list, optional
         Runtime environment command (e.g., Docker).
 
@@ -40,10 +39,13 @@ def get_version_string(db_config=None, run_time=None):
 
     """
     return (
-        f"Database name: {get_database_version_or_name(db_config, version=False)}\n"
-        f"Database version: {get_database_version_or_name(db_config, version=True)}\n"
+        f"Database name: {get_database_version_or_name(version=False)}\n"
+        f"Database version: {get_database_version_or_name(version=True)}\n"
         f"sim_telarray version: {get_sim_telarray_version(run_time)}\n"
+        "sim_telarray exe: "
+        f"{settings.config.sim_telarray_exe if settings.config.sim_telarray_exe else 'None'}\n"
         f"CORSIKA version: {get_corsika_version(run_time)}\n"
+        f"CORSIKA exe: {settings.config.corsika_exe if settings.config.corsika_exe else 'None'}\n"
         f"Build options: {get_build_options(run_time)}\n"
         f"Runtime environment: {run_time if run_time else 'None'}\n"
     )
@@ -73,14 +75,12 @@ def get_software_version(software):
         raise ValueError(f"Unknown software: {software}") from exc
 
 
-def get_database_version_or_name(db_config, version=True):
+def get_database_version_or_name(version=True):
     """
     Get the version or name of the simulation model data base used.
 
     Parameters
     ----------
-    db_config : dict
-        Dictionary containing the database configuration.
     version : bool
         If True, return the version of the database. If False, return the name.
 
@@ -91,8 +91,10 @@ def get_database_version_or_name(db_config, version=True):
 
     """
     if version:
-        return db_config and db_config.get("db_simulation_model_version")
-    return db_config and db_config.get("db_simulation_model")
+        return settings.config.db_config and settings.config.db_config.get(
+            "db_simulation_model_version"
+        )
+    return settings.config.db_config and settings.config.db_config.get("db_simulation_model")
 
 
 def get_sim_telarray_version(run_time=None):
@@ -111,16 +113,13 @@ def get_sim_telarray_version(run_time=None):
     str
         Version of the sim_telarray package.
     """
-    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if sim_telarray_path is None:
-        _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
+    if settings.config.sim_telarray_exe is None:
+        _logger.warning("sim_telarray environment not configured.")
         return None
-    sim_telarray_path = Path(sim_telarray_path) / "sim_telarray" / "bin" / "sim_telarray"
-
     if run_time is None:
-        command = [str(sim_telarray_path), "--version"]
+        command = [str(settings.config.sim_telarray_exe), "--version"]
     else:
-        command = [*run_time, str(sim_telarray_path), "--version"]
+        command = [*run_time, str(settings.config.sim_telarray_exe), "--version"]
 
     _logger.debug(f"Running command: {command}")
     result = subprocess.run(command, capture_output=True, text=True, check=False)
@@ -151,20 +150,15 @@ def get_corsika_version(run_time=None):
     str
         Version of the CORSIKA package.
     """
-    version = None
-    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if sim_telarray_path is None:
-        _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
+    if settings.config.corsika_exe is None:
+        _logger.warning("CORSIKA environment not configured.")
         return None
-    corsika_command = Path(sim_telarray_path) / "corsika-run" / "corsika"
 
     if run_time is None:
-        command = [str(corsika_command)]
+        command = [str(settings.config.corsika_exe)]
     else:
-        command = [*run_time, str(corsika_command)]
+        command = [*run_time, str(settings.config.corsika_exe)]
 
-    # Below I do not use the standard context manager because
-    # it makes mocking in the tests significantly more difficult
     process = subprocess.Popen(  # pylint: disable=consider-using-with
         command,
         stdout=subprocess.PIPE,
@@ -173,11 +167,8 @@ def get_corsika_version(run_time=None):
         text=True,
     )
 
-    # Capture output until it waits for input
-    while True:
-        line = process.stdout.readline()
-        if not line:
-            break
+    version = None
+    for line in process.stdout:
         # Extract the version from the line "NUMBER OF VERSION :  7.7550"
         if "NUMBER OF VERSION" in line:
             version = line.split(":")[1].strip()
@@ -201,9 +192,14 @@ def get_corsika_version(run_time=None):
 
 def get_build_options(run_time=None):
     """
-    Return CORSIKA / sim_telarray build options.
+    Return CORSIKA / sim_telarray config and build options.
+
+    For CORSIKA / sim_telarray build for simtools version >0.25.0:
+    expects build_opts.yml file in each CORSIKA and sim_telarray
+    directories.
 
-    Expects a build_opts.yml file in the sim_telarray directory.
+    For CORSIKA / sim_telarray build for simtools version <=0.25.0:
+    expects a build_opts.yml file in the sim_telarray directory.
 
     Parameters
     ----------
@@ -213,28 +209,71 @@ def get_build_options(run_time=None):
     Returns
     -------
     dict
-        Build options from build_opts.yml file.
+        CORSIKA / sim_telarray build options.
     """
-    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if sim_telarray_path is None:
-        raise ValueError("SIMTOOLS_SIMTEL_PATH not defined.")
-
-    build_opts_path = Path(sim_telarray_path) / "build_opts.yml"
-
+    build_opts = {}
+    for package in ["corsika", "sim_telarray"]:
+        path = _get_package_path(package)
+        if not path:
+            continue
+        try:
+            build_opts.update(_get_build_options_from_file(path / "build_opts.yml", run_time))
+        except (FileNotFoundError, TypeError, ValueError):
+            # legacy fallback only for sim_telarray
+            if package == "sim_telarray":
+                try:
+                    legacy_path = path.parent / "build_opts.yml"
+                    build_opts.update(_get_build_options_from_file(legacy_path, run_time))
+                except (FileNotFoundError, TypeError, ValueError):
+                    _logger.debug(f"No build options found for {package}.")
+    if not build_opts:
+        raise FileNotFoundError("No build option file found.")
+
+    return build_opts
+
+
+def _get_package_path(package):
+    """Get the package path from settings or environment variables."""
+    path = getattr(settings.config, f"{package}_path")
+    if path is None:
+        path = gen.load_environment_variables().get(f"{package}_path")
+    return Path(path) if path else None
+
+
+def _get_build_options_from_file(build_opts_path, run_time=None):
+    """Read build options from file."""
     if run_time is None:
         try:
             return ascii_handler.collect_data_from_file(build_opts_path)
         except FileNotFoundError as exc:
-            raise FileNotFoundError("No build_opts.yml file found.") from exc
+            raise FileNotFoundError("No build option file found.") from exc
 
     command = [*run_time, "cat", str(build_opts_path)]
-    _logger.debug(f"Reading build_opts.yml with command: {command}")
+    _logger.debug(f"Reading build option with command: {command}")
 
     result = subprocess.run(command, capture_output=True, text=True, check=False)
     if result.returncode:
-        raise FileNotFoundError(f"No build_opts.yml file found in container: {result.stderr}")
+        raise FileNotFoundError(f"No build option file found in container: {result.stderr}")
 
     try:
         return yaml.safe_load(result.stdout)
     except yaml.YAMLError as exc:
         raise ValueError(f"Error parsing build_opts.yml from container: {exc}") from exc
+
+
+def export_build_info(output_file, run_time=None):
+    """
+    Export build and version information to a file.
+
+    Parameters
+    ----------
+    output_file : str
+        Path to the output file.
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+    """
+    build_info = get_build_options(run_time)
+    build_info["simtools"] = __version__
+    build_info["database_name"] = get_database_version_or_name(version=False)
+    build_info["database_version"] = get_database_version_or_name(version=True)
+    ascii_handler.write_data_to_file(data=build_info, output_file=Path(output_file))

simtools/io/ascii_handler.py

@@ -208,9 +208,11 @@ def write_data_to_file(data, output_file, sort_keys=False, numpy_types=False):
     """
     output_file = Path(output_file)
     if output_file.suffix.lower() == ".json":
-        return _write_to_json(data, output_file, sort_keys, numpy_types)
+        _write_to_json(data, output_file, sort_keys, numpy_types)
+        return
     if output_file.suffix.lower() in [".yml", ".yaml"]:
-        return _write_to_yaml(data, output_file, sort_keys)
+        _write_to_yaml(data, output_file, sort_keys)
+        return
 
     raise ValueError(
         f"Unsupported file type {output_file.suffix}. Only .json, .yml, and .yaml are supported."

simtools/io/table_handler.py

@@ -295,7 +295,7 @@ def write_table_in_hdf5(table, output_file, table_name):
     None
     """
     for col in table.colnames:
-        if table[col].dtype.kind == "U":  # hd5 does not support unicode
+        if table[col].dtype.kind == "U":  # hdf5 does not support unicode
             table[col] = table[col].astype("S")
 
     with h5py.File(output_file, "a") as f:

simtools/job_execution/htcondor_script_generator.py

@@ -136,8 +136,6 @@ simtools-simulate-prod \\
     --view_cone {view_cone_string} \\
     --run_number $((process_id)) \\
     --run_number_offset {run_number_offset} \\
-    --number_of_runs 1 \\
-    --data_directory /tmp/simtools-data \\
     --output_path /tmp/simtools-output \\
     --log_level {args_dict["log_level"]} \\
     --pack_for_grid_register simtools-output

simtools/layout/array_layout.py

@@ -32,8 +32,6 @@ class ArrayLayout:
 
     Parameters
     ----------
-    db_config: dict
-        Database configuration.
     site: str
         Site name or location (e.g., North/South or LaPalma/Paranal)
     model_version: str
@@ -52,7 +50,6 @@ class ArrayLayout:
 
     def __init__(
         self,
-        db_config,
         site,
         model_version,
         label=None,
@@ -67,7 +64,6 @@ class ArrayLayout:
         self.model_version = model_version
         self.label = label
         self.name = name
-        self.db_config = db_config
         self.site = None if site is None else names.validate_site_name(site)
         self.site_model = None
         self.io_handler = io_handler.IOHandler()
@@ -95,14 +91,11 @@ class ArrayLayout:
     def _initialize_site_parameters_from_db(self):
         """Initialize site parameters required for transformations using the database."""
         self._logger.debug("Initialize parameters from DB")
-        if self.db_config is None:
-            raise ValueError("No database configuration provided")
 
-        self.site_model = SiteModel(
-            site=self.site,
-            model_version=self.model_version,
-            db_config=self.db_config,
-        )
+        try:
+            self.site_model = SiteModel(site=self.site, model_version=self.model_version)
+        except RuntimeError as e:
+            raise ValueError("No database configuration provided") from e
         self._corsika_observation_level = self.site_model.get_corsika_site_parameters().get(
             "corsika_observation_level", None
         )
@@ -419,7 +412,6 @@ class ArrayLayout:
             site=self.site,
             telescope_name=telescope_name,
             model_version=self.model_version,
-            db_config=self.db_config,
             label=self.label,
         )