gammasimtools 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

simtools/data_model/validate_data.py

@@ -14,6 +14,7 @@ import simtools.utils.general as gen
 from simtools.data_model import schema
 from simtools.io import ascii_handler
 from simtools.utils import names, value_conversion
+from simtools.version import is_valid_semantic_version
 
 
 class DataValidator:
@@ -46,7 +47,7 @@ class DataValidator:
         check_exact_data_type=True,
     ):
         """Initialize validation class and read required reference data columns."""
-        self._logger = logging.getLogger(__name__)
+        self.logger = logging.getLogger(__name__)
 
         self.data_file_name = data_file
         self.schema_file_name = schema_file
@@ -83,7 +84,7 @@ class DataValidator:
             return self._validate_data_dict(is_model_parameter, lists_as_strings)
         if isinstance(self.data_table, Table):
             return self._validate_data_table()
-        self._logger.error("No data or data table to validate")
+        self.logger.error("No data or data table to validate")
         raise TypeError
 
     def validate_data_file(self, is_model_parameter=None):
@@ -100,10 +101,10 @@ class DataValidator:
         try:
             if Path(self.data_file_name).suffix in (".yml", ".yaml", ".json"):
                 self.data_dict = ascii_handler.collect_data_from_file(self.data_file_name)
-                self._logger.info(f"Validating data from: {self.data_file_name}")
+                self.logger.info(f"Validating data from: {self.data_file_name}")
             else:
                 self.data_table = Table.read(self.data_file_name, guess=True, delimiter=r"\s")
-                self._logger.info(f"Validating tabled data from: {self.data_file_name}")
+                self.logger.info(f"Validating tabled data from: {self.data_file_name}")
         except (AttributeError, TypeError):
             pass
         if is_model_parameter:
@@ -129,7 +130,7 @@ class DataValidator:
             raise ValueError(f"Mismatch: version '{param_version}' vs. file '{file_stem}'.")
 
         if param_version is None:
-            self._logger.warning(f"File '{file_stem}' has no parameter version defined.")
+            self.logger.warning(f"File '{file_stem}' has no parameter version defined.")
 
     @staticmethod
     def validate_model_parameter(par_dict):
@@ -153,6 +154,46 @@ class DataValidator:
         )
         return data_validator.validate_and_transform(is_model_parameter=True)
 
+    @staticmethod
+    def validate_data_files(
+        file_directory=None,
+        file_name=None,
+        is_model_parameter=True,
+        check_exact_data_type=False,
+        schema_file=None,
+    ):
+        """
+        Validate data or model parameters in files in a directory or a single file.
+
+        Parameters
+        ----------
+        file_directory: str or Path
+            Directory with files to be validated.
+        file_name: str or Path
+            Name of the file to be validated.
+        is_model_parameter: bool
+            This is a model parameter (add some data preparation).
+        check_exact_data_type: bool
+            Require exact data type for validation.
+        """
+        if file_directory:
+            file_list = sorted(Path(file_directory).rglob("*.json"))
+        elif file_name:
+            file_list = [Path(file_name)]
+        else:
+            return
+
+        for data_file in file_list:
+            parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", data_file.stem)
+            schema_path = schema_file or schema.get_model_parameter_schema_file(f"{parameter_name}")
+            data_validator = DataValidator(
+                schema_file=schema_path,
+                data_file=data_file,
+                check_exact_data_type=check_exact_data_type,
+            )
+            data_validator.validate_and_transform(is_model_parameter)
+            data_validator.logger.info(f"Validated data file {data_file} with schema {schema_path}")
+
     def _validate_data_dict(self, is_model_parameter=False, lists_as_strings=False):
         """
         Validate values in a dictionary.
@@ -208,8 +249,10 @@ class DataValidator:
                     self.data_dict.get("instrument"), self.data_dict.get("site")
                 )
 
-        for version_string in ("version", "parameter_version", "model_version"):
-            self._check_version_string(self.data_dict.get(version_string))
+        for version_type in ("version", "parameter_version", "model_version"):
+            version_string = self.data_dict.get(version_type, "0.0.0")
+            if not is_valid_semantic_version(version_string):
+                raise ValueError(f"Invalid version string '{version_string}'")
 
         if lists_as_strings:
             self._convert_results_to_model_format()
@@ -278,7 +321,7 @@ class DataValidator:
                 "table_columns", None
             )
         except IndexError:
-            self._logger.error(f"Error reading validation schema from {self.schema_file_name}")
+            self.logger.error(f"Error reading validation schema from {self.schema_file_name}")
             raise
 
         if self._data_description is not None:
@@ -327,7 +370,7 @@ class DataValidator:
         for entry in self._data_description:
             if entry.get("required", False):
                 if entry["name"] in self.data_table.columns:
-                    self._logger.debug(f"Found required data column {entry['name']}")
+                    self.logger.debug(f"Found required data column {entry['name']}")
                 else:
                     raise KeyError(f"Missing required column {entry['name']}")
 
@@ -353,18 +396,18 @@ class DataValidator:
                     _columns_by_which_to_reverse_sort.append(entry["name"])
 
         if len(_columns_by_which_to_sort) > 0:
-            self._logger.debug(f"Sorting data columns: {_columns_by_which_to_sort}")
+            self.logger.debug(f"Sorting data columns: {_columns_by_which_to_sort}")
             try:
                 self.data_table.sort(_columns_by_which_to_sort)
             except AttributeError:
-                self._logger.error("No data table defined for sorting")
+                self.logger.error("No data table defined for sorting")
                 raise
         elif len(_columns_by_which_to_reverse_sort) > 0:
-            self._logger.debug(f"Reverse sorting data columns: {_columns_by_which_to_reverse_sort}")
+            self.logger.debug(f"Reverse sorting data columns: {_columns_by_which_to_reverse_sort}")
             try:
                 self.data_table.sort(_columns_by_which_to_reverse_sort, reverse=True)
             except AttributeError:
-                self._logger.error("No data table defined for reverse sorting")
+                self.logger.error("No data table defined for reverse sorting")
                 raise
 
     def _check_data_for_duplicates(self):
@@ -379,7 +422,7 @@ class DataValidator:
         """
         _column_with_unique_requirement = self._get_unique_column_requirement()
         if len(_column_with_unique_requirement) == 0:
-            self._logger.debug("No data columns with unique value requirement")
+            self.logger.debug("No data columns with unique value requirement")
             return
         _data_table_unique_for_key_column = unique(
             self.data_table, keys=_column_with_unique_requirement
@@ -412,10 +455,10 @@ class DataValidator:
 
         for entry in self._data_description:
             if "input_processing" in entry and "remove_duplicates" in entry["input_processing"]:
-                self._logger.debug(f"Removing duplicates for column {entry['name']}")
+                self.logger.debug(f"Removing duplicates for column {entry['name']}")
                 _unique_required_column.append(entry["name"])
 
-        self._logger.debug(f"Unique required columns: {_unique_required_column}")
+        self.logger.debug(f"Unique required columns: {_unique_required_column}")
         return _unique_required_column
 
     def _get_reference_unit(self, column_name):
@@ -470,7 +513,7 @@ class DataValidator:
             dtype=dtype,
             allow_subtypes=(not self.check_exact_data_type),
         ):
-            self._logger.error(
+            self.logger.error(
                 f"Invalid data type in column '{column_name}'. "
                 f"Expected type '{reference_dtype}', found '{dtype}' "
                 f"(exact type: {self.check_exact_data_type})"
@@ -505,9 +548,9 @@ class DataValidator:
             data = np.array(data)
 
         if np.isnan(data).any():
-            self._logger.info(f"Column {col_name} contains NaN.")
+            self.logger.info(f"Column {col_name} contains NaN.")
         if np.isinf(data).any():
-            self._logger.info(f"Column {col_name} contains infinite value.")
+            self.logger.info(f"Column {col_name} contains infinite value.")
 
         entry = self._get_data_description(col_name)
         if "allow_nan" in entry.get("input_processing", {}):
@@ -593,7 +636,7 @@ class DataValidator:
             # ensure that the data type is preserved (e.g., integers)
             return (type(data)(u.Unit(column_unit).to(reference_unit) * data), reference_unit)
         except (u.core.UnitConversionError, ValueError) as exc:
-            self._logger.error(
+            self.logger.error(
                 f"Invalid unit in data column '{col_name}'. "
                 f"Expected type '{reference_unit}', found '{column_unit}'"
             )
@@ -696,9 +739,9 @@ class DataValidator:
         try:
             col_index = int(col_name)
             if col_index < max_logs:
-                self._logger.debug(message)
+                self.logger.debug(message)
         except (ValueError, TypeError):
-            self._logger.debug(message)
+            self.logger.debug(message)
 
     @staticmethod
     def _interval_check(data, axis_range, range_type):
@@ -817,7 +860,7 @@ class DataValidator:
         except IndexError as exc:
             if len(self._data_description) == 1:  # all columns are described by the same schema
                 return self._data_description[0]
-            self._logger.error(
+            self.logger.error(
                 f"Data column '{column_name}' not found in reference column definition"
             )
             raise exc
@@ -835,7 +878,7 @@ class DataValidator:
         try:
             return _entry[_index]
         except IndexError:
-            self._logger.error(
+            self.logger.error(
                 f"Data column '{column_name}' not found in reference column definition"
             )
             raise
@@ -868,42 +911,33 @@ class DataValidator:
         if isinstance(self.data_dict["unit"], list):
             self.data_dict["unit"] = gen.convert_list_to_string(self.data_dict["unit"])
 
-    def _check_version_string(self, version):
-        """
-        Check that version string follows semantic versioning.
-
-        Parameters
-        ----------
-        version: str
-            version string
-
-        Raises
-        ------
-        ValueError
-            if version string does not follow semantic versioning
-
-        """
-        if version is None:
-            return
-        semver_regex = r"^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$"
-        if not re.match(semver_regex, version):
-            raise ValueError(f"Invalid version string '{version}'")
-        self._logger.debug(f"Valid version string '{version}'")
-
     def _check_site_and_array_element_consistency(self, instrument, site):
         """
         Check that site and array element names are consistent.
 
         An example for an inconsistency is 'LSTN' at site 'South'
         """
-        if not all([instrument, site]) or "OBS" in instrument:
+        if not (instrument and site):
+            return
+
+        instruments = [instrument] if isinstance(instrument, str) else instrument
+        if any(inst.startswith("OBS") for inst in instruments):
             return
 
         def to_sorted_list(value):
             """Return value as sorted list."""
             return [value] if isinstance(value, str) else sorted(value)
 
-        instrument_site = to_sorted_list(names.get_site_from_array_element_name(instrument))
+        instrument_sites = []
+        for inst in instruments:
+            instrument_sites.append(names.get_site_from_array_element_name(inst))
+        # names.get_site_from_array_element_name might return a list
+        flat_sites = [
+            s
+            for sublist in instrument_sites
+            for s in (sublist if isinstance(sublist, list) else [sublist])
+        ]
+        instrument_site = to_sorted_list(set(flat_sites))
         site = to_sorted_list(site)
 
         if instrument_site != site:

simtools/db/db_model_upload.py

@@ -186,7 +186,8 @@ def _read_production_tables(model_path):
     models = [model_path.name]
     if (model_path / "info.yml").exists():
         info = ascii_handler.collect_data_from_file(file_name=model_path / "info.yml")
-        models.extend(info.get("model_version_history", []))
+        if info.get("model_update") == "patch_update":
+            models.extend(info.get("model_version_history", []))
     # sort oldest --> newest
     models = sorted(set(models), key=Version, reverse=False)
     for model in models:

simtools/db/mongo_db.py

@@ -10,7 +10,7 @@ import gridfs
 import jsonschema
 from astropy.table import Table
 from bson.objectid import ObjectId
-from pymongo import MongoClient
+from pymongo import MongoClient, monitoring
 
 from simtools.io import ascii_handler
 
@@ -57,6 +57,61 @@ jsonschema_db_dict = {
 }
 
 
+class IdleConnectionMonitor(monitoring.ConnectionPoolListener):
+    """
+    A listener to track MongoDB connection pool activity.
+
+    Used to monitor idle connections and log connection events.
+    Switched on in debug mode.
+    """
+
+    def __init__(self):
+        self._logger = logging.getLogger("IdleConnectionMonitor")
+        self.open_connections = 0
+
+    def connection_created(self, event):
+        """Handle connection creation event."""
+        self.open_connections += 1
+        self._logger.debug(
+            f"MongoDB connection Created: {event.address}. Total in Pool: {self.open_connections}"
+        )
+
+    def connection_closed(self, event):
+        """Handle connection closure event."""
+        self.open_connections -= 1
+        self._logger.debug(
+            f"MongoDB connection Closed: {event.address}. Reason: {event.reason}. "
+            f"Total in Pool: {self.open_connections}"
+        )
+
+    def connection_check_out_started(self, event):
+        """Handle connection check out started event."""
+
+    def connection_check_out_failed(self, event):
+        """Handle connection check out failure event."""
+
+    def connection_checked_out(self, event):
+        """Handle connection checked out event."""
+
+    def connection_checked_in(self, event):
+        """Handle connection checked in event."""
+
+    def connection_ready(self, event):
+        """Handle connection ready event."""
+
+    def pool_created(self, event):
+        """Handle connection pool creation event."""
+
+    def pool_ready(self, event):
+        """Handle connection pool ready event."""
+
+    def pool_cleared(self, event):
+        """Handle connection pool cleared event."""
+
+    def pool_closed(self, event):
+        """Handle connection pool closure event."""
+
+
 class MongoDBHandler:  # pylint: disable=unsubscriptable-object
     """
     MongoDBHandler provides low-level interface to MongoDB operations.
@@ -70,19 +125,90 @@ class MongoDBHandler:  # pylint: disable=unsubscriptable-object
         Dictionary with the MongoDB configuration (see jsonschema_db_dict for details).
     """
 
-    db_client: MongoClient | None = None
+    db_client: MongoClient = None
     _lock = Lock()
+    _logger = logging.getLogger(__name__)
 
     def __init__(self, db_config=None):
         """Initialize the MongoDBHandler class."""
-        self._logger = logging.getLogger(__name__)
         self.db_config = MongoDBHandler.validate_db_config(db_config)
         self.list_of_collections = {}
 
-        if self.db_config and MongoDBHandler.db_client is None:
-            with MongoDBHandler._lock:
-                if MongoDBHandler.db_client is None:
-                    MongoDBHandler.db_client = self._open_db()
+        if self.db_config:
+            self._initialize_client(self.db_config)
+
+    @classmethod
+    def _initialize_client(cls, db_config):
+        """
+        Initialize the MongoDB client in a thread-safe manner.
+
+        Only initializes if it hasn't been done yet. Uses double-checked locking
+        to ensure thread safety.
+
+        Parameters
+        ----------
+        db_config: dict
+            Dictionary with the MongoDB configuration.
+        """
+        if cls.db_client is not None:
+            return
+        with cls._lock:
+            if cls.db_client is None:
+                try:
+                    uri = cls._build_uri(db_config)
+                    client_kwargs = {"maxIdleTimeMS": 10000}
+
+                    if cls._logger.isEnabledFor(logging.DEBUG):
+                        client_kwargs["event_listeners"] = [IdleConnectionMonitor()]
+
+                    cls.db_client = MongoClient(uri, **client_kwargs)
+                    cls._logger.debug("MongoDB client initialized successfully.")
+                except Exception as e:
+                    cls._logger.error(f"Failed to initialize MongoDB client: {e}")
+                    raise
+
+    @staticmethod
+    def _build_uri(db_config):
+        """
+        Build MongoDB URI from configuration.
+
+        Parameters
+        ----------
+        db_config: dict
+            Dictionary with the MongoDB configuration.
+
+        Returns
+        -------
+        str
+            MongoDB connection URI.
+        """
+        direct_connection = db_config["db_server"] in (
+            "localhost",
+            "simtools-mongodb",
+            "mongodb",
+        )
+        auth_source = (
+            db_config.get("db_api_authentication_database")
+            if db_config.get("db_api_authentication_database")
+            else "admin"
+        )
+
+        username = db_config["db_api_user"]
+        password = db_config["db_api_pw"]
+        server = db_config["db_server"]
+        port = db_config["db_api_port"]
+
+        uri_base = f"mongodb://{username}:{password}@{server}:{port}/"
+        params = [f"authSource={auth_source}"]
+
+        if direct_connection:
+            params.append("directConnection=true")
+        else:
+            params.append("ssl=true")
+            params.append("tlsAllowInvalidHostnames=true")
+            params.append("tlsAllowInvalidCertificates=true")
+
+        return f"{uri_base}?{'&'.join(params)}"
 
     @staticmethod
     def validate_db_config(db_config):
@@ -112,41 +238,6 @@ class MongoDBHandler:  # pylint: disable=unsubscriptable-object
         except jsonschema.exceptions.ValidationError as err:
             raise ValueError("Invalid MongoDB configuration") from err
 
-    def _open_db(self):
-        """
-        Open a connection to MongoDB and return the client.
-
-        Returns
-        -------
-        MongoClient
-            A PyMongo DB client
-
-        Raises
-        ------
-        KeyError
-            If the DB configuration is invalid
-        """
-        direct_connection = self.db_config["db_server"] in (
-            "localhost",
-            "simtools-mongodb",
-            "mongodb",
-        )
-        return MongoClient(
-            self.db_config["db_server"],
-            port=self.db_config["db_api_port"],
-            username=self.db_config["db_api_user"],
-            password=self.db_config["db_api_pw"],
-            authSource=(
-                self.db_config.get("db_api_authentication_database")
-                if self.db_config.get("db_api_authentication_database")
-                else "admin"
-            ),
-            directConnection=direct_connection,
-            ssl=not direct_connection,
-            tlsallowinvalidhostnames=True,
-            tlsallowinvalidcertificates=True,
-        )
-
     @staticmethod
     def get_db_name(db_name=None, db_simulation_model_version=None, model_name=None):
         """

simtools/dependencies.py

@@ -151,12 +151,11 @@ def get_corsika_version(run_time=None):
     str
         Version of the CORSIKA package.
     """
-    version = None
-    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if sim_telarray_path is None:
+    corsika_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
+    if corsika_path is None:
         _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
         return None
-    corsika_command = Path(sim_telarray_path) / "corsika-run" / "corsika"
+    corsika_command = Path(corsika_path) / "corsika-run" / "corsika"
 
     if run_time is None:
         command = [str(corsika_command)]
@@ -173,11 +172,8 @@ def get_corsika_version(run_time=None):
         text=True,
     )
 
-    # Capture output until it waits for input
-    while True:
-        line = process.stdout.readline()
-        if not line:
-            break
+    version = None
+    for line in process.stdout:
         # Extract the version from the line "NUMBER OF VERSION :  7.7550"
         if "NUMBER OF VERSION" in line:
             version = line.split(":")[1].strip()

simtools/io/eventio_handler.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+"""Read file info and run headers from eventio (CORSIKA IACT, sim_telarray) files."""
+
+import warnings
+
+from eventio import EventIOFile, iact
+from eventio.simtel import MCRunHeader, MCShower, RunHeader
+
+# Suppress all UserWarnings from corsikaio - no CORSIKA versions <7.7 are supported anyway
+warnings.filterwarnings("ignore", category=UserWarning, module=r"corsikaio\.subblocks\..*")
+
+
+def get_corsika_run_number(file):
+    """
+    Return the CORSIKA run number from an eventio (CORSIKA IACT or sim_telarray) file.
+
+    Parameters
+    ----------
+    file: str
+        Path to the eventio file.
+
+    Returns
+    -------
+    int, None
+        CORSIKA run number. Returns None if not found.
+    """
+    run_header = get_combined_corsika_run_header(file)
+    if run_header and "run" in run_header:
+        return run_header["run"]
+    run_header, _ = get_corsika_run_and_event_headers(file)
+    try:
+        return int(run_header["run_number"])
+    except (TypeError, KeyError, ValueError):
+        return None
+
+
+def get_combined_corsika_run_header(sim_telarray_file):
+    """
+    Return the CORSIKA run header information from an sim_telarray file.
+
+    Reads both RunHeader and MCRunHeader object from file and returns a merged dictionary.
+    Adds primary id from the first event.
+
+    Parameters
+    ----------
+    sim_telarray_file: str
+        Path to the sim_telarray file.
+
+    Returns
+    -------
+    dict, None
+        CORSIKA run header. Returns None if not found.
+    """
+    run_header = mc_run_header = None
+    primary_id = None
+
+    with EventIOFile(sim_telarray_file) as f:
+        for o in f:
+            if isinstance(o, RunHeader) and run_header is None:
+                run_header = o.parse()
+            elif isinstance(o, MCRunHeader) and mc_run_header is None:
+                mc_run_header = o.parse()
+            elif isinstance(o, MCShower):  # get primary_id from first MCShower
+                primary_id = o.parse().get("primary_id")
+            if run_header and mc_run_header and primary_id is not None:
+                break
+
+    run_header = run_header or {}
+    mc_run_header = mc_run_header or {}
+    if primary_id is not None:
+        mc_run_header["primary_id"] = primary_id
+    return run_header | mc_run_header or None
+
+
+def get_corsika_run_and_event_headers(corsika_iact_file):
+    """
+    Return the CORSIKA run and event headers from a CORSIKA IACT eventio file.
+
+    Parameters
+    ----------
+    corsika_iact_file: str, Path
+        Path to the CORSIKA IACT eventio file.
+
+    Returns
+    -------
+    tuple
+        CORSIKA run header and event header as dictionaries.
+    """
+    run_header = event_header = None
+
+    with EventIOFile(corsika_iact_file) as f:
+        for o in f:
+            if isinstance(o, iact.RunHeader) and run_header is None:
+                run_header = o.parse()
+            elif isinstance(o, iact.EventHeader) and event_header is None:
+                event_header = o.parse()
+            if run_header and event_header:
+                break
+
+    return run_header, event_header
+
+
+def get_simulated_events(event_io_file):
+    """
+    Return the number of shower and MC events from a simulation (eventio) file.
+
+    For a sim_telarray file, the number of simulated showers and MC events is
+    determined by counting the number of MCShower (type id 2020) and MCEvent
+    objects (type id 2021). For a CORSIKA IACT file, the number of simulated
+    showers is determined by counting the number of IACTShower (type id 1202).
+
+    Parameters
+    ----------
+    event_io_file: str, Path
+        Path to the eventio file.
+
+    Returns
+    -------
+    tuple
+        Number of showers and number of MC events (MC events for sim_telarray files only).
+    """
+    counts = {1202: 0, 2020: 0, 2021: 0}
+    with EventIOFile(event_io_file) as f:
+        for o in f:
+            t = o.header.type
+            if t in counts:
+                counts[t] += 1
+    return counts[2020] if counts[2020] else counts[1202], counts[2021]

simtools/job_execution/htcondor_script_generator.py

@@ -136,8 +136,6 @@ simtools-simulate-prod \\
     --view_cone {view_cone_string} \\
     --run_number $((process_id)) \\
     --run_number_offset {run_number_offset} \\
-    --number_of_runs 1 \\
-    --data_directory /tmp/simtools-data \\
     --output_path /tmp/simtools-output \\
     --log_level {args_dict["log_level"]} \\
     --pack_for_grid_register simtools-output

simtools/layout/array_layout_utils.py

@@ -162,7 +162,7 @@ def write_array_layouts(array_layouts, args_dict, db_config):
     ModelDataWriter.dump_model_parameter(
         parameter_name="array_layouts",
         value=array_layouts["value"],
-        instrument=site,
+        instrument=f"OBS-{site}",
         parameter_version=args_dict.get("updated_parameter_version"),
         output_file=output_file,
         db_config=db_config,