gammasimtools 0.25.0__py3-none-any.whl → 0.27.0__py3-none-any.whl

simtools/corsika/primary_particle.py

@@ -185,7 +185,7 @@ class PrimaryParticle:
             "silicon": {"corsika7_id": 2814},
             "iron": {"corsika7_id": 5626},
         }
-        for _, ids in particles.items():
+        for ids in particles.values():
             ids["pdg_id"] = Corsika7ID(ids["corsika7_id"]).to_pdgid().numerator
             ids["pdg_name"] = Particle.findall(pdgid=ids["pdg_id"])[0].name
 

simtools/data_model/metadata_collector.py

@@ -360,6 +360,9 @@ class MetadataCollector:
                     "Metadata extraction from sim_telarray files is not supported yet."
                 )
                 continue
+            elif Path(metadata_file).name.endswith((".corsika.zst", ".corsika")):
+                self._logger.warning("Metadata extraction from CORSIKA files is not supported yet.")
+                continue
             else:
                 raise ValueError(f"Unknown metadata file format: {metadata_file}")
 
@@ -422,7 +425,7 @@ class MetadataCollector:
         product_dict["creation_time"] = gen.now_date_time_in_isoformat()
         product_dict["description"] = self.schema_dict.get("description", None)
 
-        # DATA:CATEGORY
+        # metadata DATA:CATEGORY
         product_dict["data"]["category"] = "SIM"
         product_dict["data"]["level"] = "R1"
         product_dict["data"]["type"] = "Service"
@@ -431,7 +434,7 @@ class MetadataCollector:
         except KeyError:
             pass
 
-        # DATA:MODEL
+        # metadata DATA:MODEL
         product_dict["data"]["model"]["name"] = (
             self.schema_dict.get("name")
             or self.args_dict.get("metadata_product_data_name")

simtools/data_model/metadata_model.py

@@ -8,13 +8,9 @@ Follows CTAO top-level data model definition.
 
 """
 
-import logging
-
 import simtools.data_model.schema
 import simtools.utils.general as gen
 
-_logger = logging.getLogger(__name__)
-
 
 def get_default_metadata_dict(
     schema_file=None, observatory="CTA", schema_version="latest", lower_case=True

simtools/data_model/model_data_writer.py

@@ -7,6 +7,7 @@ import packaging.version
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
+from simtools import settings
 from simtools.data_model import schema, validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.db import db_handler
@@ -20,69 +21,62 @@ class ModelDataWriter:
 
     Parameters
     ----------
-    product_data_file: str
+    output_file: str
         Name of output file.
-    product_data_format: str
+    output_file_format: str
         Format of output file.
-    args_dict: Dictionary
-        Dictionary with configuration parameters.
     output_path: str or Path
         Path to output file.
-    args_dict: dict
-        Dictionary with configuration parameters.
-
     """
 
-    def __init__(
-        self,
-        product_data_file=None,
-        product_data_format=None,
-        output_path=None,
-        args_dict=None,
-    ):
+    def __init__(self, output_file=None, output_file_format=None, output_path=None):
         """Initialize model data writer."""
         self._logger = logging.getLogger(__name__)
         self.io_handler = io_handler.IOHandler()
         self.schema_dict = {}
-        if args_dict is not None:
-            output_path = args_dict.get("output_path", output_path)
-        if output_path is not None:
-            self.io_handler.set_paths(output_path=output_path)
+        self.output_label = "model_data_writer"
+        self.io_handler.set_paths(
+            output_path=output_path or settings.config.args.get("output_path"),
+            output_path_label=self.output_label,
+        )
         try:
-            self.product_data_file = self.io_handler.get_output_file(file_name=product_data_file)
+            self.output_file = self.io_handler.get_output_file(
+                file_name=output_file, output_path_label=self.output_label
+            )
         except TypeError:
-            self.product_data_file = None
-        self.product_data_format = self._astropy_data_format(product_data_format)
+            self.output_file = None
+        self.output_file_format = self._derive_data_format(output_file_format, self.output_file)
 
     @staticmethod
     def dump(
-        args_dict, output_file=None, metadata=None, product_data=None, validate_schema_file=None
+        output_file=None,
+        metadata=None,
+        product_data=None,
+        output_file_format="ascii.ecsv",
+        validate_schema_file=None,
     ):
         """
         Write model data and metadata (as static method).
 
         Parameters
         ----------
-        args_dict: dict
-            Dictionary with configuration parameters (including output file name and path).
         output_file: string or Path
             Name of output file (args["output_file"] is used if this parameter is not set).
         metadata: MetadataCollector object
             Metadata to be written.
         product_data: astropy Table
             Model data to be written
+        output_file_format: str
+            Format of output file.
         validate_schema_file: str
             Schema file used in validation of output data.
-
         """
         writer = ModelDataWriter(
-            product_data_file=(
-                args_dict.get("output_file", None) if output_file is None else output_file
-            ),
-            product_data_format=args_dict.get("output_file_format", "ascii.ecsv"),
-            args_dict=args_dict,
+            output_file=output_file,
+            output_file_format=output_file_format,
         )
-        if validate_schema_file is not None and not args_dict.get("skip_output_validation", True):
+        skip_output_validation = settings.config.args.get("skip_output_validation", True)
+        if validate_schema_file is not None and not skip_output_validation:
             product_data = writer.validate_and_transform(
                 product_data_table=product_data,
                 validate_schema_file=validate_schema_file,
@@ -98,10 +92,10 @@ class ModelDataWriter:
         output_file,
         output_path=None,
         metadata_input_dict=None,
-        db_config=None,
         unit=None,
         meta_parameter=False,
         model_parameter_schema_version=None,
+        check_db_for_existing_parameter=True,
     ):
         """
         Generate DB-style model parameter dict and write it to json file.
@@ -122,14 +116,14 @@ class ModelDataWriter:
             Path to output file.
         metadata_input_dict: dict
             Input to metadata collector.
-        db_config: dict
-            Database configuration. If not None, check if parameter with the same version exists.
         unit: str
             Unit of the parameter value (if applicable and value is not of type astropy Quantity).
         meta_parameter: bool
             Setting for meta parameter flag.
         model_parameter_schema_version: str, None
             Version of the model parameter schema (if None, use schema version from schema dict).
+        check_db_for_existing_parameter: bool
+            If True, check if parameter with same version exists in DB before writing.
 
         Returns
         -------
@@ -137,15 +131,12 @@ class ModelDataWriter:
             Validated parameter dictionary.
         """
         writer = ModelDataWriter(
-            product_data_file=output_file,
-            product_data_format="json",
-            args_dict=None,
+            output_file=output_file,
+            output_file_format="json",
             output_path=output_path,
         )
-        if db_config is not None:
-            writer.check_db_for_existing_parameter(
-                parameter_name, instrument, parameter_version, db_config
-            )
+        if check_db_for_existing_parameter:
+            writer.check_db_for_existing_parameter(parameter_name, instrument, parameter_version)
 
         unique_id = None
         if metadata_input_dict is not None:
@@ -170,9 +161,7 @@ class ModelDataWriter:
         writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         return _json_dict
 
-    def check_db_for_existing_parameter(
-        self, parameter_name, instrument, parameter_version, db_config
-    ):
+    def check_db_for_existing_parameter(self, parameter_name, instrument, parameter_version):
         """
         Check if a parameter with the same version exists in the simulation model database.
 
@@ -184,15 +173,15 @@ class ModelDataWriter:
             Name of the instrument.
         parameter_version: str
             Version of the parameter.
-        db_config: dict
-            Database configuration.
 
         Raises
         ------
         ValueError
             If parameter with the same version exists in the database.
         """
-        db = db_handler.DatabaseHandler(db_config=db_config)
+        db = db_handler.DatabaseHandler()
+        if not db.is_configured():
+            return
         try:
             db.get_model_parameter(
                 parameter=parameter_name,
@@ -373,8 +362,10 @@ class ModelDataWriter:
         """
         try:
             unit_list = []
-            for data in self.schema_dict["data"]:
-                unit_list.append(data["unit"] if data["unit"] != "dimensionless" else None)
+            unit_list = [
+                data["unit"] if data["unit"] != "dimensionless" else None
+                for data in self.schema_dict["data"]
+            ]
             return unit_list if len(unit_list) > 1 else unit_list[0]
         except (KeyError, IndexError):
             pass
@@ -436,19 +427,17 @@ class ModelDataWriter:
                 gen.change_dict_keys_case(metadata.get_top_level_metadata(), True)
             )
 
-        self._logger.info(f"Writing data to {self.product_data_file}")
-        if isinstance(product_data, dict) and Path(self.product_data_file).suffix == ".json":
-            self.write_dict_to_model_parameter_json(self.product_data_file, product_data)
+        self._logger.info(f"Writing data to {self.output_file}")
+        if isinstance(product_data, dict) and Path(self.output_file).suffix == ".json":
+            self.write_dict_to_model_parameter_json(self.output_file, product_data)
             return
         try:
-            product_data.write(
-                self.product_data_file, format=self.product_data_format, overwrite=True
-            )
+            product_data.write(self.output_file, format=self.output_file_format, overwrite=True)
         except IORegistryError:
-            self._logger.error(f"Error writing model data to {self.product_data_file}.")
+            self._logger.error(f"Error writing model data to {self.output_file}.")
             raise
         if metadata is not None:
-            metadata.write(self.product_data_file, add_activity_name=True)
+            metadata.write(self.output_file, add_activity_name=True)
 
     def write_dict_to_model_parameter_json(self, file_name, data_dict):
         """
@@ -467,10 +456,13 @@ class ModelDataWriter:
             if data writing was not successful.
         """
         data_dict = ModelDataWriter.prepare_data_dict_for_writing(data_dict)
-        self._logger.info(f"Writing data to {self.io_handler.get_output_file(file_name)}")
+        output_file = self.io_handler.get_output_file(
+            file_name, output_path_label=self.output_label
+        )
+        self._logger.info(f"Writing data to {output_file}")
         ascii_handler.write_data_to_file(
             data=data_dict,
-            output_file=self.io_handler.get_output_file(file_name),
+            output_file=output_file,
             sort_keys=True,
             numpy_types=True,
         )
@@ -510,9 +502,12 @@ class ModelDataWriter:
         return data_dict
 
     @staticmethod
-    def _astropy_data_format(product_data_format):
+    def _derive_data_format(product_data_format, output_file=None):
         """
-        Ensure conformance with astropy data format naming.
+        Derive data format and ensure conformance with astropy data format naming.
+
+        If product_data_format is None and output_file is given, derive format
+        from output_file suffix.
 
         Parameters
         ----------
@@ -520,6 +515,6 @@ class ModelDataWriter:
             format identifier
 
         """
-        if product_data_format == "ecsv":
-            product_data_format = "ascii.ecsv"
-        return product_data_format
+        if product_data_format is None and output_file is not None:
+            product_data_format = Path(output_file).suffix.lstrip(".")
+        return "ascii.ecsv" if product_data_format == "ecsv" else product_data_format

simtools/data_model/schema.py

@@ -1,6 +1,7 @@
 """Module providing functionality to read and validate dictionaries using schema."""
 
 import logging
+from functools import lru_cache
 from pathlib import Path
 
 import jsonschema
@@ -153,6 +154,7 @@ def _validate_meta_schema_url(data):
         raise FileNotFoundError(f"Meta schema URL does not exist: {data['meta_schema_url']}")
 
 
+@lru_cache
 def _retrieve_yaml_schema_from_uri(uri):
     """Load schema from a file URI."""
     path = SCHEMA_PATH / Path(uri.removeprefix("file:/"))

simtools/data_model/validate_data.py

@@ -928,9 +928,7 @@ class DataValidator:
             """Return value as sorted list."""
             return [value] if isinstance(value, str) else sorted(value)
 
-        instrument_sites = []
-        for inst in instruments:
-            instrument_sites.append(names.get_site_from_array_element_name(inst))
+        instrument_sites = [names.get_site_from_array_element_name(inst) for inst in instruments]
         # names.get_site_from_array_element_name might return a list
         flat_sites = [
             s

simtools/db/db_handler.py

@@ -4,6 +4,7 @@ import logging
 from collections import defaultdict
 from pathlib import Path
 
+from simtools import settings
 from simtools.data_model import validate_data
 from simtools.db.mongo_db import MongoDBHandler
 from simtools.io import io_handler
@@ -20,11 +21,6 @@ class DatabaseHandler:
 
     - db_simulation_model_version (from db_config): version of the simulation model database
     - model_version (from production_tables): version of the model contained in the database
-
-    Parameters
-    ----------
-    db_config: dict
-        Dictionary with the DB configuration.
     """
 
     ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"]
@@ -33,13 +29,17 @@ class DatabaseHandler:
     model_parameters_cached = {}
     model_versions_cached = {}
 
-    def __init__(self, db_config=None):
+    def __init__(self):
         """Initialize the DatabaseHandler class."""
         self._logger = logging.getLogger(__name__)
 
-        self.db_config = MongoDBHandler.validate_db_config(db_config)
+        self.db_config = (
+            MongoDBHandler.validate_db_config(dict(settings.config.db_config))
+            if settings.config.db_config
+            else None
+        )
         self.io_handler = io_handler.IOHandler()
-        self.mongo_db_handler = MongoDBHandler(db_config) if self.db_config else None
+        self.mongo_db_handler = MongoDBHandler(self.db_config) if self.db_config else None
 
         self.db_name = (
             MongoDBHandler.get_db_name(
@@ -50,6 +50,17 @@ class DatabaseHandler:
             else None
         )
 
+    def is_configured(self):
+        """
+        Check if the DatabaseHandler is configured.
+
+        Returns
+        -------
+        bool
+            True if the DatabaseHandler is configured, False otherwise.
+        """
+        return self.mongo_db_handler is not None
+
     def get_db_name(self, db_name=None, db_simulation_model_version=None, model_name=None):
         """Build DB name from configuration."""
         if db_name:
@@ -249,9 +260,7 @@ class DatabaseHandler:
             collection,
         )
         if cache_dict:
-            self._logger.debug(f"Found {array_element} in cache (key: {cache_key})")
             return cache_dict
-        self._logger.debug(f"Did not find {array_element} in cache (key: {cache_key})")
 
         try:
             parameter_version_table = production_table["parameters"][array_element]
@@ -862,6 +871,10 @@ class DatabaseHandler:
                 array_element_name,
             ]
         except KeyError as exc:
+            # simplified model definitions when e.g. adding new telescopes without design model
+            if settings.config.args.get("ignore_missing_design_model", False):
+                element_type = names.get_array_element_type_from_name(array_element_name)
+                return [array_element_name, f"{element_type}-01", f"{element_type}-design"]
             raise KeyError(
                 f"Failed generated array element list for db query for {array_element_name}"
             ) from exc

simtools/db/mongo_db.py

@@ -4,7 +4,7 @@ import io
 import logging
 import re
 from pathlib import Path
-from threading import Lock
+from threading import Lock as _Lock
 
 import gridfs
 import jsonschema
@@ -126,7 +126,7 @@ class MongoDBHandler:  # pylint: disable=unsubscriptable-object
     """
 
     db_client: MongoClient = None
-    _lock = Lock()
+    _lock = _Lock()
     _logger = logging.getLogger(__name__)
 
     def __init__(self, db_config=None):

simtools/dependencies.py

@@ -9,27 +9,26 @@ This modules provides two main functionalities:
 """
 
 import logging
-import os
 import re
 import subprocess
 from pathlib import Path
 
 import yaml
 
+from simtools import settings
 from simtools.io import ascii_handler
+from simtools.utils import general as gen
 from simtools.version import __version__
 
 _logger = logging.getLogger(__name__)
 
 
-def get_version_string(db_config=None, run_time=None):
+def get_version_string(run_time=None):
     """
     Print the versions of the dependencies.
 
     Parameters
     ----------
-    db_config : dict, optional
-        Database configuration dictionary.
     run_time : list, optional
         Runtime environment command (e.g., Docker).
 
@@ -40,10 +39,13 @@ def get_version_string(db_config=None, run_time=None):
 
     """
     return (
-        f"Database name: {get_database_version_or_name(db_config, version=False)}\n"
-        f"Database version: {get_database_version_or_name(db_config, version=True)}\n"
+        f"Database name: {get_database_version_or_name(version=False)}\n"
+        f"Database version: {get_database_version_or_name(version=True)}\n"
         f"sim_telarray version: {get_sim_telarray_version(run_time)}\n"
+        "sim_telarray exe: "
+        f"{settings.config.sim_telarray_exe if settings.config.sim_telarray_exe else 'None'}\n"
         f"CORSIKA version: {get_corsika_version(run_time)}\n"
+        f"CORSIKA exe: {settings.config.corsika_exe if settings.config.corsika_exe else 'None'}\n"
         f"Build options: {get_build_options(run_time)}\n"
         f"Runtime environment: {run_time if run_time else 'None'}\n"
     )
@@ -73,14 +75,12 @@ def get_software_version(software):
         raise ValueError(f"Unknown software: {software}") from exc
 
 
-def get_database_version_or_name(db_config, version=True):
+def get_database_version_or_name(version=True):
     """
     Get the version or name of the simulation model data base used.
 
     Parameters
     ----------
-    db_config : dict
-        Dictionary containing the database configuration.
     version : bool
         If True, return the version of the database. If False, return the name.
 
@@ -91,8 +91,10 @@ def get_database_version_or_name(db_config, version=True):
 
     """
     if version:
-        return db_config and db_config.get("db_simulation_model_version")
-    return db_config and db_config.get("db_simulation_model")
+        return settings.config.db_config and settings.config.db_config.get(
+            "db_simulation_model_version"
+        )
+    return settings.config.db_config and settings.config.db_config.get("db_simulation_model")
 
 
 def get_sim_telarray_version(run_time=None):
@@ -111,16 +113,13 @@ def get_sim_telarray_version(run_time=None):
     str
         Version of the sim_telarray package.
     """
-    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if sim_telarray_path is None:
-        _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
+    if settings.config.sim_telarray_exe is None:
+        _logger.warning("sim_telarray environment not configured.")
         return None
-    sim_telarray_path = Path(sim_telarray_path) / "sim_telarray" / "bin" / "sim_telarray"
-
     if run_time is None:
-        command = [str(sim_telarray_path), "--version"]
+        command = [str(settings.config.sim_telarray_exe), "--version"]
     else:
-        command = [*run_time, str(sim_telarray_path), "--version"]
+        command = [*run_time, str(settings.config.sim_telarray_exe), "--version"]
 
     _logger.debug(f"Running command: {command}")
     result = subprocess.run(command, capture_output=True, text=True, check=False)
@@ -151,19 +150,15 @@ def get_corsika_version(run_time=None):
     str
         Version of the CORSIKA package.
     """
-    corsika_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if corsika_path is None:
-        _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
+    if settings.config.corsika_exe is None:
+        _logger.warning("CORSIKA environment not configured.")
         return None
-    corsika_command = Path(corsika_path) / "corsika-run" / "corsika"
 
     if run_time is None:
-        command = [str(corsika_command)]
+        command = [str(settings.config.corsika_exe)]
     else:
-        command = [*run_time, str(corsika_command)]
+        command = [*run_time, str(settings.config.corsika_exe)]
 
-    # Below I do not use the standard context manager because
-    # it makes mocking in the tests significantly more difficult
     process = subprocess.Popen(  # pylint: disable=consider-using-with
         command,
         stdout=subprocess.PIPE,
@@ -197,9 +192,14 @@ def get_corsika_version(run_time=None):
 
 def get_build_options(run_time=None):
     """
-    Return CORSIKA / sim_telarray build options.
+    Return CORSIKA / sim_telarray config and build options.
+
+    For CORSIKA / sim_telarray build for simtools version >0.25.0:
+    expects build_opts.yml file in each CORSIKA and sim_telarray
+    directories.
 
-    Expects a build_opts.yml file in the sim_telarray directory.
+    For CORSIKA / sim_telarray build for simtools version <=0.25.0:
+    expects a build_opts.yml file in the sim_telarray directory.
 
     Parameters
     ----------
@@ -209,28 +209,71 @@ def get_build_options(run_time=None):
     Returns
     -------
     dict
-        Build options from build_opts.yml file.
+        CORSIKA / sim_telarray build options.
     """
-    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
-    if sim_telarray_path is None:
-        raise ValueError("SIMTOOLS_SIMTEL_PATH not defined.")
-
-    build_opts_path = Path(sim_telarray_path) / "build_opts.yml"
-
+    build_opts = {}
+    for package in ["corsika", "sim_telarray"]:
+        path = _get_package_path(package)
+        if not path:
+            continue
+        try:
+            build_opts.update(_get_build_options_from_file(path / "build_opts.yml", run_time))
+        except (FileNotFoundError, TypeError, ValueError):
+            # legacy fallback only for sim_telarray
+            if package == "sim_telarray":
+                try:
+                    legacy_path = path.parent / "build_opts.yml"
+                    build_opts.update(_get_build_options_from_file(legacy_path, run_time))
+                except (FileNotFoundError, TypeError, ValueError):
+                    _logger.debug(f"No build options found for {package}.")
+    if not build_opts:
+        raise FileNotFoundError("No build option file found.")
+
+    return build_opts
+
+
+def _get_package_path(package):
+    """Get the package path from settings or environment variables."""
+    path = getattr(settings.config, f"{package}_path")
+    if path is None:
+        path = gen.load_environment_variables().get(f"{package}_path")
+    return Path(path) if path else None
+
+
+def _get_build_options_from_file(build_opts_path, run_time=None):
+    """Read build options from file."""
     if run_time is None:
         try:
             return ascii_handler.collect_data_from_file(build_opts_path)
         except FileNotFoundError as exc:
-            raise FileNotFoundError("No build_opts.yml file found.") from exc
+            raise FileNotFoundError("No build option file found.") from exc
 
     command = [*run_time, "cat", str(build_opts_path)]
-    _logger.debug(f"Reading build_opts.yml with command: {command}")
+    _logger.debug(f"Reading build option with command: {command}")
 
     result = subprocess.run(command, capture_output=True, text=True, check=False)
     if result.returncode:
-        raise FileNotFoundError(f"No build_opts.yml file found in container: {result.stderr}")
+        raise FileNotFoundError(f"No build option file found in container: {result.stderr}")
 
     try:
         return yaml.safe_load(result.stdout)
     except yaml.YAMLError as exc:
         raise ValueError(f"Error parsing build_opts.yml from container: {exc}") from exc
+
+
+def export_build_info(output_file, run_time=None):
+    """
+    Export build and version information to a file.
+
+    Parameters
+    ----------
+    output_file : str
+        Path to the output file.
+    run_time : list, optional
+        Runtime environment command (e.g., Docker).
+    """
+    build_info = get_build_options(run_time)
+    build_info["simtools"] = __version__
+    build_info["database_name"] = get_database_version_or_name(version=False)
+    build_info["database_version"] = get_database_version_or_name(version=True)
+    ascii_handler.write_data_to_file(data=build_info, output_file=Path(output_file))