gammasimtools 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/camera/single_photon_electron_spectrum.py

@@ -0,0 +1,168 @@
+"""Single photon electron spectral analysis."""
+
+import logging
+import re
+import subprocess
+import tempfile
+from io import BytesIO
+from pathlib import Path
+
+from astropy.table import Table
+
+import simtools.data_model.model_data_writer as writer
+from simtools.constants import SCHEMA_PATH
+from simtools.data_model import validate_data
+from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.io_operations import io_handler
+
+
+class SinglePhotonElectronSpectrum:
+    """
+    Single photon electron spectral analysis.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Dictionary with input arguments.
+    """
+
+    prompt_column = "frequency (prompt)"
+    prompt_plus_afterpulse_column = "frequency (prompt+afterpulsing)"
+    afterpulse_column = "frequency (afterpulsing)"
+
+    input_schema = SCHEMA_PATH / "input" / "single_pe_spectrum.schema.yml"
+
+    def __init__(self, args_dict):
+        """Initialize SinglePhotonElectronSpectrum class."""
+        self._logger = logging.getLogger(__name__)
+        self._logger.debug("Initialize SinglePhotonElectronSpectrum class.")
+
+        self.args_dict = args_dict
+        # default output is of ecsv format
+        self.args_dict["output_file"] = str(
+            Path(self.args_dict["output_file"]).with_suffix(".ecsv")
+        )
+        self.io_handler = io_handler.IOHandler()
+        self.data = ""  # Single photon electron spectrum data (as string)
+        self.metadata = MetadataCollector(args_dict=self.args_dict)
+
+    def derive_single_pe_spectrum(self):
+        """Derive single photon electron spectrum."""
+        if self.args_dict.get("use_norm_spe"):
+            return self._derive_spectrum_norm_spe()
+
+        raise NotImplementedError(
+            "Derivation of single photon electron spectrum using a simtool is not yet implemented."
+        )
+
+    def write_single_pe_spectrum(self):
+        """
+        Write single photon electron spectrum plus metadata to disk.
+
+        Includes writing in simtel and simtools (ecsv) formats.
+
+        """
+        simtel_file = self.io_handler.get_output_directory() / Path(
+            self.args_dict["output_file"]
+        ).with_suffix(".dat")
+        self._logger.debug(f"norm_spe output file: {simtel_file}")
+        with open(simtel_file, "w", encoding="utf-8") as simtel:
+            simtel.write(self.data)
+
+        cleaned_data = re.sub(r"%%%.+", "", self.data)  # remove norm_spe row metadata
+        table = Table.read(
+            BytesIO(cleaned_data.encode("utf-8")),
+            format="ascii.no_header",
+            comment="#",
+            delimiter="\t",
+        )
+        table.rename_columns(
+            ["col1", "col2", "col3"],
+            ["amplitude", self.prompt_column, self.prompt_plus_afterpulse_column],
+        )
+
+        writer.ModelDataWriter.dump(
+            args_dict=self.args_dict,
+            metadata=self.metadata.top_level_meta,
+            product_data=table,
+            validate_schema_file=None,
+        )
+
+    def _derive_spectrum_norm_spe(self):
+        """
+        Derive single photon electron spectrum using sim_telarray tool 'norm_spe'.
+
+        Returns
+        -------
+        int
+            Return code of the executed command
+
+        Raises
+        ------
+        subprocess.CalledProcessError
+            If the command execution fails.
+        """
+        tmp_input_file = self._get_input_data(
+            input_file=self.args_dict["input_spectrum"],
+            frequency_column=self.prompt_column,
+        )
+        tmp_ap_file = self._get_input_data(
+            input_file=self.args_dict.get("afterpulse_spectrum"),
+            frequency_column=self.afterpulse_column,
+        )
+
+        command = [
+            f"{self.args_dict['simtel_path']}/sim_telarray/bin/norm_spe",
+            "-r",
+            f"{self.args_dict['step_size']},{self.args_dict['max_amplitude']}",
+            tmp_input_file.name,
+        ]
+        if tmp_ap_file:
+            command.insert(1, "-a")
+            command.insert(2, f"{tmp_ap_file.name}")
+
+        self._logger.debug(f"Running norm_spe command: {' '.join(command)}")
+        try:
+            result = subprocess.run(command, capture_output=True, text=True, check=True)
+        except subprocess.CalledProcessError as exc:
+            self._logger.error(f"Error running norm_spe: {exc}")
+            self._logger.error(f"stderr: {exc.stderr}")
+            raise exc
+        finally:
+            for tmp_file in [tmp_input_file, tmp_ap_file]:
+                try:
+                    Path(tmp_file.name).unlink()
+                except (AttributeError, FileNotFoundError):
+                    pass
+
+        self.data = result.stdout
+        return result.returncode
+
+    def _get_input_data(self, input_file, frequency_column):
+        """
+        Return input data for norm_spe command.
+
+        Input data need to be space separated values of the amplitude spectrum.
+        """
+        input_data = ""
+        if not input_file:
+            return None
+        input_file = Path(input_file)
+
+        if input_file.suffix == ".ecsv":
+            data_validator = validate_data.DataValidator(
+                schema_file=self.input_schema, data_file=input_file
+            )
+            table = data_validator.validate_and_transform()
+            input_data = "\n".join(f"{row['amplitude']} {row[frequency_column]}" for row in table)
+        else:
+            with open(input_file, encoding="utf-8") as f:
+                input_data = (
+                    f.read().replace(",", " ")
+                    if frequency_column == self.prompt_column
+                    else f.read()
+                )
+
+        with tempfile.NamedTemporaryFile(delete=False, mode="w", encoding="utf-8") as tmpfile:
+            tmpfile.write(input_data)
+        return tmpfile

simtools/configuration/commandline_parser.py

@@ -198,13 +198,6 @@ class CommandLineParser(argparse.ArgumentParser):
             required=False,
             default=None,
         )
-        _job_group.add_argument(
-            "--db_simulation_model_url",
-            help="simulation model repository URL",
-            type=str,
-            required=False,
-            default=None,
-        )
 
     def initialize_job_submission_arguments(self):
         """Initialize job submission arguments for simulator."""
@@ -243,12 +236,20 @@ class CommandLineParser(argparse.ArgumentParser):
             return
 
         _job_group = self.add_argument_group("simulation model")
-        _job_group.add_argument(
-            "--model_version",
-            help="model version",
-            type=str,
-            default=None,
-        )
+        if "model_version" in model_options:
+            _job_group.add_argument(
+                "--model_version",
+                help="production model version",
+                type=str,
+                default=None,
+            )
+        if "parameter_version" in model_options:
+            _job_group.add_argument(
+                "--parameter_version",
+                help="model parameter version",
+                type=str,
+                default=None,
+            )
         if any(
             option in model_options for option in ["site", "telescope", "layout", "layout_file"]
         ):

simtools/configuration/configurator.py

@@ -10,6 +10,7 @@ import astropy.units as u
 from dotenv import load_dotenv
 
 import simtools.configuration.commandline_parser as argparser
+from simtools.db.db_handler import jsonschema_db_dict
 from simtools.io_operations import io_handler
 from simtools.utils import general as gen
 
@@ -438,24 +439,10 @@ class Configurator:
         """
         Return parameters for DB configuration.
 
-        Parameters
-        ----------
+        Returns
+        -------
         dict
-            Dictionary with DB parameters
+            Dictionary with DB parameters.
         """
-        _db_dict = {}
-        _db_para = (
-            "db_api_user",
-            "db_api_pw",
-            "db_api_port",
-            "db_server",
-            "db_simulation_model",
-            "db_simulation_model_url",
-        )
-        try:
-            for _para in _db_para:
-                _db_dict[_para] = self.config[_para]
-        except KeyError:
-            pass
-
-        return _db_dict
+        db_params = jsonschema_db_dict["properties"].keys()
+        return {param: self.config.get(param) for param in db_params if param in self.config}

simtools/constants.py

@@ -2,8 +2,15 @@
 
 from importlib.resources import files
 
+# Schema path
+SCHEMA_PATH = files("simtools") / "schemas"
 # Path to metadata jsonschema
-METADATA_JSON_SCHEMA = files("simtools") / "schemas/metadata.metaschema.yml"
-
+METADATA_JSON_SCHEMA = SCHEMA_PATH / "metadata.metaschema.yml"
+# Path to model parameter metaschema
+MODEL_PARAMETER_METASCHEMA = SCHEMA_PATH / "model_parameter.metaschema.yml"
+# Path to model parameter description metaschema
+MODEL_PARAMETER_DESCRIPTION_METASCHEMA = (
+    SCHEMA_PATH / "model_parameter_and_data_schema.metaschema.yml"
+)
 # Path to model parameter schema files
-MODEL_PARAMETER_SCHEMA_PATH = files("simtools") / "schemas/model_parameters"
+MODEL_PARAMETER_SCHEMA_PATH = SCHEMA_PATH / "model_parameters"

simtools/corsika/corsika_config.py

@@ -6,7 +6,6 @@ from pathlib import Path
 import numpy as np
 from astropy import units as u
 
-import simtools.utils.general as gen
 from simtools.corsika.primary_particle import PrimaryParticle
 from simtools.io_operations import io_handler
 from simtools.model.model_parameter import ModelParameter
@@ -111,8 +110,6 @@ class CorsikaConfig:
         if args_dict is None:
             return {}
 
-        self._logger.debug("Setting CORSIKA parameters ")
-
         self._is_file_updated = False
         self.azimuth_angle = int(args_dict["azimuth_angle"].to("deg").value)
         self.zenith_angle = args_dict["zenith_angle"].to("deg").value
@@ -243,7 +240,7 @@ class CorsikaConfig:
 
     def _input_config_corsika_particle_kinetic_energy_cutoff(self, entry):
         """Return ECUTS parameter CORSIKA format."""
-        e_cuts = gen.convert_string_to_list(entry["value"])
+        e_cuts = entry["value"]
         return [
             f"{e_cuts[0]*u.Unit(entry['unit']).to('GeV')} "
             f"{e_cuts[1]*u.Unit(entry['unit']).to('GeV')} "
@@ -280,7 +277,7 @@ class CorsikaConfig:
 
     def _input_config_corsika_cherenkov_wavelength(self, entry):
         """Return CWAVLG parameter CORSIKA format."""
-        wavelength_range = gen.convert_string_to_list(entry["value"])
+        wavelength_range = entry["value"]
         return [
             f"{wavelength_range[0]*u.Unit(entry['unit']).to('nm')}",
             f"{wavelength_range[1]*u.Unit(entry['unit']).to('nm')}",
@@ -318,8 +315,12 @@ class CorsikaConfig:
         }
 
     def _input_config_io_buff(self, entry):
-        """Return IO_BUFFER parameter CORSIKA format."""
-        return f"{entry['value']}{entry['unit']}"
+        """Return IO_BUFFER parameter CORSIKA format (Byte or MB required)."""
+        value = entry["value"] * u.Unit(entry["unit"]).to("Mbyte")
+        # check if value is integer-like
+        if value.is_integer():
+            return f"{int(value)}MB"
+        return f"{int(entry['value'] * u.Unit(entry['unit']).to('byte'))}"
 
     def _rotate_azimuth_by_180deg(self, az, correct_for_geomagnetic_field_alignment=True):
         """

simtools/corsika/corsika_histograms.py

@@ -678,7 +678,7 @@ class CorsikaHistograms:
         ----------
         new_individual_telescopes: bool
             if False, the histograms are supposed to be filled for all telescopes.
-            if True, one histogram is set for each telescope sepparately.
+            if True, one histogram is set for each telescope separately.
         """
         if new_individual_telescopes is None:
             self._individual_telescopes = False

simtools/data_model/data_reader.py

@@ -112,9 +112,6 @@ def read_value_from_file(file_name, schema_file=None, validate=False):
     _logger.info("Reading data from %s", file_name)
 
     if validate:
-        if schema_file is None and "meta_schema_url" in data:
-            schema_file = data["meta_schema_url"]
-            _logger.debug(f"Using schema from meta_schema_url: {schema_file}")
         if schema_file is None:
             _collector = MetadataCollector(None, metadata_file_name=file_name)
             schema_file = _collector.get_data_model_schema_file_name()

simtools/data_model/metadata_collector.py

@@ -10,13 +10,12 @@ import datetime
 import getpass
 import logging
 import uuid
-from importlib.resources import files
 from pathlib import Path
 
 import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
-from simtools.data_model import metadata_model
+from simtools.data_model import metadata_model, schema
 from simtools.io_operations import io_handler
 from simtools.utils import names
 
@@ -86,6 +85,24 @@ class MetadataCollector:
             except AttributeError:
                 self._logger.debug(f"Method _fill_{meta_type}_meta not implemented")
 
+    def get_top_level_metadata(self):
+        """
+        Return top level metadata dictionary (with updated activity end time).
+
+        Returns
+        -------
+        dict
+            Top level metadata dictionary.
+
+        """
+        try:
+            self.top_level_meta[self.observatory]["activity"][
+                "end"
+            ] = datetime.datetime.now().isoformat(timespec="seconds")
+        except KeyError:
+            pass
+        return self.top_level_meta
+
     def get_data_model_schema_file_name(self):
         """
         Return data model schema file name.
@@ -117,7 +134,7 @@ class MetadataCollector:
         # from data model name
         if self.data_model_name:
             self._logger.debug(f"Schema file from data model name: {self.data_model_name}")
-            return f"{files('simtools')}/schemas/model_parameters/{self.data_model_name}.schema.yml"
+            return str(schema.get_model_parameter_schema_file(self.data_model_name))
 
         # from input metadata
         try:
@@ -249,7 +266,7 @@ class MetadataCollector:
             self._logger.error("Unknown metadata file format: %s", metadata_file_name)
             raise gen.InvalidConfigDataError
 
-        metadata_model.validate_schema(_input_metadata, None)
+        schema.validate_dict_using_schema(_input_metadata, None)
 
         return gen.change_dict_keys_case(
             self._process_metadata_from_file(_input_metadata),

simtools/data_model/metadata_model.py

@@ -9,45 +9,12 @@ Follows CTAO top-level data model definition.
 """
 
 import logging
-from importlib.resources import files
 
-import jsonschema
-
-import simtools.constants
-import simtools.utils.general as gen
-from simtools.data_model import format_checkers
-from simtools.utils import names
+import simtools.data_model.schema
 
 _logger = logging.getLogger(__name__)
 
 
-def validate_schema(data, schema_file):
-    """
-    Validate dictionary against schema.
-
-    Parameters
-    ----------
-    data
-        dictionary to be validated
-    schema_file (dict)
-        schema used for validation
-
-    Raises
-    ------
-    jsonschema.exceptions.ValidationError
-        if validation fails
-
-    """
-    schema, schema_file = _load_schema(schema_file)
-
-    try:
-        jsonschema.validate(data, schema=schema, format_checker=format_checkers.format_checker)
-    except jsonschema.exceptions.ValidationError:
-        _logger.error(f"Failed using {schema}")
-        raise
-    _logger.debug(f"Successful validation of data using schema from {schema_file}")
-
-
 def get_default_metadata_dict(schema_file=None, observatory="CTA"):
     """
     Return metadata schema with default values.
@@ -68,80 +35,10 @@ def get_default_metadata_dict(schema_file=None, observatory="CTA"):
 
 
     """
-    schema, _ = _load_schema(schema_file)
+    schema = simtools.data_model.schema.load_schema(schema_file)
     return _fill_defaults(schema["definitions"], observatory)
 
 
-def _load_schema(schema_file=None):
-    """
-    Load parameter schema from file from simpipe metadata schema.
-
-    Returns
-    -------
-    schema_file dict
-        Schema used for validation.
-    schema_file str
-        File name schema is loaded from. If schema_file is not given,
-        the default schema file name is returned.
-
-    Raises
-    ------
-    FileNotFoundError
-        if schema file is not found
-
-    """
-    if schema_file is None:
-        schema_file = files("simtools").joinpath(simtools.constants.METADATA_JSON_SCHEMA)
-
-    try:
-        schema = gen.collect_data_from_file(file_name=schema_file)
-    except FileNotFoundError:
-        schema_file = files("simtools").joinpath("schemas") / schema_file
-        schema = gen.collect_data_from_file(file_name=schema_file)
-    _logger.debug(f"Loading schema from {schema_file}")
-    _add_array_elements("InstrumentTypeElement", schema)
-
-    return schema, schema_file
-
-
-def _add_array_elements(key, schema):
-    """
-    Add list of array elements to schema.
-
-    This assumes an element [key]['enum'] is a list of elements.
-
-    Parameters
-    ----------
-    key: str
-        Key in schema dictionary
-    schema: dict
-        Schema dictionary
-
-    Returns
-    -------
-    dict
-        Schema dictionary with added array elements.
-
-    """
-    _list_of_array_elements = sorted(names.array_elements().keys())
-
-    def recursive_search(sub_schema, key):
-        if key in sub_schema:
-            if "enum" in sub_schema[key] and isinstance(sub_schema[key]["enum"], list):
-                sub_schema[key]["enum"] = list(
-                    set(sub_schema[key]["enum"] + _list_of_array_elements)
-                )
-            else:
-                sub_schema[key]["enum"] = _list_of_array_elements
-        else:
-            for _, v in sub_schema.items():
-                if isinstance(v, dict):
-                    recursive_search(v, key)
-
-    recursive_search(schema, key)
-    return schema
-
-
 def _resolve_references(yaml_data, observatory="CTA"):
     """
     Resolve references in yaml data and expand the received dictionary accordingly.
@@ -214,21 +111,21 @@ def _fill_defaults(schema, observatory="CTA"):
     return defaults
 
 
-def _fill_defaults_recursive(subschema, current_dict):
+def _fill_defaults_recursive(sub_schema, current_dict):
     """
-    Recursively fill default values from the subschema into the current dictionary.
+    Recursively fill default values from the sub_schema into the current dictionary.
 
     Parameters
     ----------
-    subschema: dict
-        Subschema describing part of the input data.
+    sub_schema: dict
+        Sub schema describing part of the input data.
     current_dict: dict
         Current dictionary to fill with default values.
     """
-    if "properties" not in subschema:
+    if "properties" not in sub_schema:
         _raise_missing_properties_error()
 
-    for prop, prop_schema in subschema["properties"].items():
+    for prop, prop_schema in sub_schema["properties"].items():
         _process_property(prop, prop_schema, current_dict)