gammasimtools 0.12.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

simtools/camera/single_photon_electron_spectrum.py

@@ -7,10 +7,12 @@ import tempfile
 from io import BytesIO
 from pathlib import Path
 
+import numpy as np
 from astropy.table import Table
+from scipy.optimize import curve_fit
 
 import simtools.data_model.model_data_writer as writer
-from simtools.constants import SCHEMA_PATH
+from simtools.constants import MODEL_PARAMETER_SCHEMA_URL, SCHEMA_PATH
 from simtools.data_model import validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io_operations import io_handler
@@ -44,12 +46,24 @@ class SinglePhotonElectronSpectrum:
         )
         self.io_handler = io_handler.IOHandler()
         self.data = ""  # Single photon electron spectrum data (as string)
+        self.args_dict["metadata_product_data_name"] = "single_pe_spectrum"
+        self.args_dict["metadata_product_data_url"] = (
+            MODEL_PARAMETER_SCHEMA_URL + "/pm_photoelectron_spectrum.schema.yml"
+        )
         self.metadata = MetadataCollector(args_dict=self.args_dict)
 
     def derive_single_pe_spectrum(self):
         """Derive single photon electron spectrum."""
+        afterpulse_fitted_spectrum = (
+            self.fit_afterpulse_spectrum() if self.args_dict.get("fit_afterpulse") else None
+        )
+
         if self.args_dict.get("use_norm_spe"):
-            return self._derive_spectrum_norm_spe()
+            return self._derive_spectrum_norm_spe(
+                input_spectrum=self.args_dict["input_spectrum"],
+                afterpulse_spectrum=self.args_dict.get("afterpulse_spectrum"),
+                afterpulse_fitted_spectrum=afterpulse_fitted_spectrum,
+            )
 
         raise NotImplementedError(
             "Derivation of single photon electron spectrum using a simtool is not yet implemented."
@@ -83,15 +97,27 @@ class SinglePhotonElectronSpectrum:
 
         writer.ModelDataWriter.dump(
             args_dict=self.args_dict,
-            metadata=self.metadata.top_level_meta,
+            metadata=self.metadata,
             product_data=table,
             validate_schema_file=None,
         )
 
-    def _derive_spectrum_norm_spe(self):
+    def _derive_spectrum_norm_spe(
+        self, input_spectrum, afterpulse_spectrum, afterpulse_fitted_spectrum
+    ):
         """
         Derive single photon electron spectrum using sim_telarray tool 'norm_spe'.
 
+        Parameters
+        ----------
+        input_spectrum : str
+            Input file with amplitude spectrum
+            (prompt spectrum only if afterpulse spectrum is given).
+        afterpulse_spectrum : str
+            Input file with afterpulse spectrum.
+        afterpulse_fitted_spectrum : astro.Table
+            Fitted afterpulse spectrum data.
+
         Returns
         -------
         int
@@ -103,11 +129,13 @@ class SinglePhotonElectronSpectrum:
             If the command execution fails.
         """
         tmp_input_file = self._get_input_data(
-            input_file=self.args_dict["input_spectrum"],
+            input_file=input_spectrum,
+            input_table=None,
             frequency_column=self.prompt_column,
         )
         tmp_ap_file = self._get_input_data(
-            input_file=self.args_dict.get("afterpulse_spectrum"),
+            input_file=afterpulse_spectrum,
+            input_table=afterpulse_fitted_spectrum,
             frequency_column=self.afterpulse_column,
         )
 
@@ -115,13 +143,14 @@ class SinglePhotonElectronSpectrum:
             f"{self.args_dict['simtel_path']}/sim_telarray/bin/norm_spe",
             "-r",
             f"{self.args_dict['step_size']},{self.args_dict['max_amplitude']}",
-            tmp_input_file.name,
         ]
         if tmp_ap_file:
-            command.insert(1, "-a")
-            command.insert(2, f"{tmp_ap_file.name}")
+            command.extend(["-a", f"{tmp_ap_file.name}"])
+            command.extend(["-s", f"{self.args_dict['scale_afterpulse_spectrum']}"])
+            command.extend(["-t", f"{self.args_dict['afterpulse_amplitude_range'][0]}"])
+        command.append(tmp_input_file.name)
 
-        self._logger.debug(f"Running norm_spe command: {' '.join(command)}")
+        self._logger.info(f"Running norm_spe command: {' '.join(command)}")
         try:
             result = subprocess.run(command, capture_output=True, text=True, check=True)
         except subprocess.CalledProcessError as exc:
@@ -138,24 +167,37 @@ class SinglePhotonElectronSpectrum:
         self.data = result.stdout
         return result.returncode
 
-    def _get_input_data(self, input_file, frequency_column):
+    def _get_input_data(self, input_file, input_table, frequency_column):
         """
-        Return input data for norm_spe command.
+        Return input data in the format required by the norm_spe tool as temporary file.
+
+        The norm_spe tool requires the data to be space separated values of the amplitude spectrum,
+        with two columns: amplitude and frequency.
+        Input is validated using the single_pe_spectrum schema (legacy input is not validated).
 
-        Input data need to be space separated values of the amplitude spectrum.
+        Parameters
+        ----------
+        input_file : str
+            Input file with amplitude spectrum.
+        input_table : astro.Table
+            Input table with amplitude spectrum.
+        frequency_column : str
+            Column name of the frequency data.
         """
-        input_data = ""
         if not input_file:
             return None
         input_file = Path(input_file)
 
-        if input_file.suffix == ".ecsv":
+        input_data = ""
+        if input_file.suffix == ".ecsv" or input_table:
             data_validator = validate_data.DataValidator(
-                schema_file=self.input_schema, data_file=input_file
+                schema_file=self.input_schema,
+                data_table=input_table,
+                data_file=input_file if input_table is None else None,
             )
             table = data_validator.validate_and_transform()
             input_data = "\n".join(f"{row['amplitude']} {row[frequency_column]}" for row in table)
-        else:
+        else:  # legacy format
             with open(input_file, encoding="utf-8") as f:
                 input_data = (
                     f.read().replace(",", " ")
@@ -166,3 +208,113 @@ class SinglePhotonElectronSpectrum:
         with tempfile.NamedTemporaryFile(delete=False, mode="w", encoding="utf-8") as tmpfile:
             tmpfile.write(input_data)
         return tmpfile
+
+    def fit_afterpulse_spectrum(self):
+        """
+        Fit afterpulse spectrum with a exponential decay function.
+
+        Assume input to be in ecsv format with columns 'amplitude', 'frequency (afterpulsing)',
+        and 'frequency stdev (afterpulsing)'.
+
+        Returns
+        -------
+        astro.Table
+            Table with fitted afterpulse spectrum data.
+        """
+        ap_min = self.args_dict["afterpulse_amplitude_range"][0]
+        fix_k = self.args_dict.get("afterpulse_decay_factor_fixed_value")
+
+        x, y, y_err = self._read_afterpulse_spectrum_for_fit(
+            self.args_dict.get("afterpulse_spectrum"), ap_min
+        )
+        fit_func, p0, bounds = self.afterpulse_fit_function(fix_k=fix_k)
+
+        result = curve_fit(fit_func, x, y, sigma=y_err, p0=p0, bounds=bounds, absolute_sigma=True)
+        params, covariance = result[0], result[1]
+        param_errors = np.sqrt(np.diag(covariance))
+        predicted = fit_func(x, *params)
+        self._afterpulse_fit_statistics(x, y, y_err, params, param_errors, predicted, fix_k)
+
+        # table with fitted afterpulse spectrum
+        x_fit = np.arange(
+            ap_min, self.args_dict["afterpulse_amplitude_range"][1], self.args_dict["step_size"]
+        )
+        y_fit = fit_func(x_fit, *params)
+        return Table([x_fit, y_fit], names=["amplitude", self.afterpulse_column])
+
+    def afterpulse_fit_function(self, fix_k):
+        """
+        Afterpulse fit function: exponential decay with linear term in the exponent.
+
+        Starting values and bounds are set for the other parameters using values typical
+        for LSTN-design. Allows to fix the K parameter.
+
+        Parameters
+        ----------
+        fix_K : float
+            Fixed value for K parameter.
+
+        Returns
+        -------
+        function
+            Exponential decay function with linear term in the exponent.
+        """
+
+        def exp_decay(x, a, b, k):
+            return a * np.exp(-1.0 / (b * (k / (x + k))) * x)
+
+        p0 = [1e-5, 8.0]  # Initial guess for [A, B] typical LSTN values
+        bounds_lower = [0, 0]
+        bounds_upper = [1.0, 20.0]
+
+        if fix_k is None:
+            p0.append(25.0)
+            bounds_lower.append(5.0)
+            bounds_upper.append(35.0)
+            return exp_decay, p0, (bounds_lower, bounds_upper)
+
+        def exp_decay_fixed_k(x, a, b):
+            return exp_decay(x, a, b, k=fix_k)
+
+        return exp_decay_fixed_k, p0, (bounds_lower, bounds_upper)
+
+    def _afterpulse_fit_statistics(self, x, y, y_err, params, param_errors, predicted, fix_k):
+        """Print and return afterpulse fit statistics."""
+        chi2 = np.sum(((y - predicted) / y_err) ** 2)
+        ndf = len(x) - len(params)
+
+        result = {
+            "params": params.tolist(),
+            "errors": param_errors.tolist(),
+            "chi2_ndf": chi2 / ndf if ndf > 0 else np.nan,
+        }
+        if fix_k is not None:
+            result["params"].append(fix_k)
+            result["errors"].append(0.0)
+
+        self._logger.info(f"Fit results: {result}")
+        return result
+
+    def _read_afterpulse_spectrum_for_fit(self, afterpulse_spectrum, fit_min_pe):
+        """
+        Read afterpulse spectrum data for fitting.
+
+        Parameters
+        ----------
+        afterpulse_spectrum : str
+            Afterpulse spectrum data file.
+        fit_min_pe : float
+            Minimum amplitude for fitting.
+
+        Returns
+        -------
+        tuple
+            Tuple with x, y, y_err data for fitting.
+        """
+        table = Table.read(afterpulse_spectrum, format="ascii.ecsv")
+        x = table["amplitude"]
+        y = table[self.afterpulse_column]
+        y_err = table["frequency stdev (afterpulsing)"]
+        mask = (x >= fit_min_pe) & (y > 0)
+        x_fit, y_fit, y_err_fit = x[mask], y[mask], y_err[mask]
+        return x_fit, y_fit, y_err_fit

simtools/constants.py

@@ -6,6 +6,8 @@ from importlib.resources import files
 SCHEMA_PATH = files("simtools") / "schemas"
 # Path to metadata jsonschema
 METADATA_JSON_SCHEMA = SCHEMA_PATH / "metadata.metaschema.yml"
+# Path to plotting configuration json schema
+PLOT_CONFIG_SCHEMA = SCHEMA_PATH / "plot_configuration.metaschema.yml"
 # Path to model parameter metaschema
 MODEL_PARAMETER_METASCHEMA = SCHEMA_PATH / "model_parameter.metaschema.yml"
 # Path to model parameter description metaschema
@@ -14,3 +16,8 @@ MODEL_PARAMETER_DESCRIPTION_METASCHEMA = (
 )
 # Path to model parameter schema files
 MODEL_PARAMETER_SCHEMA_PATH = SCHEMA_PATH / "model_parameters"
+# URL to model parameter schema files
+MODEL_PARAMETER_SCHEMA_URL = (
+    "https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/"
+    "/model_parameters"
+)

simtools/data_model/metadata_collector.py

@@ -11,6 +11,8 @@ import logging
 import uuid
 from pathlib import Path
 
+import yaml
+
 import simtools.constants
 import simtools.utils.general as gen
 import simtools.version
@@ -68,9 +70,7 @@ class MetadataCollector:
         self.top_level_meta = gen.change_dict_keys_case(
             data_dict=metadata_model.get_default_metadata_dict(), lower_case=True
         )
-        self.input_metadata = self._read_input_metadata_from_file(
-            metadata_file_name=metadata_file_name
-        )
+        self.input_metadata = self._read_input_metadata_from_file(metadata_file_name)
         self.collect_meta_data()
         if clean_meta:
             self.top_level_meta = self.clean_meta_data(self.top_level_meta)
@@ -103,6 +103,74 @@ class MetadataCollector:
             pass
         return self.top_level_meta
 
+    @staticmethod
+    def dump(args_dict, output_file, add_activity_name=False):
+        """
+        Write metadata to file (static method).
+
+        Parameters
+        ----------
+        args_dict: dict
+            Command line parameters
+        output_file: str or Path
+            Name of output file.
+        add_activity_name: bool
+            Add activity name to file name.
+        """
+        collector = MetadataCollector(args_dict)
+        collector.write(output_file, add_activity_name=add_activity_name)
+
+    def write(self, yml_file=None, keys_lower_case=False, add_activity_name=False):
+        """
+        Write toplevel metadata to file (yaml file format).
+
+        Parameters
+        ----------
+        metadata: dict
+            Metadata to be stored
+        yml_file: str
+            Name of output file.
+        keys_lower_case: bool
+            Write yaml keys in lower case.
+        add_activity_name: bool
+            Add activity name to file name.
+
+        Returns
+        -------
+        str
+            Name of output file
+
+        Raises
+        ------
+        FileNotFoundError
+            If yml_file not found.
+        TypeError
+            If yml_file is not defined.
+        """
+        metadata = self.get_top_level_metadata()
+        activity_name = metadata.get("cta", {}).get("activity", {}).get("name", "").rstrip(".")
+        suffix = f".{activity_name}.meta.yml" if add_activity_name else ".meta.yml"
+
+        if yml_file is None:
+            raise TypeError("No output file for metadata defined")
+
+        try:
+            yml_file = names.file_name_with_version(yml_file, suffix)
+            with open(yml_file, "w", encoding="UTF-8") as file:
+                yaml.safe_dump(
+                    gen.change_dict_keys_case(
+                        gen.remove_substring_recursively_from_dict(metadata, substring="\n"),
+                        keys_lower_case,
+                    ),
+                    file,
+                    sort_keys=False,
+                    explicit_start=True,
+                )
+            self._logger.info(f"Writing metadata to {yml_file}")
+            return yml_file
+        except FileNotFoundError as exc:
+            raise FileNotFoundError(f"Error writing metadata to {yml_file}") from exc
+
     def get_data_model_schema_file_name(self):
         """
         Return data model schema file name.
@@ -136,12 +204,12 @@ class MetadataCollector:
             self._logger.debug(f"Schema file from data model name: {self.data_model_name}")
             return str(schema.get_model_parameter_schema_file(self.data_model_name))
 
-        # from input metadata
+        # from first entry in input metadata (least preferred)
         try:
-            url = self.input_metadata[self.observatory]["product"]["data"]["model"]["url"]
+            url = self.input_metadata[0][self.observatory]["product"]["data"]["model"]["url"]
             self._logger.debug(f"Schema file from input metadata: {url}")
             return url
-        except KeyError:
+        except (KeyError, TypeError):
             pass
 
         self._logger.warning("No schema file found.")
@@ -170,7 +238,7 @@ class MetadataCollector:
         Parameters
         ----------
         from_input_meta: bool
-            Get site from input metadata (default: False)
+            Get site from first entry of input metadata (default: False)
 
         Returns
         -------
@@ -182,11 +250,11 @@ class MetadataCollector:
             _site = (
                 self.top_level_meta[self.observatory]["instrument"]["site"]
                 if not from_input_meta
-                else self.input_metadata[self.observatory]["instrument"]["site"]
+                else self.input_metadata[0][self.observatory]["instrument"]["site"]
             )
             if _site is not None:
                 return names.validate_site_name(_site)
-        except KeyError:
+        except (KeyError, TypeError):
             pass
         return None
 
@@ -202,7 +270,13 @@ class MetadataCollector:
         contact_dict["name"] = contact_dict.get("name") or self.args_dict.get("user_name")
         if contact_dict["name"] is None:
             self._logger.warning("No user name provided, take user info from system level.")
-            contact_dict["name"] = getpass.getuser()
+            try:
+                contact_dict["name"] = getpass.getuser()
+            except Exception as exc:  # pylint: disable=broad-except
+                contact_dict["name"] = "UNKNOWN_USER"
+                self._logger.warning(
+                    f"Failed to get user name: {exc}, setting it to {contact_dict['name']} "
+                )
         meta_dict = {
             "email": "user_mail",
             "orcid": "user_orcid",
@@ -221,17 +295,28 @@ class MetadataCollector:
             Dictionary for context metadata fields.
 
         """
-        try:  # wide try..except as for some cases we expect that there is no product metadata
-            reduced_product_meta = {
-                key: value
-                for key, value in self.input_metadata[self.observatory]["product"].items()
-                if key in {"description", "id", "creation_time", "valid", "format", "filename"}
-            }
-            self._fill_context_sim_list(context_dict["associated_data"], reduced_product_meta)
-        except (KeyError, TypeError):
-            self._logger.debug("No input product metadata appended to associated data.")
+        input_metadata = (
+            self.input_metadata if isinstance(self.input_metadata, list) else [self.input_metadata]
+        )
 
-    def _read_input_metadata_from_file(self, metadata_file_name=None):
+        for metadata in input_metadata:
+            try:  # wide try..except as for some cases we expect that there is no product metadata
+                reduced_product_meta = {
+                    key: value
+                    for key, value in metadata[self.observatory]["product"].items()
+                    if key in {"description", "id", "creation_time", "valid", "format", "filename"}
+                }
+                if metadata[self.observatory].get("activity", {}).get("name"):
+                    reduced_product_meta["activity_name"] = metadata[self.observatory][
+                        "activity"
+                    ].get("name")
+                context_dict["associated_data"] = self._fill_context_sim_list(
+                    context_dict["associated_data"], reduced_product_meta
+                )
+            except (KeyError, TypeError):
+                self._logger.debug("No input product metadata appended to associated data.")
+
+    def _read_input_metadata_from_file(self, metadata_file_name_expression=None):
         """
         Read and validate input metadata from file.
 
@@ -240,8 +325,8 @@ class MetadataCollector:
 
         Parameter
         ---------
-        metadata_file_name: str or Path
-            Name of metadata file.
+        metadata_file_name_expression: str or Path
+            Name of metadata file (regular expressions allowed).
 
         Returns
         -------
@@ -256,31 +341,32 @@ class MetadataCollector:
             if metadata does not exist
 
         """
-        metadata_file_name = (
-            self.args_dict.get("input_meta", None) or self.args_dict.get("input", None)
-            if metadata_file_name is None
-            else metadata_file_name
+        metadata_file_names = (
+            metadata_file_name_expression
+            or self.args_dict.get("input_meta")
+            or self.args_dict.get("input")
         )
 
-        if metadata_file_name is None:
+        try:
+            metadata_files = gen.resolve_file_patterns(metadata_file_names)
+        except ValueError:
             self._logger.debug("No input metadata file defined.")
-            return {}
-
-        self._logger.debug("Reading meta data from %s", metadata_file_name)
-        if Path(metadata_file_name).suffix in (".yaml", ".yml", ".json"):
-            _input_metadata = self._read_input_metadata_from_yml_or_json(metadata_file_name)
-        elif Path(metadata_file_name).suffix == ".ecsv":
-            _input_metadata = self._read_input_metadata_from_ecsv(metadata_file_name)
-        else:
-            self._logger.error("Unknown metadata file format: %s", metadata_file_name)
-            raise gen.InvalidConfigDataError
+            return None
+
+        metadata = []
+        for metadata_file in metadata_files:
+            self._logger.debug(f"Reading meta data from {metadata_file}")
+            if Path(metadata_file).suffix in (".yaml", ".yml", ".json"):
+                _input_metadata = self._read_input_metadata_from_yml_or_json(metadata_file)
+            elif Path(metadata_file).suffix == ".ecsv":
+                _input_metadata = self._read_input_metadata_from_ecsv(metadata_file)
+            else:
+                raise gen.InvalidConfigDataError(f"Unknown metadata file format: {metadata_file}")
 
-        schema.validate_dict_using_schema(_input_metadata, schema_file=METADATA_JSON_SCHEMA)
+            schema.validate_dict_using_schema(_input_metadata, schema_file=METADATA_JSON_SCHEMA)
+            metadata.append(gen.change_dict_keys_case(_input_metadata, lower_case=True))
 
-        return gen.change_dict_keys_case(
-            self._process_metadata_from_file(_input_metadata),
-            lower_case=True,
-        )
+        return metadata
 
     def _read_input_metadata_from_ecsv(self, metadata_file_name):
         """Read input metadata from ecsv file."""
@@ -348,13 +434,23 @@ class MetadataCollector:
             pass
 
         # DATA:MODEL
-        helper_dict = {"name": "name", "version": "version", "type": "meta_schema"}
-        for key, value in helper_dict.items():
-            product_dict["data"]["model"][key] = self.schema_dict.get(value, None)
-        product_dict["data"]["model"]["url"] = self.schema_file
+        product_dict["data"]["model"]["name"] = (
+            self.schema_dict.get("name")
+            or self.args_dict.get("metadata_product_data_name")
+            or "undefined_model_name"
+        )
+        product_dict["data"]["model"]["version"] = self.schema_dict.get("version", "0.0.0")
+        product_dict["data"]["model"]["type"] = self.schema_dict.get("meta_schema", None)
+        product_dict["data"]["model"]["url"] = self.schema_file or self.args_dict.get(
+            "metadata_product_data_url"
+        )
 
-        product_dict["format"] = self.args_dict.get("output_file_format", None)
-        product_dict["filename"] = str(self.args_dict.get("output_file", None))
+        product_dict["filename"] = str(self.args_dict.get("output_file", ""))
+        product_dict["format"] = (
+            self.args_dict.get("output_file_format")
+            or Path(product_dict["filename"]).suffix.lstrip(".")
+            or None
+        )
 
     def _fill_instrument_meta(self, instrument_dict):
         """
@@ -369,14 +465,17 @@ class MetadataCollector:
             Dictionary for instrument metadata fields.
 
         """
-        instrument_dict["site"] = self.args_dict.get("site", None)
-        instrument_dict["ID"] = self.args_dict.get("instrument") or self.args_dict.get(
-            "telescope", None
-        )
+        instrument_dict["site"] = self.args_dict.get("site")
+        instrument_dict["ID"] = self.args_dict.get("instrument") or self.args_dict.get("telescope")
         if instrument_dict["ID"]:
             instrument_dict["class"] = names.get_collection_name_from_array_element_name(
                 instrument_dict["ID"], False
             )
+            instrument_dict["type"] = (
+                names.get_array_element_type_from_name(instrument_dict["ID"])
+                if not instrument_dict.get("type")
+                else instrument_dict["type"]
+            )
 
     def _fill_process_meta(self, process_dict):
         """
@@ -460,15 +559,14 @@ class MetadataCollector:
             Updated meta list.
 
         """
-        if len(new_entry_dict) == 0:
+        if not new_entry_dict:
             return []
-        try:
-            if self._all_values_none(meta_list[0]):
-                meta_list[0] = new_entry_dict
-            else:
-                meta_list.append(new_entry_dict)
-        except (TypeError, IndexError):
-            meta_list = [new_entry_dict]
+        if meta_list is None or not meta_list:
+            return [new_entry_dict]
+        if self._all_values_none(meta_list[0]):
+            meta_list[0] = new_entry_dict
+        else:
+            meta_list.append(new_entry_dict)
         return meta_list
 
     def _process_metadata_from_file(self, meta_dict):
@@ -533,7 +631,7 @@ class MetadataCollector:
         """
         try:
             for document in _input_metadata["context"][key]:
-                self._fill_context_sim_list(context_dict[key], document)
+                context_dict[key] = self._fill_context_sim_list(context_dict[key], document)
         except KeyError:
             pass