gammasimtools 0.13.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

simtools/camera/single_photon_electron_spectrum.py

@@ -7,7 +7,9 @@ import tempfile
 from io import BytesIO
 from pathlib import Path
 
+import numpy as np
 from astropy.table import Table
+from scipy.optimize import curve_fit
 
 import simtools.data_model.model_data_writer as writer
 from simtools.constants import MODEL_PARAMETER_SCHEMA_URL, SCHEMA_PATH
@@ -52,8 +54,16 @@ class SinglePhotonElectronSpectrum:
 
     def derive_single_pe_spectrum(self):
         """Derive single photon electron spectrum."""
+        afterpulse_fitted_spectrum = (
+            self.fit_afterpulse_spectrum() if self.args_dict.get("fit_afterpulse") else None
+        )
+
         if self.args_dict.get("use_norm_spe"):
-            return self._derive_spectrum_norm_spe()
+            return self._derive_spectrum_norm_spe(
+                input_spectrum=self.args_dict["input_spectrum"],
+                afterpulse_spectrum=self.args_dict.get("afterpulse_spectrum"),
+                afterpulse_fitted_spectrum=afterpulse_fitted_spectrum,
+            )
 
         raise NotImplementedError(
             "Derivation of single photon electron spectrum using a simtool is not yet implemented."
@@ -92,10 +102,22 @@ class SinglePhotonElectronSpectrum:
             validate_schema_file=None,
         )
 
-    def _derive_spectrum_norm_spe(self):
+    def _derive_spectrum_norm_spe(
+        self, input_spectrum, afterpulse_spectrum, afterpulse_fitted_spectrum
+    ):
         """
         Derive single photon electron spectrum using sim_telarray tool 'norm_spe'.
 
+        Parameters
+        ----------
+        input_spectrum : str
+            Input file with amplitude spectrum
+            (prompt spectrum only if afterpulse spectrum is given).
+        afterpulse_spectrum : str
+            Input file with afterpulse spectrum.
+        afterpulse_fitted_spectrum : astro.Table
+            Fitted afterpulse spectrum data.
+
         Returns
         -------
         int
@@ -107,11 +129,13 @@ class SinglePhotonElectronSpectrum:
             If the command execution fails.
         """
         tmp_input_file = self._get_input_data(
-            input_file=self.args_dict["input_spectrum"],
+            input_file=input_spectrum,
+            input_table=None,
             frequency_column=self.prompt_column,
         )
         tmp_ap_file = self._get_input_data(
-            input_file=self.args_dict.get("afterpulse_spectrum"),
+            input_file=afterpulse_spectrum,
+            input_table=afterpulse_fitted_spectrum,
             frequency_column=self.afterpulse_column,
         )
 
@@ -119,13 +143,14 @@ class SinglePhotonElectronSpectrum:
             f"{self.args_dict['simtel_path']}/sim_telarray/bin/norm_spe",
             "-r",
             f"{self.args_dict['step_size']},{self.args_dict['max_amplitude']}",
-            tmp_input_file.name,
         ]
         if tmp_ap_file:
-            command.insert(1, "-a")
-            command.insert(2, f"{tmp_ap_file.name}")
+            command.extend(["-a", f"{tmp_ap_file.name}"])
+            command.extend(["-s", f"{self.args_dict['scale_afterpulse_spectrum']}"])
+            command.extend(["-t", f"{self.args_dict['afterpulse_amplitude_range'][0]}"])
+        command.append(tmp_input_file.name)
 
-        self._logger.debug(f"Running norm_spe command: {' '.join(command)}")
+        self._logger.info(f"Running norm_spe command: {' '.join(command)}")
         try:
             result = subprocess.run(command, capture_output=True, text=True, check=True)
         except subprocess.CalledProcessError as exc:
@@ -142,24 +167,37 @@ class SinglePhotonElectronSpectrum:
         self.data = result.stdout
         return result.returncode
 
-    def _get_input_data(self, input_file, frequency_column):
+    def _get_input_data(self, input_file, input_table, frequency_column):
         """
-        Return input data for norm_spe command.
+        Return input data in the format required by the norm_spe tool as temporary file.
+
+        The norm_spe tool requires the data to be space separated values of the amplitude spectrum,
+        with two columns: amplitude and frequency.
+        Input is validated using the single_pe_spectrum schema (legacy input is not validated).
 
-        Input data need to be space separated values of the amplitude spectrum.
+        Parameters
+        ----------
+        input_file : str
+            Input file with amplitude spectrum.
+        input_table : astro.Table
+            Input table with amplitude spectrum.
+        frequency_column : str
+            Column name of the frequency data.
         """
-        input_data = ""
         if not input_file:
             return None
         input_file = Path(input_file)
 
-        if input_file.suffix == ".ecsv":
+        input_data = ""
+        if input_file.suffix == ".ecsv" or input_table:
             data_validator = validate_data.DataValidator(
-                schema_file=self.input_schema, data_file=input_file
+                schema_file=self.input_schema,
+                data_table=input_table,
+                data_file=input_file if input_table is None else None,
             )
             table = data_validator.validate_and_transform()
             input_data = "\n".join(f"{row['amplitude']} {row[frequency_column]}" for row in table)
-        else:
+        else:  # legacy format
             with open(input_file, encoding="utf-8") as f:
                 input_data = (
                     f.read().replace(",", " ")
@@ -170,3 +208,113 @@ class SinglePhotonElectronSpectrum:
         with tempfile.NamedTemporaryFile(delete=False, mode="w", encoding="utf-8") as tmpfile:
             tmpfile.write(input_data)
         return tmpfile
+
+    def fit_afterpulse_spectrum(self):
+        """
+        Fit afterpulse spectrum with a exponential decay function.
+
+        Assume input to be in ecsv format with columns 'amplitude', 'frequency (afterpulsing)',
+        and 'frequency stdev (afterpulsing)'.
+
+        Returns
+        -------
+        astro.Table
+            Table with fitted afterpulse spectrum data.
+        """
+        ap_min = self.args_dict["afterpulse_amplitude_range"][0]
+        fix_k = self.args_dict.get("afterpulse_decay_factor_fixed_value")
+
+        x, y, y_err = self._read_afterpulse_spectrum_for_fit(
+            self.args_dict.get("afterpulse_spectrum"), ap_min
+        )
+        fit_func, p0, bounds = self.afterpulse_fit_function(fix_k=fix_k)
+
+        result = curve_fit(fit_func, x, y, sigma=y_err, p0=p0, bounds=bounds, absolute_sigma=True)
+        params, covariance = result[0], result[1]
+        param_errors = np.sqrt(np.diag(covariance))
+        predicted = fit_func(x, *params)
+        self._afterpulse_fit_statistics(x, y, y_err, params, param_errors, predicted, fix_k)
+
+        # table with fitted afterpulse spectrum
+        x_fit = np.arange(
+            ap_min, self.args_dict["afterpulse_amplitude_range"][1], self.args_dict["step_size"]
+        )
+        y_fit = fit_func(x_fit, *params)
+        return Table([x_fit, y_fit], names=["amplitude", self.afterpulse_column])
+
+    def afterpulse_fit_function(self, fix_k):
+        """
+        Afterpulse fit function: exponential decay with linear term in the exponent.
+
+        Starting values and bounds are set for the other parameters using values typical
+        for LSTN-design. Allows to fix the K parameter.
+
+        Parameters
+        ----------
+        fix_K : float
+            Fixed value for K parameter.
+
+        Returns
+        -------
+        function
+            Exponential decay function with linear term in the exponent.
+        """
+
+        def exp_decay(x, a, b, k):
+            return a * np.exp(-1.0 / (b * (k / (x + k))) * x)
+
+        p0 = [1e-5, 8.0]  # Initial guess for [A, B] typical LSTN values
+        bounds_lower = [0, 0]
+        bounds_upper = [1.0, 20.0]
+
+        if fix_k is None:
+            p0.append(25.0)
+            bounds_lower.append(5.0)
+            bounds_upper.append(35.0)
+            return exp_decay, p0, (bounds_lower, bounds_upper)
+
+        def exp_decay_fixed_k(x, a, b):
+            return exp_decay(x, a, b, k=fix_k)
+
+        return exp_decay_fixed_k, p0, (bounds_lower, bounds_upper)
+
+    def _afterpulse_fit_statistics(self, x, y, y_err, params, param_errors, predicted, fix_k):
+        """Print and return afterpulse fit statistics."""
+        chi2 = np.sum(((y - predicted) / y_err) ** 2)
+        ndf = len(x) - len(params)
+
+        result = {
+            "params": params.tolist(),
+            "errors": param_errors.tolist(),
+            "chi2_ndf": chi2 / ndf if ndf > 0 else np.nan,
+        }
+        if fix_k is not None:
+            result["params"].append(fix_k)
+            result["errors"].append(0.0)
+
+        self._logger.info(f"Fit results: {result}")
+        return result
+
+    def _read_afterpulse_spectrum_for_fit(self, afterpulse_spectrum, fit_min_pe):
+        """
+        Read afterpulse spectrum data for fitting.
+
+        Parameters
+        ----------
+        afterpulse_spectrum : str
+            Afterpulse spectrum data file.
+        fit_min_pe : float
+            Minimum amplitude for fitting.
+
+        Returns
+        -------
+        tuple
+            Tuple with x, y, y_err data for fitting.
+        """
+        table = Table.read(afterpulse_spectrum, format="ascii.ecsv")
+        x = table["amplitude"]
+        y = table[self.afterpulse_column]
+        y_err = table["frequency stdev (afterpulsing)"]
+        mask = (x >= fit_min_pe) & (y > 0)
+        x_fit, y_fit, y_err_fit = x[mask], y[mask], y_err[mask]
+        return x_fit, y_fit, y_err_fit

simtools/data_model/model_data_writer.py

@@ -269,6 +269,7 @@ class ModelDataWriter:
             "unit": unit,
             "type": self._get_parameter_type(),
             "file": self._parameter_is_a_file(),
+            "meta_parameter": False,
         }
         return self.validate_and_transform(
             product_data_dict=data_dict,
@@ -426,7 +427,7 @@ class ModelDataWriter:
         """
         Prepare data dictionary for writing to json file.
 
-        Ensure sim_telarray style lists as strings.
+        Ensure sim_telarray style lists as strings 'type' and 'unit' entries.
         Replace "None" with "null" for unit field.
 
         Parameters
@@ -441,13 +442,13 @@ class ModelDataWriter:
 
         """
         try:
-            data_dict["value"] = gen.convert_list_to_string(data_dict["value"])
-            data_dict["unit"] = gen.convert_list_to_string(data_dict["unit"], comma_separated=True)
-            data_dict["type"] = gen.convert_list_to_string(
-                data_dict["type"], comma_separated=True, collapse_list=True
-            )
             if isinstance(data_dict["unit"], str):
                 data_dict["unit"] = data_dict["unit"].replace("None", "null")
+            elif isinstance(data_dict["unit"], list):
+                data_dict["unit"] = [
+                    unit.replace("None", "null") if isinstance(unit, str) else unit
+                    for unit in data_dict["unit"]
+                ]
         except KeyError:
             pass
         return data_dict

simtools/db/db_handler.py

@@ -15,7 +15,7 @@ from pymongo import MongoClient
 from simtools.data_model import validate_data
 from simtools.io_operations import io_handler
 from simtools.simtel import simtel_table_reader
-from simtools.utils import names, value_conversion
+from simtools.utils import general, names, value_conversion
 
 __all__ = ["DatabaseHandler"]
 
@@ -273,8 +273,6 @@ class DatabaseHandler:
         """
         Get model parameters for all model versions.
 
-        Queries parameters for design and for the specified array element (if necessary).
-
         Parameters
         ----------
         site: str
@@ -290,10 +288,16 @@ class DatabaseHandler:
         """
         pars = defaultdict(dict)
         for _model_version in self.get_model_versions(collection):
-            parameter_data = self.get_model_parameters(
-                site, array_element_name, collection, _model_version
-            )
-            pars[_model_version].update(parameter_data)
+            try:
+                parameter_data = self.get_model_parameters(
+                    site, array_element_name, collection, _model_version
+                )
+                pars[_model_version].update(parameter_data)
+            except KeyError:
+                self._logger.debug(
+                    f"Skipping model version {_model_version} - {array_element_name} not found"
+                )
+                continue
         return pars
 
     def _get_parameter_for_model_version(
@@ -795,10 +799,12 @@ class DatabaseHandler:
                     f"corresponding to the {par_dict['parameter']} parameter, must be provided."
                 )
             file_path = Path(file_prefix).joinpath(par_dict["value"])
+            if not general.is_utf8_file(file_path):
+                raise ValueError(f"File is not UTF-8 encoded: {file_path}")
             files_to_add_to_db.add(f"{file_path}")
 
         self._logger.info(
-            f"Adding a new entry to DB {db_name} and collection {db_name}:\n{par_dict}"
+            f"Adding a new entry to DB {db_name} and collection {collection_name}:\n{par_dict}"
         )
         collection.insert_one(par_dict)
 

simtools/dependencies.py

@@ -81,18 +81,53 @@ def get_sim_telarray_version():
 
 def get_corsika_version():
     """
-    Get the version of the corsika package.
+    Get the version of the CORSIKA package.
 
     Returns
     -------
     str
-        Version of the corsika package.
+        Version of the CORSIKA package.
     """
+    version = None
+    sim_telarray_path = os.getenv("SIMTOOLS_SIMTEL_PATH")
+    if sim_telarray_path is None:
+        _logger.warning("Environment variable SIMTOOLS_SIMTEL_PATH is not set.")
+        return None
+    corsika_command = Path(sim_telarray_path) / "corsika-run" / "corsika"
+
+    # Below I do not use the standard context manager because
+    # it makes mocking in the tests significantly more difficult
+    process = subprocess.Popen(  # pylint: disable=consider-using-with
+        corsika_command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        stdin=subprocess.PIPE,
+        text=True,
+    )
+
+    # Capture output until it waits for input
+    while True:
+        line = process.stdout.readline()
+        if not line:
+            break
+        # Extract the version from the line "NUMBER OF VERSION :  7.7550"
+        if "NUMBER OF VERSION" in line:
+            version = line.split(":")[1].strip()
+            break
+        # Check for a specific prompt or indication that the program is waiting for input
+        if "DATA CARDS FOR RUN STEERING ARE EXPECTED FROM STANDARD INPUT" in line:
+            break
+
+    process.terminate()
+    # Check it's a valid version string
+    if version and re.match(r"\d+\.\d+", version):
+        return version
     try:
         build_opts = get_build_options()
     except (FileNotFoundError, TypeError):
-        _logger.warning("CORSIKA version not implemented yet.")
+        _logger.warning("Could not get CORSIKA version.")
         return None
+    _logger.debug("Getting the CORSIKA version from the build options.")
     return build_opts.get("corsika_version")
 
 

simtools/layout/array_layout.py

@@ -641,6 +641,7 @@ class ArrayLayout:
             "unit": "m",
             "type": "float64",
             "file": False,
+            "meta_parameter": False,
         }
 
     def get_number_of_telescopes(self):

simtools/layout/ctao_array_layouts.py

@@ -0,0 +1,172 @@
+"""Retrieve, merge, and write layouts from CTAO common identifiers repository."""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.data_model.model_data_writer import ModelDataWriter
+from simtools.io_operations import io_handler
+from simtools.utils import names
+
+_logger = logging.getLogger(__name__)
+
+
+def retrieve_array_layouts(site, repository_url, branch_name="main"):
+    """
+    Retrieve array layouts from CTAO common identifiers repository.
+
+    Parameters
+    ----------
+    site : str
+        Site identifier.
+    repository_url : str
+        URL or path to CTAO common identifiers
+    branch_name : str
+        Repository branch to use for CTAO common identifiers.
+
+    Returns
+    -------
+    dict
+        Array layouts for all CTAO sites.
+    """
+    _logger.info(f"Retrieving array layouts from {repository_url} on branch {branch_name}.")
+
+    if gen.is_url(repository_url):
+        array_element_ids = gen.collect_data_from_http(
+            url=f"{repository_url}/{branch_name}/array-element-ids.json"
+        )
+        sub_arrays = gen.collect_data_from_http(
+            url=f"{repository_url}/{branch_name}/subarray-ids.json"
+        )
+    else:
+        array_element_ids = gen.collect_data_from_file(
+            Path(repository_url) / "array-element-ids.json"
+        )
+        sub_arrays = gen.collect_data_from_file(Path(repository_url) / "subarray-ids.json")
+
+    return _get_layouts_per_site(site, sub_arrays, array_element_ids)
+
+
+def _get_layouts_per_site(site, sub_arrays, array_element_ids):
+    """
+    Get array layouts for CTAO sites.
+
+    Parameters
+    ----------
+    site : str
+        Site identifier.
+    sub_arrays : dict
+        Sub-array definitions.
+    array_element_ids : dict
+        Array element definitions.
+
+    Returns
+    -------
+    dict
+        Array layouts for CTAO sites.
+    """
+    layouts_per_site = []
+
+    for array in sub_arrays.get("subarrays", []):
+        elements = []
+        for ids in array.get("array_element_ids", []):
+            element_name = _get_array_element_name(ids, array_element_ids)
+            if names.get_site_from_array_element_name(element_name) != site:
+                break
+            elements.append(element_name)
+        if len(elements) > 0:
+            array_layout = {
+                "name": array.get("name"),
+                "elements": elements,
+            }
+            layouts_per_site.append(array_layout)
+
+    _logger.info(f"CTAO array layout definition: {layouts_per_site}")
+    return layouts_per_site
+
+
+def _get_array_element_name(ids, array_element_ids):
+    """Return array element name for common identifier."""
+    for element in array_element_ids.get("array_elements", []):
+        if element.get("id") == ids:
+            return element.get("name")
+    return None
+
+
+def merge_array_layouts(layouts_1, layouts_2):
+    """
+    Compare array layout dictionaries and merge them.
+
+    Parameters
+    ----------
+    layouts_1 : dict
+        Array layout dictionary 1.
+    layouts_2 : dict
+        Array layout dictionary 2.
+
+    Returns
+    -------
+    dict
+        Merged array layout dictionary based on layout_1.
+    """
+    merged_layout = layouts_1
+    for layout_2 in layouts_2:
+        layout_found = False
+        for layout_1 in layouts_1.get("value", {}):
+            if sorted(layout_1["elements"]) == sorted(layout_2["elements"]):
+                print(
+                    f"Equal telescope list: simtools '{layout_1['name']}' "
+                    f"and CTAO '{layout_2['name']}'"
+                )
+                layout_1["name"] = layout_2["name"]
+                layout_found = True
+        if not layout_found:
+            merged_layout["value"].append(
+                {
+                    "name": layout_2["name"],
+                    "elements": layout_2["elements"],
+                }
+            )
+            _logger.info(f"Adding {layout_2['name']} with {layout_2['elements']}")
+    return merged_layout
+
+
+def write_array_layouts(array_layouts, args_dict, db_config):
+    """
+    Write array layouts as model parameter.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    array_layouts : dict
+        Array layouts to be written.
+    db_config : dict
+        Database configuration.
+    """
+    _logger.info(f"Writing updated array layouts to the database for site {args_dict['site']}.")
+
+    io_handler_instance = io_handler.IOHandler()
+    io_handler_instance.set_paths(
+        output_path=args_dict["output_path"],
+        use_plain_output_path=args_dict["use_plain_output_path"],
+    )
+    output_file = io_handler_instance.get_output_file(
+        f"array-layouts-{args_dict['updated_parameter_version']}.json"
+    )
+
+    ModelDataWriter.dump_model_parameter(
+        parameter_name="array_layouts",
+        value=array_layouts["value"],
+        instrument=args_dict["site"],
+        parameter_version=args_dict.get("updated_parameter_version"),
+        output_file=output_file,
+        use_plain_output_path=args_dict["use_plain_output_path"],
+        db_config=db_config,
+    )
+    MetadataCollector.dump(
+        args_dict,
+        output_file,
+        add_activity_name=True,
+    )

simtools/model/array_model.py

@@ -12,7 +12,7 @@ from simtools.io_operations import io_handler
 from simtools.model.site_model import SiteModel
 from simtools.model.telescope_model import TelescopeModel
 from simtools.simtel.simtel_config_writer import SimtelConfigWriter
-from simtools.utils import general, names
+from simtools.utils import names
 
 __all__ = ["ArrayModel"]
 
@@ -326,12 +326,11 @@ class ArrayModel:
             "site": site,
             "parameter_version": parameter_version,
             "unique_id": None,
-            "value": general.convert_list_to_string(
-                [x.to("m").value, y.to("m").value, z.to("m").value]
-            ),
+            "value": [x.to("m").value, y.to("m").value, z.to("m").value],
             "unit": "m",
             "type": "float64",
             "file": False,
+            "meta_parameter": False,
         }
 
     def _get_array_elements_from_list(self, array_elements_list: list[str]) -> dict: