gammasimtools 0.8.2__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/simtel/simulator_camera_efficiency.py

@@ -154,10 +154,12 @@ class SimulatorCameraEfficiency(SimtelRunner):
         command += " 300"  # Xmax
         command += f" {self._telescope_model.get_parameter_value('atmospheric_profile')}"
         command += f" {self.zenith_angle}"
-        command += f" 2>{self._file_log}"
-        command += f" >{self._file_simtel}"
 
-        return f"cd {self._simtel_path.joinpath('sim_telarray')} && {command}"
+        return (
+            f"cd {self._simtel_path.joinpath('sim_telarray')} && {command}",
+            self._file_simtel,
+            self._file_log,
+        )
 
     def _check_run_result(self, run_number=None):  # pylint: disable=unused-argument
         """Check run results.
@@ -169,7 +171,7 @@ class SimulatorCameraEfficiency(SimtelRunner):
         """
         # Checking run
         if not self._file_simtel.exists():
-            msg = "Camera efficiency simulation results file does not exist"
+            msg = f"Camera efficiency simulation results file does not exist ({self._file_simtel})."
             self._logger.error(msg)
             raise RuntimeError(msg)
 

simtools/simtel/simulator_light_emission.py

@@ -1,7 +1,7 @@
 """Simulation using the light emission package for calibration."""
 
 import logging
-import os
+import stat
 from pathlib import Path
 
 import astropy.units as u
@@ -469,5 +469,5 @@ class SimulatorLightEmission(SimtelRunner):
                 file.write(f"{command_plot}\n\n")
                 file.write("# End\n\n")
 
-        os.system(f"chmod ug+x {_script_file}")
+        _script_file.chmod(_script_file.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP)
         return _script_file

simtools/simtel/simulator_ray_tracing.py

@@ -191,9 +191,8 @@ class SimulatorRayTracing(SimtelRunner):
             command += super().get_config_option("mirror_align_random_distance", "0.")
             command += super().get_config_option("mirror_align_random_vertical", "0.,28.,0.,0.")
         command += " " + str(self._corsika_file)
-        command += f" 2>&1 > {self._log_file} 2>&1"
 
-        return command
+        return command, self._log_file, self._log_file
 
     def _check_run_result(self, run_number=None):  # pylint: disable=unused-argument
         """

simtools/simulator.py

@@ -2,6 +2,8 @@
 
 import logging
 import re
+import shutil
+import tarfile
 from collections import defaultdict
 from pathlib import Path
 
@@ -68,7 +70,7 @@ class Simulator:
         self.runs = self._initialize_run_list()
         self._results = defaultdict(list)
         self._test = self.args_dict.get("test", False)
-        self._submit_engine = self.args_dict.get("submit_engine", "local")
+        self.submit_engine = self.args_dict.get("submit_engine", "local")
         self._submit_options = self.args_dict.get("submit_options", None)
         self._extra_commands = extra_commands
 
@@ -265,7 +267,7 @@ class Simulator:
         input_file_list: str or list of str
             Single file or list of files of shower simulations.
         """
-        self._logger.info(f"Submission command: {self._submit_engine}")
+        self._logger.info(f"Submission command: {self.submit_engine}")
 
         runs_and_files_to_submit = self._get_runs_and_files_to_submit(
             input_file_list=input_file_list
@@ -281,7 +283,7 @@ class Simulator:
             )
 
             job_manager = JobManager(
-                submit_engine=self._submit_engine,
+                submit_engine=self.submit_engine,
                 submit_options=self._submit_options,
                 test=self._test,
             )
@@ -389,50 +391,46 @@ class Simulator:
             run number
 
         """
-        self._results["output"].append(
-            str(self._simulation_runner.get_file_name(file_type="output", run_number=run_number))
+        keys = ["output", "sub_out", "log", "input", "hist", "corsika_log"]
+        defaults = {key: None for key in keys}
+        results = {key: defaults[key] for key in keys}
+        results["output"] = str(
+            self._simulation_runner.get_file_name(file_type="output", run_number=run_number)
         )
-        self._results["sub_out"].append(
-            str(
-                self._simulation_runner.get_file_name(
-                    file_type="sub_log", run_number=run_number, mode="out"
-                )
+        results["sub_out"] = str(
+            self._simulation_runner.get_file_name(
+                file_type="sub_log", mode="out", run_number=run_number
             )
         )
 
-        if self.simulation_software in ["simtel", "corsika_simtel"]:
-            self._results["log"].append(
-                str(
-                    self._simulation_runner.get_file_name(
-                        simulation_software="simtel", file_type="log", run_number=run_number
-                    )
+        if "simtel" in self.simulation_software:
+            results["log"] = str(
+                self._simulation_runner.get_file_name(
+                    file_type="log", simulation_software="simtel", run_number=run_number
                 )
             )
-            self._results["input"].append(str(file))
-            self._results["hist"].append(
-                str(
-                    self._simulation_runner.get_file_name(
-                        simulation_software="simtel", file_type="histogram", run_number=run_number
-                    )
+            results["input"] = str(file)
+            results["hist"] = str(
+                self._simulation_runner.get_file_name(
+                    file_type="histogram", simulation_software="simtel", run_number=run_number
                 )
             )
-        else:
-            self._results["corsika_autoinputs_log"].append(
-                str(
-                    self._simulation_runner.get_file_name(
-                        simulation_software="corsika", file_type="log", run_number=run_number
-                    )
+
+        if "corsika" in self.simulation_software:
+            results["corsika_log"] = str(
+                self._simulation_runner.get_file_name(
+                    file_type="corsika_log", simulation_software="corsika", run_number=run_number
                 )
             )
-            self._results["input"].append(None)
-            self._results["hist"].append(None)
-            self._results["log"].append(None)
+
+        for key in keys:
+            self._results[key].append(results[key])
 
     def get_file_list(self, file_type="output"):
         """
         Get list of files generated by simulations.
 
-        Options are "input", "output", "hist", "log".
+        Options are "input", "output", "hist", "log", "corsika_log".
         Not all file types are available for all simulation types.
         Returns an empty list for an unknown file type.
 
@@ -548,7 +546,7 @@ class Simulator:
 
     def save_file_lists(self):
         """Save files lists for output and log files."""
-        for file_type in ["output", "log", "hist"]:
+        for file_type in ["output", "log", "corsika_log", "hist"]:
             file_name = self.io_handler.get_output_directory(label=self.label).joinpath(
                 f"{file_type}_files.txt"
             )
@@ -560,3 +558,52 @@ class Simulator:
                         f.write(f"{line}\n")
             else:
                 self._logger.debug(f"No files to save for {file_type} files.")
+
+    def pack_for_register(self, directory_for_grid_upload=None):
+        """
+        Pack simulation output files for registering on the grid.
+
+        Parameters
+        ----------
+        directory_for_grid_upload: str
+            Directory for the tarball with output files.
+
+        """
+        self._logger.info(
+            f"Packing the output files for registering on the grid ({directory_for_grid_upload})"
+        )
+        output_files = self.get_file_list(file_type="output")
+        log_files = self.get_file_list(file_type="log")
+        corsika_log_files = self.get_file_list(file_type="corsika_log")
+        histogram_files = self.get_file_list(file_type="hist")
+        tar_file_name = Path(log_files[0]).name.replace("log.gz", "log_hist.tar.gz")
+        directory_for_grid_upload = (
+            Path(directory_for_grid_upload)
+            if directory_for_grid_upload
+            else self.io_handler.get_output_directory(label=self.label).joinpath(
+                "directory_for_grid_upload"
+            )
+        )
+        directory_for_grid_upload.mkdir(parents=True, exist_ok=True)
+
+        tar_file_name = directory_for_grid_upload.joinpath(tar_file_name)
+
+        with tarfile.open(tar_file_name, "w:gz") as tar:
+            files_to_tar = (
+                (log_files[:1] if log_files else [])
+                + (histogram_files[:1] if histogram_files else [])
+                + (corsika_log_files[:1] if corsika_log_files else [])
+            )
+            for file_to_tar in files_to_tar:
+                tar.add(file_to_tar, arcname=Path(file_to_tar).name)
+
+        for file_to_move in [*output_files]:
+            source_file = Path(file_to_move)
+            destination_file = directory_for_grid_upload / source_file.name
+            if destination_file.exists():
+                self._logger.warning(f"Overwriting existing file: {destination_file}")
+            # Note that this will overwrite previous files which exist in the directory
+            # It should be fine for normal production since each run is on a separate node
+            # so no files are expected there.
+            shutil.move(source_file, destination_file)
+        self._logger.info(f"Output files for the grid placed in {directory_for_grid_upload!s}")

simtools/testing/configuration.py

@@ -3,7 +3,6 @@
 import logging
 from pathlib import Path
 
-import pytest
 import yaml
 
 import simtools.utils.general as gen
@@ -11,6 +10,10 @@ import simtools.utils.general as gen
 _logger = logging.getLogger(__name__)
 
 
+class VersionError(Exception):
+    """Raise if model version requested is not supported."""
+
+
 def get_list_of_test_configurations(config_files):
     """
     Return list of test configuration dictionaries or test names.
@@ -121,7 +124,7 @@ def _skip_test_for_model_version(config, model_version_requested):
         return
     model_version_config = config["CONFIGURATION"]["MODEL_VERSION"]
     if model_version_requested != model_version_config:
-        pytest.skip(
+        raise VersionError(
             f"Model version requested {model_version_requested} not supported for this test"
         )
 
@@ -158,13 +161,14 @@ def _prepare_test_options(config, output_path, model_version=None):
 
     tmp_config_file = output_path / "tmp_config.yml"
     config_file_model_version = config.get("MODEL_VERSION")
-    if model_version is not None and "MODEL_VERSION" in config:
+    if model_version and "MODEL_VERSION" in config:
         config.update({"MODEL_VERSION": model_version})
-    if "OUTPUT_PATH" in config:
-        config.update({"OUTPUT_PATH": str(Path(output_path).joinpath(config["OUTPUT_PATH"]))})
-        config.update({"USE_PLAIN_OUTPUT_PATH": True})
-    if "DATA_DIRECTORY" in config:
-        config.update({"DATA_DIRECTORY": str(Path(output_path).joinpath(config["DATA_DIRECTORY"]))})
+
+    for key in ["OUTPUT_PATH", "DATA_DIRECTORY", "PACK_FOR_GRID_REGISTER"]:
+        if key in config:
+            config[key] = str(Path(output_path).joinpath(config[key]))
+            if key == "OUTPUT_PATH":
+                config["USE_PLAIN_OUTPUT_PATH"] = True
 
     _logger.info(f"Writing config file: {tmp_config_file}")
     with open(tmp_config_file, "w", encoding="utf-8") as file:

simtools/testing/helpers.py

@@ -1,25 +1,25 @@
 """Helper functions for integration testing."""
 
 import os
-
-import pytest
+from pathlib import Path
 
 
 def skip_camera_efficiency(config):
     """Skip camera efficiency tests if the old version of testeff is used."""
     if "camera-efficiency" in config["APPLICATION"]:
         if not _new_testeff_version():
-            pytest.skip(
+            return (
                 "Any applications calling the old version of testeff are skipped "
                 "due to a limitation of the old testeff not allowing to specify "
                 "the include directory. Please update your sim_telarray tarball."
             )
         full_test_name = f"{config['APPLICATION']}_{config['TEST_NAME']}"
         if "simtools-validate-camera-efficiency_SSTS" == full_test_name:
-            pytest.skip(
+            return (
                 "The test simtools-validate-camera-efficiency_SSTS is skipped "
                 "since the fake SST mirrors are not yet implemented (#1155)"
             )
+    return None
 
 
 def _new_testeff_version():
@@ -28,7 +28,7 @@ def _new_testeff_version():
 
     This test checks if the new version is used.
     """
-    testeff_path = os.path.join(os.getenv("SIMTOOLS_SIMTEL_PATH"), "sim_telarray/testeff.c")
+    testeff_path = Path(os.getenv("SIMTOOLS_SIMTEL_PATH")) / "sim_telarray/testeff.c"
     try:
         with open(testeff_path, encoding="utf-8") as file:
             file_content = file.read()

simtools/testing/validate_output.py

@@ -55,8 +55,13 @@ def validate_application_output(config):
         if "REFERENCE_OUTPUT_FILE" in integration_test:
             _validate_reference_output_file(config, integration_test)
 
+        if "TEST_OUTPUT_FILES" in integration_test:
+            _validate_output_path_and_file(config, integration_test["TEST_OUTPUT_FILES"])
         if "OUTPUT_FILE" in integration_test:
-            _validate_output_path_and_file(config, integration_test)
+            _validate_output_path_and_file(
+                config,
+                [{"PATH_DESCRIPTOR": "OUTPUT_PATH", "FILE": integration_test["OUTPUT_FILE"]}],
+            )
 
         if "FILE_TYPE" in integration_test:
             assert assertions.assert_file_type(
@@ -79,31 +84,25 @@ def _validate_reference_output_file(config, integration_test):
     )
 
 
-def _validate_output_path_and_file(config, integration_test):
-    """Check if output path and file exist."""
-    _logger.info(f"PATH {config['CONFIGURATION']['OUTPUT_PATH']}")
-    _logger.info(f"File {integration_test['OUTPUT_FILE']}")
-
-    data_path = config["CONFIGURATION"].get(
-        "DATA_DIRECTORY", config["CONFIGURATION"]["OUTPUT_PATH"]
-    )
-    output_file_path = Path(data_path) / integration_test["OUTPUT_FILE"]
-
-    _logger.info(f"Checking path: {output_file_path}")
-    assert output_file_path.exists()
-
-    expected_output = [
-        d["EXPECTED_OUTPUT"] for d in config["INTEGRATION_TESTS"] if "EXPECTED_OUTPUT" in d
-    ]
-    if expected_output and "log_hist" not in integration_test["OUTPUT_FILE"]:
-        # Get the expected output from the configuration file
-        expected_output = expected_output[0]
-        _logger.info(
-            f"Checking the output of {integration_test['OUTPUT_FILE']} "
-            "complies with the expected output: "
-            f"{expected_output}"
-        )
-        assert assertions.check_output_from_sim_telarray(output_file_path, expected_output)
+def _validate_output_path_and_file(config, integration_file_tests):
+    """Check if output paths and files exist."""
+    for file_test in integration_file_tests:
+        try:
+            output_path = config["CONFIGURATION"][file_test["PATH_DESCRIPTOR"]]
+        except KeyError as exc:
+            raise KeyError(
+                f"Path {file_test['PATH_DESCRIPTOR']} not found in integration test configuration."
+            ) from exc
+
+        output_file_path = Path(output_path) / file_test["FILE"]
+        _logger.info(f"Checking path: {output_file_path}")
+        assert output_file_path.exists()
+
+        if "EXPECTED_OUTPUT" in file_test:
+            assert assertions.check_output_from_sim_telarray(
+                output_file_path,
+                file_test["EXPECTED_OUTPUT"],
+            )
 
 
 def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None):
@@ -235,6 +234,7 @@ def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None):
 
         if np.issubdtype(table1_masked[col_name].dtype, np.floating):
             if not np.allclose(table1_masked[col_name], table2_masked[col_name], rtol=tolerance):
+                _logger.warning(f"Column {col_name} outside of relative tolerance {tolerance}")
                 return False
 
     return True

simtools/utils/general.py

@@ -164,6 +164,9 @@ def collect_data_from_file(file_name):
             return yaml.safe_load(file)
         except yaml.constructor.ConstructorError:
             return _load_yaml_using_astropy(file)
+        except yaml.composer.ComposerError:
+            file.seek(0)
+            return list(yaml.safe_load_all(file))
 
 
 def collect_kwargs(label, in_kwargs):
@@ -333,9 +336,10 @@ def program_is_executable(program):
     Follows https://stackoverflow.com/questions/377017/
 
     """
+    program = Path(program)
 
     def is_exe(fpath):
-        return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+        return fpath.is_file() and os.access(fpath, os.X_OK)
 
     fpath, _ = os.path.split(program)
     if fpath:
@@ -344,7 +348,7 @@ def program_is_executable(program):
     else:
         try:
             for path in os.environ["PATH"].split(os.pathsep):
-                exe_file = os.path.join(path, program)
+                exe_file = Path(path) / program
                 if is_exe(exe_file):
                     return exe_file
         except KeyError:
@@ -442,7 +446,7 @@ def get_file_age(file_path):
     if not Path(file_path).is_file():
         raise FileNotFoundError(f"'{file_path}' does not exist or is not a file.")
 
-    file_stats = os.stat(file_path)
+    file_stats = Path(file_path).stat()
     modification_time = file_stats.st_mtime
     current_time = time.time()
 
@@ -785,3 +789,46 @@ def read_file_encoded_in_utf_or_latin(file_name):
             raise UnicodeDecodeError("Unable to decode file using UTF-8 or Latin-1.") from exc
 
     return lines
+
+
+def get_structure_array_from_table(table, column_names):
+    """
+    Get a structured array from an astropy table for a selected list of columns.
+
+    Parameters
+    ----------
+    table: astropy.table.Table
+        Table to be converted.
+    column_names: list
+        List of column names to be included in the structured array.
+
+    Returns
+    -------
+    numpy.ndarray
+        Structured array containing the table data.
+    """
+    return np.array(
+        list(zip(*[np.array(table[col]) for col in column_names])),
+        dtype=[(col, np.array(table[col]).dtype) for col in column_names],
+    )
+
+
+def convert_keys_in_dict_to_lowercase(data):
+    """
+    Recursively convert all dictionary keys to lowercase.
+
+    Parameters
+    ----------
+    data: dict
+        Dictionary to be converted.
+
+    Returns
+    -------
+    dict
+        Dictionary with all keys converted to lowercase.
+    """
+    if isinstance(data, dict):
+        return {k.lower(): convert_keys_in_dict_to_lowercase(v) for k, v in data.items()}
+    if isinstance(data, list):
+        return [convert_keys_in_dict_to_lowercase(i) for i in data]
+    return data

simtools/utils/names.py

@@ -1,10 +1,10 @@
 """Validation of names."""
 
-import glob
 import logging
 import re
 from functools import cache
 from importlib.resources import files
+from pathlib import Path
 
 import yaml
 
@@ -59,7 +59,7 @@ def site_names():
 @cache
 def load_model_parameters(class_key_list):
     model_parameters = {}
-    schema_files = glob.glob(str(files("simtools") / "schemas/model_parameters") + "/*.yml")
+    schema_files = list(Path(files("simtools") / "schemas/model_parameters").rglob("*.yml"))
     for schema_file in schema_files:
         with open(schema_file, encoding="utf-8") as f:
             data = yaml.safe_load(f)

simtools/utils/value_conversion.py

@@ -84,7 +84,7 @@ def get_value_unit_type(value, unit_str=None):
             pass
         base_unit = unit_str
 
-    return base_value, base_unit, base_type
+    return base_value, _unit_as_string(base_unit), base_type
 
 
 def split_value_and_unit(value):
@@ -174,3 +174,11 @@ def get_value_as_quantity(value, unit):
             _logger.error(f"Cannot convert {value.unit} to {unit}.")
             raise
     return value * unit
+
+
+def _unit_as_string(unit):
+    """Return the string representation of a unit. Collapse if it is a list of identical items."""
+    if not isinstance(unit, list):
+        unit = [unit]
+    unit = [str(element) if element is not None else None for element in unit]
+    return unit[0] if len(set(unit)) == 1 else unit

simtools/visualization/plot_tables.py

@@ -0,0 +1,106 @@
+#!/usr/bin/python3
+"""Plot tabular data."""
+
+import numpy as np
+
+import simtools.utils.general as gen
+from simtools.io_operations import legacy_data_handler
+from simtools.model.site_model import SiteModel
+from simtools.model.telescope_model import TelescopeModel
+from simtools.visualization import visualize
+
+
+def plot(config, output_file, db_config=None):
+    """
+    Plot tabular data from data or from model parameter files.
+
+    Parameters
+    ----------
+    config: dict
+        Configuration dictionary for plotting.
+    output_file: str
+        Output file.
+    """
+    data = read_table_data(config, db_config)
+
+    fig = visualize.plot_1d(
+        data,
+        **config,
+    )
+    visualize.save_figure(fig, output_file)
+
+
+def read_table_data(config, db_config):
+    """
+    Read table data from file or parameter database.
+
+    Parameters
+    ----------
+    config: dict
+        Configuration dictionary for plotting.
+
+    Returns
+    -------
+    Dict
+        Dict with table data (astropy tables).
+    """
+    data = {}
+
+    for _config in config["tables"]:
+        if "parameter" in _config:
+            table = _read_table_from_model_database(_config, db_config)
+        elif "file_name" in _config:
+            table = legacy_data_handler.read_legacy_data_as_table(
+                _config["file_name"], _config["type"]
+            )
+        else:
+            raise ValueError("No table data defined in configuration.")
+        if _config.get("normalize_y"):
+            table[_config["column_y"]] = (
+                table[_config["column_y"]] / table[_config["column_y"]].max()
+            )
+        if _config.get("select_values"):
+            table = _select_values_from_table(
+                table,
+                _config["select_values"]["column_name"],
+                _config["select_values"]["value"],
+            )
+        data[_config["label"]] = gen.get_structure_array_from_table(
+            table, [_config["column_x"], _config["column_y"]]
+        )
+    return data
+
+
+def _read_table_from_model_database(table_config, db_config):
+    """
+    Read table data from model parameter database.
+
+    Parameters
+    ----------
+    table_config: dict
+        Configuration dictionary for table data.
+
+    Returns
+    -------
+    Table
+        Astropy table.
+    """
+    if "telescope" in table_config:
+        model = TelescopeModel(
+            site=table_config["site"],
+            telescope_name=table_config["telescope"],
+            model_version=table_config["model_version"],
+            mongo_db_config=db_config,
+        )
+    else:
+        model = SiteModel(
+            site=table_config["site"],
+            model_version=table_config["model_version"],
+            mongo_db_config=db_config,
+        )
+    return model.get_model_file_as_table(table_config["parameter"])
+
+
+def _select_values_from_table(table, column_name, value):
+    """Return a table with only the rows where column_name == value."""
+    return table[np.isclose(table[column_name], value)]