gammasimtools 0.20.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

simtools/applications/validate_cumulative_psf.py

@@ -132,11 +132,10 @@ def main():  # noqa: D103
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    # Output directory to save files related directly to this app
     _io_handler = io_handler.IOHandler()
-    output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
+    output_dir = _io_handler.get_output_directory()
 
-    tel_model, site_model = initialize_simulation_models(
+    tel_model, site_model, _ = initialize_simulation_models(
         label=label,
         db_config=db_config,
         site=args_dict["site"],

simtools/applications/validate_file_using_schema.py

@@ -61,12 +61,15 @@ def _parse(label, description):
 
     """
     config = configurator.Configurator(label=label, description=description)
-    group = config.parser.add_mutually_exclusive_group(required=True)
-    group.add_argument("--file_name", help="File to be validated")
-    group.add_argument(
+    config.parser.add_argument(
+        "--file_name",
+        help="File to be validated (full path or name pattern, e.g., '*.json')",
+        default="*.json",
+    )
+    config.parser.add_argument(
         "--file_directory",
         help=(
-            "Directory with json files to be validated. "
+            "Directory with files to be validated. "
             "If no schema file is provided, the assumption is that model "
             "parameters are validated and the schema files are taken from "
             f"{MODEL_PARAMETER_SCHEMA_PATH}."
@@ -84,6 +87,11 @@ def _parse(label, description):
         help="Require exact data type for validation",
         action="store_true",
     )
+    config.parser.add_argument(
+        "--ignore_software_version",
+        help="Ignore software version check.",
+        action="store_true",
+    )
     return config.initialize(paths=False)
 
 
@@ -115,11 +123,11 @@ def _get_schema_file_name(args_dict, data_dict=None):
     return schema_file
 
 
-def _get_json_file_list(file_directory=None, file_name=None):
-    """Return list of json files in a directory."""
+def _get_file_list(file_directory=None, file_name="*.json"):
+    """Return list of files in a directory."""
     file_list = []
     if file_directory is not None:
-        file_list = list(Path(file_directory).rglob("*.json"))
+        file_list = list(Path(file_directory).rglob(file_name))
         if not file_list:
             raise FileNotFoundError(f"No files found in {file_directory}")
     elif file_name is not None:
@@ -137,9 +145,7 @@ def validate_dict_using_schema(args_dict, logger):
     the metadata section of the data dictionary.
 
     """
-    for file_name in _get_json_file_list(
-        args_dict.get("file_directory"), args_dict.get("file_name")
-    ):
+    for file_name in _get_file_list(args_dict.get("file_directory"), args_dict.get("file_name")):
         try:
             data = ascii_handler.collect_data_from_file(file_name=file_name)
         except FileNotFoundError as exc:
@@ -148,7 +154,9 @@ def validate_dict_using_schema(args_dict, logger):
         try:
             for data_dict in data:
                 schema.validate_dict_using_schema(
-                    data_dict, _get_schema_file_name(args_dict, data_dict)
+                    data_dict,
+                    _get_schema_file_name(args_dict, data_dict),
+                    ignore_software_version=args_dict.get("ignore_software_version", False),
                 )
         except Exception as exc:
             raise ValueError(f"Validation of file {file_name} failed") from exc
@@ -159,7 +167,7 @@ def validate_data_files(args_dict, logger):
     """Validate data files."""
     if args_dict.get("file_directory") is not None:
         tmp_args_dict = {}
-        for file_name in _get_json_file_list(args_dict.get("file_directory")):
+        for file_name in _get_file_list(args_dict.get("file_directory")):
             tmp_args_dict["file_name"] = file_name
             parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem)
             schema_file = schema.get_model_parameter_schema_file(f"{parameter_name}")

simtools/applications/validate_optics.py

@@ -126,9 +126,9 @@ def main():  # noqa: D103
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     _io_handler = io_handler.IOHandler()
-    output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
+    output_dir = _io_handler.get_output_directory()
 
-    tel_model, site_model = initialize_simulation_models(
+    tel_model, site_model, _ = initialize_simulation_models(
         label=label,
         db_config=db_config,
         site=args_dict["site"],

simtools/camera/camera_efficiency.py

@@ -40,14 +40,14 @@ class CameraEfficiency:
         self.label = label
 
         self.io_handler = io_handler.IOHandler()
-        self.telescope_model, self.site_model = initialize_simulation_models(
-            self.label,
-            db_config,
-            config_data["site"],
-            config_data["telescope"],
-            config_data["model_version"],
+        self.telescope_model, self.site_model, _ = initialize_simulation_models(
+            label=self.label,
+            db_config=db_config,
+            model_version=config_data["model_version"],
+            site=config_data["site"],
+            telescope_name=config_data["telescope"],
         )
-        self.output_dir = self.io_handler.get_output_directory(self.label, sub_dir="plots")
+        self.output_dir = self.io_handler.get_output_directory()
 
         self._results = None
         self._has_results = False
@@ -101,10 +101,7 @@ class CameraEfficiency:
                 label=self.label,
             )
 
-            _file[label] = self.io_handler.get_output_directory(
-                label=self.label,
-                sub_dir="camera_efficiency",
-            ).joinpath(file_name)
+            _file[label] = self.io_handler.get_output_directory().joinpath(file_name)
         return _file
 
     def simulate(self):

simtools/configuration/commandline_parser.py

@@ -21,7 +21,7 @@ class CommandLineParser(argparse.ArgumentParser):
 
     Wrapper around standard python argparse.ArgumentParser.
 
-    Command line arguments should be given in snake_case, e.g.  input_meta.
+    Command line arguments should be given in snake_case, e.g. 'input_meta'.
 
     Parameters
     ----------
@@ -102,12 +102,6 @@ class CommandLineParser(argparse.ArgumentParser):
             default="./simtools-output/",
             required=False,
         )
-        _job_group.add_argument(
-            "--use_plain_output_path",
-            help="use plain output path (without the tool name and dates)",
-            action="store_true",
-            required=False,
-        )
         _job_group.add_argument(
             "--model_path",
             help="path pointing towards simulation model file directory",

simtools/configuration/configurator.py

@@ -340,8 +340,6 @@ class Configurator:
         _io_handler = io_handler.IOHandler()
         _io_handler.set_paths(
             output_path=self.config.get("output_path", None),
-            use_plain_output_path=self.config.get("use_plain_output_path", False),
-            data_path=self.config.get("data_path", None),
             model_path=self.config.get("model_path", None),
         )
 

simtools/corsika/corsika_config.py

@@ -114,8 +114,8 @@ class CorsikaConfig:
             return {}
 
         self.is_file_updated = False
-        self.azimuth_angle = int(args_dict["azimuth_angle"].to("deg").value)
-        self.zenith_angle = args_dict["zenith_angle"].to("deg").value
+        self.azimuth_angle = int(args_dict.get("azimuth_angle", 0.0 * u.deg).to("deg").value)
+        self.zenith_angle = int(args_dict.get("zenith_angle", 0.0 * u.deg).to("deg").value)
 
         self._logger.debug(
             f"Setting CORSIKA parameters from database ({args_dict['model_version']})"
@@ -563,8 +563,9 @@ class CorsikaConfig:
         if self.is_file_updated:
             self._logger.debug(f"CORSIKA input file already updated: {self.config_file_path}")
             return self.config_file_path
-        _output_generic_file_name = self.set_output_file_and_directory(use_multipipe=use_multipipe)
         self._logger.info(f"Exporting CORSIKA input file to {self.config_file_path}")
+        _output_generic_file_name = self.set_output_file_and_directory(use_multipipe=use_multipipe)
+        self._logger.info(f"Output generic file name: {_output_generic_file_name}")
 
         with open(self.config_file_path, "w", encoding="utf-8") as file:
             file.write("\n* [ RUN PARAMETERS ]\n")
@@ -712,13 +713,10 @@ class CorsikaConfig:
         str
             Output file name.
         """
-        sub_dir = "corsika_sim_telarray" if use_multipipe else "corsika"
-        config_file_name = self.get_corsika_config_file_name(file_type="config")
-        file_directory = self.io_handler.get_output_directory(label=self.label, sub_dir=sub_dir)
-        self._logger.debug(f"Creating directory {file_directory}")
-        file_directory.mkdir(parents=True, exist_ok=True)
-        self.config_file_path = file_directory.joinpath(config_file_name)
-
+        self.config_file_path = self.io_handler.get_output_file(
+            file_name=self.get_corsika_config_file_name(file_type="config"),
+            sub_dir="corsika_sim_telarray" if use_multipipe else "corsika",
+        )
         return self.get_corsika_config_file_name(file_type="output_generic")
 
     def _write_seeds(self, file, use_test_seeds=False):
@@ -748,7 +746,7 @@ class CorsikaConfig:
             Piece of text to be added to the CORSIKA input file.
         """
         corsika_input_list = ""
-        for telescope_name, telescope in self.array_model.telescope_model.items():
+        for telescope_name, telescope in self.array_model.telescope_models.items():
             positions = telescope.get_parameter_value_with_unit("array_element_position_ground")
             corsika_input_list += "TELESCOPE"
             for pos in positions:

simtools/corsika/corsika_histograms.py

@@ -22,6 +22,7 @@ from simtools.io.ascii_handler import collect_data_from_file
 from simtools.io.hdf5_handler import fill_hdf5_table
 from simtools.utils.geometry import convert_2d_to_radial_distr, rotate
 from simtools.utils.names import sanitize_name
+from simtools.visualization import plot_corsika_histograms as visualize
 
 X_AXIS_STRING = "x axis"
 Y_AXIS_STRING = "y axis"
@@ -66,7 +67,7 @@ class CorsikaHistograms:
             raise FileNotFoundError
 
         self.io_handler = io_handler.IOHandler()
-        _default_output_path = self.io_handler.get_output_directory(self.label, "corsika")
+        _default_output_path = self.io_handler.get_output_directory("corsika")
 
         if output_path is None:
             self.output_path = _default_output_path
@@ -111,6 +112,86 @@ class CorsikaHistograms:
         self.read_event_information()
         self._initialize_header()
 
+    def parse_telescope_indices(self, indices_arg):
+        """Return telescope indices as ndarray[int] or None.
+
+        Accepts None, a sequence of strings/ints. Raises ValueError on invalid input.
+        """
+        if indices_arg is None:
+            return None
+        try:
+            return np.array(indices_arg).astype(int)
+        except ValueError as exc:
+            msg = (
+                f"{indices_arg} not a valid input. Please use integer numbers for telescope_indices"
+            )
+            self._logger.error(msg)
+            raise ValueError(msg) from exc
+
+    def should_overwrite(
+        self, write_hdf5: bool, event1d: list | None, event2d: list | None
+    ) -> bool:
+        """Return True if output HDF5 exists and any writing flag is requested."""
+        exists = Path(self.hdf5_file_name).exists()
+        if exists and (write_hdf5 or bool(event1d) or bool(event2d)):
+            self._logger.warning(
+                f"Output hdf5 file {self.hdf5_file_name} already exists. Overwriting it."
+            )
+            return True
+        return False
+
+    def run_export_pipeline(
+        self,
+        *,
+        individual_telescopes: bool,
+        hist_config,
+        indices_arg,
+        write_pdf: bool,
+        write_hdf5: bool,
+        event1d: list | None,
+        event2d: list | None,
+        test: bool = False,
+    ) -> dict:
+        """Run the full histogram export pipeline and return output artifact paths.
+
+        Returns a dict with optional keys: pdf_photons, pdf_event1d, pdf_event2d.
+        """
+        outputs: dict[str, Path | None] = {
+            "pdf_photons": None,
+            "pdf_event_1d": None,
+            "pdf_event_2d": None,
+        }
+
+        indices = self.parse_telescope_indices(indices_arg)
+        overwrite = self.should_overwrite(write_hdf5, event1d, event2d)
+
+        self.set_histograms(
+            telescope_indices=indices,
+            individual_telescopes=individual_telescopes,
+            hist_config=hist_config,
+        )
+
+        if write_pdf:
+            pdf_path = visualize.export_all_photon_figures_pdf(self, test=test)
+            outputs["pdf_photons"] = pdf_path
+        if write_hdf5:
+            self.export_histograms(overwrite=overwrite)
+
+        if event1d is not None:
+            outputs["pdf_event_1d"] = visualize.derive_event_1d_histograms(
+                self, event1d, pdf=write_pdf, hdf5=write_hdf5, overwrite=not write_hdf5
+            )
+        if event2d is not None:
+            outputs["pdf_event_2d"] = visualize.derive_event_2d_histograms(
+                self,
+                event2d,
+                pdf=write_pdf,
+                hdf5=write_hdf5,
+                overwrite=not (write_hdf5 or bool(event1d)),
+            )
+
+        return outputs
+
     @property
     def hdf5_file_name(self):
         """

simtools/data_model/model_data_writer.py

@@ -3,6 +3,7 @@
 import logging
 from pathlib import Path
 
+import packaging.version
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
@@ -29,8 +30,6 @@ class ModelDataWriter:
         Dictionary with configuration parameters.
     output_path: str or Path
         Path to output file.
-    use_plain_output_path: bool
-        Use plain output path.
     args_dict: dict
         Dictionary with configuration parameters.
 
@@ -41,7 +40,6 @@ class ModelDataWriter:
         product_data_file=None,
         product_data_format=None,
         output_path=None,
-        use_plain_output_path=True,
         args_dict=None,
     ):
         """Initialize model data writer."""
@@ -50,11 +48,8 @@ class ModelDataWriter:
         self.schema_dict = {}
         if args_dict is not None:
             output_path = args_dict.get("output_path", output_path)
-            use_plain_output_path = args_dict.get("use_plain_output_path", use_plain_output_path)
         if output_path is not None:
-            self.io_handler.set_paths(
-                output_path=output_path, use_plain_output_path=use_plain_output_path
-            )
+            self.io_handler.set_paths(output_path=output_path)
         try:
             self.product_data_file = self.io_handler.get_output_file(file_name=product_data_file)
         except TypeError:
@@ -104,9 +99,10 @@ class ModelDataWriter:
         parameter_version,
         output_file,
         output_path=None,
-        use_plain_output_path=False,
         metadata_input_dict=None,
         db_config=None,
+        unit=None,
+        meta_parameter=False,
     ):
         """
         Generate DB-style model parameter dict and write it to json file.
@@ -125,12 +121,12 @@ class ModelDataWriter:
             Name of output file.
         output_path: str or Path
             Path to output file.
-        use_plain_output_path: bool
-            Use plain output path.
         metadata_input_dict: dict
             Input to metadata collector.
         db_config: dict
             Database configuration. If not None, check if parameter with the same version exists.
+        unit: str
+            Unit of the parameter value (if applicable and value is not of type astropy Quantity).
 
         Returns
         -------
@@ -142,7 +138,6 @@ class ModelDataWriter:
             product_data_format="json",
             args_dict=None,
             output_path=output_path,
-            use_plain_output_path=use_plain_output_path,
         )
         if db_config is not None:
             writer.check_db_for_existing_parameter(
@@ -160,7 +155,13 @@ class ModelDataWriter:
             )
 
         _json_dict = writer.get_validated_parameter_dict(
-            parameter_name, value, instrument, parameter_version, unique_id
+            parameter_name,
+            value,
+            instrument,
+            parameter_version,
+            unique_id,
+            unit=unit,
+            meta_parameter=meta_parameter,
         )
         writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         return _json_dict
@@ -210,6 +211,8 @@ class ModelDataWriter:
         parameter_version,
         unique_id=None,
         schema_version=None,
+        unit=None,
+        meta_parameter=False,
     ):
         """
         Get validated parameter dictionary.
@@ -226,6 +229,12 @@ class ModelDataWriter:
             Version of the parameter.
         schema_version: str
             Version of the schema.
+        unique_id: str
+            Unique ID of the parameter set (from metadata).
+        unit: str
+            Unit of the parameter value (if applicable and value is not an astropy Quantity).
+        meta_parameter: bool
+            Setting for meta parameter flag.
 
         Returns
         -------
@@ -233,10 +242,10 @@ class ModelDataWriter:
             Validated parameter dictionary.
         """
         self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
-        schema_file = schema.get_model_parameter_schema_file(parameter_name)
-        self.schema_dict = ascii_handler.collect_data_from_file(schema_file)
+        self.schema_dict, schema_file = self._read_schema_dict(parameter_name, schema_version)
 
-        value, unit = value_conversion.split_value_and_unit(value)
+        if unit is None:
+            value, unit = value_conversion.split_value_and_unit(value)
 
         data_dict = {
             "schema_version": schema.get_model_parameter_schema_version(schema_version),
@@ -249,10 +258,8 @@ class ModelDataWriter:
             "unit": unit,
             "type": self._get_parameter_type(),
             "file": self._parameter_is_a_file(),
-            "meta_parameter": False,
-            "model_parameter_schema_version": self.schema_dict.get(
-                "model_parameter_schema_version", "0.1.0"
-            ),
+            "meta_parameter": meta_parameter,
+            "model_parameter_schema_version": self.schema_dict.get("schema_version", "0.1.0"),
         }
         return self.validate_and_transform(
             product_data_dict=data_dict,
@@ -260,19 +267,74 @@ class ModelDataWriter:
             is_model_parameter=True,
         )
 
+    def _read_schema_dict(self, parameter_name, schema_version):
+        """
+        Read schema dict for given parameter name and version.
+
+        Use newest schema version if schema_version is None.
+
+        Parameters
+        ----------
+        parameter_name: str
+            Name of the parameter.
+        schema_version: str
+            Schema version.
+
+        Returns
+        -------
+        dict
+            Schema dictionary.
+        """
+        schema_file = schema.get_model_parameter_schema_file(parameter_name)
+        schemas = ascii_handler.collect_data_from_file(schema_file)
+        if isinstance(schemas, list):
+            if schema_version is None:
+                return self._find_highest_schema_version(schemas), schema_file
+            for entry in schemas:
+                if entry.get("schema_version") == schema_version:
+                    return entry, schema_file
+        else:
+            return schemas, schema_file
+
+        raise ValueError(f"Schema version {schema_version} not found for {parameter_name}")
+
+    def _find_highest_schema_version(self, schema_list):
+        """
+        Find entry with highest schema_version in a list of schema dicts.
+
+        Parameters
+        ----------
+        schema_list: list
+            List of schema dictionaries.
+
+        Returns
+        -------
+        dict
+            Schema dictionary with highest schema_version.
+        """
+        try:
+            valid_entries = [entry for entry in schema_list if "schema_version" in entry]
+        except TypeError as exc:
+            raise TypeError("No valid schema versions found in the list.") from exc
+        return max(valid_entries, key=lambda e: packaging.version.Version(e["schema_version"]))
+
     def _get_parameter_type(self):
         """
         Return parameter type from schema.
 
+        Reduce list of types to single type if all types are the same.
+
         Returns
         -------
-        str
+        str or list[str]
             Parameter type
         """
-        _parameter_type = []
-        for data in self.schema_dict["data"]:
-            _parameter_type.append(data["type"])
-        return _parameter_type if len(_parameter_type) > 1 else _parameter_type[0]
+        _parameter_type = [data["type"] for data in self.schema_dict["data"]]
+        return (
+            _parameter_type[0]
+            if all(t == _parameter_type[0] for t in _parameter_type)
+            else _parameter_type
+        )
 
     def _parameter_is_a_file(self):
         """
@@ -399,7 +461,7 @@ class ModelDataWriter:
         ascii_handler.write_data_to_file(
             data=data_dict,
             output_file=self.io_handler.get_output_file(file_name),
-            sort_keys=False,
+            sort_keys=True,
             numpy_types=True,
         )
 

simtools/data_model/schema.py

@@ -4,9 +4,12 @@ import logging
 from pathlib import Path
 
 import jsonschema
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
 from referencing import Registry, Resource
 
 import simtools.utils.general as gen
+from simtools import version
 from simtools.constants import (
     METADATA_JSON_SCHEMA,
     MODEL_PARAMETER_METASCHEMA,
@@ -66,7 +69,7 @@ def get_model_parameter_schema_file(parameter):
 
 def get_model_parameter_schema_version(schema_version=None):
     """
-    Validate  and return schema versions.
+    Validate and return schema versions.
 
     If no schema_version is given, the most recent version is provided.
 
@@ -92,7 +95,9 @@ def get_model_parameter_schema_version(schema_version=None):
     raise ValueError(f"Schema version {schema_version} not found in {MODEL_PARAMETER_METASCHEMA}.")
 
 
-def validate_dict_using_schema(data, schema_file=None, json_schema=None):
+def validate_dict_using_schema(
+    data, schema_file=None, json_schema=None, ignore_software_version=False
+):
     """
     Validate a data dictionary against a schema.
 
@@ -102,6 +107,10 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
         dictionary to be validated
     schema_file (dict)
         schema used for validation
+    json_schema (dict)
+        schema used for validation
+    ignore_software_version: bool
+        If True, ignore software version check.
 
     Raises
     ------
@@ -115,6 +124,8 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     if json_schema is None:
         json_schema = load_schema(schema_file, get_schema_version_from_data(data))
 
+    _validate_deprecation_and_version(data, ignore_software_version)
+
     validator = jsonschema.Draft6Validator(
         schema=json_schema,
         format_checker=format_checkers.format_checker,
@@ -294,3 +305,51 @@ def _add_array_elements(key, schema):
 
     recursive_search(schema, key)
     return schema
+
+
+def _validate_deprecation_and_version(
+    data, software_name="simtools", ignore_software_version=False
+):
+    """
+    Check if data contains deprecated parameters or version mismatches.
+
+    Parameters
+    ----------
+    data: dict
+        Data dictionary to check.
+    software_name: str
+        Name of the software to check version against.
+    ignore_software_version: bool
+        If True, ignore software version check.
+    """
+    if not isinstance(data, dict):
+        return
+
+    if data.get("deprecated", False):
+        note = data.get("deprecation_note", "(no deprecation note provided)")
+        _logger.warning(f"Data is deprecated. Note: {note}")
+
+    def check_version(sw):
+        constraint = sw.get("version")
+        if constraint is None:
+            return
+        constraint = constraint.strip()
+        spec = SpecifierSet(constraint, prereleases=True)
+        if Version(version.__version__) in spec:
+            _logger.debug(
+                f"Version {version.__version__} of {software_name} matches constraint {constraint}."
+            )
+        else:
+            msg = (
+                f"Version {version.__version__} of {software_name} "
+                f"does not match constraint {constraint}."
+            )
+            if ignore_software_version:
+                _logger.warning(f"{msg}, but version check is ignored.")
+                return
+            raise ValueError(msg)
+
+    for sw in data.get("simulation_software", []):
+        if sw.get("name") == software_name:
+            check_version(sw)
+            break

simtools/data_model/validate_data.py

@@ -232,7 +232,7 @@ class DataValidator:
         else:
             self._check_data_type(np.array(value).dtype, index, value)
 
-        if self.data_dict.get("type") not in ("string", "dict", "file"):
+        if self._get_data_description(index).get("type", None) not in ("string", "dict", "file"):
             self._check_for_not_a_number(value, index)
             value, unit = self._check_and_convert_units(value, unit, index)
             for range_type in ("allowed_range", "required_range"):