gammasimtools 0.21.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

simtools/applications/validate_optics.py

@@ -65,7 +65,6 @@ r"""
 
 """
 
-import logging
 from pathlib import Path
 
 import astropy.units as u
@@ -73,18 +72,17 @@ import matplotlib.pyplot as plt
 import numpy as np
 from matplotlib.backends.backend_pdf import PdfPages
 
-import simtools.utils.general as gen
+from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
-from simtools.io import io_handler
 from simtools.model.model_utils import initialize_simulation_models
 from simtools.ray_tracing.ray_tracing import RayTracing
 from simtools.visualization import visualize
 
 
-def _parse(label):
+def _parse():
     """Parse command line configuration."""
     config = configurator.Configurator(
-        label=label,
+        label=get_application_label(__file__),
         description=(
             "Calculate and plot the PSF and effective mirror area as a function of off-axis angle "
             "of the telescope requested."
@@ -118,22 +116,16 @@ def _parse(label):
     return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
-def main():  # noqa: D103
-    label = Path(__file__).stem
-    args_dict, db_config = _parse(label)
+def main():
+    """Validate the optical model parameters through ray tracing simulations."""
+    app_context = startup_application(_parse, setup_io_handler=True)
 
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
-
-    _io_handler = io_handler.IOHandler()
-    output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
-
-    tel_model, site_model = initialize_simulation_models(
-        label=label,
-        db_config=db_config,
-        site=args_dict["site"],
-        telescope_name=args_dict["telescope"],
-        model_version=args_dict["model_version"],
+    tel_model, site_model, _ = initialize_simulation_models(
+        label=Path(__file__).stem,
+        db_config=app_context.db_config,
+        site=app_context.args["site"],
+        telescope_name=app_context.args["telescope"],
+        model_version=app_context.args["model_version"],
     )
 
     ######################################################################
@@ -147,22 +139,24 @@ def main():  # noqa: D103
     # }
     # tel_model.change_multiple_parameters(**pars_to_change)
 
-    print(f"\nValidating telescope optics with ray tracing simulations for {tel_model.name}\n")
+    app_context.logger.info(
+        f"\nValidating telescope optics with ray tracing simulations for {tel_model.name}\n"
+    )
 
     ray = RayTracing(
         telescope_model=tel_model,
         site_model=site_model,
-        simtel_path=args_dict["simtel_path"],
-        zenith_angle=args_dict["zenith"] * u.deg,
-        source_distance=args_dict["src_distance"] * u.km,
+        simtel_path=app_context.args["simtel_path"],
+        zenith_angle=app_context.args["zenith"] * u.deg,
+        source_distance=app_context.args["src_distance"] * u.km,
         off_axis_angle=np.linspace(
             0,
-            args_dict["max_offset"],
-            int(args_dict["max_offset"] / args_dict["offset_steps"]) + 1,
+            app_context.args["max_offset"],
+            int(app_context.args["max_offset"] / app_context.args["offset_steps"]) + 1,
         )
         * u.deg,
     )
-    ray.simulate(test=args_dict["test"], force=False)
+    ray.simulate(test=app_context.args["test"], force=False)
     ray.analyze(force=True)
 
     # Plotting
@@ -171,17 +165,17 @@ def main():  # noqa: D103
 
         ray.plot(key, marker="o", linestyle=":", color="k")
 
-        plot_file_name = "_".join((label, tel_model.name, key))
-        plot_file = output_dir.joinpath(plot_file_name)
+        plot_file_name = "_".join((app_context.args.get("label"), tel_model.name, key))
+        plot_file = app_context.io_handler.get_output_file(plot_file_name)
         visualize.save_figure(plt, plot_file)
 
     # Plotting images
-    if args_dict["plot_images"]:
-        plot_file_name = "_".join((label, tel_model.name, "images.pdf"))
-        plot_file = output_dir.joinpath(plot_file_name)
+    if app_context.args["plot_images"]:
+        plot_file_name = "_".join((app_context.args.get("label"), tel_model.name, "images.pdf"))
+        plot_file = app_context.io_handler.get_output_file(plot_file_name)
         pdf_pages = PdfPages(plot_file)
 
-        logger.info(f"Plotting images into {plot_file}")
+        app_context.logger.info(f"Plotting images into {plot_file}")
 
         for image in ray.images():
             fig = plt.figure(figsize=(8, 6), tight_layout=True)

simtools/camera/camera_efficiency.py

@@ -15,8 +15,6 @@ from simtools.simtel.simulator_camera_efficiency import SimulatorCameraEfficienc
 from simtools.utils import names
 from simtools.visualization import visualize
 
-__all__ = ["CameraEfficiency"]
-
 
 class CameraEfficiency:
     """
@@ -40,14 +38,14 @@ class CameraEfficiency:
         self.label = label
 
         self.io_handler = io_handler.IOHandler()
-        self.telescope_model, self.site_model = initialize_simulation_models(
-            self.label,
-            db_config,
-            config_data["site"],
-            config_data["telescope"],
-            config_data["model_version"],
+        self.telescope_model, self.site_model, _ = initialize_simulation_models(
+            label=self.label,
+            db_config=db_config,
+            model_version=config_data["model_version"],
+            site=config_data["site"],
+            telescope_name=config_data["telescope"],
         )
-        self.output_dir = self.io_handler.get_output_directory(self.label, sub_dir="plots")
+        self.output_dir = self.io_handler.get_output_directory()
 
         self._results = None
         self._has_results = False
@@ -101,10 +99,7 @@ class CameraEfficiency:
                 label=self.label,
             )
 
-            _file[label] = self.io_handler.get_output_directory(
-                label=self.label,
-                sub_dir="camera_efficiency",
-            ).joinpath(file_name)
+            _file[label] = self.io_handler.get_output_directory().joinpath(file_name)
         return _file
 
     def simulate(self):

simtools/configuration/commandline_parser.py

@@ -10,10 +10,6 @@ import astropy.units as u
 import simtools.version
 from simtools.utils import names
 
-__all__ = [
-    "CommandLineParser",
-]
-
 
 class CommandLineParser(argparse.ArgumentParser):
     """
@@ -21,7 +17,7 @@ class CommandLineParser(argparse.ArgumentParser):
 
     Wrapper around standard python argparse.ArgumentParser.
 
-    Command line arguments should be given in snake_case, e.g.  input_meta.
+    Command line arguments should be given in snake_case, e.g. 'input_meta'.
 
     Parameters
     ----------
@@ -102,12 +98,6 @@ class CommandLineParser(argparse.ArgumentParser):
             default="./simtools-output/",
             required=False,
         )
-        _job_group.add_argument(
-            "--use_plain_output_path",
-            help="use plain output path (without the tool name and dates)",
-            action="store_true",
-            required=False,
-        )
         _job_group.add_argument(
             "--model_path",
             help="path pointing towards simulation model file directory",
@@ -567,6 +557,38 @@ class CommandLineParser(argparse.ArgumentParser):
         )
         return job_group
 
+    @staticmethod
+    def scientific_int(value):
+        """
+        Convert string (including scientific notation) to integer.
+
+        Parameters
+        ----------
+        value: str or int or float
+            Value to convert to integer. Can be a regular integer,
+            float, or string in scientific notation (e.g., '1e7').
+
+        Returns
+        -------
+        int
+            Converted integer value
+
+        Raises
+        ------
+        argparse.ArgumentTypeError
+            If the value cannot be converted to an integer
+        """
+        try:
+            f = float(value)
+            if not f.is_integer():
+                raise ValueError
+            return int(f)
+        except (ValueError, TypeError) as exc:
+            raise argparse.ArgumentTypeError(
+                f"Invalid integer value: '{value}'. "
+                "Expected an integer or scientific notation like '1e7'."
+            ) from exc
+
     @staticmethod
     def site(value):
         """

simtools/configuration/configurator.py

@@ -14,11 +14,6 @@ from simtools.db.db_handler import jsonschema_db_dict
 from simtools.io import ascii_handler, io_handler
 from simtools.utils import general as gen
 
-__all__ = [
-    "Configurator",
-    "InvalidConfigurationParameterError",
-]
-
 
 class InvalidConfigurationParameterError(Exception):
     """Exception for Invalid configuration parameter."""
@@ -340,8 +335,6 @@ class Configurator:
         _io_handler = io_handler.IOHandler()
         _io_handler.set_paths(
             output_path=self.config.get("output_path", None),
-            use_plain_output_path=self.config.get("use_plain_output_path", False),
-            data_path=self.config.get("data_path", None),
             model_path=self.config.get("model_path", None),
         )
 

simtools/corsika/corsika_config.py

@@ -10,11 +10,6 @@ from simtools.corsika.primary_particle import PrimaryParticle
 from simtools.io import io_handler
 from simtools.model.model_parameter import ModelParameter
 
-__all__ = [
-    "CorsikaConfig",
-    "InvalidCorsikaInputError",
-]
-
 
 class InvalidCorsikaInputError(Exception):
     """Exception for invalid corsika input."""
@@ -114,8 +109,8 @@ class CorsikaConfig:
             return {}
 
         self.is_file_updated = False
-        self.azimuth_angle = int(args_dict["azimuth_angle"].to("deg").value)
-        self.zenith_angle = args_dict["zenith_angle"].to("deg").value
+        self.azimuth_angle = int(args_dict.get("azimuth_angle", 0.0 * u.deg).to("deg").value)
+        self.zenith_angle = int(args_dict.get("zenith_angle", 0.0 * u.deg).to("deg").value)
 
         self._logger.debug(
             f"Setting CORSIKA parameters from database ({args_dict['model_version']})"
@@ -563,8 +558,9 @@ class CorsikaConfig:
         if self.is_file_updated:
             self._logger.debug(f"CORSIKA input file already updated: {self.config_file_path}")
             return self.config_file_path
-        _output_generic_file_name = self.set_output_file_and_directory(use_multipipe=use_multipipe)
         self._logger.info(f"Exporting CORSIKA input file to {self.config_file_path}")
+        _output_generic_file_name = self.set_output_file_and_directory(use_multipipe=use_multipipe)
+        self._logger.info(f"Output generic file name: {_output_generic_file_name}")
 
         with open(self.config_file_path, "w", encoding="utf-8") as file:
             file.write("\n* [ RUN PARAMETERS ]\n")
@@ -712,13 +708,10 @@ class CorsikaConfig:
         str
             Output file name.
         """
-        sub_dir = "corsika_sim_telarray" if use_multipipe else "corsika"
-        config_file_name = self.get_corsika_config_file_name(file_type="config")
-        file_directory = self.io_handler.get_output_directory(label=self.label, sub_dir=sub_dir)
-        self._logger.debug(f"Creating directory {file_directory}")
-        file_directory.mkdir(parents=True, exist_ok=True)
-        self.config_file_path = file_directory.joinpath(config_file_name)
-
+        self.config_file_path = self.io_handler.get_output_file(
+            file_name=self.get_corsika_config_file_name(file_type="config"),
+            sub_dir="corsika_sim_telarray" if use_multipipe else "corsika",
+        )
         return self.get_corsika_config_file_name(file_type="output_generic")
 
     def _write_seeds(self, file, use_test_seeds=False):
@@ -748,7 +741,7 @@ class CorsikaConfig:
             Piece of text to be added to the CORSIKA input file.
         """
         corsika_input_list = ""
-        for telescope_name, telescope in self.array_model.telescope_model.items():
+        for telescope_name, telescope in self.array_model.telescope_models.items():
             positions = telescope.get_parameter_value_with_unit("array_element_position_ground")
             corsika_input_list += "TELESCOPE"
             for pos in positions:

simtools/corsika/corsika_histograms.py

@@ -67,7 +67,7 @@ class CorsikaHistograms:
             raise FileNotFoundError
 
         self.io_handler = io_handler.IOHandler()
-        _default_output_path = self.io_handler.get_output_directory(self.label, "corsika")
+        _default_output_path = self.io_handler.get_output_directory("corsika")
 
         if output_path is None:
             self.output_path = _default_output_path

simtools/data_model/data_reader.py

@@ -10,8 +10,6 @@ from simtools.data_model import validate_data
 from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io import ascii_handler
 
-__all__ = ["read_table_from_file", "read_value_from_file"]
-
 _logger = logging.getLogger(__name__)
 
 

simtools/data_model/metadata_collector.py

@@ -18,8 +18,6 @@ from simtools.data_model import metadata_model, schema
 from simtools.io import ascii_handler, io_handler
 from simtools.utils import names
 
-__all__ = ["MetadataCollector"]
-
 
 class MetadataCollector:
     """

simtools/data_model/model_data_writer.py

@@ -3,6 +3,7 @@
 import logging
 from pathlib import Path
 
+import packaging.version
 from astropy.io.registry.base import IORegistryError
 
 import simtools.utils.general as gen
@@ -12,8 +13,6 @@ from simtools.db import db_handler
 from simtools.io import ascii_handler, io_handler
 from simtools.utils import names, value_conversion
 
-__all__ = ["ModelDataWriter"]
-
 
 class ModelDataWriter:
     """
@@ -29,8 +28,6 @@ class ModelDataWriter:
         Dictionary with configuration parameters.
     output_path: str or Path
         Path to output file.
-    use_plain_output_path: bool
-        Use plain output path.
     args_dict: dict
         Dictionary with configuration parameters.
 
@@ -41,7 +38,6 @@ class ModelDataWriter:
         product_data_file=None,
         product_data_format=None,
         output_path=None,
-        use_plain_output_path=True,
         args_dict=None,
     ):
         """Initialize model data writer."""
@@ -50,11 +46,8 @@ class ModelDataWriter:
         self.schema_dict = {}
         if args_dict is not None:
             output_path = args_dict.get("output_path", output_path)
-            use_plain_output_path = args_dict.get("use_plain_output_path", use_plain_output_path)
         if output_path is not None:
-            self.io_handler.set_paths(
-                output_path=output_path, use_plain_output_path=use_plain_output_path
-            )
+            self.io_handler.set_paths(output_path=output_path)
         try:
             self.product_data_file = self.io_handler.get_output_file(file_name=product_data_file)
         except TypeError:
@@ -104,9 +97,10 @@ class ModelDataWriter:
         parameter_version,
         output_file,
         output_path=None,
-        use_plain_output_path=False,
         metadata_input_dict=None,
         db_config=None,
+        unit=None,
+        meta_parameter=False,
     ):
         """
         Generate DB-style model parameter dict and write it to json file.
@@ -125,12 +119,12 @@ class ModelDataWriter:
             Name of output file.
         output_path: str or Path
             Path to output file.
-        use_plain_output_path: bool
-            Use plain output path.
         metadata_input_dict: dict
             Input to metadata collector.
         db_config: dict
             Database configuration. If not None, check if parameter with the same version exists.
+        unit: str
+            Unit of the parameter value (if applicable and value is not of type astropy Quantity).
 
         Returns
         -------
@@ -142,7 +136,6 @@ class ModelDataWriter:
             product_data_format="json",
             args_dict=None,
             output_path=output_path,
-            use_plain_output_path=use_plain_output_path,
         )
         if db_config is not None:
             writer.check_db_for_existing_parameter(
@@ -160,7 +153,13 @@ class ModelDataWriter:
             )
 
         _json_dict = writer.get_validated_parameter_dict(
-            parameter_name, value, instrument, parameter_version, unique_id
+            parameter_name,
+            value,
+            instrument,
+            parameter_version,
+            unique_id,
+            unit=unit,
+            meta_parameter=meta_parameter,
         )
         writer.write_dict_to_model_parameter_json(output_file, _json_dict)
         return _json_dict
@@ -210,6 +209,8 @@ class ModelDataWriter:
         parameter_version,
         unique_id=None,
         schema_version=None,
+        unit=None,
+        meta_parameter=False,
     ):
         """
         Get validated parameter dictionary.
@@ -226,6 +227,12 @@ class ModelDataWriter:
             Version of the parameter.
         schema_version: str
             Version of the schema.
+        unique_id: str
+            Unique ID of the parameter set (from metadata).
+        unit: str
+            Unit of the parameter value (if applicable and value is not an astropy Quantity).
+        meta_parameter: bool
+            Setting for meta parameter flag.
 
         Returns
         -------
@@ -233,10 +240,10 @@ class ModelDataWriter:
             Validated parameter dictionary.
         """
         self._logger.debug(f"Getting validated parameter dictionary for {instrument}")
-        schema_file = schema.get_model_parameter_schema_file(parameter_name)
-        self.schema_dict = ascii_handler.collect_data_from_file(schema_file)
+        self.schema_dict, schema_file = self._read_schema_dict(parameter_name, schema_version)
 
-        value, unit = value_conversion.split_value_and_unit(value)
+        if unit is None:
+            value, unit = value_conversion.split_value_and_unit(value)
 
         data_dict = {
             "schema_version": schema.get_model_parameter_schema_version(schema_version),
@@ -249,10 +256,8 @@ class ModelDataWriter:
             "unit": unit,
             "type": self._get_parameter_type(),
             "file": self._parameter_is_a_file(),
-            "meta_parameter": False,
-            "model_parameter_schema_version": self.schema_dict.get(
-                "model_parameter_schema_version", "0.1.0"
-            ),
+            "meta_parameter": meta_parameter,
+            "model_parameter_schema_version": self.schema_dict.get("schema_version", "0.1.0"),
         }
         return self.validate_and_transform(
             product_data_dict=data_dict,
@@ -260,19 +265,74 @@ class ModelDataWriter:
             is_model_parameter=True,
         )
 
+    def _read_schema_dict(self, parameter_name, schema_version):
+        """
+        Read schema dict for given parameter name and version.
+
+        Use newest schema version if schema_version is None.
+
+        Parameters
+        ----------
+        parameter_name: str
+            Name of the parameter.
+        schema_version: str
+            Schema version.
+
+        Returns
+        -------
+        dict
+            Schema dictionary.
+        """
+        schema_file = schema.get_model_parameter_schema_file(parameter_name)
+        schemas = ascii_handler.collect_data_from_file(schema_file)
+        if isinstance(schemas, list):
+            if schema_version is None:
+                return self._find_highest_schema_version(schemas), schema_file
+            for entry in schemas:
+                if entry.get("schema_version") == schema_version:
+                    return entry, schema_file
+        else:
+            return schemas, schema_file
+
+        raise ValueError(f"Schema version {schema_version} not found for {parameter_name}")
+
+    def _find_highest_schema_version(self, schema_list):
+        """
+        Find entry with highest schema_version in a list of schema dicts.
+
+        Parameters
+        ----------
+        schema_list: list
+            List of schema dictionaries.
+
+        Returns
+        -------
+        dict
+            Schema dictionary with highest schema_version.
+        """
+        try:
+            valid_entries = [entry for entry in schema_list if "schema_version" in entry]
+        except TypeError as exc:
+            raise TypeError("No valid schema versions found in the list.") from exc
+        return max(valid_entries, key=lambda e: packaging.version.Version(e["schema_version"]))
+
     def _get_parameter_type(self):
         """
         Return parameter type from schema.
 
+        Reduce list of types to single type if all types are the same.
+
         Returns
         -------
-        str
+        str or list[str]
             Parameter type
         """
-        _parameter_type = []
-        for data in self.schema_dict["data"]:
-            _parameter_type.append(data["type"])
-        return _parameter_type if len(_parameter_type) > 1 else _parameter_type[0]
+        _parameter_type = [data["type"] for data in self.schema_dict["data"]]
+        return (
+            _parameter_type[0]
+            if all(t == _parameter_type[0] for t in _parameter_type)
+            else _parameter_type
+        )
 
     def _parameter_is_a_file(self):
         """
@@ -399,7 +459,7 @@ class ModelDataWriter:
         ascii_handler.write_data_to_file(
             data=data_dict,
             output_file=self.io_handler.get_output_file(file_name),
-            sort_keys=False,
+            sort_keys=True,
             numpy_types=True,
         )
 

simtools/data_model/schema.py

@@ -4,9 +4,12 @@ import logging
 from pathlib import Path
 
 import jsonschema
+from packaging.specifiers import SpecifierSet
+from packaging.version import Version
 from referencing import Registry, Resource
 
 import simtools.utils.general as gen
+from simtools import version
 from simtools.constants import (
     METADATA_JSON_SCHEMA,
     MODEL_PARAMETER_METASCHEMA,
@@ -66,7 +69,7 @@ def get_model_parameter_schema_file(parameter):
 
 def get_model_parameter_schema_version(schema_version=None):
     """
-    Validate  and return schema versions.
+    Validate and return schema versions.
 
     If no schema_version is given, the most recent version is provided.
 
@@ -92,7 +95,9 @@ def get_model_parameter_schema_version(schema_version=None):
     raise ValueError(f"Schema version {schema_version} not found in {MODEL_PARAMETER_METASCHEMA}.")
 
 
-def validate_dict_using_schema(data, schema_file=None, json_schema=None):
+def validate_dict_using_schema(
+    data, schema_file=None, json_schema=None, ignore_software_version=False
+):
     """
     Validate a data dictionary against a schema.
 
@@ -102,6 +107,10 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
         dictionary to be validated
     schema_file (dict)
         schema used for validation
+    json_schema (dict)
+        schema used for validation
+    ignore_software_version: bool
+        If True, ignore software version check.
 
     Raises
     ------
@@ -115,6 +124,8 @@ def validate_dict_using_schema(data, schema_file=None, json_schema=None):
     if json_schema is None:
         json_schema = load_schema(schema_file, get_schema_version_from_data(data))
 
+    _validate_deprecation_and_version(data, ignore_software_version)
+
     validator = jsonschema.Draft6Validator(
         schema=json_schema,
         format_checker=format_checkers.format_checker,
@@ -294,3 +305,51 @@ def _add_array_elements(key, schema):
 
     recursive_search(schema, key)
     return schema
+
+
+def _validate_deprecation_and_version(
+    data, software_name="simtools", ignore_software_version=False
+):
+    """
+    Check if data contains deprecated parameters or version mismatches.
+
+    Parameters
+    ----------
+    data: dict
+        Data dictionary to check.
+    software_name: str
+        Name of the software to check version against.
+    ignore_software_version: bool
+        If True, ignore software version check.
+    """
+    if not isinstance(data, dict):
+        return
+
+    if data.get("deprecated", False):
+        note = data.get("deprecation_note", "(no deprecation note provided)")
+        _logger.warning(f"Data is deprecated. Note: {note}")
+
+    def check_version(sw):
+        constraint = sw.get("version")
+        if constraint is None:
+            return
+        constraint = constraint.strip()
+        spec = SpecifierSet(constraint, prereleases=True)
+        if Version(version.__version__) in spec:
+            _logger.debug(
+                f"Version {version.__version__} of {software_name} matches constraint {constraint}."
+            )
+        else:
+            msg = (
+                f"Version {version.__version__} of {software_name} "
+                f"does not match constraint {constraint}."
+            )
+            if ignore_software_version:
+                _logger.warning(f"{msg}, but version check is ignored.")
+                return
+            raise ValueError(msg)
+
+    for sw in data.get("simulation_software", []):
+        if sw.get("name") == software_name:
+            check_version(sw)
+            break