gammasimtools 0.8.2__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/applications/simulate_prod.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
 r"""
-    Generate simulation configuration and run simulations (if required).
+    Generate simulation configuration and run simulations.
 
     Multipipe scripts will be produced as part of this application.
     Allows to run array layout simulation including shower and detector simulations
@@ -43,9 +43,9 @@ r"""
         The location of the output directories corsika-data and simtel-data.
         the label is added to the data_directory, such that the output
         will be written to data_directory/label/simtel-data.
-    pack_for_grid_register (bool, optional)
+    pack_for_grid_register (str, optional)
         Set whether to prepare a tarball for registering the output files on the grid.
-        The files are written to the output_path/directory_for_grid_upload directory.
+        The files are written to the specified directory.
     log_level (str, optional)
         Log level to print.
 
@@ -68,9 +68,6 @@ r"""
 """
 
 import logging
-import shutil
-import tarfile
-from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
@@ -106,10 +103,10 @@ def _parse(description=None):
     )
     config.parser.add_argument(
         "--pack_for_grid_register",
-        help="Set whether to prepare a tarball for registering the output files on the grid.",
-        action="store_true",
+        help="Directory for a tarball for registering the output files on the grid.",
+        type=str,
         required=False,
-        default=False,
+        default=None,
     )
     config.parser.add_argument(
         "--save_file_lists",
@@ -137,49 +134,11 @@ def _parse(description=None):
     return config.initialize(
         db_config=True,
         job_submission=True,
-        simulation_model=["site", "layout", "telescope"],
+        simulation_model=["site", "layout", "telescope", "model_version"],
         simulation_configuration={"software": None, "corsika_configuration": ["all"]},
     )
 
 
-def pack_for_register(logger, simulator, args_dict):
-    """
-    Pack the output files for registering on the grid.
-
-    Parameters
-    ----------
-    logger: logging.Logger
-        Logger object.
-    simulator: Simulator
-        Simulator object.
-
-    """
-    logger.info("Packing the output files for registering on the grid")
-    output_files = simulator.get_file_list(file_type="output")
-    log_files = simulator.get_file_list(file_type="log")
-    histogram_files = simulator.get_file_list(file_type="hist")
-    tar_file_name = Path(log_files[0]).name.replace("log.gz", "log_hist.tar.gz")
-    directory_for_grid_upload = Path(args_dict.get("output_path")).joinpath(
-        "directory_for_grid_upload"
-    )
-    directory_for_grid_upload.mkdir(parents=True, exist_ok=True)
-    tar_file_name = directory_for_grid_upload.joinpath(tar_file_name)
-
-    with tarfile.open(tar_file_name, "w:gz") as tar:
-        files_to_tar = log_files[:1] + histogram_files[:1]
-        for file_to_tar in files_to_tar:
-            tar.add(file_to_tar, arcname=Path(file_to_tar).name)
-
-    for file_to_move in [*output_files]:
-        source_file = Path(file_to_move)
-        destination_file = directory_for_grid_upload / source_file.name
-        # Note that this will overwrite previous files which exist in the directory
-        # It should be fine for normal production since each run is on a separate node
-        # so no files are expected there.
-        shutil.move(source_file, destination_file)
-    logger.info(f"Output files for the grid placed in {directory_for_grid_upload!s}")
-
-
 def main():  # noqa: D103
     args_dict, db_config = _parse(description="Run simulations for productions")
 
@@ -192,17 +151,23 @@ def main():  # noqa: D103
 
     simulator.simulate()
 
-    logger.info(
-        f"Production run is complete for primary {args_dict['primary']} showers "
-        f"coming from {args_dict['azimuth_angle']} azimuth and zenith angle of "
-        f"{args_dict['zenith_angle']} at the {args_dict['site']} site, "
-        f"using the {args_dict['model_version']} simulation model."
-    )
-
-    if args_dict["pack_for_grid_register"]:
-        pack_for_register(logger, simulator, args_dict)
-    if args_dict["save_file_lists"]:
-        simulator.save_file_lists()
+    if simulator.submit_engine == "local":
+        logger.info(
+            f"Production run complete for primary {args_dict['primary']} showers "
+            f"from {args_dict['azimuth_angle']} azimuth and {args_dict['zenith_angle']} zenith "
+            f"at {args_dict['site']} site, using {args_dict['model_version']} model."
+        )
+        if args_dict.get("pack_for_grid_register"):
+            simulator.pack_for_register(args_dict["pack_for_grid_register"])
+        if args_dict["save_file_lists"]:
+            simulator.save_file_lists()
+    else:
+        logger.info("Production run submitted to the workload manager")
+        if args_dict["pack_for_grid_register"] or args_dict["save_file_lists"]:
+            logger.warning(
+                "Packing for grid register or saving file lists not supported for "
+                f"{simulator.submit_engine}."
+            )
 
 
 if __name__ == "__main__":

simtools/applications/simulate_prod_htcondor_generator.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python3
+
+r"""
+    Generate a run script and submit file for HT Condor job submission of a simulation production.
+
+    This tool facilitates the submission of multiple simulations to the HT Condor batch system,
+    enabling:
+
+    - Execution of simulations using the "simtools-simulate-prod" application.
+    - 'number_of_runs' jobs are submitted to the HT Condor batch system.
+    - Utilization of an Apptainer image containing the SimPipe simulation software and tools.
+    - Packaging of data and histogram files, and writing them to a specified directory.
+
+    This tool is intended for use in an HT Condor environment. Jobs run in a container universe
+    using the Apptainer image specified in the command line ('--apptainer_image'). Output is written
+    to the 'output_path' directory, with 'simtools-output' and 'logs' subdirectories.
+
+    Requirements for the 'simtools-simulate-prod-htcondor-generator' application:
+
+    - Availability of an Apptainer image 'simtools-prod' (obtainable from the package registry on
+      GitHub, e.g., via 'apptainer pull --force docker://ghcr.io/gammasim/simtools-prod:latest').
+    - Environment parameters required to run CORSIKA and sim_telarray, as well as DB access
+      credentials.  These should be listed similarly to a '.env' file and copied to
+      'output_path/env.txt'.  Ensure that the path to the simulation software is correctly set to
+      'SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray'.
+
+
+    Command line arguments
+    ----------------------
+    output_path (str, required)
+        Directory where the output and the simulation data files will be written.
+    apptainer_image (str, optional)
+        Apptainer image to use for the simulation (full path).
+    priority (int, optional)
+        Job priority (default: 1).
+
+    (all other command line arguments are identical to those of :ref:`simulate_prod`).
+
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.job_execution import htcondor_script_generator
+
+
+def _parse(description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    description: str
+        Application description.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+
+    """
+    config = configurator.Configurator(description=description)
+    config.parser.add_argument(
+        "--apptainer_image",
+        help="Apptainer image to use for the simulation (full path).",
+        type=str,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--priority",
+        help="Job priority.",
+        type=int,
+        required=False,
+        default=1,
+    )
+    return config.initialize(
+        db_config=False,
+        job_submission=False,
+        simulation_model=["site", "layout", "telescope", "model_version"],
+        simulation_configuration={"software": None, "corsika_configuration": ["all"]},
+    )
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse(description="Prepare simulations production for HT Condor job submission")
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    htcondor_script_generator.generate_submission_script(args_dict)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/submit_data_from_external.py

@@ -104,7 +104,7 @@ def main():  # noqa: D103
 
     writer.ModelDataWriter.dump(
         args_dict=args_dict,
-        metadata=_metadata.top_level_meta,
+        metadata=_metadata.get_top_level_metadata(),
         product_data=data_validator.validate_and_transform(),
     )
 

simtools/applications/validate_camera_efficiency.py

@@ -86,7 +86,7 @@ def _parse(label):
     )
     _args_dict, _db_config = config.initialize(
         db_config=True,
-        simulation_model="telescope",
+        simulation_model=["telescope", "model_version"],
         simulation_configuration={"corsika_configuration": ["zenith_angle", "azimuth_angle"]},
     )
     if _args_dict["site"] is None or _args_dict["telescope"] is None:

simtools/applications/validate_camera_fov.py

@@ -54,7 +54,7 @@ import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
-from simtools.visualization import plot_camera
+from simtools.visualization import plot_camera, visualize
 
 
 def _parse():
@@ -83,7 +83,7 @@ def _parse():
         ),
         default=50,
     )
-    return config.initialize(db_config=True, simulation_model="telescope")
+    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
 def main():  # noqa: D103
@@ -133,11 +133,7 @@ def main():  # noqa: D103
             ) from exc
     fig = plot_camera.plot_pixel_layout(camera, args_dict["camera_in_sky_coor"], pixel_ids_to_print)
     plot_file_prefix = output_dir.joinpath(f"{label}_{tel_model.name}_pixel_layout")
-    for suffix in ["pdf", "png"]:
-        file_name = f"{plot_file_prefix!s}.{suffix}"
-        fig.savefig(file_name, format=suffix, bbox_inches="tight")
-        print(f"\nSaved camera plot in {file_name}\n")
-    fig.clf()
+    visualize.save_figure(fig, f"{plot_file_prefix!s}", log_title="camera")
 
 
 if __name__ == "__main__":

simtools/applications/validate_cumulative_psf.py

@@ -110,7 +110,7 @@ def _parse(label):
         help="Data file name with the measured PSF vs radius [cm]",
         type=str,
     )
-    return config.initialize(db_config=True, simulation_model="telescope")
+    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
 def load_data(datafile):
@@ -182,9 +182,7 @@ def main():  # noqa: D103
 
     plot_file_name = label + "_" + tel_model.name + "_cumulative_PSF"
     plot_file = output_dir.joinpath(plot_file_name)
-    for f in ["pdf", "png"]:
-        plt.savefig(str(plot_file) + "." + f, format=f, bbox_inches="tight")
-    fig.clf()
+    visualize.save_figure(fig, plot_file)
 
     # Plotting image
     data_to_plot = im.get_image_data()
@@ -195,9 +193,7 @@ def main():  # noqa: D103
 
     plot_file_name = label + "_" + tel_model.name + "_image"
     plot_file = output_dir.joinpath(plot_file_name)
-    for f in ["pdf", "png"]:
-        fig.savefig(str(plot_file) + "." + f, format=f, bbox_inches="tight")
-    fig.clf()
+    visualize.save_figure(fig, plot_file)
 
 
 if __name__ == "__main__":

simtools/applications/validate_file_using_schema.py

@@ -33,11 +33,14 @@ r"""
 """
 
 import logging
-from importlib.resources import files
+import re
 from pathlib import Path
 
+import jsonschema
+
 import simtools.utils.general as gen
 from simtools.configuration import configurator
+from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
 from simtools.data_model import metadata_collector, metadata_model, validate_data
 
 
@@ -62,13 +65,12 @@ def _parse(label, description):
     group = config.parser.add_mutually_exclusive_group(required=True)
     group.add_argument("--file_name", help="File to be validated")
     group.add_argument(
-        "--model_parameters_directory",
+        "--file_directory",
         help=(
-            "Directory with json files with model parameters to be validated."
-            "All *.json files in the directory will be validated."
-            "Schema files will be taken from simtools/schemas/model_parameters/."
-            "Note that in this case the data_type argument is ignored"
-            "and data_type=model_parameter is always used."
+            "Directory with json files to be validated. "
+            "If no schema file is provided, the assumption is that model "
+            "parameters are validated and the schema files are taken from "
+            f"{MODEL_PARAMETER_SCHEMA_PATH}."
         ),
     )
     config.parser.add_argument("--schema", help="Json schema file", required=False)
@@ -116,31 +118,39 @@ def _get_schema_file_name(args_dict, data_dict=None):
 
 def validate_schema(args_dict, logger):
     """
-    Validate a schema file given in yaml or json format.
+    Validate a schema file (or several files) given in yaml or json format.
 
     Schema is either given as command line argument, read from the meta_schema_url or from
     the metadata section of the data dictionary.
 
     """
-    try:
-        data = gen.collect_data_from_file(file_name=args_dict["file_name"])
-    except FileNotFoundError as exc:
-        logger.error(f"Error reading schema file from {args_dict['file_name']}")
-        raise exc
-    metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data))
-    logger.info(f"Successful validation of schema file {args_dict['file_name']}")
+    if args_dict.get("file_directory") is not None:
+        file_list = list(Path(args_dict["file_directory"]).rglob("*.json"))
+    else:
+        file_list = [args_dict["file_name"]]
+    for file_name in file_list:
+        try:
+            data = gen.collect_data_from_file(file_name=file_name)
+        except FileNotFoundError as exc:
+            logger.error(f"Error reading schema file from {file_name}")
+            raise exc
+        try:
+            metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data))
+        except jsonschema.exceptions.ValidationError as exc:
+            logger.error(f"Failed validation of file {file_name}")
+            raise exc
+        logger.info(f"Successful validation of file {file_name}")
 
 
 def validate_data_files(args_dict, logger):
     """Validate data files."""
-    model_parameters_directory = args_dict.get("model_parameters_directory")
-    if model_parameters_directory is not None:
+    file_directory = args_dict.get("file_directory")
+    if file_directory is not None:
         tmp_args_dict = {}
-        for file_name in Path(model_parameters_directory).rglob("*.json"):
+        for file_name in Path(file_directory).rglob("*.json"):
             tmp_args_dict["file_name"] = file_name
-            schema_file = (
-                files("simtools") / "schemas/model_parameters" / f"{file_name.stem}.schema.yml"
-            )
+            parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem)
+            schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml"
             tmp_args_dict["schema"] = schema_file
             tmp_args_dict["data_type"] = "model_parameter"
             tmp_args_dict["require_exact_data_type"] = args_dict["require_exact_data_type"]

simtools/applications/validate_optics.py

@@ -78,6 +78,7 @@ from simtools.configuration import configurator
 from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 from simtools.ray_tracing.ray_tracing import RayTracing
+from simtools.visualization import visualize
 
 
 def _parse(label):
@@ -114,7 +115,7 @@ def _parse(label):
         help="Produce a multiple pages pdf file with the image plots.",
         action="store_true",
     )
-    return config.initialize(db_config=True, simulation_model=["telescope"])
+    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
 def main():  # noqa: D103
@@ -171,9 +172,7 @@ def main():  # noqa: D103
 
         plot_file_name = "_".join((label, tel_model.name, key))
         plot_file = output_dir.joinpath(plot_file_name)
-        plt.savefig(str(plot_file) + ".pdf", format="pdf", bbox_inches="tight")
-        plt.savefig(str(plot_file) + ".png", format="png", bbox_inches="tight")
-        plt.clf()
+        visualize.save_figure(plt, plot_file)
 
     # Plotting images
     if args_dict["plot_images"]:

simtools/camera_efficiency.py

@@ -514,7 +514,4 @@ class CameraEfficiency:
         plot_file = self.output_dir.joinpath(
             self.label + "_" + self.telescope_model.name + "_" + plot_title
         )
-        for f in ["pdf", "png"]:
-            fig.savefig(str(plot_file) + "." + f, format=f, bbox_inches="tight")
-        self._logger.info(f"Plotted {plot_title} efficiency in {plot_file}")
-        fig.clf()
+        visualize.save_figure(fig, plot_file, log_title=f"{plot_title} efficiency")

simtools/configuration/commandline_parser.py

@@ -198,13 +198,6 @@ class CommandLineParser(argparse.ArgumentParser):
             required=False,
             default=None,
         )
-        _job_group.add_argument(
-            "--db_simulation_model_url",
-            help="simulation model repository URL",
-            type=str,
-            required=False,
-            default=None,
-        )
 
     def initialize_job_submission_arguments(self):
         """Initialize job submission arguments for simulator."""
@@ -243,12 +236,13 @@ class CommandLineParser(argparse.ArgumentParser):
             return
 
         _job_group = self.add_argument_group("simulation model")
-        _job_group.add_argument(
-            "--model_version",
-            help="model version",
-            type=str,
-            default=None,
-        )
+        if "model_version" in model_options:
+            _job_group.add_argument(
+                "--model_version",
+                help="model version",
+                type=str,
+                default=None,
+            )
         if any(
             option in model_options for option in ["site", "telescope", "layout", "layout_file"]
         ):

simtools/configuration/configurator.py

@@ -10,6 +10,7 @@ import astropy.units as u
 from dotenv import load_dotenv
 
 import simtools.configuration.commandline_parser as argparser
+from simtools.db.db_handler import jsonschema_db_dict
 from simtools.io_operations import io_handler
 from simtools.utils import general as gen
 
@@ -438,24 +439,10 @@ class Configurator:
         """
         Return parameters for DB configuration.
 
-        Parameters
-        ----------
+        Returns
+        -------
         dict
-            Dictionary with DB parameters
+            Dictionary with DB parameters.
         """
-        _db_dict = {}
-        _db_para = (
-            "db_api_user",
-            "db_api_pw",
-            "db_api_port",
-            "db_server",
-            "db_simulation_model",
-            "db_simulation_model_url",
-        )
-        try:
-            for _para in _db_para:
-                _db_dict[_para] = self.config[_para]
-        except KeyError:
-            pass
-
-        return _db_dict
+        db_params = jsonschema_db_dict["properties"].keys()
+        return {param: self.config.get(param) for param in db_params if param in self.config}

simtools/data_model/metadata_collector.py

@@ -86,6 +86,24 @@ class MetadataCollector:
             except AttributeError:
                 self._logger.debug(f"Method _fill_{meta_type}_meta not implemented")
 
+    def get_top_level_metadata(self):
+        """
+        Return top level metadata dictionary (with updated activity end time).
+
+        Returns
+        -------
+        dict
+            Top level metadata dictionary.
+
+        """
+        try:
+            self.top_level_meta[self.observatory]["activity"][
+                "end"
+            ] = datetime.datetime.now().isoformat(timespec="seconds")
+        except KeyError:
+            pass
+        return self.top_level_meta
+
     def get_data_model_schema_file_name(self):
         """
         Return data model schema file name.

simtools/data_model/metadata_model.py

@@ -38,7 +38,10 @@ def validate_schema(data, schema_file):
         if validation fails
 
     """
-    schema, schema_file = _load_schema(schema_file)
+    schema, schema_file = _load_schema(
+        schema_file,
+        data.get("schema_version", "0.1.0"),  # default version to ensure backward compatibility
+    )
 
     try:
         jsonschema.validate(data, schema=schema, format_checker=format_checkers.format_checker)
@@ -72,17 +75,17 @@ def get_default_metadata_dict(schema_file=None, observatory="CTA"):
     return _fill_defaults(schema["definitions"], observatory)
 
 
-def _load_schema(schema_file=None):
+def _load_schema(schema_file=None, schema_version=None):
     """
     Load parameter schema from file from simpipe metadata schema.
 
     Returns
     -------
-    schema_file dict
-        Schema used for validation.
-    schema_file str
+    schema_file: str
         File name schema is loaded from. If schema_file is not given,
         the default schema file name is returned.
+    schema_version: str
+        Schema version.
 
     Raises
     ------
@@ -98,6 +101,16 @@ def _load_schema(schema_file=None):
     except FileNotFoundError:
         schema_file = files("simtools").joinpath("schemas") / schema_file
         schema = gen.collect_data_from_file(file_name=schema_file)
+
+    if isinstance(schema, list):  # schema file with several schemas defined
+        if schema_version is None:
+            raise ValueError(f"Schema version not given in {schema_file}.")
+        schema = next((doc for doc in schema if doc.get("version") == schema_version), None)
+        if schema is None:
+            raise ValueError(f"Schema version {schema_version} not found in {schema_file}.")
+    elif schema_version is not None and schema_version != schema.get("version"):
+        _logger.warning(f"Schema version {schema_version} does not match {schema.get('version')}")
+
     _logger.debug(f"Loading schema from {schema_file}")
     _add_array_elements("InstrumentTypeElement", schema)
 

simtools/data_model/model_data_writer.py

@@ -170,7 +170,7 @@ class ModelDataWriter:
             metadata_input_dict["output_file"] = output_file
             metadata_input_dict["output_file_format"] = Path(output_file).suffix.lstrip(".")
             writer.write_metadata_to_yml(
-                metadata=MetadataCollector(args_dict=metadata_input_dict).top_level_meta,
+                metadata=MetadataCollector(args_dict=metadata_input_dict).get_top_level_metadata(),
                 yml_file=output_path / f"{Path(output_file).stem}",
             )
         return _json_dict