gammasimtools 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/applications/simulate_prod.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
 r"""
-    Generate simulation configuration and run simulations (if required).
+    Generate simulation configuration and run simulations.
 
     Multipipe scripts will be produced as part of this application.
     Allows to run array layout simulation including shower and detector simulations
@@ -31,7 +31,7 @@ r"""
         Zenith angle in degrees.
     nshow (int, optional)
         Number of showers to simulate.
-        The Number of simulated events depends on the number of times a shower is re-used in the
+        The Number of simulated events depends on the number of times a shower is reused in the
         telescope simulation. The number provided here is before any reuse factors.
     start_run (int, required)
         Start run number such that the actual run number will be 'start_run' + 'run'.
@@ -43,9 +43,9 @@ r"""
         The location of the output directories corsika-data and simtel-data.
         the label is added to the data_directory, such that the output
         will be written to data_directory/label/simtel-data.
-    pack_for_grid_register (bool, optional)
+    pack_for_grid_register (str, optional)
         Set whether to prepare a tarball for registering the output files on the grid.
-        The files are written to the output_path/directory_for_grid_upload directory.
+        The files are written to the specified directory.
     log_level (str, optional)
         Log level to print.
 
@@ -68,9 +68,6 @@ r"""
 """
 
 import logging
-import shutil
-import tarfile
-from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
@@ -106,10 +103,10 @@ def _parse(description=None):
     )
     config.parser.add_argument(
         "--pack_for_grid_register",
-        help="Set whether to prepare a tarball for registering the output files on the grid.",
-        action="store_true",
+        help="Directory for a tarball for registering the output files on the grid.",
+        type=str,
         required=False,
-        default=False,
+        default=None,
     )
     config.parser.add_argument(
         "--save_file_lists",
@@ -137,49 +134,11 @@ def _parse(description=None):
     return config.initialize(
         db_config=True,
         job_submission=True,
-        simulation_model=["site", "layout", "telescope"],
+        simulation_model=["site", "layout", "telescope", "model_version"],
         simulation_configuration={"software": None, "corsika_configuration": ["all"]},
     )
 
 
-def pack_for_register(logger, simulator, args_dict):
-    """
-    Pack the output files for registering on the grid.
-
-    Parameters
-    ----------
-    logger: logging.Logger
-        Logger object.
-    simulator: Simulator
-        Simulator object.
-
-    """
-    logger.info("Packing the output files for registering on the grid")
-    output_files = simulator.get_file_list(file_type="output")
-    log_files = simulator.get_file_list(file_type="log")
-    histogram_files = simulator.get_file_list(file_type="hist")
-    tar_file_name = Path(log_files[0]).name.replace("log.gz", "log_hist.tar.gz")
-    directory_for_grid_upload = Path(args_dict.get("output_path")).joinpath(
-        "directory_for_grid_upload"
-    )
-    directory_for_grid_upload.mkdir(parents=True, exist_ok=True)
-    tar_file_name = directory_for_grid_upload.joinpath(tar_file_name)
-
-    with tarfile.open(tar_file_name, "w:gz") as tar:
-        files_to_tar = log_files[:1] + histogram_files[:1]
-        for file_to_tar in files_to_tar:
-            tar.add(file_to_tar, arcname=Path(file_to_tar).name)
-
-    for file_to_move in [*output_files]:
-        source_file = Path(file_to_move)
-        destination_file = directory_for_grid_upload / source_file.name
-        # Note that this will overwrite previous files which exist in the directory
-        # It should be fine for normal production since each run is on a separate node
-        # so no files are expected there.
-        shutil.move(source_file, destination_file)
-    logger.info(f"Output files for the grid placed in {directory_for_grid_upload!s}")
-
-
 def main():  # noqa: D103
     args_dict, db_config = _parse(description="Run simulations for productions")
 
@@ -192,17 +151,23 @@ def main():  # noqa: D103
 
     simulator.simulate()
 
-    logger.info(
-        f"Production run is complete for primary {args_dict['primary']} showers "
-        f"coming from {args_dict['azimuth_angle']} azimuth and zenith angle of "
-        f"{args_dict['zenith_angle']} at the {args_dict['site']} site, "
-        f"using the {args_dict['model_version']} simulation model."
-    )
-
-    if args_dict["pack_for_grid_register"]:
-        pack_for_register(logger, simulator, args_dict)
-    if args_dict["save_file_lists"]:
-        simulator.save_file_lists()
+    if simulator.submit_engine == "local":
+        logger.info(
+            f"Production run complete for primary {args_dict['primary']} showers "
+            f"from {args_dict['azimuth_angle']} azimuth and {args_dict['zenith_angle']} zenith "
+            f"at {args_dict['site']} site, using {args_dict['model_version']} model."
+        )
+        if args_dict.get("pack_for_grid_register"):
+            simulator.pack_for_register(args_dict["pack_for_grid_register"])
+        if args_dict["save_file_lists"]:
+            simulator.save_file_lists()
+    else:
+        logger.info("Production run submitted to the workload manager")
+        if args_dict["pack_for_grid_register"] or args_dict["save_file_lists"]:
+            logger.warning(
+                "Packing for grid register or saving file lists not supported for "
+                f"{simulator.submit_engine}."
+            )
 
 
 if __name__ == "__main__":

simtools/applications/simulate_prod_htcondor_generator.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python3
+
+r"""
+Generate a run script and submit file for HT Condor job submission of a simulation production.
+
+This tool facilitates the submission of multiple simulations to the HT Condor batch system,
+enabling:
+
+- Execution of simulations using the "simtools-simulate-prod" application.
+- 'number_of_runs' jobs are submitted to the HT Condor batch system.
+- Utilization of an Apptainer image containing the SimPipe simulation software and tools.
+- Packaging of data and histogram files, and writing them to a specified directory.
+
+This tool is intended for use in an HT Condor environment. Jobs run in a container universe
+using the Apptainer image specified in the command line ('--apptainer_image'). Output is written
+to the 'output_path' directory, with 'simtools-output' and 'logs' subdirectories.
+
+Requirements for the 'simtools-simulate-prod-htcondor-generator' application:
+
+- Availability of an Apptainer image 'simtools-prod' (obtainable from the package registry on
+  GitHub, e.g., via 'apptainer pull --force docker://ghcr.io/gammasim/simtools-prod:latest').
+- Environment parameters required to run CORSIKA and sim_telarray, as well as DB access
+  credentials.  These should be listed similarly to a '.env' file and copied to
+  'output_path/env.txt'.  Ensure that the path to the simulation software is correctly set to
+  'SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray'.
+
+
+Command line arguments
+----------------------
+output_path (str, required)
+    Directory where the output and the simulation data files will be written.
+apptainer_image (str, optional)
+    Apptainer image to use for the simulation (full path).
+priority (int, optional)
+    Job priority (default: 1).
+
+(all other command line arguments are identical to those of :ref:`simulate_prod`).
+
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.job_execution import htcondor_script_generator
+
+
+def _parse(description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    description: str
+        Application description.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+
+    """
+    config = configurator.Configurator(description=description)
+    config.parser.add_argument(
+        "--apptainer_image",
+        help="Apptainer image to use for the simulation (full path).",
+        type=str,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--priority",
+        help="Job priority.",
+        type=int,
+        required=False,
+        default=1,
+    )
+    return config.initialize(
+        db_config=False,
+        job_submission=False,
+        simulation_model=["site", "layout", "telescope", "model_version"],
+        simulation_configuration={"software": None, "corsika_configuration": ["all"]},
+    )
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse(description="Prepare simulations production for HT Condor job submission")
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    htcondor_script_generator.generate_submission_script(args_dict)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/submit_data_from_external.py

@@ -24,7 +24,7 @@ r"""
         simtools-submit-data-from-external \\
             --input_meta ./tests/resources/MLTdata-preproduction.meta.yml \\
             --input ./tests/resources/MLTdata-preproduction.ecsv \\
-            --schema ./tests/resources/schema_MST_mirror_2f_measurements.yml \\
+            --schema src/simtools/schemas/input/MST_mirror_2f_measurements.schema.yml \\
             --output_file TEST-submit_data_from_external.ecsv
 
     Expected final print-out message:
@@ -83,6 +83,12 @@ def _parse(label, description):
         type=str,
         required=False,
     )
+    config.parser.add_argument(
+        "--ignore_metadata",
+        help="Ignore metadata",
+        action="store_true",
+        required=False,
+    )
     return config.initialize(output=True)
 
 
@@ -95,16 +101,18 @@ def main():  # noqa: D103
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    _metadata = MetadataCollector(args_dict=args_dict)
+    _metadata = None if args_dict.get("ignore_metadata") else MetadataCollector(args_dict)
 
     data_validator = validate_data.DataValidator(
-        schema_file=_metadata.get_data_model_schema_file_name(),
+        schema_file=(
+            _metadata.get_data_model_schema_file_name() if _metadata else args_dict.get("schema")
+        ),
         data_file=args_dict["input"],
     )
 
     writer.ModelDataWriter.dump(
         args_dict=args_dict,
-        metadata=_metadata.top_level_meta,
+        metadata=_metadata.get_top_level_metadata() if _metadata else None,
         product_data=data_validator.validate_and_transform(),
     )
 

simtools/applications/submit_model_parameter_from_external.py

@@ -16,8 +16,8 @@ r"""
         instrument name.
     site (str)
         site location.
-    model_version (str)
-        Model version.
+    parameter_version (str)
+        Parameter version.
     input_meta (str, optional)
         input meta data file (yml format)
 
@@ -33,7 +33,7 @@ r"""
             --value 2 \\
             --instrument LSTN-design \\
             --site North \\
-            --model_version 6.0.0 \\
+            --parameter_version 0.1.0 \\
             --input_meta num_gains.metadata.yml
 
 """
@@ -70,7 +70,9 @@ def _parse(label, description):
     )
     config.parser.add_argument("--instrument", type=str, required=True, help="Instrument name")
     config.parser.add_argument("--site", type=str, required=True, help="Site location")
-    config.parser.add_argument("--model_version", type=str, required=True, help="Model version")
+    config.parser.add_argument(
+        "--parameter_version", type=str, required=True, help="Parameter version"
+    )
 
     config.parser.add_argument(
         "--value",
@@ -102,7 +104,7 @@ def main():  # noqa: D103
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     output_path = (
-        Path(args_dict["output_path"]) / args_dict["model_version"] / args_dict["instrument"]
+        Path(args_dict["output_path"]) / args_dict["parameter_version"] / args_dict["instrument"]
         if args_dict.get("output_path")
         else None
     )
@@ -110,7 +112,7 @@ def main():  # noqa: D103
         parameter_name=args_dict["parameter"],
         value=args_dict["value"],
         instrument=args_dict["instrument"],
-        model_version=args_dict["model_version"],
+        parameter_version=args_dict["parameter_version"],
         output_file=Path(args_dict["parameter"]).with_suffix(".json"),
         output_path=output_path,
         use_plain_output_path=args_dict.get("use_plain_output_path"),

simtools/applications/validate_camera_efficiency.py

@@ -47,7 +47,7 @@ import logging
 from pathlib import Path
 
 import simtools.utils.general as gen
-from simtools.camera_efficiency import CameraEfficiency
+from simtools.camera.camera_efficiency import CameraEfficiency
 from simtools.configuration import configurator
 
 
@@ -86,7 +86,7 @@ def _parse(label):
     )
     _args_dict, _db_config = config.initialize(
         db_config=True,
-        simulation_model="telescope",
+        simulation_model=["telescope", "model_version"],
         simulation_configuration={"corsika_configuration": ["zenith_angle", "azimuth_angle"]},
     )
     if _args_dict["site"] is None or _args_dict["telescope"] is None:
@@ -106,7 +106,7 @@ def main():  # noqa: D103
     ce = CameraEfficiency(
         db_config=_db_config,
         simtel_path=args_dict["simtel_path"],
-        label=label,
+        label=args_dict.get("label", label),
         config_data=args_dict,
     )
     ce.simulate()

simtools/applications/validate_camera_fov.py

@@ -54,7 +54,7 @@ import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
-from simtools.visualization import plot_camera
+from simtools.visualization import plot_camera, visualize
 
 
 def _parse():
@@ -83,7 +83,7 @@ def _parse():
         ),
         default=50,
     )
-    return config.initialize(db_config=True, simulation_model="telescope")
+    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
 def main():  # noqa: D103
@@ -133,11 +133,7 @@ def main():  # noqa: D103
             ) from exc
     fig = plot_camera.plot_pixel_layout(camera, args_dict["camera_in_sky_coor"], pixel_ids_to_print)
     plot_file_prefix = output_dir.joinpath(f"{label}_{tel_model.name}_pixel_layout")
-    for suffix in ["pdf", "png"]:
-        file_name = f"{plot_file_prefix!s}.{suffix}"
-        fig.savefig(file_name, format=suffix, bbox_inches="tight")
-        print(f"\nSaved camera plot in {file_name}\n")
-    fig.clf()
+    visualize.save_figure(fig, f"{plot_file_prefix!s}", log_title="camera")
 
 
 if __name__ == "__main__":

simtools/applications/validate_cumulative_psf.py

@@ -110,7 +110,7 @@ def _parse(label):
         help="Data file name with the measured PSF vs radius [cm]",
         type=str,
     )
-    return config.initialize(db_config=True, simulation_model="telescope")
+    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
 def load_data(datafile):
@@ -182,9 +182,7 @@ def main():  # noqa: D103
 
     plot_file_name = label + "_" + tel_model.name + "_cumulative_PSF"
     plot_file = output_dir.joinpath(plot_file_name)
-    for f in ["pdf", "png"]:
-        plt.savefig(str(plot_file) + "." + f, format=f, bbox_inches="tight")
-    fig.clf()
+    visualize.save_figure(fig, plot_file)
 
     # Plotting image
     data_to_plot = im.get_image_data()
@@ -195,9 +193,7 @@ def main():  # noqa: D103
 
     plot_file_name = label + "_" + tel_model.name + "_image"
     plot_file = output_dir.joinpath(plot_file_name)
-    for f in ["pdf", "png"]:
-        fig.savefig(str(plot_file) + "." + f, format=f, bbox_inches="tight")
-    fig.clf()
+    visualize.save_figure(fig, plot_file)
 
 
 if __name__ == "__main__":

simtools/applications/validate_file_using_schema.py

@@ -33,12 +33,13 @@ r"""
 """
 
 import logging
-from importlib.resources import files
+import re
 from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.data_model import metadata_collector, metadata_model, validate_data
+from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
+from simtools.data_model import metadata_collector, schema, validate_data
 
 
 def _parse(label, description):
@@ -62,13 +63,12 @@ def _parse(label, description):
     group = config.parser.add_mutually_exclusive_group(required=True)
     group.add_argument("--file_name", help="File to be validated")
     group.add_argument(
-        "--model_parameters_directory",
+        "--file_directory",
         help=(
-            "Directory with json files with model parameters to be validated."
-            "All *.json files in the directory will be validated."
-            "Schema files will be taken from simtools/schemas/model_parameters/."
-            "Note that in this case the data_type argument is ignored"
-            "and data_type=model_parameter is always used."
+            "Directory with json files to be validated. "
+            "If no schema file is provided, the assumption is that model "
+            "parameters are validated and the schema files are taken from "
+            f"{MODEL_PARAMETER_SCHEMA_PATH}."
         ),
     )
     config.parser.add_argument("--schema", help="Json schema file", required=False)
@@ -114,33 +114,47 @@ def _get_schema_file_name(args_dict, data_dict=None):
     return schema_file
 
 
-def validate_schema(args_dict, logger):
+def _get_json_file_list(file_directory=None, file_name=None):
+    """Return list of json files in a directory."""
+    file_list = []
+    if file_directory is not None:
+        file_list = list(Path(file_directory).rglob("*.json"))
+        if not file_list:
+            raise FileNotFoundError(f"No files found in {file_directory}")
+    elif file_name is not None:
+        file_list = [file_name]
+
+    return file_list
+
+
+def validate_dict_using_schema(args_dict, logger):
     """
-    Validate a schema file given in yaml or json format.
+    Validate a schema file (or several files) given in yaml or json format.
 
     Schema is either given as command line argument, read from the meta_schema_url or from
     the metadata section of the data dictionary.
 
     """
-    try:
-        data = gen.collect_data_from_file(file_name=args_dict["file_name"])
-    except FileNotFoundError as exc:
-        logger.error(f"Error reading schema file from {args_dict['file_name']}")
-        raise exc
-    metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data))
-    logger.info(f"Successful validation of schema file {args_dict['file_name']}")
+    for file_name in _get_json_file_list(
+        args_dict.get("file_directory"), args_dict.get("file_name")
+    ):
+        try:
+            data = gen.collect_data_from_file(file_name=file_name)
+        except FileNotFoundError as exc:
+            logger.error(f"Error reading schema file from {file_name}")
+            raise exc
+        schema.validate_dict_using_schema(data, _get_schema_file_name(args_dict, data))
+        logger.info(f"Successful validation of file {file_name}")
 
 
 def validate_data_files(args_dict, logger):
     """Validate data files."""
-    model_parameters_directory = args_dict.get("model_parameters_directory")
-    if model_parameters_directory is not None:
+    if args_dict.get("file_directory") is not None:
         tmp_args_dict = {}
-        for file_name in Path(model_parameters_directory).rglob("*.json"):
+        for file_name in _get_json_file_list(args_dict.get("file_directory")):
             tmp_args_dict["file_name"] = file_name
-            schema_file = (
-                files("simtools") / "schemas/model_parameters" / f"{file_name.stem}.schema.yml"
-            )
+            parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem)
+            schema_file = schema.get_model_parameter_schema_file(f"{parameter_name}")
             tmp_args_dict["schema"] = schema_file
             tmp_args_dict["data_type"] = "model_parameter"
             tmp_args_dict["require_exact_data_type"] = args_dict["require_exact_data_type"]
@@ -182,7 +196,7 @@ def main():  # noqa: D103
     if args_dict["data_type"].lower() == "metadata":
         validate_metadata(args_dict, logger)
     elif args_dict["data_type"].lower() == "schema":
-        validate_schema(args_dict, logger)
+        validate_dict_using_schema(args_dict, logger)
     else:
         validate_data_files(args_dict, logger)
 

simtools/applications/validate_optics.py

@@ -78,6 +78,7 @@ from simtools.configuration import configurator
 from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 from simtools.ray_tracing.ray_tracing import RayTracing
+from simtools.visualization import visualize
 
 
 def _parse(label):
@@ -114,7 +115,7 @@ def _parse(label):
         help="Produce a multiple pages pdf file with the image plots.",
         action="store_true",
     )
-    return config.initialize(db_config=True, simulation_model=["telescope"])
+    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
 def main():  # noqa: D103
@@ -171,9 +172,7 @@ def main():  # noqa: D103
 
         plot_file_name = "_".join((label, tel_model.name, key))
         plot_file = output_dir.joinpath(plot_file_name)
-        plt.savefig(str(plot_file) + ".pdf", format="pdf", bbox_inches="tight")
-        plt.savefig(str(plot_file) + ".png", format="png", bbox_inches="tight")
-        plt.clf()
+        visualize.save_figure(plt, plot_file)
 
     # Plotting images
     if args_dict["plot_images"]:

simtools/{camera_efficiency.py → camera/camera_efficiency.py}

@@ -514,7 +514,4 @@ class CameraEfficiency:
         plot_file = self.output_dir.joinpath(
             self.label + "_" + self.telescope_model.name + "_" + plot_title
         )
-        for f in ["pdf", "png"]:
-            fig.savefig(str(plot_file) + "." + f, format=f, bbox_inches="tight")
-        self._logger.info(f"Plotted {plot_title} efficiency in {plot_file}")
-        fig.clf()
+        visualize.save_figure(fig, plot_file, log_title=f"{plot_title} efficiency")