gammasimtools 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

simtools/applications/generate_array_config.py

@@ -5,10 +5,6 @@
     The applications generates the sim_telarray configuration files for a given array, site,
     and model_version using the model parameters stored in the database.
 
-    To change model parameters, clone the model parameters repository and apply the changes.
-    Forward the path to the repository to the application using the ``--db_simulation_model_url``
-    argument.
-
     Command line arguments
     ----------------------
     site : str
@@ -54,7 +50,7 @@ def _parse(label, description):
         Command line parser object.
     """
     config = configurator.Configurator(label=label, description=description)
-    return config.initialize(db_config=True, simulation_model=["site", "layout"])
+    return config.initialize(db_config=True, simulation_model=["site", "layout", "model_version"])
 
 
 def main():

simtools/applications/generate_regular_arrays.py

@@ -25,7 +25,6 @@
 """
 
 import logging
-import os
 from pathlib import Path
 
 import astropy.units as u
@@ -52,7 +51,9 @@ def _parse():
             f"  SST: {telescope_distance['SST']}\n"
         ),
     )
-    return config.initialize(db_config=False, simulation_model="site", output=True)
+    return config.initialize(
+        db_config=False, simulation_model=["site", "model_version"], output=True
+    )
 
 
 def main():
@@ -101,10 +102,12 @@ def main():
         table.sort("telescope_name")
         table.pprint()
 
-        output_file = args_dict.get("output_file", None)
-        if output_file is not None:
-            base_name, file_extension = os.path.splitext(output_file)
-            output_file = f"{base_name}-{args_dict['site']}-{array_name}{file_extension}"
+        output_file = args_dict.get("output_file")
+        if output_file:
+            output_path = Path(output_file)
+            output_file = output_path.with_name(
+                f"{output_path.stem}-{args_dict['site']}-{array_name}{output_path.suffix}"
+            )
         writer.ModelDataWriter.dump(
             args_dict=args_dict,
             output_file=output_file,

simtools/applications/generate_simtel_array_histograms.py

@@ -154,7 +154,7 @@ def check_and_log_overwrite(config_parser, logger):
     Returns
     -------
     bool
-        True if the hdf5 file exists and should be overwritten, False otherwise.
+        True if the hdf5 file exists and should be overwritten.
     """
     if Path(f"{config_parser['output_file_name']}.hdf5").exists() and config_parser["hdf5"]:
         msg = (

simtools/applications/plot_array_layout.py

@@ -146,7 +146,9 @@ def _parse(label, description, usage):
         required=False,
         default=None,
     )
-    return config.initialize(db_config=True, simulation_model=["site", "layout", "layout_file"])
+    return config.initialize(
+        db_config=True, simulation_model=["site", "model_version", "layout", "layout_file"]
+    )
 
 
 def _get_site_from_telescope_list_name(telescope_list_file):

simtools/applications/plot_tabular_data.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python3
+"""
+Plot tabular data read from file or from model parameter database.
+
+Uses a configuration file to define the data to be plotted and all
+plotting details.
+
+Command line arguments
+----------------------
+config_file (str, required)
+    Configuration file name for plotting.
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.io_operations import io_handler
+from simtools.visualization import plot_tables
+
+
+def _parse(label, description, usage):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing the application.
+    description : str
+        Description of the application.
+    usage : str
+        Example on how to use the application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description, usage=usage)
+
+    config.parser.add_argument(
+        "--plot_config",
+        help="Plotting configuration file name.",
+        type=str,
+        required=True,
+        default=None,
+    )
+    config.parser.add_argument(
+        "--output_file",
+        help="Output file name (without suffix)",
+        type=str,
+        required=True,
+    )
+    return config.initialize(db_config=True)
+
+
+def main():
+    """Plot tabular data."""
+    args_dict, db_config_ = _parse(
+        label=Path(__file__).stem,
+        description="Plots tabular data.",
+        usage="""simtools-plot-tabular-data --plot_config config_file_name "
+                 --output_file output_file_name""",
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict.get("log_level", "INFO")))
+    io_handler_instance = io_handler.IOHandler()
+
+    plot_config = gen.convert_keys_in_dict_to_lowercase(
+        gen.collect_data_from_file(args_dict["plot_config"])
+    )
+
+    plot_tables.plot(
+        config=plot_config["cta_simpipe"]["plot"],
+        output_file=io_handler_instance.get_output_file(args_dict["output_file"]),
+        db_config=db_config_,
+    )
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/production_scale_events.py

@@ -36,7 +36,6 @@ The output will display the scaled events for the specified grid point.
 
 import json
 import logging
-import os
 from pathlib import Path
 
 import astropy.units as u
@@ -129,7 +128,7 @@ def main():
                 # Build file path based on base_path, zenith, and offset
                 file_name = f"prod6_LaPalma-{int(zenith.value)}deg_"
                 file_name += "gamma_cone.N.Am-4LSTs09MSTs_ID0_reduced.fits"
-                file_path = os.path.join(args_dict["base_path"], file_name)
+                file_path = Path(args_dict["base_path"]).joinpath(file_name)
 
                 evaluator = StatisticalErrorEvaluator(
                     file_path,

simtools/applications/simulate_light_emission.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-"""
+r"""
     Simulate calibration devices using the light emission package.
 
     Run the application in the command line.
@@ -235,7 +235,7 @@ def _parse(label):
     )
     return config.initialize(
         db_config=True,
-        simulation_model="telescope",
+        simulation_model=["telescope", "model_version"],
         require_command_line=True,
     )
 

simtools/applications/simulate_prod.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
 r"""
-    Generate simulation configuration and run simulations (if required).
+    Generate simulation configuration and run simulations.
 
     Multipipe scripts will be produced as part of this application.
     Allows to run array layout simulation including shower and detector simulations
@@ -43,9 +43,9 @@ r"""
         The location of the output directories corsika-data and simtel-data.
         the label is added to the data_directory, such that the output
         will be written to data_directory/label/simtel-data.
-    pack_for_grid_register (bool, optional)
+    pack_for_grid_register (str, optional)
         Set whether to prepare a tarball for registering the output files on the grid.
-        The files are written to the output_path/directory_for_grid_upload directory.
+        The files are written to the specified directory.
     log_level (str, optional)
         Log level to print.
 
@@ -68,9 +68,6 @@ r"""
 """
 
 import logging
-import shutil
-import tarfile
-from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
@@ -106,10 +103,10 @@ def _parse(description=None):
     )
     config.parser.add_argument(
         "--pack_for_grid_register",
-        help="Set whether to prepare a tarball for registering the output files on the grid.",
-        action="store_true",
+        help="Directory for a tarball for registering the output files on the grid.",
+        type=str,
         required=False,
-        default=False,
+        default=None,
     )
     config.parser.add_argument(
         "--save_file_lists",
@@ -137,49 +134,11 @@ def _parse(description=None):
     return config.initialize(
         db_config=True,
         job_submission=True,
-        simulation_model=["site", "layout", "telescope"],
+        simulation_model=["site", "layout", "telescope", "model_version"],
         simulation_configuration={"software": None, "corsika_configuration": ["all"]},
     )
 
 
-def pack_for_register(logger, simulator, args_dict):
-    """
-    Pack the output files for registering on the grid.
-
-    Parameters
-    ----------
-    logger: logging.Logger
-        Logger object.
-    simulator: Simulator
-        Simulator object.
-
-    """
-    logger.info("Packing the output files for registering on the grid")
-    output_files = simulator.get_file_list(file_type="output")
-    log_files = simulator.get_file_list(file_type="log")
-    histogram_files = simulator.get_file_list(file_type="hist")
-    tar_file_name = Path(log_files[0]).name.replace("log.gz", "log_hist.tar.gz")
-    directory_for_grid_upload = Path(args_dict.get("output_path")).joinpath(
-        "directory_for_grid_upload"
-    )
-    directory_for_grid_upload.mkdir(parents=True, exist_ok=True)
-    tar_file_name = directory_for_grid_upload.joinpath(tar_file_name)
-
-    with tarfile.open(tar_file_name, "w:gz") as tar:
-        files_to_tar = log_files[:1] + histogram_files[:1]
-        for file_to_tar in files_to_tar:
-            tar.add(file_to_tar, arcname=Path(file_to_tar).name)
-
-    for file_to_move in [*output_files]:
-        source_file = Path(file_to_move)
-        destination_file = directory_for_grid_upload / source_file.name
-        # Note that this will overwrite previous files which exist in the directory
-        # It should be fine for normal production since each run is on a separate node
-        # so no files are expected there.
-        shutil.move(source_file, destination_file)
-    logger.info(f"Output files for the grid placed in {directory_for_grid_upload!s}")
-
-
 def main():  # noqa: D103
     args_dict, db_config = _parse(description="Run simulations for productions")
 
@@ -192,17 +151,23 @@ def main():  # noqa: D103
 
     simulator.simulate()
 
-    logger.info(
-        f"Production run is complete for primary {args_dict['primary']} showers "
-        f"coming from {args_dict['azimuth_angle']} azimuth and zenith angle of "
-        f"{args_dict['zenith_angle']} at the {args_dict['site']} site, "
-        f"using the {args_dict['model_version']} simulation model."
-    )
-
-    if args_dict["pack_for_grid_register"]:
-        pack_for_register(logger, simulator, args_dict)
-    if args_dict["save_file_lists"]:
-        simulator.save_file_lists()
+    if simulator.submit_engine == "local":
+        logger.info(
+            f"Production run complete for primary {args_dict['primary']} showers "
+            f"from {args_dict['azimuth_angle']} azimuth and {args_dict['zenith_angle']} zenith "
+            f"at {args_dict['site']} site, using {args_dict['model_version']} model."
+        )
+        if args_dict.get("pack_for_grid_register"):
+            simulator.pack_for_register(args_dict["pack_for_grid_register"])
+        if args_dict["save_file_lists"]:
+            simulator.save_file_lists()
+    else:
+        logger.info("Production run submitted to the workload manager")
+        if args_dict["pack_for_grid_register"] or args_dict["save_file_lists"]:
+            logger.warning(
+                "Packing for grid register or saving file lists not supported for "
+                f"{simulator.submit_engine}."
+            )
 
 
 if __name__ == "__main__":

simtools/applications/simulate_prod_htcondor_generator.py

@@ -0,0 +1,95 @@
+#!/usr/bin/python3
+
+r"""
+    Generate a run script and submit file for HT Condor job submission of a simulation production.
+
+    This tool facilitates the submission of multiple simulations to the HT Condor batch system,
+    enabling:
+
+    - Execution of simulations using the "simtools-simulate-prod" application.
+    - 'number_of_runs' jobs are submitted to the HT Condor batch system.
+    - Utilization of an Apptainer image containing the SimPipe simulation software and tools.
+    - Packaging of data and histogram files, and writing them to a specified directory.
+
+    This tool is intended for use in an HT Condor environment. Jobs run in a container universe
+    using the Apptainer image specified in the command line ('--apptainer_image'). Output is written
+    to the 'output_path' directory, with 'simtools-output' and 'logs' subdirectories.
+
+    Requirements for the 'simtools-simulate-prod-htcondor-generator' application:
+
+    - Availability of an Apptainer image 'simtools-prod' (obtainable from the package registry on
+      GitHub, e.g., via 'apptainer pull --force docker://ghcr.io/gammasim/simtools-prod:latest').
+    - Environment parameters required to run CORSIKA and sim_telarray, as well as DB access
+      credentials.  These should be listed similarly to a '.env' file and copied to
+      'output_path/env.txt'.  Ensure that the path to the simulation software is correctly set to
+      'SIMTOOLS_SIMTEL_PATH=/workdir/sim_telarray'.
+
+
+    Command line arguments
+    ----------------------
+    output_path (str, required)
+        Directory where the output and the simulation data files will be written.
+    apptainer_image (str, optional)
+        Apptainer image to use for the simulation (full path).
+    priority (int, optional)
+        Job priority (default: 1).
+
+    (all other command line arguments are identical to those of :ref:`simulate_prod`).
+
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.job_execution import htcondor_script_generator
+
+
+def _parse(description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    description: str
+        Application description.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+
+    """
+    config = configurator.Configurator(description=description)
+    config.parser.add_argument(
+        "--apptainer_image",
+        help="Apptainer image to use for the simulation (full path).",
+        type=str,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--priority",
+        help="Job priority.",
+        type=int,
+        required=False,
+        default=1,
+    )
+    return config.initialize(
+        db_config=False,
+        job_submission=False,
+        simulation_model=["site", "layout", "telescope", "model_version"],
+        simulation_configuration={"software": None, "corsika_configuration": ["all"]},
+    )
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse(description="Prepare simulations production for HT Condor job submission")
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    htcondor_script_generator.generate_submission_script(args_dict)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/submit_data_from_external.py

@@ -104,7 +104,7 @@ def main():  # noqa: D103
 
     writer.ModelDataWriter.dump(
         args_dict=args_dict,
-        metadata=_metadata.top_level_meta,
+        metadata=_metadata.get_top_level_metadata(),
         product_data=data_validator.validate_and_transform(),
     )
 

simtools/applications/validate_camera_efficiency.py

@@ -86,7 +86,7 @@ def _parse(label):
     )
     _args_dict, _db_config = config.initialize(
         db_config=True,
-        simulation_model="telescope",
+        simulation_model=["telescope", "model_version"],
         simulation_configuration={"corsika_configuration": ["zenith_angle", "azimuth_angle"]},
     )
     if _args_dict["site"] is None or _args_dict["telescope"] is None:

simtools/applications/validate_camera_fov.py

@@ -34,7 +34,7 @@ r"""
     .. code-block:: console
 
         simtools-validate-camera-fov --site North \\
-            --telescope LST-1 --model_version 5.0.0
+            --telescope LSTN-01 --model_version 5.0.0
 
     The output is saved in simtools-output/validate_camera_fov.
 
@@ -54,7 +54,7 @@ import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
-from simtools.visualization import plot_camera
+from simtools.visualization import plot_camera, visualize
 
 
 def _parse():
@@ -83,7 +83,7 @@ def _parse():
         ),
         default=50,
     )
-    return config.initialize(db_config=True, simulation_model="telescope")
+    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
 def main():  # noqa: D103
@@ -133,11 +133,7 @@ def main():  # noqa: D103
             ) from exc
     fig = plot_camera.plot_pixel_layout(camera, args_dict["camera_in_sky_coor"], pixel_ids_to_print)
     plot_file_prefix = output_dir.joinpath(f"{label}_{tel_model.name}_pixel_layout")
-    for suffix in ["pdf", "png"]:
-        file_name = f"{plot_file_prefix!s}.{suffix}"
-        fig.savefig(file_name, format=suffix, bbox_inches="tight")
-        print(f"\nSaved camera plot in {file_name}\n")
-    fig.clf()
+    visualize.save_figure(fig, f"{plot_file_prefix!s}", log_title="camera")
 
 
 if __name__ == "__main__":

simtools/applications/validate_cumulative_psf.py

@@ -110,7 +110,7 @@ def _parse(label):
         help="Data file name with the measured PSF vs radius [cm]",
         type=str,
     )
-    return config.initialize(db_config=True, simulation_model="telescope")
+    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
 def load_data(datafile):
@@ -182,9 +182,7 @@ def main():  # noqa: D103
 
     plot_file_name = label + "_" + tel_model.name + "_cumulative_PSF"
     plot_file = output_dir.joinpath(plot_file_name)
-    for f in ["pdf", "png"]:
-        plt.savefig(str(plot_file) + "." + f, format=f, bbox_inches="tight")
-    fig.clf()
+    visualize.save_figure(fig, plot_file)
 
     # Plotting image
     data_to_plot = im.get_image_data()
@@ -195,9 +193,7 @@ def main():  # noqa: D103
 
     plot_file_name = label + "_" + tel_model.name + "_image"
     plot_file = output_dir.joinpath(plot_file_name)
-    for f in ["pdf", "png"]:
-        fig.savefig(str(plot_file) + "." + f, format=f, bbox_inches="tight")
-    fig.clf()
+    visualize.save_figure(fig, plot_file)
 
 
 if __name__ == "__main__":

simtools/applications/validate_file_using_schema.py

@@ -33,10 +33,14 @@ r"""
 """
 
 import logging
+import re
 from pathlib import Path
 
+import jsonschema
+
 import simtools.utils.general as gen
 from simtools.configuration import configurator
+from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
 from simtools.data_model import metadata_collector, metadata_model, validate_data
 
 
@@ -61,13 +65,12 @@ def _parse(label, description):
     group = config.parser.add_mutually_exclusive_group(required=True)
     group.add_argument("--file_name", help="File to be validated")
     group.add_argument(
-        "--model_parameters_directory",
+        "--file_directory",
         help=(
-            "Directory with json files with model parameters to be validated."
-            "All *.json files in the directory will be validated."
-            "Schema files will be taken from simtools/schemas/model_parameters/."
-            "Note that in this case the data_type argument is ignored"
-            "and data_type=model_parameter is always used."
+            "Directory with json files to be validated. "
+            "If no schema file is provided, the assumption is that model "
+            "parameters are validated and the schema files are taken from "
+            f"{MODEL_PARAMETER_SCHEMA_PATH}."
         ),
     )
     config.parser.add_argument("--schema", help="Json schema file", required=False)
@@ -115,34 +118,39 @@ def _get_schema_file_name(args_dict, data_dict=None):
 
 def validate_schema(args_dict, logger):
     """
-    Validate a schema file given in yaml or json format.
+    Validate a schema file (or several files) given in yaml or json format.
 
     Schema is either given as command line argument, read from the meta_schema_url or from
     the metadata section of the data dictionary.
 
     """
-    try:
-        data = gen.collect_data_from_file(file_name=args_dict["file_name"])
-    except FileNotFoundError as exc:
-        logger.error(f"Error reading schema file from {args_dict['file_name']}")
-        raise exc
-    metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data))
-    logger.info(f"Successful validation of schema file {args_dict['file_name']}")
+    if args_dict.get("file_directory") is not None:
+        file_list = list(Path(args_dict["file_directory"]).rglob("*.json"))
+    else:
+        file_list = [args_dict["file_name"]]
+    for file_name in file_list:
+        try:
+            data = gen.collect_data_from_file(file_name=file_name)
+        except FileNotFoundError as exc:
+            logger.error(f"Error reading schema file from {file_name}")
+            raise exc
+        try:
+            metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data))
+        except jsonschema.exceptions.ValidationError as exc:
+            logger.error(f"Failed validation of file {file_name}")
+            raise exc
+        logger.info(f"Successful validation of file {file_name}")
 
 
 def validate_data_files(args_dict, logger):
     """Validate data files."""
-    model_parameters_directory = args_dict.get("model_parameters_directory")
-    if model_parameters_directory is not None:
+    file_directory = args_dict.get("file_directory")
+    if file_directory is not None:
         tmp_args_dict = {}
-        for file_name in Path(model_parameters_directory).rglob("*.json"):
+        for file_name in Path(file_directory).rglob("*.json"):
             tmp_args_dict["file_name"] = file_name
-            schema_file = (
-                Path(__file__).parent
-                / "../schemas"
-                / "model_parameters"
-                / f"{file_name.stem}.schema.yml"
-            )
+            parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem)
+            schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml"
             tmp_args_dict["schema"] = schema_file
             tmp_args_dict["data_type"] = "model_parameter"
             tmp_args_dict["require_exact_data_type"] = args_dict["require_exact_data_type"]

simtools/applications/validate_optics.py

@@ -78,6 +78,7 @@ from simtools.configuration import configurator
 from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
 from simtools.ray_tracing.ray_tracing import RayTracing
+from simtools.visualization import visualize
 
 
 def _parse(label):
@@ -114,7 +115,7 @@ def _parse(label):
         help="Produce a multiple pages pdf file with the image plots.",
         action="store_true",
     )
-    return config.initialize(db_config=True, simulation_model=["telescope"])
+    return config.initialize(db_config=True, simulation_model=["telescope", "model_version"])
 
 
 def main():  # noqa: D103
@@ -171,9 +172,7 @@ def main():  # noqa: D103
 
         plot_file_name = "_".join((label, tel_model.name, key))
         plot_file = output_dir.joinpath(plot_file_name)
-        plt.savefig(str(plot_file) + ".pdf", format="pdf", bbox_inches="tight")
-        plt.savefig(str(plot_file) + ".png", format="png", bbox_inches="tight")
-        plt.clf()
+        visualize.save_figure(plt, plot_file)
 
     # Plotting images
     if args_dict["plot_images"]:

simtools/camera_efficiency.py

@@ -514,7 +514,4 @@ class CameraEfficiency:
         plot_file = self.output_dir.joinpath(
             self.label + "_" + self.telescope_model.name + "_" + plot_title
         )
-        for f in ["pdf", "png"]:
-            fig.savefig(str(plot_file) + "." + f, format=f, bbox_inches="tight")
-        self._logger.info(f"Plotted {plot_title} efficiency in {plot_file}")
-        fig.clf()
+        visualize.save_figure(fig, plot_file, log_title=f"{plot_title} efficiency")