gammasimtools 0.6.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

simtools/applications/simulate_prod.py

@@ -1,35 +1,27 @@
 #!/usr/bin/python3
 
-"""
-    Summary
-    -------
-    This application is used to run simulations for productions (typically on the grid).
-    It allows to run a Paranal (CTAO-South) or La Palma (CTAO-North) array layout simulation
-    with the provided "prod_tag" simulation configuration (e.g., Prod6)
-    for a given primary particle, azimuth, and zenith angle.
+r"""
+    Generate simulation configuration and run simulations (if required).
 
-    The entire simulation chain is performed, i.e., shower simulations with CORSIKA
-    which are piped directly to sim_telarray using the sim_telarray multipipe mechanism.
-    This script produces all the necessary configuration files for CORSIKA and
-    sim_telarray before running simulation.
-    The multipipe scripts will be produced as part of this script.
+    Multipipe scripts will be produced as part of this application.
+    Allows to run array layout simulation including shower and detector simulations
 
-    This script does not provide a mechanism to submit jobs to a batch system like others
-    in simtools since it is meant to be executed on a grid node
-    (distributed to it by the workload management system).
+    The entire simulation chain, parts of it, or nothing is executed:
+
+    - shower simulations with CORSIKA only
+    - shower simulations with CORSIKA which are piped directly to sim_telarray using
+      the sim_telarray multipipe mechanism.
 
     Command line arguments
     ----------------------
-    production_config (str, Path, required)
-        Simulation configuration file
-        (contains the default setup which can be overwritten by the command line options).
     model_version (str, required)
-        The telescope model version to use (e.g., Prod5).
+        The telescope model version to use (e.g., 5.0.0).
     site (str, required)
         North or South (case insensitive).
     primary (str, required)
-        Name of the primary particle to simulate. The available options are
-        gamma, gamma_diffuse, electron, proton, muon, helium, nitrogen, silicon, and iron.
+        Name or ID of the primary particle to simulate. Allowed are common names like gamma, proton,
+        or IDs for CORSIKA7 (e.g. 14 for proton) and PDG (e.g. 2212 for proton). Use the
+        'primary_id_type' option to specify the type of ID.
     azimuth_angle (str or float, required)
         Telescope pointing direction in azimuth.
         It can be in degrees between 0 and 360 or one of north, south, east or west
@@ -50,10 +42,10 @@
     data_directory (str, optional)
         The location of the output directories corsika-data and simtel-data.
         the label is added to the data_directory, such that the output
-        will be written to `data_directory/label/simtel-data`.
+        will be written to data_directory/label/simtel-data.
     pack_for_grid_register (bool, optional)
         Set whether to prepare a tarball for registering the output files on the grid.
-        The files are written to the `output_path/directory_for_grid_upload` directory.
+        The files are written to the output_path/directory_for_grid_upload directory.
     log_level (str, optional)
         Log level to print.
 
@@ -63,35 +55,16 @@
 
     .. code-block:: console
 
-        simtools-simulate-prod \
-        --production_config tests/resources/prod_multi_config_test.yml --model_version Prod5 \
-        --site north --primary gamma --azimuth_angle north --zenith_angle 20 \
-        --start_run 0 --run 1
+        simtools-simulate-prod \\
+        --model_version 5.0.0 --site north --primary gamma --azimuth_angle north \\
+        --zenith_angle 20 --start_run 0 --run 1
 
     By default the configuration is saved in simtools-output/test-production
     together with the actual simulation output in corsika-data and simtel-data within.
     The location of the latter directories can be set
     to a different location via the option --data_directory,
     but the label is always added to the data_directory, such that the output
-    will be written to `data_directory/label/simtel-data`.
-
-    Expected final print-out message:
-
-    .. code-block:: console
-
-        INFO::array_layout(l569)::read_telescope_list_file::Reading array elements from ...
-        WARNING::corsika_runner(l127)::_load_corsika_config_data::data_directory not given
-        in corsika_config - default output directory will be set.
-        INFO::array_layout(l569)::read_telescope_list_file::Reading array elements from ...
-        INFO::corsika_config(l493)::_set_output_file_and_directory::Creating directory
-        INFO::simulator(l405)::simulate::Submission command: local
-        INFO::simulator(l410)::simulate::Starting submission for 1 run
-        INFO::array_model(l315)::export_simtel_array_config_file::Writing array config file into
-        INFO::job_manager(l95)::submit::Submitting script
-        INFO::job_manager(l96)::submit::Job output stream
-        INFO::job_manager(l97)::submit::Job error stream
-        INFO::job_manager(l98)::submit::Job log stream
-        INFO::job_manager(l119)::_submit_local::Running script locally
+    will be written to data_directory/label/simtel-data.
 """
 
 import logging
@@ -99,22 +72,19 @@ import shutil
 import tarfile
 from pathlib import Path
 
-from astropy.io.misc import yaml
-
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.configuration.commandline_parser import CommandLineParser
 from simtools.simulator import Simulator
 
 
 def _parse(description=None):
     """
-    Parse the command line configuration.
+    Parse command line configuration.
 
     Parameters
     ----------
     description: str
-        Description of the application.
+        Application description.
 
     Returns
     -------
@@ -122,165 +92,117 @@ def _parse(description=None):
         Command line parser object.
 
     """
-
     config = configurator.Configurator(description=description)
     config.parser.add_argument(
-        "--production_config",
+        "--data_directory",
         help=(
-            "Simulation configuration file "
-            "(contains the default setup which can be overwritten by the command line options)"
+            "The directory where to save the corsika-data and simtel-data output directories."
+            "the label is added to the data_directory, such that the output"
+            "will be written to data_directory/label/simtel-data."
         ),
-        type=str,
-        required=True,
-    )
-    config.parser.add_argument(
-        "--primary",
-        help="Primary particle to simulate.",
         type=str.lower,
-        required=True,
-        choices=[
-            "gamma",
-            "gamma_diffuse",
-            "electron",
-            "proton",
-            "muon",
-            "helium",
-            "nitrogen",
-            "silicon",
-            "iron",
-        ],
-    )
-    config.parser.add_argument(
-        "--azimuth_angle",
-        help=(
-            "Telescope pointing direction in azimuth. "
-            "It can be in degrees between 0 and 360 or one of north, south, east or west "
-            "(case insensitive). Note that North is 0 degrees and "
-            "the azimuth grows clockwise, so East is 90 degrees."
-        ),
-        type=CommandLineParser.azimuth_angle,
-        required=True,
-    )
-    config.parser.add_argument(
-        "--zenith_angle",
-        help="Zenith angle in degrees (between 0 and 180).",
-        type=CommandLineParser.zenith_angle,
-        required=True,
+        required=False,
+        default="./simtools-output/",
     )
     config.parser.add_argument(
-        "--nshow",
-        help="Number of showers to simulate.",
-        type=int,
+        "--pack_for_grid_register",
+        help="Set whether to prepare a tarball for registering the output files on the grid.",
+        action="store_true",
         required=False,
+        default=False,
     )
     config.parser.add_argument(
-        "--start_run",
-        help=(
-            "Start run number such that the actual run number will be 'start_run' + 'run'. "
-            "This is useful in case a new transform is submitted for the same production. "
-            "It allows the transformation system to keep using sequential run numbers without "
-            "repetition."
-        ),
-        type=int,
-        required=True,
+        "--save_file_lists",
+        help="Save lists of output and log files.",
+        action="store_true",
+        required=False,
+        default=False,
     )
     config.parser.add_argument(
-        "--run",
-        help="Run number (actual run number will be 'start_run' + 'run')",
-        type=int,
-        required=True,
+        "--corsika_test_seeds",
+        help="Use predefined random seeds for CORSIKA for testing purposes.",
+        action="store_true",
+        required=False,
+        default=False,
     )
     config.parser.add_argument(
-        "--data_directory",
+        "--sim_telarray_seeds",
         help=(
-            "The directory where to save the corsika-data and simtel-data output directories."
-            "the label is added to the data_directory, such that the output"
-            "will be written to `data_directory/label/simtel-data`."
+            "Use the provided random seed for sim_telarray instrument setup."
+            "Used only for testing purposes for now."
         ),
-        type=str.lower,
+        type=str,
         required=False,
-        default="./simtools-output/",
     )
-    config.parser.add_argument(
-        "--pack_for_grid_register",
-        help="Set whether to prepare a tarball for registering the output files on the grid.",
-        action="store_true",
-        required=False,
-        default=False,
+    return config.initialize(
+        db_config=True,
+        job_submission=True,
+        simulation_model=["site", "layout", "telescope"],
+        simulation_configuration={"software": None, "corsika_configuration": ["all"]},
     )
-    return config.initialize(db_config=True, telescope_model=True)
 
 
-def main():
-    args_dict, db_config = _parse(description="Run simulations for productions")
+def pack_for_register(logger, simulator, args_dict):
+    """
+    Pack the output files for registering on the grid.
 
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+    Parameters
+    ----------
+    logger: logging.Logger
+        Logger object.
+    simulator: Simulator
+        Simulator object.
 
-    try:
-        with open(args_dict["production_config"], encoding="utf-8") as file:
-            config_data = yaml.load(file)
-    except FileNotFoundError:
-        logger.error(
-            f"Error loading simulation configuration file from {args_dict['production_config']}"
-        )
-        raise
+    """
+    logger.info("Packing the output files for registering on the grid")
+    output_files = simulator.get_file_list(file_type="output")
+    log_files = simulator.get_file_list(file_type="log")
+    histogram_files = simulator.get_file_list(file_type="hist")
+    tar_file_name = Path(log_files[0]).name.replace("log.gz", "log_hist.tar.gz")
+    directory_for_grid_upload = Path(args_dict.get("output_path")).joinpath(
+        "directory_for_grid_upload"
+    )
+    directory_for_grid_upload.mkdir(parents=True, exist_ok=True)
+    tar_file_name = directory_for_grid_upload.joinpath(tar_file_name)
+
+    with tarfile.open(tar_file_name, "w:gz") as tar:
+        files_to_tar = log_files[:1] + histogram_files[:1]
+        for file_to_tar in files_to_tar:
+            tar.add(file_to_tar, arcname=Path(file_to_tar).name)
 
-    # Overwrite default and optional settings
-    config_data["showers"]["run_list"] = args_dict["run"] + args_dict["start_run"]
-    config_data["showers"]["primary"] = args_dict["primary"]
-    config_data["common"]["site"] = args_dict["site"]
-    config_data["common"]["zenith"] = args_dict["zenith_angle"]
-    config_data["common"]["phi"] = args_dict["azimuth_angle"]
-    label = config_data["common"].pop("label", "test-production")
-    config_data["common"]["data_directory"] = Path(args_dict["data_directory"]) / label
+    for file_to_move in [*output_files]:
+        source_file = Path(file_to_move)
+        destination_file = directory_for_grid_upload / source_file.name
+        # Note that this will overwrite previous files which exist in the directory
+        # It should be fine for normal production since each run is on a separate node
+        # so no files are expected there.
+        shutil.move(source_file, destination_file)
+    logger.info(f"Output files for the grid placed in {directory_for_grid_upload!s}")
 
-    if args_dict["nshow"] is not None:
-        config_data["showers"]["nshow"] = args_dict["nshow"]
-    if args_dict["label"] is not None:
-        label = args_dict["label"]
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse(description="Run simulations for productions")
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     simulator = Simulator(
-        label=label,
-        simulator="corsika_simtel",
-        simulator_source_path=args_dict["simtel_path"],
-        config_data=config_data,
-        submit_command="local",
-        test=args_dict["test"],
-        mongo_db_config=db_config,
+        label=args_dict.get("label"), args_dict=args_dict, mongo_db_config=db_config
     )
 
     simulator.simulate()
 
     logger.info(
-        f"Production run is complete for primary {config_data['showers']['primary']} showers "
-        f"coming from {config_data['common']['phi']} azimuth and zenith angle of "
-        f"{config_data['common']['zenith']} at the {args_dict['site']} site, "
-        f"using the {config_data['array']['model_version']} telescope model."
+        f"Production run is complete for primary {args_dict['primary']} showers "
+        f"coming from {args_dict['azimuth_angle']} azimuth and zenith angle of "
+        f"{args_dict['zenith_angle']} at the {args_dict['site']} site, "
+        f"using the {args_dict['model_version']} simulation model."
     )
 
     if args_dict["pack_for_grid_register"]:
-        logger.info("Packing the output files for registering on the grid")
-        output_files = simulator.get_list_of_output_files()
-        log_files = simulator.get_list_of_log_files()
-        histogram_files = simulator.get_list_of_histogram_files()
-        tar_file_name = Path(log_files[0]).name.replace("log.gz", "log_hist.tar.gz")
-        with tarfile.open(tar_file_name, "w:gz") as tar:
-            files_to_tar = log_files[:1] + histogram_files[:1]
-            for file_to_tar in files_to_tar:
-                tar.add(file_to_tar, arcname=Path(file_to_tar).name)
-        directory_for_grid_upload = Path(args_dict.get("output_path")).joinpath(
-            "directory_for_grid_upload"
-        )
-        directory_for_grid_upload.mkdir(parents=True, exist_ok=True)
-        for file_to_move in [*output_files, tar_file_name]:
-            source_file = Path(file_to_move)
-            destination_file = directory_for_grid_upload / source_file.name
-            # Note that this will overwrite previous files which exist in the directory
-            # It should be fine for normal production since each run is on a separate node
-            # so no files are expected there.
-            shutil.move(source_file, destination_file)
-        logger.info(f"Output files for the grid placed in {str(directory_for_grid_upload)}")
+        pack_for_register(logger, simulator, args_dict)
+    if args_dict["save_file_lists"]:
+        simulator.save_file_lists()
 
 
 if __name__ == "__main__":

simtools/applications/submit_data_from_external.py

@@ -1,8 +1,6 @@
 #!/usr/bin/python3
-"""
-    Summary
-    -------
-    Submit model parameter (value, table) through the command line.
+r"""
+    Submit data file through the command line.
 
     Input data and metadata is validated, and if necessary enriched
     and converted following a pre-described schema.
@@ -23,17 +21,17 @@
 
     .. code-block:: console
 
-        simtools-submit-data-from-external \
-            --input_meta ./tests/resources/MLTdata-preproduction.meta.yml \
-            --input ./tests/resources/MLTdata-preproduction.ecsv \
-            --schema ./tests/resources/schema_MST_mirror_2f_measurements.yml \
+        simtools-submit-data-from-external \\
+            --input_meta ./tests/resources/MLTdata-preproduction.meta.yml \\
+            --input ./tests/resources/MLTdata-preproduction.ecsv \\
+            --schema ./tests/resources/schema_MST_mirror_2f_measurements.yml \\
             --output_file TEST-submit_data_from_external.ecsv
 
     Expected final print-out message:
 
     .. code-block:: console
 
-        INFO::model_data_writer(l70)::write_data::Writing data to \
+        INFO::model_data_writer(l70)::write_data::Writing data to \\
             /simtools/simtools-output/d-2023-07-31/TEST-submit_data_from_external.ecsv
 
 """
@@ -50,7 +48,7 @@ from simtools.data_model.metadata_collector import MetadataCollector
 
 def _parse(label, description):
     """
-    Parse command line configuration
+    Parse command line configuration.
 
     Parameters
     ----------
@@ -65,7 +63,6 @@ def _parse(label, description):
         Command line parser object
 
     """
-
     config = configurator.Configurator(label=label, description=description)
 
     config.parser.add_argument(
@@ -89,7 +86,7 @@ def _parse(label, description):
     return config.initialize(output=True)
 
 
-def main():
+def main():  # noqa: D103
     args_dict, _ = _parse(
         label=Path(__file__).stem,
         description="Submit and validate data (e.g., input data to tools, model parameters).",

simtools/applications/submit_model_parameter_from_external.py

@@ -0,0 +1,122 @@
+#!/usr/bin/python3
+r"""
+    Submit a model parameter value and corresponding metadata through the command line.
+
+    Input and metadata is validated, and if necessary enriched and converted following
+    the model parameter schemas. Model parameter data is written in the simtools-style
+    json format, metadata as a yaml file.
+
+    Command line arguments
+    ----------------------
+    parameter (str)
+        model parameter name
+    value (str, value)
+        input value (number, string, string-type lists)
+    instrument (str)
+        instrument name.
+    site (str)
+        site location.
+    model_version (str)
+        Model version.
+    input_meta (str, optional)
+        input meta data file (yml format)
+
+    Example
+    -------
+
+    Submit the number of gains for the LSTN-design readout chain:
+
+    .. code-block:: console
+
+        simtools-submit-model-parameter-from-external \
+            --parameter num_gains \\
+            --value 2 \\
+            --instrument LSTN-design \\
+            --site North \\
+            --model_version 6.0.0 \\
+            --input_meta num_gains.metadata.yml
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.data_model.model_data_writer as writer
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+
+
+def _parse(label, description):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label: str
+        Label describing application.
+    description: str
+        Description of application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object
+
+    """
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--parameter", type=str, required=True, help="Parameter for simulation model"
+    )
+    config.parser.add_argument("--instrument", type=str, required=True, help="Instrument name")
+    config.parser.add_argument("--site", type=str, required=True, help="Site location")
+    config.parser.add_argument("--model_version", type=str, required=True, help="Model version")
+
+    config.parser.add_argument(
+        "--value",
+        type=str,
+        required=True,
+        help=(
+            "Model parameter value. "
+            "Can be a single number, a number with a unit, or a list of values with units. "
+            'Examples: "--value=5", "--value=\'5 km\'", "--value=\'5 cm, 0.5 deg\'"'
+        ),
+    )
+
+    config.parser.add_argument(
+        "--input_meta",
+        help="meta data file associated to input data",
+        type=str,
+        required=False,
+    )
+    return config.initialize(output=True)
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse(
+        label=Path(__file__).stem,
+        description="Submit and validate a model parameters).",
+    )
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    output_path = (
+        Path(args_dict["output_path"]) / args_dict["model_version"] / args_dict["instrument"]
+        if args_dict.get("output_path")
+        else None
+    )
+    writer.ModelDataWriter.dump_model_parameter(
+        parameter_name=args_dict["parameter"],
+        value=args_dict["value"],
+        instrument=args_dict["instrument"],
+        model_version=args_dict["model_version"],
+        output_file=Path(args_dict["parameter"]).with_suffix(".json"),
+        output_path=output_path,
+        use_plain_output_path=args_dict.get("use_plain_output_path"),
+        metadata_input_dict=args_dict,
+    )
+
+
+if __name__ == "__main__":
+    main()