gammasimtools 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

simtools/applications/print_array_elements.py

@@ -2,61 +2,80 @@
 """
     Summary
     -------
-    Print a list of array element positions in different CTAO coordinate \
-    systems.
+    Convert and print a list of array element positions in different coordinate \
+    systems relevant for CTAO.
+
+    Description
+    -----------
+
+    This application converts a list of array element positions in different CTAO \
+    coordinate systems.
 
     Available coordinate systems are:
+
     1. UTM system
-    2. CORSIKA coordinates
+    2. ground system (similar to sim_telarray system with x-axis pointing toward \
+        geographic north and y-axis pointing towards the west)
     3. Mercator system
 
     Command line arguments
     ----------------------
-    array_element_list (str)
-        File name with list of array element positions (ecsv format)
+    input (str)
+        File name with list of array element positions
     compact (str)
-        Compact output (in requested coordinate system; possible are corsika,utm,mercator)
+        Compact output in requested coordinate system; possible are corsika,utm,mercator
     export (str)
-        Export array element list to file (in requested coordinate system; \
-            possible are corsika,utm,mercator)
+        Export array element list to file in requested coordinate system; \
+            possible are ground, utm, mercator
     use_corsika_telescope_height (bool)
-        Use CORSIKA coordinates for telescope heights (requires CORSIKA observeration level)
+        Use CORSIKA coordinates for telescope heights (requires CORSIKA observation level)
+    select_assets (str)
+        Select a subset of array elements / telescopes (e.g., MSTN, LSTN)
 
 
     Example
     -------
     Print a list of array elements using a list of telescope positions in UTM coordinates.
 
-    Example:
-
-    Run the application:
-
     .. code-block:: console
 
-        simtools-print-array-elements \
-            --array_element_list tests/resources/telescope_positions-South-4MST.ecsv \
-            --compact corsika
+        simtools-print-array-elements \\
+            --input tests/resources/telescope_positions-South-4MST.ecsv \\
+            --compact ground
 
     Expected final print-out message:
 
     .. code-block:: console
 
-    telescope_name pos_x pos_y altitude
-    MST-01      -0.02      -0.00    2162.00
-    MST-02       1.43     151.02    2163.00
-    MST-03      -1.47    -151.02    2169.00
-    MST-04     150.72      73.57    2159.00
+        telescope_name pos_x pos_y altitude
+        MST-01      -0.02      -0.00    2162.00
+        MST-02       1.43     151.02    2163.00
+        MST-03      -1.47    -151.02    2169.00
+        MST-04     150.72      73.57    2159.00
 
+    The following example converts a list of telescope positions in UTM coordinates \
+    and writes the output to a file in ground (sim_telarray) coordinates. Also selects \
+    only a subset of the array elements (telescopes; ignore calibration devices):
 
+    .. code-block:: console
+
+        simtools-print-array-elements \\
+            --input tests/resources/telescope_positions-North-utm.ecsv \\
+            --export ground --use_corsika_telescope_height \\
+            --select_assets LSTN, MSTN, SSTN
+
+    Expected output is a ecsv file in the directory printed to the screen.
 
 """
 
 import logging
 from pathlib import Path
 
+import simtools.data_model.model_data_writer as writer
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.layout import layout_array
+from simtools.data_model.metadata_collector import MetadataCollector
+from simtools.layout import array_layout
 
 
 def _parse(label=None, description=None):
@@ -80,9 +99,15 @@ def _parse(label=None, description=None):
     config = configurator.Configurator(label=label, description=description)
 
     config.parser.add_argument(
-        "--array_element_list",
-        help="list of array element positions (ecsv format)",
-        required=True,
+        "--input",
+        help="list of array element positions",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--input_meta",
+        help="meta data file associated to input data",
+        type=str,
+        required=False,
     )
     config.parser.add_argument(
         "--compact",
@@ -90,7 +115,7 @@ def _parse(label=None, description=None):
         required=False,
         default="",
         choices=[
-            "corsika",
+            "ground",
             "utm",
             "mercator",
         ],
@@ -101,14 +126,14 @@ def _parse(label=None, description=None):
         required=False,
         default=None,
         choices=[
-            "corsika",
+            "ground",
             "utm",
             "mercator",
         ],
     )
     config.parser.add_argument(
         "--use_corsika_telescope_height",
-        help="Use CORSIKA coordinates for telescope heights (requires CORSIKA observeration level)",
+        help="Use CORSIKA coordinates for telescope heights (requires CORSIKA observation level)",
         required=False,
         default=False,
         action="store_true",
@@ -120,23 +145,45 @@ def _parse(label=None, description=None):
         default=None,
         nargs="+",
     )
-    return config.initialize()
+    config.parser.add_argument(
+        "--skip_input_validation",
+        help="skip input data validation against schema",
+        default=False,
+        required=False,
+        action="store_true",
+    )
+    return config.initialize(output=True, require_command_line=True)
 
 
 def main():
     label = Path(__file__).stem
-    args_dict, _ = _parse(label, description="Print a list of array element positions")
+    data_model_name = "array_coordinates"
+    args_dict, _ = _parse(
+        label,
+        description=f"Print a list of array element positions ({data_model_name})",
+    )
 
     _logger = logging.getLogger()
     _logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    layout = layout_array.LayoutArray(telescope_list_file=args_dict["array_element_list"])
+    layout = array_layout.ArrayLayout(
+        telescope_list_file=args_dict["input"],
+        telescope_list_metadata_file=args_dict["input_meta"],
+        validate=not args_dict["skip_input_validation"],
+    )
     layout.select_assets(args_dict["select_assets"])
     layout.convert_coordinates()
+
     if args_dict["export"] is not None:
-        layout.export_telescope_list(
-            crs_name=args_dict["export"],
-            corsika_z=args_dict["use_corsika_telescope_height"],
+        _metadata = MetadataCollector(args_dict=args_dict, data_model_name=data_model_name)
+        writer.ModelDataWriter.dump(
+            args_dict=args_dict,
+            metadata=_metadata.top_level_meta,
+            product_data=layout.export_telescope_list_table(
+                crs_name=args_dict["export"],
+                corsika_z=args_dict["use_corsika_telescope_height"],
+            ),
+            validate_schema_file=_metadata.get_data_model_schema_file_name(),
         )
     else:
         layout.print_telescope_list(

simtools/applications/produce_array_config.py

@@ -65,7 +65,7 @@
     array_config (str, required)
         Path to a yaml file with the array config data.
     verbosity (str, optional)
-        Log level to print (default=INFO).
+        Log level to print.
 
     Example
     -------
@@ -75,7 +75,7 @@
 
     .. code-block:: console
 
-        python applications/get_file_from_db.py --file_name array_config_test.yml
+        simtools-get-file-from-db --file_name array_config_test.yml
 
     Run the application. Runtime < 1 min.
 

simtools/applications/production.py

@@ -43,9 +43,14 @@
         If activated, no job will be submitted, but all configuration files \
         and run scripts will be created.
     data_directory (str, optional)
-        The location of the output directories corsika-data and simtel-data
+        The location of the output directories corsika-data and simtel-data.
+    corsika_files (str, optional)
+        The CORSIKA files to pass to simtel_array.
+        If it is provided, these CORSIKA files are used and the application does not search for them
+        in the data directory.
+        This option should only be used in combination with the `showers_only` option.
     verbosity (str, optional)
-        Log level to print (default=INFO).
+        Log level to print.
 
     Example
     -------
@@ -55,7 +60,7 @@
 
     .. code-block:: console
 
-        simtools-get-file-from_db --file_name prod_config_test.yml
+        simtools-get-file-from-db --file_name prod_config_test.yml
 
     Run the application:
 
@@ -76,6 +81,7 @@
 
 import logging
 from copy import copy
+from pathlib import Path
 
 from astropy.io.misc import yaml
 
@@ -142,6 +148,13 @@ def _parse(description=None):
         required=False,
         default="./",
     )
+    config.parser.add_argument(
+        "--corsika_files",
+        help="The CORSIKA files to pass to simtel_array.",
+        type=str,
+        required=False,
+        default=None,
+    )
     group = config.parser.add_mutually_exclusive_group(required=True)
     group.add_argument(
         "--showers_only",
@@ -221,7 +234,6 @@ def _proccess_simulation_config_file(config_file, primary_config, logger):
         for key, value in this_default.items():
             config_showers[primary][key] = value
             config_arrays[primary][key] = value
-
     return label, config_showers, config_arrays
 
 
@@ -237,6 +249,11 @@ def main():
     if args_dict["label"] is None:
         args_dict["label"] = label
 
+    if args_dict["corsika_files"] is not None and args_dict["array_only"] is False:
+        msg = "`--corsika_files` option should be used only with `--array_only` argument."
+        logger.error(msg)
+        raise ValueError
+
     shower_simulators = {}
     for primary, config_data in shower_configs.items():
         if "data_directory" in args_dict:
@@ -270,7 +287,24 @@ def main():
                 mongo_db_config=db_config,
             )
         for primary, array in array_simulators.items():
-            input_list = shower_simulators[primary].get_list_of_output_files()
+            if args_dict["corsika_files"] is None:
+                input_list = shower_simulators[primary].get_list_of_output_files()
+            else:
+                if not isinstance(args_dict["corsika_files"], list):
+                    args_dict["corsika_files"] = [args_dict["corsika_files"]]
+                input_list = args_dict["corsika_files"]
+
+            for corsika_file in input_list:
+                if not Path(corsika_file).exists():
+                    msg = (
+                        f"CORSIKA file {corsika_file} does not exist. Please run the "
+                        "production with the `--showers_only` option first or point the "
+                        "tool to the correct path to the corsika files with "
+                        "`--corsika_directory`."
+                    )
+                    logger.error(msg)
+                    raise FileNotFoundError
+            logger.info(f"Getting CORSIKA files: {input_list}.")
             _task_function = getattr(array, args_dict["task"])
             _task_function(input_file_list=input_list)
 

simtools/applications/sim_showers_for_trigger_rates.py

@@ -25,21 +25,22 @@
     primary (str, required)
         Name of the primary particle (proton, helium ...).
     nruns (int, optional)
-        Number of runs to be simulated (default=100).
+        Number of runs to be simulated.
     nevents (int, optional)
-        Number of events simulated per run (default=100000).
+        Number of events simulated per run.
     zenith (float, optional)
-        Zenith angle in deg (default=20).
+        Zenith angle in deg.
     azimuth (float, optional)
-        Azimuth angle in deg (default=0).
-    output (str, optional)
-        Path of the directory to store the output simulations. By default, \
-        the standard output directory defined by config will be used.
+        Azimuth angle in deg.
+    data_directory (str, optional)
+        The location of the output directories corsika-data.
+        the label is added to the data_directory, such that the output
+        will be written to `data_directory/label/corsika-data`.
     test (activation mode, optional)
         If activated, no job will be submitted. Instead, an example of the \
         run script will be printed.
     verbosity (str, optional)
-        Log level to print (default=INFO).
+        Log level to print.
 
     Example
     -------
@@ -47,7 +48,7 @@
 
     .. code-block:: console
 
-        simtools-sim-showers-for-trigger_rates --array 4LST --site North --primary \
+        simtools-sim-showers-for-trigger-rates --array 4LST --site North --primary \
         proton --nruns 2 --nevents 10000 --test --submit_command local
 
     The output is saved in simtools-output/sim_showers_for_trigger_rates.
@@ -68,8 +69,8 @@ from pathlib import Path
 
 import astropy.units as u
 
-from simtools import io_handler
 from simtools.configuration import configurator
+from simtools.io_operations import io_handler
 from simtools.simulator import Simulator
 from simtools.utils import general as gen
 
@@ -105,24 +106,20 @@ def _parse(label=None, description=None):
         type=str,
         required=True,
     )
+    config.parser.add_argument("--nruns", help="Number of runs", type=int, default=100)
+    config.parser.add_argument("--nevents", help="Number of events/run", type=int, default=100000)
+    config.parser.add_argument("--zenith", help="Zenith angle in deg", type=float, default=20)
+    config.parser.add_argument("--azimuth", help="Azimuth angle in deg", type=float, default=0)
     config.parser.add_argument(
-        "--nruns", help="Number of runs (default=100)", type=int, default=100
-    )
-    config.parser.add_argument(
-        "--nevents", help="Number of events/run (default=100)", type=int, default=100000
-    )
-    config.parser.add_argument(
-        "--zenith", help="Zenith angle in deg (default=20)", type=float, default=20
-    )
-    config.parser.add_argument(
-        "--azimuth", help="Azimuth angle in deg (default=0)", type=float, default=0
-    )
-    # TODO confusing with output_path?
-    config.parser.add_argument(
-        "--output",
-        help="Path of the output directory where the simulations will be saved.",
-        type=str,
-        default=None,
+        "--data_directory",
+        help=(
+            "The directory where to save the corsika-data output directories."
+            "the label is added to the data_directory, such that the output"
+            "will be written to `data_directory/label/corsika-data`."
+        ),
+        type=str.lower,
+        required=False,
+        default="./simtools-output/",
     )
     return config.initialize(telescope_model=True, job_submission=True, db_config=True)
 
@@ -138,10 +135,9 @@ def main():
 
     # Output directory to save files related directly to this app
     _io_handler = io_handler.IOHandler()
-    output_dir = _io_handler.get_output_directory(label, dir_type="application-plots")
-    print(output_dir)
+    output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
     shower_config_data = {
-        "data_directory": args_dict["output"],
+        "data_directory": Path(args_dict["data_directory"]) / label,
         "site": args_dict["site"],
         "layout_name": args_dict["array"],
         "run_range": [1, args_dict["nruns"] + 1],

simtools/applications/simulate_prod.py

@@ -10,8 +10,8 @@
 
     The entire simulation chain is performed, i.e., shower simulations with CORSIKA
     which are piped directly to sim_telarray using the sim_telarray multipipe mechanism.
-    This script assumes that all the necessary configuration files for CORISKA and
-    sim_telarray are available. FIXME - This is not true at the moment, need to fix I guess.
+    This script produces all the necessary configuration files for CORSIKA and
+    sim_telarray before running simulation.
     The multipipe scripts will be produced as part of this script.
 
     This script does not provide a mechanism to submit jobs to a batch system like others
@@ -48,9 +48,14 @@
     run (int, required)
         Run number (actual run number will be 'start_run' + 'run').
     data_directory (str, optional)
-        The location of the output directories corsika-data and simtel-data
+        The location of the output directories corsika-data and simtel-data.
+        the label is added to the data_directory, such that the output
+        will be written to `data_directory/label/simtel-data`.
+    pack_for_grid_register (bool, optional)
+        Set whether to prepare a tarball for registering the output files on the grid.
+        The files are written to the `output_path/directory_for_grid_upload` directory.
     log_level (str, optional)
-        Log level to print (default=INFO).
+        Log level to print.
 
     Example
     -------
@@ -61,20 +66,23 @@
         simtools-simulate-prod \
         --production_config tests/resources/prod_multi_config_test.yml --model_version Prod5 \
         --site north --primary gamma --azimuth_angle north --zenith_angle 20 \
-         --start_run 0 --run 1
+        --start_run 0 --run 1
 
     By default the configuration is saved in simtools-output/test-production
-    and the output in corsika-data and simtel-data. The location of the latter directories
-    can be set to a different location via the option --data_directory.
+    together with the actual simulation output in corsika-data and simtel-data within.
+    The location of the latter directories can be set
+    to a different location via the option --data_directory,
+    but the label is always added to the data_directory, such that the output
+    will be written to `data_directory/label/simtel-data`.
 
     Expected final print-out message:
 
     .. code-block:: console
 
-        INFO::layout_array(l569)::read_telescope_list_file::Reading array elements from ...
+        INFO::array_layout(l569)::read_telescope_list_file::Reading array elements from ...
         WARNING::corsika_runner(l127)::_load_corsika_config_data::data_directory not given
         in corsika_config - default output directory will be set.
-        INFO::layout_array(l569)::read_telescope_list_file::Reading array elements from ...
+        INFO::array_layout(l569)::read_telescope_list_file::Reading array elements from ...
         INFO::corsika_config(l493)::_set_output_file_and_directory::Creating directory
         INFO::simulator(l405)::simulate::Submission command: local
         INFO::simulator(l410)::simulate::Starting submission for 1 run
@@ -87,8 +95,8 @@
 """
 
 import logging
+import shutil
 import tarfile
-from copy import copy
 from pathlib import Path
 
 from astropy.io.misc import yaml
@@ -184,10 +192,14 @@ def _parse(description=None):
     )
     config.parser.add_argument(
         "--data_directory",
-        help="The directory where to save the corsika-data and simtel-data output directories.",
+        help=(
+            "The directory where to save the corsika-data and simtel-data output directories."
+            "the label is added to the data_directory, such that the output"
+            "will be written to `data_directory/label/simtel-data`."
+        ),
         type=str.lower,
         required=False,
-        default="./",
+        default="./simtools-output/",
     )
     config.parser.add_argument(
         "--pack_for_grid_register",
@@ -199,112 +211,40 @@ def _parse(description=None):
     return config.initialize(db_config=True, telescope_model=True)
 
 
-def _proccess_simulation_config_file(config_file, primary_config, logger):
-    """
-    Read the simulation configuration file with all the details
-    on shower and array simulations.
-
-    Parameters
-    ----------
-    config_file: str
-        Name of simulation configuration file.
-    primary_config: str
-        Name of the primary selected from the configuration file.
-    logger: logging.logger
-        The logger to use to record log calls.
-
-    Returns
-    -------
-    str
-        Label of simulation configuration.
-    dict
-        Configuration of shower simulations.
-    dict
-        Configuration of array simulations.
-
-    Raises
-    ------
-    FileNotFoundError
-    """
-
-    try:
-        with open(config_file, encoding="utf-8") as file:
-            config_data = yaml.load(file)
-    except FileNotFoundError:
-        logger.error(f"Error loading simulation configuration file from {config_file}")
-        raise
-
-    label = config_data.pop("label", "test_run")
-    default_data = config_data.pop("default", {})
-
-    for primary, primary_data in config_data.items():
-        if primary_config is not None and primary != primary_config:
-            continue
-
-        this_default = copy(default_data)
-
-        config_showers = copy(this_default.pop("showers", {}))
-        config_arrays = copy(this_default.pop("array", {}))
-
-        # Grabbing common entries for showers and array
-        for key, value in primary_data.items():
-            if key in ["showers", "array"]:
-                continue
-            config_showers[key] = value
-            config_arrays[key] = value
-
-        # Grabbing showers entries
-        for key, value in primary_data.get("showers", {}).items():
-            config_showers[key] = value
-        config_showers["primary"] = primary
-
-        # Grabbing array entries
-        for key, value in primary_data.get("array", {}).items():
-            config_arrays[key] = value
-        config_arrays["primary"] = primary
-
-        # Filling in the remaining default keys
-        for key, value in this_default.items():
-            config_showers[key] = value
-            config_arrays[key] = value
-
-    config_arrays["data_directory"] = config_showers["data_directory"]
-    config_arrays["site"] = config_showers["site"]
-    config_arrays["layout_name"] = config_showers["layout_name"]
-
-    return label, config_showers, config_arrays
-
-
 def main():
     args_dict, db_config = _parse(description="Run simulations for productions")
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    label, shower_configs, array_configs = _proccess_simulation_config_file(
-        args_dict["production_config"], args_dict["primary"], logger
-    )
+    try:
+        with open(args_dict["production_config"], encoding="utf-8") as file:
+            config_data = yaml.load(file)
+    except FileNotFoundError:
+        logger.error(
+            f"Error loading simulation configuration file from {args_dict['production_config']}"
+        )
+        raise
 
     # Overwrite default and optional settings
-    shower_configs["run_list"] = args_dict["run"] + args_dict["start_run"]
-    array_configs["site"] = shower_configs["site"] = args_dict["site"]
-    array_configs["zenith"] = shower_configs["zenith"] = args_dict["zenith_angle"]
-    array_configs["phi"] = shower_configs["phi"] = args_dict["azimuth_angle"]
+    config_data["showers"]["run_list"] = args_dict["run"] + args_dict["start_run"]
+    config_data["showers"]["primary"] = args_dict["primary"]
+    config_data["common"]["site"] = args_dict["site"]
+    config_data["common"]["zenith"] = args_dict["zenith_angle"]
+    config_data["common"]["phi"] = args_dict["azimuth_angle"]
+    label = config_data["common"].pop("label", "test-production")
+    config_data["common"]["data_directory"] = Path(args_dict["data_directory"]) / label
 
     if args_dict["nshow"] is not None:
-        shower_configs["nshow"] = args_dict["nshow"]
+        config_data["showers"]["nshow"] = args_dict["nshow"]
     if args_dict["label"] is not None:
         label = args_dict["label"]
-    if "data_directory" in args_dict:
-        array_configs["data_directory"] = shower_configs["data_directory"] = args_dict[
-            "data_directory"
-        ]
 
     simulator = Simulator(
         label=label,
         simulator="corsika_simtel",
         simulator_source_path=args_dict["simtel_path"],
-        config_data=shower_configs | array_configs,
+        config_data=config_data,
         submit_command="local",
         test=args_dict["test"],
         mongo_db_config=db_config,
@@ -313,10 +253,10 @@ def main():
     simulator.simulate()
 
     logger.info(
-        f"Production run is complete for primary {shower_configs['primary']} showers "
-        f"coming from {shower_configs['phi']} azimuth and zenith angle of "
-        f"{shower_configs['zenith']} at the {args_dict['site']} site, "
-        f"using the {array_configs['model_version']} telescope model."
+        f"Production run is complete for primary {config_data['showers']['primary']} showers "
+        f"coming from {config_data['common']['phi']} azimuth and zenith angle of "
+        f"{config_data['common']['zenith']} at the {args_dict['site']} site, "
+        f"using the {config_data['array']['model_version']} telescope model."
     )
 
     if args_dict["pack_for_grid_register"]:
@@ -329,7 +269,9 @@ def main():
             files_to_tar = log_files[:1] + histogram_files[:1]
             for file_to_tar in files_to_tar:
                 tar.add(file_to_tar, arcname=Path(file_to_tar).name)
-        directory_for_grid_upload = Path("directory_for_grid_upload")
+        directory_for_grid_upload = Path(args_dict.get("output_path")).joinpath(
+            "directory_for_grid_upload"
+        )
         directory_for_grid_upload.mkdir(parents=True, exist_ok=True)
         for file_to_move in [*output_files, tar_file_name]:
             source_file = Path(file_to_move)
@@ -337,7 +279,7 @@ def main():
             # Note that this will overwrite previous files which exist in the directory
             # It should be fine for normal production since each run is on a separate node
             # so no files are expected there.
-            source_file.replace(destination_file)
+            shutil.move(source_file, destination_file)
         logger.info(f"Output files for the grid placed in {str(directory_for_grid_upload)}")