gammasimtools 0.6.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

simtools/applications/calculate_trigger_rate.py

@@ -0,0 +1,210 @@
+#!/usr/bin/python3
+
+r"""
+Calculates array or single-telescope trigger rates.
+
+The applications reads from a simtel_array output file, a list of
+simtel_array output files ou from a file containing a list of simtel_array files.
+
+
+Command line arguments
+----------------------
+simtel_file_names (str or list):
+    Path to the simtel_array file or a list of simtel_array output files.
+    Files can be generated in `simulate_prod` using the ``--save_file_lists`` option.
+save_tables (bool):
+    If true, save the tables with the energy-dependent trigger rate to a ecsv file.
+area_from_distribution (bool):
+    If true, the area thrown (the area in which the simulated events are distributed)
+    in the trigger rate calculation is estimated based on the event distribution.
+    The expected shape of the distribution of events as function of the core distance is triangular
+    up to the maximum distance. The weighted mean radius of the triangular distribution is 2/3 times
+    the upper edge. Therefore, when using the ``area_from_distribution`` flag, the mean distance
+    times 3/2, returns just the position of the upper edge in the triangle distribution with little
+    impact of the binning and little dependence on the scatter area defined in the simulation.
+    This is special useful when calculating trigger rate for individual telescopes.
+    If false, the area thrown is estimated based on the maximum distance as given in
+    the simulation configuration.
+
+Example
+-------
+Calculate trigger rate from simtel_array file
+
+.. code-block:: console
+
+    simtools-calculate-trigger-rate --simtel_file_names tests/resources/ \\
+    run201_proton_za20deg_azm0deg_North_test_layout_test-prod.simtel.zst
+
+Expected final print-out message:
+
+.. code-block:: console
+
+    System trigger rate (Hz): 9.0064e+03 pm 9.0087e+03 Hz
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.io_operations import io_handler
+from simtools.simtel.simtel_io_histograms import SimtelIOHistograms
+
+
+def _parse(label, description):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label: str
+        Label describing the application.
+    description: str
+        Description of the application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object
+
+    """
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--simtel_file_names",
+        help="Name of the simtel_array output files to be calculate the trigger rate from  or the "
+        "text file containing the list of simtel_array output files.",
+        nargs="+",
+        required=True,
+        type=str,
+    )
+
+    config.parser.add_argument(
+        "--save_tables",
+        help="If true, saves the trigger rates per energy bin into ECSV files.",
+        action="store_true",
+    )
+
+    config.parser.add_argument(
+        "--area_from_distribution",
+        help="If true, calculates the trigger rates using the event distribution.",
+        action="store_true",
+    )
+
+    config.parser.add_argument(
+        "--stack_files",
+        help="If true, stacks all the histograms.",
+        action="store_true",
+    )
+
+    config_parser, _ = config.initialize(
+        db_config=False,
+        paths=True,
+        simulation_configuration={"corsika_configuration": ["energy_range", "view_cone"]},
+    )
+
+    return config_parser
+
+
+def _get_simulation_parameters(config_parser):
+    """
+    Get energy range and view cone in the correct form to use in the simtel classes.
+
+    Parameters
+    ----------
+    CommandLineParser:
+        Command line parser object as defined by the _parse function.
+
+    Returns
+    -------
+    list:
+        The energy range used in the simulation.
+    list:
+        The view cone used in the simulation.
+
+    """
+    if config_parser["energy_range"] is not None:
+        energy_range = [
+            config_parser["energy_range"][0].to("TeV").value,
+            config_parser["energy_range"][1].to("TeV").value,
+        ]
+    else:
+        energy_range = None
+    if config_parser["view_cone"] is not None:
+        view_cone = [
+            config_parser["view_cone"][0].to("deg").value,
+            config_parser["view_cone"][1].to("deg").value,
+        ]
+    else:
+        view_cone = None
+
+    return energy_range, view_cone
+
+
+def main():  # noqa: D103
+    label = Path(__file__).stem
+    description = (
+        "Calculates the simulated and triggered event rate based on simtel_array output files."
+    )
+    config_parser = _parse(label, description)
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(config_parser["log_level"]))
+
+    # Building list of simtel_array files from the input files
+    simtel_array_files = []
+    for one_file in config_parser["simtel_file_names"]:
+        try:
+            if Path(one_file).suffix in [".zst", ".simtel", ".hdata"]:
+                simtel_array_files.append(one_file)
+            else:
+                # Collecting hist files
+                with open(one_file, encoding="utf-8") as file:
+                    for line in file:
+                        # Removing '\n' from filename, in case it is left there.
+                        simtel_array_files.append(line.replace("\n", ""))
+        except FileNotFoundError as exc:
+            msg = f"{one_file} is not a file."
+            logger.error(msg)
+            raise FileNotFoundError from exc
+
+    energy_range, view_cone = _get_simulation_parameters(config_parser)
+
+    histograms = SimtelIOHistograms(
+        simtel_array_files,
+        area_from_distribution=config_parser["area_from_distribution"],
+        energy_range=energy_range,
+        view_cone=view_cone,
+    )
+
+    logger.info("Calculating simulated and triggered event rate")
+    (
+        sim_event_rates,
+        triggered_event_rates,
+        triggered_event_rate_uncertainties,
+        trigger_rate_in_tables,
+    ) = histograms.calculate_trigger_rates(
+        print_info=True, stack_files=config_parser["stack_files"]
+    )
+
+    # Print out results
+    for i_hist, _ in enumerate(sim_event_rates):
+        print(f"\nFile {histograms.histogram_files[i_hist]}\n")
+        print(
+            f"System trigger rate (Hz): {triggered_event_rates[i_hist].value:.4e} \u00B1 "
+            f"{triggered_event_rate_uncertainties[i_hist].value:.4e} Hz"
+        )
+    if config_parser["save_tables"]:
+        io_handler_instance = io_handler.IOHandler()
+        output_path = io_handler_instance.get_output_directory(label, sub_dir="application-plots")
+        for i_table, table in enumerate(trigger_rate_in_tables):
+            output_file = (
+                str(output_path.joinpath(Path(simtel_array_files[i_table]).stem)) + ".ecsv"
+            )
+            logger.info(f"Writing table {i_table + 1} to {output_file}")
+            table.write(output_file, overwrite=True)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/convert_all_model_parameters_from_simtel.py

@@ -0,0 +1,372 @@
+#!/usr/bin/python3
+r"""
+    Convert all simulation model parameters exported from sim_telarray format using schema files.
+
+    Check value, type, and range, convert units, and write json files
+    ready to be submitted to the model database. Prints out parameters which are not found
+    in simtel configuration file and parameters which are not found in simtools schema files.
+
+    Command line arguments
+    ----------------------
+    simtel_cfg_file (str)
+        File name of sim_telarray configuration file containing all simulation model parameters.
+
+    simtel_telescope_name (str)
+        Name of the telescope in the sim_telarray configuration file.
+
+    telescope (str, optional)
+        Telescope model name (e.g. LST-1, SST-D, ...)
+
+    Example
+    -------
+
+    Extract model parameters with schema files from simtel configuration file
+    (requires access to the model parameter repository)
+
+    .. code-block:: console
+
+       simtools-convert-all-model-parameters-from-simtel \\
+          --schema_directory ../model_parameters/schema\\
+          --simtel_cfg_file all_telescope_config_la_palma.cfg\\
+          --simtel_telescope_name CT1\\
+          --telescope LSTN-01\\
+          --model_version "2024-03-06"
+
+    The export of the model parameters from sim_telarray for 6.0.0 can be done e.g., as follows:
+
+    .. code-block:: console
+
+        ./sim_telarray/bin/sim_telarray -c sim_telarray/cfg/CTA/CTA-PROD6-LaPalma.cfg \\
+            -C limits=no-internal -C initlist=no-internal -C list=no-internal \\
+            -C typelist=no-internal -C maximum_telescopes=30 -DNSB_AUTOSCALE \\
+            -DNECTARCAM -DHYPER_LAYOUT -DNUM_TELESCOPES=30 /dev/null \\
+            2>|/dev/null | grep '(@cfg)'  >| all_telescope_config_la_palma.cfg
+
+        ./sim_telarray/bin/sim_telarray -c sim_telarray/cfg/CTA/CTA-PROD6-Paranal.cfg \\
+            -C limits=no-internal -C initlist=no-internal -C list=no-internal \\
+            -C typelist=no-internal -C maximum_telescopes=87 -DNSB_AUTOSCALE \\
+            -DFLASHCAM -DHYPER_LAYOUT -DNUM_TELESCOPES=87 /dev/null \\
+            2>|/dev/null | grep '(@cfg)'  >| all_telescope_config_paranal.cfg
+
+"""
+
+import logging
+import re
+from pathlib import Path
+
+import numpy as np
+
+import simtools.data_model.model_data_writer as writer
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
+from simtools.io_operations.io_handler import IOHandler
+from simtools.simtel.simtel_config_reader import SimtelConfigReader
+
+
+def _parse(label=None, description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label: str
+        Label describing application.
+    description: str
+        Description of application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object
+
+    """
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--schema_directory",
+        help=(
+            "Directory with schema files for model parameter validation "
+            "(default: simtools schema directory)"
+        ),
+        required=False,
+    )
+    config.parser.add_argument(
+        "--simtel_cfg_file",
+        help="File name for simtel_array configuration",
+        type=str,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--simtel_telescope_name",
+        help="Name of the telescope in the sim_telarray configuration file",
+        type=str,
+        required=True,
+    )
+    return config.initialize(simulation_model="telescope")
+
+
+def get_list_of_parameters_and_schema_files(schema_directory):
+    """
+    Return list of parameters and schema files located in schema file directory.
+
+    Returns
+    -------
+    list
+        List of parameters found in schema file directory.
+    list
+        List of schema files found in schema file directory.
+
+    """
+    schema_files = sorted(Path(schema_directory).rglob("*.schema.yml"))
+    parameters = []
+    for schema_file in schema_files:
+        schema_dict = gen.collect_data_from_file(file_name=schema_file)
+        parameters.append(schema_dict.get("name"))
+    return parameters, schema_files
+
+
+def get_list_of_simtel_parameters(simtel_config_file, logger):
+    """
+    Return list of simtel parameters found in simtel configuration file.
+
+    Parameters
+    ----------
+    simtel_config_file: str
+        File name for sim_telarray configuration
+    logger: logging.Logger
+        Logger object
+
+    Returns
+    -------
+    list
+        List of parameters found in simtel configuration file.
+
+    """
+    simtel_parameter_set = set()
+    with open(simtel_config_file, encoding="utf-8") as file:
+        for line in file:
+            parts_of_lines = re.split(r",\s*|\s+", line.strip())
+            simtel_parameter_set.add(parts_of_lines[1].lower())
+    logger.info(f"Found {len(simtel_parameter_set)} parameters in simtel configuration file.")
+    return list(simtel_parameter_set)
+
+
+def read_simtel_config_file(args_dict, logger, schema_file, camera_pixels=None):
+    """
+    Read the simtel configuration file.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Dictionary with command line arguments.
+    logger: logging.Logger
+        Logger object
+    schema_file: str
+        Schema path name.
+    camera_pixels: int
+        Number of camera pixels.
+
+    """
+    simtel_config_reader = SimtelConfigReader(
+        schema_file=schema_file,
+        simtel_config_file=args_dict["simtel_cfg_file"],
+        simtel_telescope_name=args_dict["simtel_telescope_name"],
+        camera_pixels=camera_pixels,
+    )
+    logger.info(f"Simtel parameter: {simtel_config_reader.parameter_dict}")
+
+    if simtel_config_reader.parameter_dict is None or len(simtel_config_reader.parameter_dict) == 0:
+        return None
+    return simtel_config_reader
+
+
+def get_number_of_camera_pixel(args_dict, logger):
+    """
+    Get the number of camera pixels from the simtel configuration file.
+
+    Required to set the dimension some of the parameter correctly, as simtel
+    in some cases does not provide the dimension ('all:' in the parameter files).
+
+    Parameters
+    ----------
+    args_dict: dict
+        Dictionary with command line arguments.
+    logger: logging.Logger
+        Logger object
+
+    Returns
+    -------
+    int, None
+        Number of camera pixels (None if file is not found)
+
+    """
+    try:
+        simtel_config_reader = SimtelConfigReader(
+            schema_file=MODEL_PARAMETER_SCHEMA_PATH / "camera_pixels.schema.yml",
+            simtel_config_file=args_dict["simtel_cfg_file"],
+            simtel_telescope_name=args_dict["simtel_telescope_name"],
+        )
+        _camera_pixel = simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"])
+    except (FileNotFoundError, AttributeError):
+        logger.warning("Failed to read camera pixel parameter.")
+        _camera_pixel = None
+    logger.info(f"Number of camera pixels: {_camera_pixel}")
+    return _camera_pixel
+
+
+def read_and_export_parameters(args_dict, logger):
+    """
+    Read and export parameters from simtel configuration file to json files.
+
+    Only applicable parameters are exported to json.
+    Provide extensive logging information on the parameters found in the simtel
+    configuration file.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Dictionary with command line arguments.
+    logger: logging.Logger
+        Logger object
+
+    Returns
+    -------
+    list
+        List of simtel parameters not found in schema files.
+    list
+        List of simtools parameter not found in simtel configuration file.
+
+    """
+    _parameters, _schema_files = get_list_of_parameters_and_schema_files(
+        args_dict.get("schema_directory", MODEL_PARAMETER_SCHEMA_PATH)
+    )
+    _simtel_parameters = get_list_of_simtel_parameters(args_dict["simtel_cfg_file"], logger)
+
+    io_handler = IOHandler()
+    io_handler.set_paths(output_path=args_dict["output_path"], use_plain_output_path=True)
+
+    _camera_pixel = get_number_of_camera_pixel(args_dict, logger)
+
+    _parameters_not_in_simtel = []
+
+    for _parameter, _schema_file in zip(_parameters, _schema_files):
+
+        logger.info(f"Parameter: {_parameter} Schema file: {_schema_file}")
+        simtel_config_reader = read_simtel_config_file(
+            args_dict, logger, _schema_file, _camera_pixel
+        )
+
+        if simtel_config_reader is None:
+            logger.info("Parameter not found in sim_telarray configuration file.")
+            _parameters_not_in_simtel.append(_parameter)
+            continue
+
+        _json_dict = writer.ModelDataWriter.dump_model_parameter(
+            parameter_name=_parameter,
+            value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
+            instrument=args_dict["telescope"],
+            model_version=args_dict["model_version"],
+            output_file=io_handler.get_output_file(f"{_parameter}.json"),
+        )
+
+        simtel_config_reader.compare_simtel_config_with_schema()
+
+        if simtel_config_reader.simtel_parameter_name.lower() in _simtel_parameters:
+            _simtel_parameters.remove(simtel_config_reader.simtel_parameter_name.lower())
+
+        if _json_dict["file"]:
+            logger.info(f"File name for {_parameter} is {_json_dict['value']}")
+
+    return _parameters_not_in_simtel, _simtel_parameters
+
+
+def print_parameters_not_found(_parameters_not_in_simtel, _simtel_parameters, args_dict, logger):
+    """
+    Print simtel/simtools parameter not found in schema and ocnfiguration files.
+
+    For simtel parameters not found, check if the setting for the chose
+    telescope is different from the default values.
+
+    Parameters
+    ----------
+    _parameters_not_in_simtel: list
+        List of simtel parameters not found in schema files.
+    _simtel_parameters: list
+        List of simtel parameters not found in simtools schema files.
+    args_dict: dict
+        Dictionary with command line arguments.
+    logger: logging.Logger
+        Logger object
+
+    """
+    logger.info(
+        f"Parameters not found in simtools schema files ({len(_parameters_not_in_simtel)}):"
+    )
+    for para in sorted(_parameters_not_in_simtel):
+        logger.info(f"  {para}")
+
+    logger.info(f"Simtel parameters not found in schema files ({len(_simtel_parameters)}):")
+    for para in sorted(_simtel_parameters):
+        logger.info(f"Simtel parameter: {para}")
+        simtel_config_reader = SimtelConfigReader(
+            schema_file=None,
+            simtel_config_file=args_dict["simtel_cfg_file"],
+            simtel_telescope_name=args_dict["simtel_telescope_name"],
+            parameter_name=para,
+        )
+        _default = simtel_config_reader.parameter_dict.get("default")
+        _tel_value = simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"])
+        # simple comparison of default value and telescope values, does not work for lists
+        try:
+            if _default == _tel_value or np.isclose(_default, _tel_value):
+                logger.info(f"    Default and telescope values are equal: {_default}")
+                continue
+        except (ValueError, TypeError):
+            pass
+        if isinstance(_default, np.ndarray):
+            logger.warning(f"    Default value: {_default} (length: {_default.size})")
+        else:
+            logger.warning(f"    Default value: {_default}")
+        if isinstance(_tel_value, np.ndarray):
+            logger.warning(f"    Telescope value: {_tel_value} (length: {_tel_value.size})")
+        else:
+            logger.warning(f"    Telescope value: {_tel_value}")
+
+
+def print_list_of_files(args_dict, logger):
+    """
+    Print model parameters which describe a file name.
+
+    This is useful to find files which are part of the model.
+
+    Parameters
+    ----------
+    args_dict: dict
+        Dictionary with command line arguments.
+    logger: logging.Logger
+        Logger object
+
+    """
+    model_files = sorted(Path(args_dict["output_path"]).rglob("*.json"))
+    for file in model_files:
+        model_dict = gen.collect_data_from_file(file_name=file)
+        if model_dict.get("file"):
+            logger.info(f"{file.name}: {model_dict['value']}")
+
+
+def main():  # noqa: D103
+    args_dict, _ = _parse(
+        label=Path(__file__).stem,
+        description="Convert simulation model parameters from sim_telarray to simtools format.",
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    _parameters_not_in_simtel, _simtel_parameters = read_and_export_parameters(args_dict, logger)
+    print_parameters_not_found(_parameters_not_in_simtel, _simtel_parameters, args_dict, logger)
+    print_list_of_files(args_dict, logger)
+
+
+if __name__ == "__main__":
+    main()