gammasimtools 0.12.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

simtools/applications/derive_ctao_array_layouts.py

@@ -0,0 +1,120 @@
+#!/usr/bin/python3
+
+r"""
+    Derive CTAO array layouts definitions using CTAO common identifiers repository.
+
+    Retrieves the CTAO array layouts definitions from the CTAO common identifiers repository
+    and synchronizes them with the model parameter definition in 'array_layouts' for all
+    CTAO sites.
+
+    Requires access to the CTAO common identifiers repository. Future versions of the
+    common identifiers might be stored in a CTAO technical database.
+
+    Command line arguments
+    ----------------------
+    site (str)
+        CTAO site (North or South).
+    parameter_version (str)
+        Model parameter version.
+    repository_url (str)
+        URL or path of the CTAO common identifiers repository.
+    repository_branch (str )
+        Repository branch to use for CTAO common identifiers.
+    updated_parameter_version (str)
+        Updated parameter version.
+
+    Example
+    -------
+
+    Derive the CTAO array layouts definitions for the North site using the CTAO common identifiers
+    repository. Merge the retrieved array layouts with the model parameter definition in
+    'array_layouts' (in this example for site North and parameter version 2.0.0). Write the
+    updated array layouts and metadata using the parameter version indicated by the
+    'updated_parameter_version' argument.
+
+    .. code-block:: console
+
+        simtools-derive-ctao-array-layouts --site North \
+            --repository_url "https://gitlab.cta-observatory.org/" \
+            "cta-computing/common/identifiers/-/raw/" \
+            --repository_branch main \
+            --site North --parameter_version 2.0.0 \
+            --updated_parameter_version 3.0.0
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler
+from simtools.layout.ctao_array_layouts import (
+    merge_array_layouts,
+    retrieve_array_layouts,
+    write_array_layouts,
+)
+
+
+def _parse(label):
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        label=label,
+        description="Derive CTAO array layouts from CTAO common identifiers repository.",
+    )
+    config.parser.add_argument(
+        "--repository_url",
+        help="URL or path of the CTAO common identifiers repository.",
+        type=str,
+        default="https://gitlab.cta-observatory.org/cta-computing/common/identifiers/-/raw/",
+    )
+    config.parser.add_argument(
+        "--repository_branch",
+        help="Repository branch to use for CTAO common identifiers.",
+        type=str,
+        default="main",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--updated_parameter_version",
+        help="Updated parameter version.",
+        type=str,
+        required=False,
+    )
+    return config.initialize(
+        db_config=True, output=True, simulation_model=["site", "parameter_version", "model_version"]
+    )
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse(Path(__file__).stem)
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    ctao_array_layouts = retrieve_array_layouts(
+        site=args_dict["site"],
+        repository_url=args_dict["repository_url"],
+        branch_name=args_dict["repository_branch"],
+    )
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    db_array_layouts = db.get_model_parameter(
+        parameter="array_layouts",
+        site=args_dict["site"],
+        array_element_name=None,
+        parameter_version=args_dict.get("parameter_version"),
+        model_version=args_dict.get("model_version"),
+    )
+    db_array_layouts["array_layouts"].pop("_id", None)
+    db_array_layouts["array_layouts"].pop("entry_date", None)
+    logger.info(f"Layouts from DB: {db_array_layouts}")
+
+    write_array_layouts(
+        array_layouts=merge_array_layouts(db_array_layouts["array_layouts"], ctao_array_layouts),
+        args_dict=args_dict,
+        db_config=db_config,
+    )
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/derive_photon_electron_spectrum.py

@@ -69,11 +69,39 @@ def _parse(label):
     )
     config.parser.add_argument(
         "--max_amplitude",
-        help="Maximum amplitude in amplitude spectrum",
+        help="Maximum amplitude for single p.e. for amplitude spectrum",
         type=float,
         default=42.0,
         required=False,
     )
+    config.parser.add_argument(
+        "--scale_afterpulse_spectrum",
+        help="Scale afterpulse spectrum by the given factor",
+        type=float,
+        default=1.0,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--afterpulse_amplitude_range",
+        help="Amplitude range in pe for afterpulse calculation",
+        type=float,
+        nargs=2,
+        default=[0.0, 42.0],
+        required=False,
+    )
+    config.parser.add_argument(
+        "--fit_afterpulse",
+        help="Fit afterpulse spectrum with an exponential decay function.",
+        action="store_true",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--afterpulse_decay_factor_fixed_value",
+        help="Fix decay factor in afterpulse fit (free fit parameter if not set set).",
+        type=float,
+        default=15.0,
+        required=False,
+    )
     config.parser.add_argument(
         "--use_norm_spe",
         help="Use sim_telarray tool 'norm_spe' to normalize the spectrum.",
@@ -81,7 +109,7 @@ def _parse(label):
         required=False,
     )
 
-    return config.initialize(db_config=False, output=True)
+    return config.initialize(db_config=False, output=True, simulation_model=["telescope"])
 
 
 def main():  # noqa: D103

simtools/applications/docs_produce_array_element_report.py

@@ -9,10 +9,11 @@ Currently only implemented for telescopes.
 """
 
 import logging
+from pathlib import Path
 
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
-from simtools.model.telescope_model import TelescopeModel
+from simtools.reporting.docs_auto_report_generator import ReportGenerator
 from simtools.reporting.docs_read_parameters import ReadParameters
 from simtools.utils import general as gen
 
@@ -25,46 +26,61 @@ def _parse(label):
     )
 
     config.parser.add_argument(
-        "--parameter",
+        "--all_telescopes",
         action="store_true",
-        help="Compare all parameters across model versions for one telescope.",
+        help="Produce reports for all telescopes.",
+    )
+
+    config.parser.add_argument(
+        "--all_model_versions",
+        action="store_true",
+        help="Produce reports for all model versions.",
+    )
+
+    config.parser.add_argument(
+        "--all_sites", action="store_true", help="Produce reports for all sites."
+    )
+
+    config.parser.add_argument(
+        "--observatory",
+        action="store_true",
+        help="Produce reports for an observatory at a given site.",
     )
 
     return config.initialize(
-        db_config=True, simulation_model=["site", "telescope", "model_version", "parameter_version"]
+        db_config=True, simulation_model=["site", "telescope", "model_version"]
     )
 
 
 def main():  # noqa: D103
     label_name = "reports"
     args, db_config = _parse(label_name)
+
     io_handler_instance = io_handler.IOHandler()
-    output_path = io_handler_instance.get_output_directory(
-        label=label_name, sub_dir=f"productions/{args['model_version']}"
-    )
+    output_path = io_handler_instance.get_output_directory(label=label_name, sub_dir="productions")
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args["log_level"]))
 
-    telescope_model = TelescopeModel(
-        site=args["site"],
-        telescope_name=args["telescope"],
-        model_version=args["model_version"],
-        label=label_name,
-        mongo_db_config=db_config,
-    )
-
-    ReadParameters(
-        db_config,
-        telescope_model,
-        output_path,
-    ).produce_array_element_report()
-
-    logger.info(
-        f"Markdown report generated for {args['site']}"
-        f" Telescope {args['telescope']} (v{args['model_version']}):"
-        f" {output_path}"
-    )
+    if any([args.get("all_telescopes"), args.get("all_sites"), args.get("all_model_versions")]):
+        ReportGenerator(
+            db_config,
+            args,
+            output_path,
+        ).auto_generate_array_element_reports()
+
+    else:
+        ReadParameters(
+            db_config,
+            args,
+            Path(output_path / f"{args['model_version']}"),
+        ).produce_array_element_report()
+
+        logger.info(
+            f"Markdown report generated for {args['site']}"
+            f" Telescope {args['telescope']} (v{args['model_version']}):"
+            f" {output_path}"
+        )
 
 
 if __name__ == "__main__":

simtools/applications/docs_produce_model_parameter_reports.py

@@ -1,7 +1,7 @@
 #!/usr/bin/python3
 
 r"""
-Produce one file per model parameter for a given array element.
+Produce a model parameter report per array element.
 
 The markdown reports include detailed information on each parameter,
 comparing their values over various model versions.
@@ -12,7 +12,7 @@ import logging
 
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
-from simtools.model.telescope_model import TelescopeModel
+from simtools.reporting.docs_auto_report_generator import ReportGenerator
 from simtools.reporting.docs_read_parameters import ReadParameters
 from simtools.utils import general as gen
 
@@ -24,10 +24,18 @@ def _parse(label):
         description=("Produce a markdown report for model parameters."),
     )
 
-    return config.initialize(
-        db_config=True, simulation_model=["site", "telescope", "model_version"]
+    config.parser.add_argument(
+        "--all_telescopes",
+        action="store_true",
+        help="Produce reports for all telescopes.",
     )
 
+    config.parser.add_argument(
+        "--all_sites", action="store_true", help="Produce reports for all sites."
+    )
+
+    return config.initialize(db_config=True, simulation_model=["site", "telescope"])
+
 
 def main():  # noqa: D103
     label_name = "reports"
@@ -38,25 +46,24 @@ def main():  # noqa: D103
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args["log_level"]))
 
-    telescope_model = TelescopeModel(
-        site=args["site"],
-        telescope_name=args["telescope"],
-        model_version=args["model_version"],
-        label=label_name,
-        mongo_db_config=db_config,
-    )
+    if any([args.get("all_telescopes"), args.get("all_sites")]):
+        ReportGenerator(
+            db_config,
+            args,
+            output_path,
+        ).auto_generate_parameter_reports()
 
-    ReadParameters(
-        db_config,
-        telescope_model,
-        output_path,
-    ).produce_model_parameter_reports()
+    else:
+        ReadParameters(
+            db_config,
+            args,
+            output_path,
+        ).produce_model_parameter_reports()
 
-    logger.info(
-        f"Markdown report generated for {args['site']}"
-        f" Telescope {args['telescope']} (v{args['model_version']}):"
-        f" {output_path}"
-    )
+        logger.info(
+            f"Markdown report generated for {args['site']}"
+            f"Telescope {args['telescope']}: {output_path}"
+        )
 
 
 if __name__ == "__main__":

simtools/applications/generate_simtel_event_data.py

@@ -0,0 +1,117 @@
+#!/usr/bin/python3
+
+"""
+Reduces and compiles event data from multiple input files into a structured dataset with event info.
+
+Command line arguments
+----------------------
+prefix (str, required)
+    Path prefix for the input files.
+output_file (str, required)
+    Path to save the output file.
+max_files (int, optional, default=100)
+    Maximum number of files to process.
+print_dataset_information (flag)
+    Print information about the datasets in the generated reduced event dataset.
+
+Example
+-------
+Generate a reduced dataset from input files and save the result.
+
+.. code-block:: console
+
+    simtools-production-extract-mc-event-data \
+    simtools-generate-simtel-event-data \
+        --prefix path/to/input_files/ \
+        --wildcard 'gamma_*dark*.simtel.zst' \
+        --output_file output_file.hdf5 \
+        --max_files 50 \
+        --print_dataset_information
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.io_operations import io_handler
+from simtools.simtel.simtel_io_event_reader import SimtelIOEventDataReader
+from simtools.simtel.simtel_io_event_writer import SimtelIOEventDataWriter
+
+
+def _parse(label, description):
+    """
+    Parse command line arguments.
+
+    Returns
+    -------
+    dict
+        Parsed command-line arguments.
+    """
+    config = configurator.Configurator(label=label, description=description)
+
+    config.parser.add_argument(
+        "--prefix", type=str, required=True, help="Prefix path for input files."
+    )
+    config.parser.add_argument(
+        "--wildcard",
+        type=str,
+        required=True,
+        help="Wildcard for querying the files in the directory (e.g., 'gamma_*dark*.simtel.zst')",
+    )
+    config.parser.add_argument("--output_file", type=str, required=True, help="Output filename.")
+    config.parser.add_argument(
+        "--max_files", type=int, default=100, help="Maximum number of files to process."
+    )
+
+    config.parser.add_argument(
+        "--print_dataset_information",
+        type=int,
+        help="Print given number of rows of the dataset.",
+        default=0,
+    )
+
+    return config.initialize(db_config=False)
+
+
+def main():
+    """
+    Process event data files and store data in reduced dataset.
+
+    The reduced dataset contains shower information, array information and triggered telescopes.
+    """
+    label = Path(__file__).stem
+
+    args_dict, _ = _parse(
+        label=label,
+        description=(
+            "Process files and store reduced dataset with event information, "
+            "array information and triggered telescopes."
+        ),
+    )
+
+    _logger = logging.getLogger()
+    _logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+    _logger.info(f"Loading input files with prefix: {args_dict['prefix']}")
+
+    input_path = Path(args_dict["prefix"])
+    files = list(input_path.glob(args_dict["wildcard"]))
+    if not files:
+        _logger.warning("No matching input files found.")
+        return
+
+    output_path = io_handler.IOHandler().get_output_directory(label)
+    output_filepath = Path(output_path).joinpath(f"{args_dict['output_file']}")
+
+    output_filepath.parent.mkdir(parents=True, exist_ok=True)
+    generator = SimtelIOEventDataWriter(files, output_filepath, args_dict["max_files"])
+    generator.process_files()
+    _logger.info(f"reduced dataset saved to: {output_filepath}")
+
+    if args_dict["print_dataset_information"] > 0:
+        reader = SimtelIOEventDataReader(output_filepath)
+        reader.print_dataset_information(args_dict.get("print_dataset_information"))
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/plot_tabular_data.py

@@ -17,6 +17,9 @@ from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
+from simtools.constants import PLOT_CONFIG_SCHEMA
+from simtools.data_model import schema
+from simtools.data_model.metadata_collector import MetadataCollector
 from simtools.io_operations import io_handler
 from simtools.visualization import plot_tables
 
@@ -54,7 +57,7 @@ def _parse(label, description, usage):
         type=str,
         required=True,
     )
-    return config.initialize(db_config=True)
+    return config.initialize(db_config=True, simulation_model=["telescope"])
 
 
 def main():
@@ -70,7 +73,10 @@ def main():
     io_handler_instance = io_handler.IOHandler()
 
     plot_config = gen.convert_keys_in_dict_to_lowercase(
-        gen.collect_data_from_file(args_dict["plot_config"])
+        schema.validate_dict_using_schema(
+            gen.collect_data_from_file(args_dict["plot_config"]),
+            PLOT_CONFIG_SCHEMA,
+        )
     )
 
     plot_tables.plot(
@@ -79,6 +85,12 @@ def main():
         db_config=db_config_,
     )
 
+    MetadataCollector.dump(
+        args_dict,
+        io_handler_instance.get_output_file(args_dict["output_file"]),
+        add_activity_name=True,
+    )
+
 
 if __name__ == "__main__":
     main()

simtools/applications/print_version.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python3
+"""
+Print the versions of the simtools software.
+
+The versions of simtools, the DB, sim_telarray, and CORSIKA are printed.
+
+"""
+
+import json
+import logging
+from pathlib import Path
+
+from simtools import dependencies, version
+from simtools.configuration import configurator
+from simtools.io_operations import io_handler
+from simtools.utils import general as gen
+
+
+def _parse(label, description, usage):
+    """
+    Parse command line configuration.
+
+    No command line arguments are required for this application,
+    but the configurator is called to set up the DB connection and
+    the structure with _parse is kept from other applications for consistency.
+
+    Parameters
+    ----------
+    label : str
+        Label describing the application.
+    description : str
+        Description of the application.
+    usage : str
+        Example on how to use the application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description, usage=usage)
+
+    return config.initialize(db_config=True, output=True)
+
+
+def main():
+    """Print the versions of the simtools software."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(
+        label=label,
+        description="Print the versions of simtools, the DB, sim_telarray and CORSIKA.",
+        usage="simtools-print-version",
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+    io_handler_instance = io_handler.IOHandler()
+
+    version_string = dependencies.get_version_string(db_config)
+    version_dict = {"simtools version": version.__version__}
+
+    print()
+    # The loop below is not necessary, there is only one entry, but it is cleaner
+    for key, value in version_dict.items():  #
+        print(f"{key}: {value}")
+    print(version_string)
+
+    version_list = version_string.strip().split("\n")
+    for version_entry in version_list:
+        key, value = version_entry.split(": ")
+        version_dict[key] = value
+
+    with open(
+        io_handler_instance.get_output_file(args_dict["output_file"], label=label),
+        "w",
+        encoding="utf-8",
+    ) as f:
+        json.dump(version_dict, f, indent=4)
+
+
+if __name__ == "__main__":
+    main()