gammasimtools 0.12.0__py3-none-any.whl → 0.14.0__py3-none-any.whl

{gammasimtools-0.12.0.dist-info → gammasimtools-0.14.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (76.0.0)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

{gammasimtools-0.12.0.dist-info → gammasimtools-0.14.0.dist-info}/entry_points.txt

@@ -10,6 +10,7 @@ simtools-db-get-array-layouts-from-db = simtools.applications.db_get_array_layou
 simtools-db-get-file-from-db = simtools.applications.db_get_file_from_db:main
 simtools-db-get-parameter-from-db = simtools.applications.db_get_parameter_from_db:main
 simtools-db-inspect-databases = simtools.applications.db_inspect_databases:main
+simtools-derive-ctao-array-layouts = simtools.applications.derive_ctao_array_layouts:main
 simtools-derive-mirror-rnda = simtools.applications.derive_mirror_rnda:main
 simtools-derive-photon-electron-spectrum = simtools.applications.derive_photon_electron_spectrum:main
 simtools-derive-psf-parameters = simtools.applications.derive_psf_parameters:main
@@ -20,9 +21,11 @@ simtools-generate-corsika-histograms = simtools.applications.generate_corsika_hi
 simtools-generate-default-metadata = simtools.applications.generate_default_metadata:main
 simtools-generate-regular-arrays = simtools.applications.generate_regular_arrays:main
 simtools-generate-simtel-array-histograms = simtools.applications.generate_simtel_array_histograms:main
+simtools-generate-simtel-event-data = simtools.applications.generate_simtel_event_data:main
 simtools-plot-array-layout = simtools.applications.plot_array_layout:main
 simtools-plot-tabular-data = simtools.applications.plot_tabular_data:main
-simtools-production-derive-limits = simtools.applications.production_derive_limits:main
+simtools-print-version = simtools.applications.print_version:main
+simtools-production-derive-corsika-limits = simtools.applications.production_derive_corsika_limits:main
 simtools-production-generate-simulation-config = simtools.applications.production_generate_simulation_config:main
 simtools-production-scale-events = simtools.applications.production_scale_events:main
 simtools-run-application = simtools.applications.run_application:main

simtools/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.12.0'
-__version_tuple__ = version_tuple = (0, 12, 0)
+__version__ = version = '0.14.0'
+__version_tuple__ = version_tuple = (0, 14, 0)

simtools/applications/convert_all_model_parameters_from_simtel.py

@@ -1,8 +1,8 @@
 #!/usr/bin/python3
 r"""
-    Convert all simulation model parameters exported from sim_telarray format using schema files.
+    Convert all simulation model parameters exported from sim_telarray format.
 
-    Check value, type, and range, convert units, and write json files
+    Check value, type, and range, convert units using schema files. Write json files
     ready to be submitted to the model database. Prints out parameters which are not found
     in simtel configuration file and parameters which are not found in simtools schema files.
 
@@ -19,22 +19,15 @@ r"""
     telescope (str, optional)
         Telescope model name (e.g. LST-1, SST-D, ...)
 
+    skip_parameter (str, optional)
+        List of parameters to be skipped (use sim_telarray names).
+
     Example
     -------
 
-    Extract model parameters with schema files from simtel configuration file
-    (requires access to the model parameter repository)
-
-    .. code-block:: console
-
-       simtools-convert-all-model-parameters-from-simtel \\
-          --schema_directory ../model_parameters/schema\\
-          --simtel_cfg_file all_telescope_config_la_palma.cfg\\
-          --simtel_telescope_name CT1\\
-          --telescope LSTN-01\\
-          --parameter_version "1.0.0"
-
-    The export of the model parameters from sim_telarray for 6.0.0 can be done e.g., as follows:
+    To export the model parameters from sim_telarray, first copy and unpack the configuration
+    tar ball from sim_telarray (usually called 'sim_telarray_config.tar.gz') to the sim_telarray
+    working directory. Extract the configuration using the following command:
 
     .. code-block:: console
 
@@ -42,18 +35,32 @@ r"""
             -C limits=no-internal -C initlist=no-internal -C list=no-internal \\
             -C typelist=no-internal -C maximum_telescopes=30 -DNSB_AUTOSCALE \\
             -DNECTARCAM -DHYPER_LAYOUT -DNUM_TELESCOPES=30 /dev/null \\
-            2>|/dev/null | grep '(@cfg)'  >| all_telescope_config_la_palma.cfg
+            2>|/dev/null | grep '(@cfg)' | sed 's/^(@cfg) //' >| all_telescope_config_la_palma.cfg
 
         ./sim_telarray/bin/sim_telarray -c sim_telarray/cfg/CTA/CTA-PROD6-Paranal.cfg \\
             -C limits=no-internal -C initlist=no-internal -C list=no-internal \\
             -C typelist=no-internal -C maximum_telescopes=87 -DNSB_AUTOSCALE \\
             -DFLASHCAM -DHYPER_LAYOUT -DNUM_TELESCOPES=87 /dev/null \\
-            2>|/dev/null | grep '(@cfg)'  >| all_telescope_config_paranal.cfg
+            2>|/dev/null | grep '(@cfg)' | sed 's/^(@cfg) //' >| all_telescope_config_paranal.cfg
+
+
+    Extract then model parameters from the simtel_array configuration file for LSTN-01
+    (telescopes are named CT1, CT2, ..., in the sim_telarray configuration file and must be
+    provided in the "simtel_telescope_name" command line argument)
+    and write json files in the same format as the model parameter database:
+
+    .. code-block:: console
+
+       simtools-convert-all-model-parameters-from-simtel \\
+          --simtel_cfg_file all_telescope_config_la_palma.cfg\\
+          --simtel_telescope_name CT1\\
+          --telescope LSTN-01\\
+          --parameter_version "1.0.0"\\
+          --output_path /path/to/output
 
 """
 
 import logging
-import re
 from pathlib import Path
 
 import numpy as np
@@ -63,7 +70,7 @@ import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.data_model import schema
 from simtools.io_operations.io_handler import IOHandler
-from simtools.simtel.simtel_config_reader import SimtelConfigReader
+from simtools.simtel import simtel_config_reader
 
 
 def _parse(label=None, description=None):
@@ -85,14 +92,6 @@ def _parse(label=None, description=None):
     """
     config = configurator.Configurator(label=label, description=description)
 
-    config.parser.add_argument(
-        "--schema_directory",
-        help=(
-            "Directory with schema files for model parameter validation "
-            "(default: simtools schema directory)"
-        ),
-        required=False,
-    )
     config.parser.add_argument(
         "--simtel_cfg_file",
         help="File name for simtel_array configuration",
@@ -105,36 +104,17 @@ def _parse(label=None, description=None):
         type=str,
         required=True,
     )
+    config.parser.add_argument(
+        "--skip_parameter",
+        help="List of parameters to be skipped.",
+        type=str,
+        nargs="*",
+        default=[],
+    )
     return config.initialize(simulation_model=["telescope", "parameter_version"])
 
 
-def get_list_of_simtel_parameters(simtel_config_file, logger):
-    """
-    Return list of simtel parameters found in simtel configuration file.
-
-    Parameters
-    ----------
-    simtel_config_file: str
-        File name for sim_telarray configuration
-    logger: logging.Logger
-        Logger object
-
-    Returns
-    -------
-    list
-        List of parameters found in simtel configuration file.
-
-    """
-    simtel_parameter_set = set()
-    with open(simtel_config_file, encoding="utf-8") as file:
-        for line in file:
-            parts_of_lines = re.split(r",\s*|\s+", line.strip())
-            simtel_parameter_set.add(parts_of_lines[1].lower())
-    logger.info(f"Found {len(simtel_parameter_set)} parameters in simtel configuration file.")
-    return list(simtel_parameter_set)
-
-
-def read_simtel_config_file(args_dict, logger, schema_file, camera_pixels=None):
+def read_simtel_config_file(args_dict, schema_file, camera_pixels=None):
     """
     Read the simtel configuration file.
 
@@ -142,25 +122,26 @@ def read_simtel_config_file(args_dict, logger, schema_file, camera_pixels=None):
     ----------
     args_dict: dict
         Dictionary with command line arguments.
-    logger: logging.Logger
-        Logger object
     schema_file: str
         Schema path name.
     camera_pixels: int
         Number of camera pixels.
 
+    Returns
+    -------
+    SimtelConfigReader, None
+        SimtelConfigReader object (None if parameter not found)
+
     """
-    simtel_config_reader = SimtelConfigReader(
+    config_reader = simtel_config_reader.SimtelConfigReader(
         schema_file=schema_file,
         simtel_config_file=args_dict["simtel_cfg_file"],
         simtel_telescope_name=args_dict["simtel_telescope_name"],
         camera_pixels=camera_pixels,
     )
-    logger.info(f"Simtel parameter: {simtel_config_reader.parameter_dict}")
-
-    if simtel_config_reader.parameter_dict is None or len(simtel_config_reader.parameter_dict) == 0:
+    if config_reader.parameter_dict is None or len(config_reader.parameter_dict) == 0:
         return None
-    return simtel_config_reader
+    return config_reader
 
 
 def get_number_of_camera_pixel(args_dict, logger):
@@ -184,12 +165,10 @@ def get_number_of_camera_pixel(args_dict, logger):
 
     """
     try:
-        simtel_config_reader = SimtelConfigReader(
-            schema_file=schema.get_model_parameter_schema_file("camera_pixels"),
-            simtel_config_file=args_dict["simtel_cfg_file"],
-            simtel_telescope_name=args_dict["simtel_telescope_name"],
+        config_reader = read_simtel_config_file(
+            args_dict, schema.get_model_parameter_schema_file("camera_pixels")
         )
-        _camera_pixel = simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"])
+        _camera_pixel = config_reader.parameter_dict.get(args_dict["simtel_telescope_name"])
     except (FileNotFoundError, AttributeError):
         logger.warning("Failed to read camera pixel parameter.")
         _camera_pixel = None
@@ -219,13 +198,14 @@ def read_and_export_parameters(args_dict, logger):
         List of simtools parameter not found in simtel configuration file.
 
     """
-    _parameters, _schema_files = schema.get_get_model_parameter_schema_files(
-        args_dict.get("schema_directory")
+    _parameters, _schema_files = schema.get_get_model_parameter_schema_files()
+    _simtel_parameters = simtel_config_reader.get_list_of_simtel_parameters(
+        args_dict["simtel_cfg_file"]
     )
-    _simtel_parameters = get_list_of_simtel_parameters(args_dict["simtel_cfg_file"], logger)
+    logger.info(f"Found {len(_simtel_parameters)} parameters in simtel configuration file.")
 
     io_handler = IOHandler()
-    io_handler.set_paths(output_path=args_dict["output_path"], use_plain_output_path=True)
+    io_handler.set_paths(output_path=args_dict["output_path"])
 
     _camera_pixel = get_number_of_camera_pixel(args_dict, logger)
 
@@ -233,27 +213,34 @@ def read_and_export_parameters(args_dict, logger):
 
     for _parameter, _schema_file in zip(_parameters, _schema_files):
         logger.info(f"Parameter: {_parameter} Schema file: {_schema_file}")
-        simtel_config_reader = read_simtel_config_file(
-            args_dict, logger, _schema_file, _camera_pixel
-        )
+        if _parameter in args_dict["skip_parameter"]:
+            logger.info(f"Skipping {_parameter}")
+            continue
+        config_reader = read_simtel_config_file(args_dict, _schema_file, _camera_pixel)
 
-        if simtel_config_reader is None:
+        if config_reader is None:
             logger.info("Parameter not found in sim_telarray configuration file.")
             _parameters_not_in_simtel.append(_parameter)
             continue
 
+        logger.info(f"Simtel parameter: {config_reader.parameter_dict}")
+
         _json_dict = writer.ModelDataWriter.dump_model_parameter(
             parameter_name=_parameter,
-            value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
+            value=config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
             instrument=args_dict["telescope"],
             parameter_version=args_dict["parameter_version"],
-            output_file=io_handler.get_output_file(f"{_parameter}.json"),
+            output_file=io_handler.get_output_file(
+                f"{_parameter}-{args_dict['parameter_version']}.json",
+                label=f"{args_dict['telescope']}",
+                sub_dir=f"{_parameter}",
+            ),
         )
 
-        simtel_config_reader.compare_simtel_config_with_schema()
+        config_reader.compare_simtel_config_with_schema()
 
-        if simtel_config_reader.simtel_parameter_name.lower() in _simtel_parameters:
-            _simtel_parameters.remove(simtel_config_reader.simtel_parameter_name.lower())
+        if config_reader.simtel_parameter_name.lower() in _simtel_parameters:
+            _simtel_parameters.remove(config_reader.simtel_parameter_name.lower())
 
         if _json_dict["file"]:
             logger.info(f"File name for {_parameter} is {_json_dict['value']}")
@@ -263,7 +250,7 @@ def read_and_export_parameters(args_dict, logger):
 
 def print_parameters_not_found(_parameters_not_in_simtel, _simtel_parameters, args_dict, logger):
     """
-    Print simtel/simtools parameter not found in schema and ocnfiguration files.
+    Print simtel/simtools parameter not found in schema and configuration files.
 
     For simtel parameters not found, check if the setting for the chose
     telescope is different from the default values.
@@ -289,29 +276,31 @@ def print_parameters_not_found(_parameters_not_in_simtel, _simtel_parameters, ar
     logger.info(f"Simtel parameters not found in schema files ({len(_simtel_parameters)}):")
     for para in sorted(_simtel_parameters):
         logger.info(f"Simtel parameter: {para}")
-        simtel_config_reader = SimtelConfigReader(
+        config_reader = simtel_config_reader.SimtelConfigReader(
             schema_file=None,
             simtel_config_file=args_dict["simtel_cfg_file"],
             simtel_telescope_name=args_dict["simtel_telescope_name"],
             parameter_name=para,
         )
-        _default = simtel_config_reader.parameter_dict.get("default")
-        _tel_value = simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"])
+        _default = config_reader.parameter_dict.get("default")
+        _tel_value = config_reader.parameter_dict.get(args_dict["simtel_telescope_name"])
         # simple comparison of default value and telescope values, does not work for lists
         try:
             if _default == _tel_value or np.isclose(_default, _tel_value):
-                logger.info(f"    Default and telescope values are equal: {_default}")
+                logger.info(f"    Default and telescope values for {para} are equal: {_default}")
                 continue
         except (ValueError, TypeError):
             pass
         if isinstance(_default, np.ndarray):
-            logger.warning(f"    Default value: {_default} (length: {_default.size})")
+            logger.warning(f"    Default value ({para}): {_default} (length: {_default.size})")
         else:
-            logger.warning(f"    Default value: {_default}")
+            logger.warning(f"    Default value ({para}): {_default}")
         if isinstance(_tel_value, np.ndarray):
-            logger.warning(f"    Telescope value: {_tel_value} (length: {_tel_value.size})")
+            logger.warning(
+                f"    Telescope value ({para}): {_tel_value} (length: {_tel_value.size})"
+            )
         else:
-            logger.warning(f"    Telescope value: {_tel_value}")
+            logger.warning(f"    Telescope value ({para}): {_tel_value}")
 
 
 def print_list_of_files(args_dict, logger):

simtools/applications/convert_geo_coordinates_of_array_elements.py

@@ -178,7 +178,7 @@ def main():
         )
         writer.ModelDataWriter.dump(
             args_dict=args_dict,
-            metadata=metadata.get_top_level_metadata() if metadata else None,
+            metadata=metadata,
             product_data=product_data,
             validate_schema_file=validate_schema_file,
         )

simtools/applications/db_add_file_to_db.py

@@ -38,6 +38,7 @@
 """
 
 import logging
+import uuid
 from pathlib import Path
 
 import simtools.utils.general as gen
@@ -69,6 +70,11 @@ def _parse():
         type=str,
         help=("The database to insert the files to."),
     )
+    config.parser.add_argument(
+        "--test_db",
+        help="Use sandbox database. Drop all data after the operation.",
+        action="store_true",
+    )
 
     return config.initialize(paths=False, db_config=True)
 
@@ -134,6 +140,10 @@ def confirm_and_insert_files(files_to_insert, args_dict, db, logger):
     """
     plural = "" if len(files_to_insert) == 1 else "s"
 
+    if args_dict.get("test_db", False):
+        args_dict["db"] = args_dict["db"] + str(uuid.uuid4())
+        logger.info(f"Using test database: {args_dict['db']}")
+
     print(f"Should the following file{plural} be inserted to the {args_dict['db']} DB?:\n")
     print(*files_to_insert, sep="\n")
     print()
@@ -145,6 +155,11 @@ def confirm_and_insert_files(files_to_insert, args_dict, db, logger):
     else:
         logger.info(f"Aborted, did not insert file{plural} to the {args_dict['db']} DB")
 
+    # drop test database; be safe and required DB name is sandbox
+    if args_dict.get("test_db", False) and "sandbox" in args_dict["db"]:
+        logger.info(f"Test database used. Dropping all data from {args_dict['db']}")
+        db.db_client.drop_database(args_dict["db"])
+
 
 def main():  # noqa: D103
     args_dict, db_config = _parse()

simtools/applications/db_add_value_from_json_to_db.py

@@ -26,6 +26,7 @@ r"""
 """
 
 import logging
+import uuid
 from pathlib import Path
 
 import simtools.utils.general as gen
@@ -45,7 +46,11 @@ def _parse():
     config.parser.add_argument(
         "--db_collection", help="DB collection to which to add new values.", required=True
     )
-    config.parser.add_argument("--db", help="Database name", type=str, required=True)
+    config.parser.add_argument(
+        "--test_db",
+        help="Use sandbox database. Drop all data after the operation.",
+        action="store_true",
+    )
     return config.initialize(db_config=True)
 
 
@@ -55,6 +60,9 @@ def main():  # noqa: D103
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
+    if args_dict.get("test_db", False):
+        db_config["db_simulation_model"] = db_config["db_simulation_model"] + str(uuid.uuid4())
+        logger.info(f"Using test database: {db_config['db_simulation_model']}")
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
 
     files_to_insert = []
@@ -74,6 +82,8 @@ def main():  # noqa: D103
     print(*files_to_insert, sep="\n")
     print()
 
+    logger.info(f"DB {args_dict['db_simulation_model']} selected.")
+
     if gen.user_confirm():
         for file_to_insert_now in files_to_insert:
             par_dict = gen.collect_data_from_file(file_name=file_to_insert_now)
@@ -91,6 +101,13 @@ def main():  # noqa: D103
     else:
         logger.info("Aborted, no change applied to the database")
 
+    # drop test database; be safe and required DB name is sandbox
+    if args_dict.get("test_db", False) and "sandbox" in args_dict["db_simulation_model"]:
+        logger.info(
+            f"Test database used. Dropping all data from {db_config['db_simulation_model']}"
+        )
+        db.db_client.drop_database(db_config["db_simulation_model"])
+
 
 if __name__ == "__main__":
     main()

simtools/applications/db_get_parameter_from_db.py

@@ -3,22 +3,36 @@
 r"""
     Get a parameter entry from DB for a specific telescope or a site.
 
-    The application receives a parameter name, a site, a telescope (if applicable) and \
-    a version. It then prints out the parameter entry.
+    The application receives a parameter name, a site, a telescope (if applicable) and
+    a version. Allow to print the parameter entry to screen or save it to a file.
+    Parameter describing a table file can be written to disk or exported as an astropy table
+    (if available).
 
     Command line arguments
     ----------------------
     parameter (str, required)
         Parameter name
 
+    parameter_version (str, optional)
+        Parameter version
+
+    model_version (str, required)
+        Model version
+
     site (str, required)
         South or North.
 
     telescope (str, optional)
         Telescope model name (e.g. LST-1, SST-D, ...)
 
-    log_level (str, optional)
-        Log level to print.
+    output_file (str, optional)
+        Output file name. If not given, print to stdout.
+
+    export_model_file (bool, optional)
+        Export model file (if parameter describes a file).
+
+    export_model_file_as_table (bool, optional)
+        Export model file as astropy table (if parameter describes a file).
 
     Raises
     ------
@@ -35,12 +49,14 @@ r"""
                 --model_version 5.0.0
 
     Get the mirror_list parameter using the parameter_version from the DB.
+    Write the mirror list to disk.
 
     .. code-block:: console
 
         simtools-db-get-parameter-from-db --parameter mirror_list \\
                 --site North --telescope LSTN-01 \\
-                --parameter_version 1.0.0
+                --parameter_version 1.0.0 \\
+                --export_model_file
 
 """
 
@@ -53,7 +69,6 @@ import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.db import db_handler
 from simtools.io_operations import io_handler
-from simtools.utils import names
 
 
 def _parse():
@@ -72,6 +87,18 @@ def _parse():
         type=str,
         required=False,
     )
+    config.parser.add_argument(
+        "--export_model_file",
+        help="Export model file (if parameter describes a file)",
+        action="store_true",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--export_model_file_as_table",
+        help="Export model file as astropy table (if parameter describes a file)",
+        action="store_true",
+        required=False,
+    )
 
     return config.initialize(
         db_config=True, simulation_model=["telescope", "parameter_version", "model_version"]
@@ -86,26 +113,29 @@ def main():  # noqa: D103
 
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
 
-    # get parameter using 'parameter_version'
-    if args_dict["parameter_version"] is not None:
-        pars = db.get_model_parameter(
+    pars = db.get_model_parameter(
+        parameter=args_dict["parameter"],
+        site=args_dict["site"],
+        array_element_name=args_dict.get("telescope"),
+        parameter_version=args_dict.get("parameter_version"),
+        model_version=args_dict.get("model_version"),
+    )
+    if args_dict["export_model_file"] or args_dict["export_model_file_as_table"]:
+        table = db.export_model_file(
             parameter=args_dict["parameter"],
-            parameter_version=args_dict["parameter_version"],
             site=args_dict["site"],
             array_element_name=args_dict["telescope"],
+            parameter_version=args_dict.get("parameter_version"),
+            model_version=args_dict.get("model_version"),
+            export_file_as_table=args_dict["export_model_file_as_table"],
         )
-    # get parameter using 'model_version'
-    elif args_dict["model_version"] is not None:
-        pars = db.get_model_parameters(
-            site=args_dict["site"],
-            array_element_name=args_dict.get("telescope"),
-            model_version=args_dict["model_version"],
-            collection=names.get_collection_name_from_parameter_name(args_dict["parameter"]),
-        )
-    else:
-        raise ValueError("Either 'parameter_version' or 'model_version' must be provided.")
-    if args_dict["parameter"] not in pars:
-        raise KeyError(f"The requested parameter, {args_dict['parameter']}, does not exist.")
+        param_value = pars[args_dict["parameter"]]["value"]
+        table_file = Path(io_handler.IOHandler().get_output_directory()) / f"{param_value}"
+        logger.info(f"Exported model file {param_value} to {table_file}")
+        if table and table_file.suffix != ".ecsv":
+            table.write(table_file.with_suffix(".ecsv"), format="ascii.ecsv", overwrite=True)
+            logger.info(f"Exported model file {param_value} to {table_file.with_suffix('.ecsv')}")
+
     if args_dict["output_file"] is not None:
         _output_file = (
             Path(io_handler.IOHandler().get_output_directory()) / args_dict["output_file"]