gammasimtools 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.17.0'
-__version_tuple__ = version_tuple = (0, 17, 0)
+__version__ = version = '0.19.0'
+__version_tuple__ = version_tuple = (0, 19, 0)
+
+__commit_id__ = commit_id = None

simtools/applications/calculate_trigger_rate.py

@@ -48,7 +48,7 @@ from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 from simtools.simtel.simtel_io_histograms import SimtelIOHistograms
 
 

simtools/applications/convert_all_model_parameters_from_simtel.py

@@ -69,7 +69,8 @@ import simtools.data_model.model_data_writer as writer
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.data_model import schema
-from simtools.io_operations.io_handler import IOHandler
+from simtools.io import ascii_handler
+from simtools.io.io_handler import IOHandler
 from simtools.simtel import simtel_config_reader
 
 
@@ -170,7 +171,7 @@ def get_number_of_camera_pixel(args_dict, logger):
         )
         _camera_pixel = config_reader.parameter_dict.get(args_dict["simtel_telescope_name"])
     except (FileNotFoundError, AttributeError):
-        logger.warning("Failed to read camera pixel parameter.")
+        logger.warning("Unable to retrieve camera pixel parameter.")
         _camera_pixel = None
     logger.info(f"Number of camera pixels: {_camera_pixel}")
     return _camera_pixel
@@ -319,7 +320,7 @@ def print_list_of_files(args_dict, logger):
     """
     model_files = sorted(Path(args_dict["output_path"]).rglob("*.json"))
     for file in model_files:
-        model_dict = gen.collect_data_from_file(file_name=file)
+        model_dict = ascii_handler.collect_data_from_file(file_name=file)
         if model_dict.get("file"):
             logger.info(f"{file.name}: {model_dict['value']}")
 

simtools/applications/convert_geo_coordinates_of_array_elements.py

@@ -140,10 +140,10 @@ def _parse(label=None, description=None):
 def main():
     """Print a list of array elements."""
     label = Path(__file__).stem
-    data_model_name = "array_coordinates"
+    model_parameter_name = "array_coordinates"
     args_dict, db_config = _parse(
         label,
-        description=f"Print a list of array element positions ({data_model_name})",
+        description=f"Print a list of array element positions ({model_parameter_name})",
     )
 
     logger = logging.getLogger()
@@ -153,7 +153,7 @@ def main():
         site = args_dict.get("site", None)
         metadata, validate_schema_file = None, None
     else:
-        metadata = MetadataCollector(args_dict=args_dict, data_model_name=data_model_name)
+        metadata = MetadataCollector(args_dict=args_dict, model_parameter_name=model_parameter_name)
         site = metadata.get_site(from_input_meta=True)
         validate_schema_file = metadata.get_data_model_schema_file_name()
 

simtools/applications/db_add_simulation_model_from_repository_to_db.py

@@ -18,7 +18,16 @@ r"""
 
     Examples
     --------
-    Upload model data repository to the DB:
+    Upload model data repository to the DB
+    Loops over all subdirectories in 'input_path' and uploads all json files to the
+    database 'new_db_name' (or updates an existing database with the same name):
+
+    * subdirectories starting with 'OBS' are uploaded to the 'sites' collection
+    * json files from the subdirectory 'configuration_sim_telarray/configuration_corsika'
+      are uploaded to the 'configuration_sim_telarray/configuration_corsika' collection
+    * 'Files' are added to the 'files' collection
+    * all other json files are uploaded to collection defined in the array element description
+      in 'simtools/schemas/array_elements.yml'
 
     .. code-block:: console
 

simtools/applications/db_add_value_from_json_to_db.py

@@ -32,6 +32,7 @@ from pathlib import Path
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.db import db_handler
+from simtools.io import ascii_handler
 
 
 def _parse():
@@ -86,7 +87,7 @@ def main():  # noqa: D103
 
     if gen.user_confirm():
         for file_to_insert_now in files_to_insert:
-            par_dict = gen.collect_data_from_file(file_name=file_to_insert_now)
+            par_dict = ascii_handler.collect_data_from_file(file_name=file_to_insert_now)
             logger.info(f"Adding the following parameter to the DB: {par_dict['parameter']}")
             db.add_new_parameter(
                 db_name=db_config["db_simulation_model"],

simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py

@@ -33,7 +33,6 @@
 
 """
 
-import json
 import logging
 from pathlib import Path
 
@@ -41,7 +40,8 @@ import astropy.table
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.data_model.model_data_writer import JsonNumpyEncoder, ModelDataWriter
+from simtools.data_model.model_data_writer import ModelDataWriter
+from simtools.io import ascii_handler
 from simtools.model.array_model import ArrayModel
 
 
@@ -145,17 +145,12 @@ def write_ground_array_elements_to_repository(args_dict, db_config, logger):
         output_path = Path(args_dict["repository_path"]) / f"{element_name}"
         output_path.mkdir(parents=True, exist_ok=True)
         logger.info(f"Writing array element positions (ground) to {output_path}")
-        with open(
-            output_path / "array_element_position_ground.json", "w", encoding="utf-8"
-        ) as file:
-            json.dump(
-                data,
-                file,
-                indent=4,
-                sort_keys=False,
-                cls=JsonNumpyEncoder,
-            )
-            file.write("\n")
+        ascii_handler.write_data_to_file(
+            data=data,
+            output_file=output_path / "array_element_position_ground.json",
+            sort_keys=False,
+            numpy_types=True,
+        )
 
 
 def main():

simtools/applications/db_generate_compound_indexes.py

@@ -0,0 +1,61 @@
+#!/usr/bin/python3
+
+"""
+Generate compound indexes for the specified database.
+
+This needs to be done once after a database has been set up.
+Significantly accelerates database querying (at least a factor
+of 5 in query time with a factor of 10 less documents examined).
+
+Command line arguments
+----------------------
+db_name (str, optional)
+    Database name (use "all" for all databases)
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler
+
+
+def _parse():
+    config = configurator.Configurator(
+        description="Generate compound indexes for a specific database"
+    )
+    config.parser.add_argument(
+        "--db_name",
+        help="Database name",
+        default="all",
+        required=True,
+    )
+    return config.initialize(db_config=True)
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    # databases without internal databases we don't have rights to modify
+    databases = [
+        d for d in db.db_client.list_database_names() if d not in ("config", "admin", "local")
+    ]
+    requested = args_dict["db_name"]
+    if requested != "all" and requested not in databases:
+        raise ValueError(
+            f"Requested database '{requested}' not found. "
+            f"Following databases are available: {', '.join(databases)}"
+        )
+
+    databases = databases if requested == "all" else [requested]
+    for db_name in databases:
+        logger.info(f"Generating compound indexes for database: {db_name}")
+        db.generate_compound_indexes(db_name=db_name)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/db_get_file_from_db.py

@@ -38,7 +38,7 @@ import logging
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.db import db_handler
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 
 
 def _parse():

simtools/applications/db_get_parameter_from_db.py

@@ -60,7 +60,6 @@ r"""
 
 """
 
-import json
 import logging
 from pathlib import Path
 from pprint import pprint
@@ -68,7 +67,7 @@ from pprint import pprint
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.db import db_handler
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 
 
 def _parse():
@@ -142,8 +141,9 @@ def main():  # noqa: D103
         )
         pars[args_dict["parameter"]].pop("_id")
         pars[args_dict["parameter"]].pop("entry_date")
-        with open(_output_file, "w", encoding="utf-8") as json_file:
-            json.dump(pars[args_dict["parameter"]], json_file, indent=4)
+        ascii_handler.write_data_to_file(
+            data=pars[args_dict["parameter"]], output_file=_output_file
+        )
     else:
         pprint(pars[args_dict["parameter"]])
 

simtools/applications/db_inspect_databases.py

@@ -6,7 +6,7 @@ Inspect databases and print (available database names and collections).
 Command line arguments
 ----------------------
 db_name (str, optional)
-    Inspect a specific database.
+    Database name (use "all" for all databases)
 """
 
 import logging
@@ -20,11 +20,11 @@ def _parse():
     config = configurator.Configurator(description="Inspect databases")
     config.parser.add_argument(
         "--db_name",
-        help="Inspect a specific database (use all to print all databases)",
+        help="Database name",
         default="all",
         required=True,
     )
-    return config.initialize(db_config=True, simulation_model="telescope")
+    return config.initialize(db_config=True)
 
 
 def main():  # noqa: D103
@@ -34,18 +34,28 @@ def main():  # noqa: D103
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
-
-    databases = db.db_client.list_database_names()
+    # databases without internal databases we don't have rights to modify
+    databases = [
+        d for d in db.db_client.list_database_names() if d not in ("config", "admin", "local")
+    ]
+    requested = args_dict["db_name"]
+    if requested != "all" and requested not in databases:
+        raise ValueError(
+            f"Requested database '{requested}' not found. "
+            f"Following databases are available: {', '.join(databases)}"
+        )
+
+    databases = databases if requested == "all" else [requested]
 
     for db_name in databases:
-        if args_dict["db_name"] != "all" and db_name != args_dict["db_name"]:
-            continue
-        # missing admin rights; skip config and admin
-        if db_name in ("config", "admin", "local"):
-            continue
         print("Database:", db_name)
         collections = db.get_collections(db_name=db_name)
         print("   Collections:", collections)
+        print("   Indexes:")
+        for collection_name in collections:
+            db_collection = db.get_collection(collection_name=collection_name, db_name=db_name)
+            for idx in db_collection.list_indexes():
+                print(f"     {collection_name}: {idx}")
 
 
 if __name__ == "__main__":

simtools/applications/derive_mirror_rnda.py

@@ -1,18 +1,15 @@
 #!/usr/bin/python3
 
 r"""
-    Derive the simulation model parameter mirror_reflection_random_angle.
-
-    This parameter, often referred to as "mirror roughness," is used to align the simulation
-    with the measured containment diameter of the optical point-spread function (PSF) for
-    individual mirror panels.
+    Derive mirror random reflection angle (mirror roughness) of a single mirror panel.
 
     Description
     -----------
 
     This application derives the value of the simulation model parameter
     *mirror_reflection_random_angle* using measurements of the focal length
-    and PSF of individual mirror panels.
+    and point-spread function (PSF) of individual mirror panels.
+    This parameter is sometimes referred to as the "mirror roughness".
 
     PSF measurements are provided by one of the following options:
 
@@ -22,7 +19,7 @@ r"""
         * file (table) with measured PSF for each mirror panel spot size (``--psf_measurement``)
 
     The containment fraction used for the PSF diameter calculation is set through
-    the argument ``--containment_fraction`` (typically 0.8 = 80%).
+    the argument ``--containment_fraction`` (typically 0.8 = 80%; called below D80).
 
     Mirror panels are simulated individually, using one of the following options to set the
     mirror panel focal length:
@@ -37,7 +34,7 @@ r"""
     taken from the Model Parameters DB (default) or can be set using the argument ``--rnda``.
 
     Ray-tracing simulations are performed for single mirror configurations for each
-    mirror given in the mirror_list. The mean simulated containment diameter for all the mirrors
+    mirror given in the mirror list. The mean simulated containment diameter for all the mirrors
     is compared with the mean measured containment diameter. The algorithm defines a new value for
     the random reflection angle based on the sign of the difference between measured and simulated
     containment diameters and a new set of simulations is performed. This process is repeated
@@ -59,7 +56,7 @@ r"""
     .. image:: images/derive_mirror_rnda_North-MST-FlashCam-D_D80-distributions.png
       :width: 49 %
 
-    This application uses the following :ref:`SimulationSoftware` tools:
+    This application uses the following software tools:
 
         - sim_telarray/bin/sim_telarray
         - sim_telarray/bin/rx (optional)
@@ -67,7 +64,7 @@ r"""
     Command line arguments
     ----------------------
     telescope (str, required)
-        Telescope name (e.g. North-LST-1, South-SST-D, ...)
+        Telescope name (e.g. LSTN-01, SSTS-25)
     model_version (str, optional)
         Model version
     psf_measurement (str, optional)
@@ -89,6 +86,8 @@ r"""
     random_focal_length (float, optional)
         Value of the random focal lengths to replace the default random_focal_length. Only used if
         'use_random_focal_length' is activated.
+    random_focal_length_seed (int, optional)
+        Seed for the random number generator used for focal length variation.
     no_tuning (activation mode, optional)
         Turn off the tuning - A single case will be simulated and plotted.
     test (activation mode, optional)
@@ -96,8 +95,8 @@ r"""
 
     Example
     -------
-    Derive mirror random reflection angle for a mid-sized telescope (MST),
-    simulation production 5.0.0
+    Derive mirror random reflection angle for a large-sized telescope (LSTS),
+    simulation production 6.0.0
 
     .. code-block:: console
 
@@ -200,6 +199,13 @@ def _parse(label):
         type=float,
         required=False,
     )
+    config.parser.add_argument(
+        "--random_focal_length_seed",
+        help="Seed for the random number generator used for focal length variation.",
+        type=int,
+        required=False,
+        default=None,
+    )
     config.parser.add_argument(
         "--no_tuning",
         help="no tuning of random_reflection_angle (a single case will be simulated).",