gammasimtools 0.18.0__py3-none-any.whl → 0.20.0__py3-none-any.whl

simtools/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '0.18.0'
-__version_tuple__ = version_tuple = (0, 18, 0)
+__version__ = version = '0.20.0'
+__version_tuple__ = version_tuple = (0, 20, 0)
+
+__commit_id__ = commit_id = None

simtools/applications/calculate_incident_angles.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python
+r"""
+Calculate photon incident angles on focal plane and primary/secondary mirrors.
+
+Creates photon files with additional columns for incident angles calculation.
+Outputs files and histograms of the incidence angles at
+the focal plane, primary mirror, and if available, secondary mirror.
+Optional debug plots can be also generated.
+Note that this application does not include a full raytracing of telescope structures,
+and their non symmetric shadowing at off-axis angles.
+
+Example usage
+-------------
+
+.. code-block:: console
+
+    simtools-calculate-incident-angles \
+        --off_axis_angles 0 1 2 3 4 \
+        --source_distance 10 \
+        --number_of_photons 1000000 \
+        --model_version 6.0.0 \
+        --telescope MSTN-04 \
+        --site North
+
+Command line arguments
+----------------------
+
+- off_axis_angles (float, optional)
+    One or more off-axis angles in degrees (space-separated). Default: [0.0].
+- source_distance (float, optional)
+    Source distance in kilometers. Default: 10.0.
+- number_of_photons (int, optional)
+    Number of photons of the light source to trace per run. Default: 10000.
+- perfect_mirror (flag, optional)
+    Assume perfect mirror shape/alignment/reflection.
+- debug_plots (flag, optional)
+    Generate additional debug plots (radius histograms, XY heatmaps, radius vs angle).
+- calculate_primary_secondary_angles / no-calculate_primary_secondary_angles
+    Include or skip angles on primary/secondary mirrors. Default: include.
+
+The application writes:
+
+- imaging list (photons) file
+- stars list file
+- a histogram of incident angles (PNG)
+- a results table in ECSV format
+
+Example of a focal-plane incident angle plot for a SST:
+
+.. _plot_calculate_incident_angles_plot:
+.. image:: images/incident_angles_multi_calculate_incident_angles_SSTS-04.png
+    :width: 49 %
+
+Example of a primary mirror incident angle plot for a SST:
+
+.. _plot_calculate_incident_angles_plot_primary:
+.. image:: images/incident_angles_primary_multi_calculate_incident_angles_SSTS-04.png
+    :width: 49 %
+
+Note also the relation between radius and primary mirror incident angles, and how this relates to
+the peak seen in the primary mirror incident angle distribution:
+
+.. _plot_calculate_incident_angles_plot_angle_vs_radius:
+.. image:: images/primary_angle_vs_radius.png
+    :width: 49 %
+
+Example of a secondary mirror incident angle plot for a SST:
+
+.. _plot_calculate_incident_angles_plot_secondary:
+.. image:: images/incident_angles_secondary_multi_calculate_incident_angles_SSTS-04.png
+    :width: 49 %
+"""
+
+import logging
+from pathlib import Path
+
+from simtools.configuration import configurator
+from simtools.ray_tracing.incident_angles import IncidentAnglesCalculator
+from simtools.visualization.plot_incident_angles import plot_incident_angles
+
+
+def _parse(label):
+    """Parse command line configuration."""
+    config = configurator.Configurator(
+        label=label,
+        description=(
+            "Calculate photon incident angles on focal plane and primary/secondary mirrors."
+        ),
+    )
+    config.parser.add_argument(
+        "--off_axis_angles",
+        help="One or more off-axis angles in degrees (space-separated)",
+        type=float,
+        nargs="+",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--source_distance",
+        help="Source distance in kilometers",
+        type=float,
+        default=10.0,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--number_of_photons",
+        help="Number of star photons to trace (per run)",
+        type=int,
+        default=10000,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--perfect_mirror",
+        help="Assume perfect mirror shape/alignment/reflection",
+        action="store_true",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--debug_plots",
+        dest="debug_plots",
+        help="Generate additional debug plots (radius histograms, XY heatmaps, radius vs angle)",
+        action="store_true",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--calculate_primary_secondary_angles",
+        dest="calculate_primary_secondary_angles",
+        help="Also compute angles of incidence on primary and secondary mirrors",
+        required=False,
+        action="store_true",
+    )
+    return config.initialize(
+        db_config=True,
+        simulation_model=["telescope", "site", "model_version"],
+    )
+
+
+def main():
+    """Application to calculate incident angles using ray tracing."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(label)
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+
+    logger.info("Starting calculation of incident angles")
+
+    output_base = Path(args_dict.get("output_path", "./"))
+    output_dir = (
+        output_base / label if not args_dict.get("use_plain_output_path", False) else output_base
+    )
+    base_label = args_dict.get("label", label)
+    telescope_name = args_dict["telescope"]
+    label_with_telescope = f"{base_label}_{telescope_name}"
+
+    calculator = IncidentAnglesCalculator(
+        simtel_path=args_dict["simtel_path"],
+        db_config=db_config,
+        config_data=args_dict,
+        output_dir=output_dir,
+        label=base_label,
+    )
+    offsets = [float(v) for v in args_dict.get("off_axis_angles", [0.0])]
+
+    results_by_offset = calculator.run_for_offsets(offsets)
+    plot_incident_angles(
+        results_by_offset,
+        output_dir,
+        label_with_telescope,
+        debug_plots=args_dict.get("debug_plots", False),
+    )
+    total = sum(len(t) for t in results_by_offset.values())
+    summary_msg = (
+        f"Calculated incident angles for {len(results_by_offset)} offsets,\n"
+        f"total photon statistics {total}"
+    )
+    if total < 1_000_000:
+        summary_msg += " (below 1e6; results may be statistically unstable)"
+    logger.info(summary_msg)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/convert_all_model_parameters_from_simtel.py

@@ -69,7 +69,8 @@ import simtools.data_model.model_data_writer as writer
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.data_model import schema
-from simtools.io_operations.io_handler import IOHandler
+from simtools.io import ascii_handler
+from simtools.io.io_handler import IOHandler
 from simtools.simtel import simtel_config_reader
 
 
@@ -170,7 +171,7 @@ def get_number_of_camera_pixel(args_dict, logger):
         )
         _camera_pixel = config_reader.parameter_dict.get(args_dict["simtel_telescope_name"])
     except (FileNotFoundError, AttributeError):
-        logger.warning("Failed to read camera pixel parameter.")
+        logger.warning("Unable to retrieve camera pixel parameter.")
         _camera_pixel = None
     logger.info(f"Number of camera pixels: {_camera_pixel}")
     return _camera_pixel
@@ -319,7 +320,7 @@ def print_list_of_files(args_dict, logger):
     """
     model_files = sorted(Path(args_dict["output_path"]).rglob("*.json"))
     for file in model_files:
-        model_dict = gen.collect_data_from_file(file_name=file)
+        model_dict = ascii_handler.collect_data_from_file(file_name=file)
         if model_dict.get("file"):
             logger.info(f"{file.name}: {model_dict['value']}")
 

simtools/applications/convert_geo_coordinates_of_array_elements.py

@@ -140,10 +140,10 @@ def _parse(label=None, description=None):
 def main():
     """Print a list of array elements."""
     label = Path(__file__).stem
-    data_model_name = "array_coordinates"
+    model_parameter_name = "array_coordinates"
     args_dict, db_config = _parse(
         label,
-        description=f"Print a list of array element positions ({data_model_name})",
+        description=f"Print a list of array element positions ({model_parameter_name})",
     )
 
     logger = logging.getLogger()
@@ -153,7 +153,7 @@ def main():
         site = args_dict.get("site", None)
         metadata, validate_schema_file = None, None
     else:
-        metadata = MetadataCollector(args_dict=args_dict, data_model_name=data_model_name)
+        metadata = MetadataCollector(args_dict=args_dict, model_parameter_name=model_parameter_name)
         site = metadata.get_site(from_input_meta=True)
         validate_schema_file = metadata.get_data_model_schema_file_name()
 

simtools/applications/db_add_simulation_model_from_repository_to_db.py

@@ -5,14 +5,16 @@ r"""
     Generates a new database with all required collections.
     Follows the structure of the CTAO gitlab simulation model repository.
 
-    This is an application for experts and should not be used by the general user.
+    This is an application for DB maintainers and should not be used by the general user.
 
     Command line arguments
 
     input_path (str, required)
         Path of local copy of model parameter repository.
-    db_name (str, required)
+    db_simulation_model (str, required)
         Name of new DB to be created.
+    db_simulation_model_version (str, required)
+        Version of the new DB to be created.
     type (str, optional)
         Type of data to be uploaded to the DB. Options are: model_parameters, production_tables.
 
@@ -20,7 +22,7 @@ r"""
     --------
     Upload model data repository to the DB
     Loops over all subdirectories in 'input_path' and uploads all json files to the
-    database 'new_db_name' (or updates an existing database with the same name):
+    database (or updates an existing database with the same name):
 
     * subdirectories starting with 'OBS' are uploaded to the 'sites' collection
     * json files from the subdirectory 'configuration_sim_telarray/configuration_corsika'
@@ -33,7 +35,8 @@ r"""
 
         simtools-db-simulation-model-from-repository-to-db \
             --input_path /path/to/repository \
-            --db_name new_db_name \
+            --db_simulation_model database name \
+            --db_simulation_model_version new database version \
             --type model_parameters
 
     Upload production tables to the DB:
@@ -42,7 +45,8 @@ r"""
 
         simtools-db-simulation-model-from-repository-to-db \
             --input_path /path/to/repository \
-            --db_name new_db_name \
+            --db_simulation_model database name \
+            --db_simulation_model_version new database version \
             --type production_tables
 
 """
@@ -78,12 +82,6 @@ def _parse(label=None, description=None):
         type=Path,
         required=True,
     )
-    config.parser.add_argument(
-        "--db_name",
-        help="Name of the new simulation model database to be created.",
-        type=str.strip,
-        required=True,
-    )
     config.parser.add_argument(
         "--type",
         help="Type of data to be uploaded to the database.",
@@ -94,7 +92,12 @@ def _parse(label=None, description=None):
     )
 
     args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True)
-    db_config["db_simulation_model"] = args_dict["db_name"]  # overwrite explicitly DB configuration
+    if args_dict.get("db_simulation_model") and args_dict.get("db_simulation_model_version"):
+        # overwrite explicitly DB configuration
+        db_config["db_simulation_model"] = args_dict["db_simulation_model"]
+        db_config["db_simulation_model_version"] = args_dict["db_simulation_model_version"]
+    else:
+        raise ValueError("Both db_simulation_model and db_simulation_model_version are required.")
     return args_dict, db_config
 
 
@@ -110,9 +113,9 @@ def main():
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
 
     if args_dict.get("type") == "model_parameters":
-        db_model_upload.add_model_parameters_to_db(args_dict, db)
+        db_model_upload.add_model_parameters_to_db(input_path=Path(args_dict["input_path"]), db=db)
     elif args_dict.get("type") == "production_tables":
-        db_model_upload.add_production_tables_to_db(args_dict, db)
+        db_model_upload.add_production_tables_to_db(input_path=Path(args_dict["input_path"]), db=db)
 
 
 if __name__ == "__main__":

simtools/applications/db_add_value_from_json_to_db.py

@@ -32,6 +32,7 @@ from pathlib import Path
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.db import db_handler
+from simtools.io import ascii_handler
 
 
 def _parse():
@@ -61,8 +62,8 @@ def main():  # noqa: D103
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     if args_dict.get("test_db", False):
-        db_config["db_simulation_model"] = db_config["db_simulation_model"] + str(uuid.uuid4())
-        logger.info(f"Using test database: {db_config['db_simulation_model']}")
+        db_config["db_simulation_model_version"] = str(uuid.uuid4())
+        logger.info(f"Using test database version {db_config['db_simulation_model_version']}")
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
 
     files_to_insert = []
@@ -82,14 +83,13 @@ def main():  # noqa: D103
     print(*files_to_insert, sep="\n")
     print()
 
-    logger.info(f"DB {args_dict['db_simulation_model']} selected.")
+    logger.info(f"DB {db.get_db_name()} selected.")
 
     if gen.user_confirm():
         for file_to_insert_now in files_to_insert:
-            par_dict = gen.collect_data_from_file(file_name=file_to_insert_now)
+            par_dict = ascii_handler.collect_data_from_file(file_name=file_to_insert_now)
             logger.info(f"Adding the following parameter to the DB: {par_dict['parameter']}")
             db.add_new_parameter(
-                db_name=db_config["db_simulation_model"],
                 par_dict=par_dict,
                 collection_name=args_dict["db_collection"],
                 file_prefix="./",
@@ -102,11 +102,9 @@ def main():  # noqa: D103
         logger.info("Aborted, no change applied to the database")
 
     # drop test database; be safe and required DB name is sandbox
-    if args_dict.get("test_db", False) and "sandbox" in args_dict["db_simulation_model"]:
-        logger.info(
-            f"Test database used. Dropping all data from {db_config['db_simulation_model']}"
-        )
-        db.db_client.drop_database(db_config["db_simulation_model"])
+    if args_dict.get("test_db", False) and "sandbox" in db.get_db_name():
+        logger.info(f"Test database used. Dropping all data from {db.get_db_name()}")
+        db.db_client.drop_database(db.get_db_name())
 
 
 if __name__ == "__main__":

simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py

@@ -33,7 +33,6 @@
 
 """
 
-import json
 import logging
 from pathlib import Path
 
@@ -41,7 +40,8 @@ import astropy.table
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.data_model.model_data_writer import JsonNumpyEncoder, ModelDataWriter
+from simtools.data_model.model_data_writer import ModelDataWriter
+from simtools.io import ascii_handler
 from simtools.model.array_model import ArrayModel
 
 
@@ -145,17 +145,12 @@ def write_ground_array_elements_to_repository(args_dict, db_config, logger):
         output_path = Path(args_dict["repository_path"]) / f"{element_name}"
         output_path.mkdir(parents=True, exist_ok=True)
         logger.info(f"Writing array element positions (ground) to {output_path}")
-        with open(
-            output_path / "array_element_position_ground.json", "w", encoding="utf-8"
-        ) as file:
-            json.dump(
-                data,
-                file,
-                indent=4,
-                sort_keys=False,
-                cls=JsonNumpyEncoder,
-            )
-            file.write("\n")
+        ascii_handler.write_data_to_file(
+            data=data,
+            output_file=output_path / "array_element_position_ground.json",
+            sort_keys=False,
+            numpy_types=True,
+        )
 
 
 def main():

simtools/applications/db_generate_compound_indexes.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python3
+
+"""
+Generate compound indexes for the specified database.
+
+This needs to be done once after a database has been set up.
+Significantly accelerates database querying (at least a factor
+of 5 in query time with a factor of 10 less documents examined).
+
+Command line arguments
+----------------------
+db_name (str, optional)
+    Database name (use "all" for all databases)
+"""
+
+import logging
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler
+
+
+def _parse():
+    config = configurator.Configurator(
+        description="Generate compound indexes for a specific database"
+    )
+    config.parser.add_argument(
+        "--db_name",
+        help="Database name",
+        default=None,
+        required=False,
+    )
+    return config.initialize(db_config=True)
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    # databases without internal databases we don't have rights to modify
+    databases = [
+        d for d in db.db_client.list_database_names() if d not in ("config", "admin", "local")
+    ]
+    requested = db.get_db_name(
+        db_name=args_dict["db_name"],
+        model_version=args_dict.get("db_simulation_model_version"),
+        model_name=args_dict.get("db_simulation_model"),
+    )
+    if requested != "all" and requested not in databases:
+        raise ValueError(
+            f"Requested database '{requested}' not found. "
+            f"Following databases are available: {', '.join(databases)}"
+        )
+
+    databases = databases if requested == "all" else [requested]
+    for db_name in databases:
+        logger.info(f"Generating compound indexes for database: {db_name}")
+        db.generate_compound_indexes(db_name=db_name)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/db_get_file_from_db.py

@@ -38,7 +38,7 @@ import logging
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.db import db_handler
-from simtools.io_operations import io_handler
+from simtools.io import io_handler
 
 
 def _parse():
@@ -65,29 +65,17 @@ def main():  # noqa: D103
     _io_handler = io_handler.IOHandler()
 
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
-    available_dbs = [
-        db_config["db_simulation_model"],
-    ]
-    file_id = {}
-    for db_name in available_dbs:
-        try:
-            file_id = db.export_model_files(
-                db_name=db_name,
-                dest=_io_handler.get_output_directory(),
-                file_names=args_dict["file_name"],
-            )
-            logger.info(
-                f"Got file {args_dict['file_name']} from DB {db_name} "
-                f"and saved into {_io_handler.get_output_directory()}"
-            )
-            break
-        except FileNotFoundError:
-            continue
-
-    for key, value in file_id.items():
-        if value is None:
-            logger.error(f"The file {key} was not found in any of the available DBs.")
-            raise FileNotFoundError
+    file_id = db.export_model_files(
+        dest=_io_handler.get_output_directory(),
+        file_names=args_dict["file_name"],
+    )
+    if file_id is None:
+        logger.error(f"The file {args_dict['file_name']} was not found in {db.db_name}.")
+        raise FileNotFoundError
+    logger.info(
+        f"Got file {args_dict['file_name']} from DB {db.db_name} "
+        f"and saved into {_io_handler.get_output_directory()}"
+    )
 
 
 if __name__ == "__main__":

simtools/applications/db_get_parameter_from_db.py

@@ -60,7 +60,6 @@ r"""
 
 """
 
-import json
 import logging
 from pathlib import Path
 from pprint import pprint
@@ -68,7 +67,7 @@ from pprint import pprint
 import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.db import db_handler
-from simtools.io_operations import io_handler
+from simtools.io import ascii_handler, io_handler
 
 
 def _parse():
@@ -142,8 +141,9 @@ def main():  # noqa: D103
         )
         pars[args_dict["parameter"]].pop("_id")
         pars[args_dict["parameter"]].pop("entry_date")
-        with open(_output_file, "w", encoding="utf-8") as json_file:
-            json.dump(pars[args_dict["parameter"]], json_file, indent=4)
+        ascii_handler.write_data_to_file(
+            data=pars[args_dict["parameter"]], output_file=_output_file
+        )
     else:
         pprint(pars[args_dict["parameter"]])
 

simtools/applications/db_inspect_databases.py

@@ -6,7 +6,7 @@ Inspect databases and print (available database names and collections).
 Command line arguments
 ----------------------
 db_name (str, optional)
-    Inspect a specific database.
+    Database name (use "all" for all databases)
 """
 
 import logging
@@ -20,11 +20,11 @@ def _parse():
     config = configurator.Configurator(description="Inspect databases")
     config.parser.add_argument(
         "--db_name",
-        help="Inspect a specific database (use all to print all databases)",
+        help="Database name",
         default="all",
         required=True,
     )
-    return config.initialize(db_config=True, simulation_model="telescope")
+    return config.initialize(db_config=True)
 
 
 def main():  # noqa: D103
@@ -34,18 +34,28 @@ def main():  # noqa: D103
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
-
-    databases = db.db_client.list_database_names()
+    # databases without internal databases we don't have rights to modify
+    databases = [
+        d for d in db.db_client.list_database_names() if d not in ("config", "admin", "local")
+    ]
+    requested = args_dict["db_name"]
+    if requested != "all" and requested not in databases:
+        raise ValueError(
+            f"Requested database '{requested}' not found. "
+            f"Following databases are available: {', '.join(databases)}"
+        )
+
+    databases = databases if requested == "all" else [requested]
 
     for db_name in databases:
-        if args_dict["db_name"] != "all" and db_name != args_dict["db_name"]:
-            continue
-        # missing admin rights; skip config and admin
-        if db_name in ("config", "admin", "local"):
-            continue
         print("Database:", db_name)
         collections = db.get_collections(db_name=db_name)
         print("   Collections:", collections)
+        print("   Indexes:")
+        for collection_name in collections:
+            db_collection = db.get_collection(collection_name=collection_name, db_name=db_name)
+            for idx in db_collection.list_indexes():
+                print(f"     {collection_name}: {idx}")
 
 
 if __name__ == "__main__":