gammasimtools 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

simtools/utils/names.py

@@ -3,11 +3,12 @@
 import logging
 import re
 from functools import cache
-from importlib.resources import files
 from pathlib import Path
 
 import yaml
 
+from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH, SCHEMA_PATH
+
 _logger = logging.getLogger(__name__)
 
 __all__ = [
@@ -34,7 +35,7 @@ def array_elements():
     dict
         Array elements.
     """
-    with open(files("simtools") / "schemas/array_elements.yml", encoding="utf-8") as file:
+    with open(Path(SCHEMA_PATH) / "array_elements.yml", encoding="utf-8") as file:
         return yaml.safe_load(file)["data"]
 
 
@@ -59,7 +60,7 @@ def site_names():
 @cache
 def load_model_parameters(class_key_list):
     model_parameters = {}
-    schema_files = list(Path(files("simtools") / "schemas/model_parameters").rglob("*.yml"))
+    schema_files = list(Path(MODEL_PARAMETER_SCHEMA_PATH).rglob("*.yml"))
     for schema_file in schema_files:
         with open(schema_file, encoding="utf-8") as f:
             data = yaml.safe_load(f)
@@ -293,7 +294,7 @@ def get_site_from_array_element_name(name):
     return array_elements()[get_array_element_type_from_name(name)]["site"]
 
 
-def get_collection_name_from_array_element_name(name):
+def get_collection_name_from_array_element_name(name, array_elements_only=True):
     """
     Get collection name (e.g., telescopes, calibration_devices, sites) of array element from name.
 
@@ -301,6 +302,8 @@ def get_collection_name_from_array_element_name(name):
     ----------
     name: str
         Array element name.
+    array_elements_only: bool
+        If True, only array elements are considered.
 
     Returns
     -------
@@ -311,18 +314,28 @@ def get_collection_name_from_array_element_name(name):
         return array_elements()[get_array_element_type_from_name(name)]["collection"]
     except ValueError:
         pass
+    if name.startswith("OBS"):
+        return "sites"
     try:
         validate_site_name(name)
         return "sites"
     except ValueError as exc:
-        raise ValueError(f"Invalid array element name {name}: {exc}") from exc
+        if array_elements_only:
+            raise ValueError(f"Invalid array element name {name}") from exc
+    if name in (
+        "configuration_sim_telarray",
+        "configuration_corsika",
+        "Files",
+        "Dummy-Telescope",
+    ):
+        return name
+
+    raise ValueError(f"Invalid array element name {name}")
 
 
 def get_simulation_software_name_from_parameter_name(
     par_name,
     simulation_software="sim_telarray",
-    search_telescope_parameters=True,
-    search_site_parameters=True,
 ):
     """
     Get the name used in the simulation software from the model parameter name.
@@ -336,21 +349,13 @@ def get_simulation_software_name_from_parameter_name(
         Model parameter name.
     simulation_software: str
         Simulation software name.
-    search_telescope_parameters: bool
-        If True, telescope model parameters are included.
-    search_site_parameters: bool
-        If True, site model parameters are included.
 
     Returns
     -------
     str
         Simtel parameter name.
     """
-    _parameter_names = {}
-    if search_telescope_parameters:
-        _parameter_names.update(telescope_parameters())
-    if search_site_parameters:
-        _parameter_names.update(site_parameters())
+    _parameter_names = {**telescope_parameters(), **site_parameters()}
 
     try:
         _parameter = _parameter_names[par_name]
@@ -575,12 +580,11 @@ def sanitize_name(name):
     ValueError:
         if the string name can not be sanitized.
     """
-    # Convert to lowercase
+    if name is None:
+        # _logger.info("The string is None and can't be sanitized.")
+        return name
     sanitized = name.lower()
-
-    # Replace spaces with underscores
     sanitized = sanitized.replace(" ", "_")
-
     # Remove characters that are not alphanumerics or underscores
     sanitized = re.sub(r"\W|^(?=\d)", "_", sanitized)
     if not sanitized.isidentifier():

simtools/applications/db_add_model_parameters_from_repository_to_db.py

@@ -1,176 +0,0 @@
-#!/usr/bin/python3
-r"""
-    Add parameters found in a model parameter repository to a new database.
-
-    Generates a new database with all required collections.
-    Follows the structure of the CTAO gitlab model parameters repository.
-    file as input.
-
-    This is an application for experts and should not be used by the general user.
-
-    Command line arguments
-
-    input_path (str, required)
-        Path of local copy of model parameter repository.
-    db_name (str, required)
-        Name of new DB to be created.
-    type (str, optional)
-        Type of data to be uploaded to the DB. Options are: model_parameters
-
-    Examples
-    --------
-    Upload model data repository to the DB:
-
-    .. code-block:: console
-
-        simtools-db-add_model-parameters-from-repository-to-db \
-            --input_path /path/to/repository \
-            --db_name new_db_name \
-            --type model_parameters
-"""
-
-import logging
-from pathlib import Path
-
-import simtools.utils.general as gen
-from simtools.configuration import configurator
-from simtools.db import db_handler
-from simtools.utils import names
-
-
-def _parse(label=None, description=None):
-    """
-    Parse command line configuration.
-
-    Parameters
-    ----------
-    label : str
-        Label describing application.
-    description : str
-        Description of application.
-
-    Returns
-    -------
-    CommandLineParser
-        Command line parser object.
-    """
-    config = configurator.Configurator(label=label, description=description)
-    config.parser.add_argument(
-        "--input_path",
-        help="Path to model parameter repository.",
-        type=Path,
-        required=True,
-    )
-    config.parser.add_argument(
-        "--db_name",
-        help="Name of the new model parameter database to be created.",
-        type=str.strip,
-        required=True,
-    )
-    config.parser.add_argument(
-        "--type",
-        help="Type of data to be uploaded to the database.",
-        type=str,
-        required=False,
-        default="model_parameters",
-        choices=["model_parameters"],
-    )
-
-    args_dict, db_config = config.initialize(
-        output=True, require_command_line=True, db_config=True, simulation_model="version"
-    )
-    db_config["db_simulation_model"] = args_dict["db_name"]  # overwrite explicitly DB configuration
-    return args_dict, db_config
-
-
-def add_values_from_json_to_db(file, collection, db, db_name, file_prefix, logger):
-    """
-    Upload data from json files to db.
-
-    Parameters
-    ----------
-    file : list
-        Json file to be uploaded to the DB.
-    collection : str
-        The DB collection to which to add the file.
-    db : DatabaseHandler
-        Database handler object.
-    db_name : str
-        Name of the database to be created.
-    file_prefix : str
-        Path to location of all additional files to be uploaded.
-    logger : logging.Logger
-        Logger object.
-    """
-    par_dict = gen.collect_data_from_file(file_name=file)
-    logger.info(
-        f"Adding the following parameter to the DB: {par_dict['parameter']} "
-        f"(collection {collection} in database {db_name})"
-    )
-    db.add_new_parameter(
-        db_name=db_name,
-        par_dict=par_dict,
-        collection_name=collection,
-        file_prefix=file_prefix,
-    )
-
-
-def _add_model_parameters_to_db(args_dict, db, logger):
-    """
-    Add model parameters to the DB.
-
-    Parameters
-    ----------
-    args_dict : dict
-        Command line arguments.
-    db : DatabaseHandler
-        Database handler object.
-    logger : logging.Logger
-        Logger object.
-
-    """
-    input_path = Path(args_dict["input_path"])
-    array_elements = [d for d in input_path.iterdir() if d.is_dir()]
-    for element in array_elements:
-        try:
-            collection = names.get_collection_name_from_array_element_name(element.name)
-        except ValueError:
-            if element.name.startswith("OBS"):
-                collection = "sites"
-            elif element.name in {"configuration_sim_telarray", "configuration_corsika"}:
-                collection = element.name
-            elif element.name == "Files":
-                logger.info("Files are uploaded with the corresponding model parameters")
-                continue
-        logger.info(f"Reading model parameters for {element.name} into collection {collection}")
-        files_to_insert = list(Path(element).rglob("*json"))
-        for file in files_to_insert:
-            if collection == "files":
-                logger.info("Not yet implemented files")
-            else:
-                add_values_from_json_to_db(
-                    file=file,
-                    collection=collection,
-                    db=db,
-                    db_name=args_dict["db_name"],
-                    file_prefix=input_path / "Files",
-                    logger=logger,
-                )
-
-
-def main():
-    """Application main."""
-    label = Path(__file__).stem
-    args_dict, db_config = _parse(
-        label, description="Add or update a model parameter database to the DB"
-    )
-    logger = logging.getLogger()
-    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
-
-    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
-
-    _add_model_parameters_to_db(args_dict, db, logger)
-
-
-if __name__ == "__main__":
-    main()

simtools/db/db_array_elements.py

@@ -1,130 +0,0 @@
-"""Retrieval of array elements from the database."""
-
-from functools import lru_cache
-
-from pymongo import ASCENDING
-
-from simtools.utils import names
-
-
-@lru_cache
-def get_array_elements(db_collection, model_version):
-    """
-    Get all array element names and their design model for a given DB collection.
-
-    Uses the 'design_model' parameter to determine the design model of the array element.
-    Assumes that a design model is defined for every array element.
-
-    Parameters
-    ----------
-    db_collection:
-        pymongo.collection.Collection
-    model_version: str
-       Model version.
-
-    Returns
-    -------
-    dict
-        Dict with array element names found and their design model
-
-    Raises
-    ------
-    ValueError
-        If query for collection name not implemented.
-    KeyError
-        If array element entry in the database is incomplete.
-
-    """
-    query = {"version": model_version}
-    results = db_collection.find(query, {"instrument": 1, "value": 1, "parameter": 1}).sort(
-        "instrument", ASCENDING
-    )
-
-    _all_available_array_elements = {}
-    for doc in results:
-        try:
-            if doc["parameter"] == "design_model":
-                _all_available_array_elements[doc["instrument"]] = doc["value"]
-        except KeyError as exc:
-            raise KeyError(f"Incomplete array element entry in the database: {doc}.") from exc
-
-    if len(_all_available_array_elements) == 0:
-        raise ValueError(f"No array elements found in DB collection {db_collection}.")
-
-    return _all_available_array_elements
-
-
-def get_array_element_list_for_db_query(array_element_name, db, model_version, collection):
-    """
-    Get array element name and design model for querying the database.
-
-    Return a list of array element names to be used for querying the database for a given array
-    element. This is in most cases the array element itself and its design model.
-    In cases of no design model available, the design model of the array element is returned.
-
-    Parameters
-    ----------
-    array_element_name: str
-        Name of the array element model (e.g. MSTN-01).
-    db: DBHandler
-        Instance of the database handler
-    model_version: str
-        Model version.
-    collection: str
-        DB collection to get the array elements from (e.g., telescopes, calibration_devices)
-
-    Returns
-    -------
-    list
-        List of array element model names as used in the DB.
-
-    """
-    try:
-        _available_array_elements = get_array_elements(
-            db.get_collection(db_name=None, collection_name=collection),
-            db.model_version(model_version),
-        )
-    except ValueError:
-        return [names.get_array_element_type_from_name(array_element_name) + "-design"]
-    try:
-        return [_available_array_elements[array_element_name], array_element_name]
-    except KeyError:
-        pass
-
-    if array_element_name in _available_array_elements.values():
-        return [array_element_name]
-
-    raise ValueError(f"Array element {array_element_name} not found in DB.")
-
-
-def get_array_elements_of_type(array_element_type, db, model_version, collection):
-    """
-    Get all array elements of a certain type in the specified collection in the DB.
-
-    Return e.g. for array_element_type='MSTS' all MSTS array elements found in the collection.
-
-    Parameters
-    ----------
-    array_element_type : str
-        Type of the array element (e.g. LSTN, MSTS)
-    model_version : str
-        Which version to get the array elements of
-    collection : str
-        Which collection to get the array elements from:
-        i.e. telescopes, calibration_devices
-    db_name : str
-        Database name
-
-    Returns
-    -------
-    list
-        Sorted list of all array element names found in collection
-
-    """
-    _available_array_elements = get_array_elements(
-        db.get_collection(db_name=None, collection_name=collection),
-        db.model_version(model_version),
-    )
-    return sorted(
-        [entry for entry in _available_array_elements if entry.startswith(array_element_type)]
-    )