gammasimtools 0.6.0__py3-none-any.whl → 0.8.1__py3-none-any.whl

simtools/applications/db_add_model_parameters_from_repository_to_db.py

@@ -0,0 +1,184 @@
+#!/usr/bin/python3
+"""
+    Add parameters found in a model parameter repository to a new database.
+
+    Generates a new database with all required collections.
+    Follows the structure of the CTAO gitlab model parameters repository.
+    file as input.
+
+    This is an application for experts and should not be used by the general user.
+
+    Command line arguments
+
+    input_path (str, required)
+        Path of local copy of model parameter repository.
+    db_name (str, required)
+        Name of new DB to be created.
+    type (str, optional)
+        Type of data to be uploaded to the DB. Options are: model_parameters
+
+    Examples
+    --------
+    Upload model data repository to the DB:
+
+    .. code-block:: console
+
+        simtools-db-add_model-parameters-from-repository-to-db \
+            --input_path /path/to/repository \
+            --db_name new_db_name \
+            --type model_parameters
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler
+from simtools.utils import names
+
+
+def _parse(label=None, description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing application.
+    description : str
+        Description of application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description)
+    config.parser.add_argument(
+        "--input_path",
+        help="Path to model parameter repository.",
+        type=Path,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--db_name",
+        help="Name of the new model parameter database to be created.",
+        type=str.strip,
+        required=True,
+    )
+    config.parser.add_argument(
+        "--type",
+        help="Type of data to be uploaded to the database.",
+        type=str,
+        required=False,
+        default="model_parameters",
+        choices=["model_parameters"],
+    )
+
+    args_dict, db_config = config.initialize(
+        output=True, require_command_line=True, db_config=True, simulation_model="version"
+    )
+    db_config["db_simulation_model"] = args_dict["db_name"]  # overwrite explicitly DB configuration
+    return args_dict, db_config
+
+
+def add_values_from_json_to_db(file, collection, db, db_name, file_prefix, logger):
+    """
+    Upload data from json files to db.
+
+    Parameters
+    ----------
+    file : list
+        Json file to be uploaded to the DB.
+    collection : str
+        The DB collection to which to add the file.
+    db : DatabaseHandler
+        Database handler object.
+    db_name : str
+        Name of the database to be created.
+    file_prefix : str
+        Path to location of all additional files to be uploaded.
+    logger : logging.Logger
+        Logger object.
+    """
+    par_dict = gen.collect_data_from_file(file_name=file)
+    logger.info(
+        f"Adding the following parameter to the DB: {par_dict['parameter']} "
+        f"(collection {collection} in database {db_name})"
+    )
+    db.add_new_parameter(
+        db_name=db_name,
+        array_element_name=par_dict["instrument"],
+        parameter=par_dict["parameter"],
+        version=par_dict["version"],
+        value=par_dict["value"],
+        site=par_dict["site"],
+        type=par_dict["type"],
+        collection_name=collection,
+        applicable=par_dict["applicable"],
+        file=par_dict["file"],
+        unit=par_dict.get("unit", None),
+        file_prefix=file_prefix,
+    )
+
+
+def _add_model_parameters_to_db(args_dict, db, logger):
+    """
+    Add model parameters to the DB.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    db : DatabaseHandler
+        Database handler object.
+    logger : logging.Logger
+        Logger object.
+
+    """
+    input_path = Path(args_dict["input_path"])
+    array_elements = [d for d in input_path.iterdir() if d.is_dir()]
+    for element in array_elements:
+        try:
+            collection = names.get_collection_name_from_array_element_name(element.name)
+        except ValueError:
+            if element.name.startswith("OBS"):
+                collection = "sites"
+            elif element.name in {"configuration_sim_telarray", "configuration_corsika"}:
+                collection = element.name
+            elif element.name == "Files":
+                logger.info("Files are uploaded with the corresponding model parameters")
+                continue
+        logger.info(f"Reading model parameters for {element.name} into collection {collection}")
+        files_to_insert = list(Path(element).rglob("*json"))
+        for file in files_to_insert:
+            if collection == "files":
+                logger.info("Not yet implemented files")
+            else:
+                add_values_from_json_to_db(
+                    file=file,
+                    collection=collection,
+                    db=db,
+                    db_name=args_dict["db_name"],
+                    file_prefix=input_path / "Files",
+                    logger=logger,
+                )
+
+
+def main():
+    """Application main."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(
+        label, description="Add or update a model parameter database to the DB"
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    _add_model_parameters_to_db(args_dict, db, logger)
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/db_add_value_from_json_to_db.py

@@ -0,0 +1,105 @@
+#!/usr/bin/python3
+
+r"""
+    Add a new parameter / value to a collection in the DB using a json file as input.
+
+    Command line arguments
+    ----------------------
+    file_name (str, required)
+        Name of the file to upload including the full path.
+    db_collection (str, required)
+        The DB collection to which to add the file.
+    db (str)
+        The DB to insert the files to. \
+        The choices are either the default CTA simulation DB or a sandbox for testing.
+
+    Example
+    -------
+
+    Upload a file to sites collection:
+
+    .. code-block:: console
+
+        simtools-add-value-from-json-to-db \\
+            --file_name new_value.json --db_collection sites
+
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler
+
+
+def _parse():
+    config = configurator.Configurator(description="Add a new parameter to the DB.")
+    group = config.parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--file_name", help="file to be added", type=str)
+    group.add_argument(
+        "--input_path",
+        help="A directory with json files to upload to the DB.",
+        type=Path,
+    )
+    config.parser.add_argument(
+        "--db_collection", help="DB collection to which to add new values.", required=True
+    )
+    config.parser.add_argument("--db", help="Database name", type=str, required=True)
+    return config.initialize(db_config=True)
+
+
+def main():  # noqa: D103
+    args_dict, db_config = _parse()
+
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+
+    files_to_insert = []
+    if args_dict.get("file_name", None) is not None:
+        files_to_insert.append(args_dict["file_name"])
+    else:
+        files_to_insert.extend(Path(args_dict["input_path"]).glob("*json"))
+
+    if len(files_to_insert) < 1:
+        raise ValueError("No files were provided to upload")
+    plural = "s" if len(files_to_insert) > 1 else ""
+
+    print(
+        f"Should the following parameter{plural} be inserted to the "
+        f"{args_dict['db_collection']} DB collection?:\n"
+    )
+    print(*files_to_insert, sep="\n")
+    print()
+
+    if gen.user_confirm():
+        for file_to_insert_now in files_to_insert:
+            par_dict = gen.collect_data_from_file(file_name=file_to_insert_now)
+            logger.info(f"Adding the following parameter to the DB: {par_dict['parameter']}")
+            db.add_new_parameter(
+                db_name=db_config["db_simulation_model"],
+                array_element_name=par_dict["instrument"],
+                parameter=par_dict["parameter"],
+                version=par_dict["version"],
+                value=par_dict["value"],
+                site=par_dict["site"],
+                type=par_dict["type"],
+                collection_name=args_dict["db_collection"],
+                applicable=par_dict["applicable"],
+                file=par_dict["file"],
+                unit=par_dict.get("unit", None),
+                file_prefix="./",
+            )
+            logger.info(
+                f"Value for {par_dict['parameter']} added to "
+                f"{args_dict['db_collection']} collection."
+            )
+    else:
+        logger.info("Aborted, no change applied to the database")
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py

@@ -0,0 +1,180 @@
+#!/usr/bin/python3
+"""
+    Read array element positions from file and add them to model repository.
+
+    This is an application for experts and should not be used by the general user.
+    Reading of input is fine-tuned to the array element files as provided by CTAO.
+
+    Command line arguments
+
+    input : str
+        File containing a table of array element positions.
+    repository_path : str
+        Path of local copy of model parameter repository.
+    model_version : str
+        Model version.
+    site : str
+        Observatory site.
+    coordinate_system : str
+        Coordinate system of array element positions (ground or utm).
+
+    Examples
+    --------
+    Add array element positions to repository:
+
+    .. code-block:: console
+
+        simtools-write-array-element-positions-to-repository \
+            --input /path/to/positions.txt \
+            --repository_path /path/to/repository \
+            --model_version 1.0.0 \
+            --coordinate_system ground \
+            --site North
+
+"""
+
+import json
+import logging
+from pathlib import Path
+
+import astropy.table
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.data_model.model_data_writer import JsonNumpyEncoder, ModelDataWriter
+from simtools.model.array_model import ArrayModel
+
+
+def _parse(label=None, description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing application.
+    description : str
+        Description of application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description)
+    config.parser.add_argument(
+        "--input",
+        help="File containing a table of array element positions.",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--repository_path",
+        help="Output path to model parameter repository.",
+        type=Path,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--coordinate_system",
+        help="Coordinate system of array element positions (utm or ground).",
+        default="ground",
+        required=False,
+        type=str,
+        choices=["ground", "utm"],
+    )
+
+    return config.initialize(db_config=True, simulation_model="site")
+
+
+def write_utm_array_elements_to_repository(args_dict, logger):
+    """
+    Write UTM position of array elements to model repository.
+
+    Read array element positions from file. The ecsv row definition might
+    include telescope_name or asset_code and sequence_number.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    logger : Logger
+        Logger object.
+
+    """
+    array_elements = astropy.table.Table.read(args_dict["input"])
+    for row in array_elements:
+        instrument = (
+            row["telescope_name"]
+            if "telescope_name" in array_elements.colnames
+            else f"{row['asset_code']}-{row['sequence_number']}"
+        )
+        output_path = Path(args_dict["repository_path"]) / f"{instrument}"
+        output_path.mkdir(parents=True, exist_ok=True)
+        logger.info(f"Writing array element positions (utm) to {output_path}")
+
+        ModelDataWriter.dump_model_parameter(
+            parameter_name="array_element_position_utm",
+            instrument=instrument,
+            value=f"{row['utm_east']} {row['utm_north']} {row['altitude']}",
+            model_version=args_dict["model_version"],
+            output_path=output_path,
+            output_file="array_element_position_utm.json",
+        )
+
+
+def write_ground_array_elements_to_repository(args_dict, db_config, logger):
+    """
+    Write ground position of array elements to model repository.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Command line arguments.
+    db_config : dict
+        Database configuration.
+    logger : Logger
+        Logger object.
+
+    """
+    array_model = ArrayModel(
+        mongo_db_config=db_config,
+        model_version=args_dict["model_version"],
+        site=args_dict["site"],
+        array_elements=args_dict["input"],
+    )
+    for element_name, data in array_model.array_elements.items():
+        output_path = Path(args_dict["repository_path"]) / f"{element_name}"
+        output_path.mkdir(parents=True, exist_ok=True)
+        logger.info(f"Writing array element positions (ground) to {output_path}")
+        with open(
+            output_path / "array_element_position_ground.json", "w", encoding="utf-8"
+        ) as file:
+            json.dump(
+                data,
+                file,
+                indent=4,
+                sort_keys=False,
+                cls=JsonNumpyEncoder,
+            )
+            file.write("\n")
+
+
+def main():
+    """Application main."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(
+        label, description="Add array element positions to model parameter repository"
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    if args_dict["coordinate_system"] == "utm":
+        write_utm_array_elements_to_repository(args_dict, logger)
+    elif args_dict["coordinate_system"] == "ground":
+        write_ground_array_elements_to_repository(args_dict, db_config, logger)
+    else:
+        logger.error("Invalid coordinate system. Allowed are 'utm' and 'ground'.")
+        raise ValueError
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/db_get_array_layouts_from_db.py

@@ -0,0 +1,162 @@
+#!/usr/bin/python3
+
+"""
+Get list of array layouts or list of elements for a given layout as defined in the db.
+
+To get the list of pre-defined array layouts, use ``--list_available_layouts``.
+
+To get the list of array elements for a given layout, use ``--array_layout_name``.
+
+To get the positions for a set of array elements, use ``--array_element_list``.
+Listing of array elements follows this logic:
+
+* explicit listing: e.g., ``-array_element_list MSTN-01, MSTN05``
+* listing of types: e.g, ``-array_element_list MSTN`` plots all telescopes of type MSTN.
+
+Command line arguments
+----------------------
+list_available_layouts : bool, optional
+    List available layouts in the database.
+array_layout_name : str
+    Name of the layout array (e.g., test_layout, alpha, 4mst, etc.).
+array_element_list : list
+    List of array elements (e.g., telescopes) to plot (e.g., ``LSTN-01 LSTN-02 MSTN``).
+coordinate_system : str, optional
+    Coordinate system for the array layout (ground or utm).
+output_file : str, optional
+    Name of the output file to be saved as astropy table (ecsv file)
+
+Examples
+--------
+List pre-defined array layouts.
+
+.. code-block:: console
+
+    simtools-db-get-array-layouts-from-db --site North --model_version "6.0.0"
+
+Retrieve telescope positions for array layout 'test_layout' from database.
+
+.. code-block:: console
+
+    simtools-db-get-array-layouts-from-db --site North --model_version "6.0.0"
+        --array_layout_name test_layout
+
+Retrieve telescope positions from database (utm coordinate system) and write to an ecsv files
+
+.. code-block:: console
+
+    simtools-db-get-array-layouts-from-db --site North --model_version "6.0.0"
+      --array_element_list LSTN-01 LSTN-02 MSTN
+      --coordinate_system utm
+      --output_file telescope_positions-test_layout.ecsv
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.data_model.model_data_writer as writer
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.model.array_model import ArrayModel
+from simtools.model.site_model import SiteModel
+
+
+def _parse(label, description):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing the application.
+    description : str
+        Description of the application.
+
+    Returns
+    -------
+    CommandLineParser
+        Command line parser object.
+    """
+    config = configurator.Configurator(label=label, description=description)
+
+    input_group = config.parser.add_mutually_exclusive_group()
+    input_group.add_argument(
+        "--list_available_layouts",
+        help="List available layouts in the database.",
+        action="store_true",
+        required=False,
+    )
+    config.parser.add_argument(
+        "--coordinate_system",
+        help="Coordinate system for the array layout.",
+        type=str,
+        required=False,
+        default="ground",
+        choices=["ground", "utm"],
+    )
+    return config.initialize(db_config=True, simulation_model=["site", "layout"], output=True)
+
+
+def _layout_from_db(args_dict, db_config):
+    """
+    Read array elements and their positions from data base using the layout name.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Dictionary with the command line arguments.
+    db_config : dict
+        Database configuration.
+
+    Returns
+    -------
+    astropy.table.Table
+        Table with array element positions.
+    """
+    array_model = ArrayModel(
+        mongo_db_config=db_config,
+        model_version=args_dict["model_version"],
+        site=args_dict["site"],
+        layout_name=args_dict.get("array_layout_name", None),
+        array_elements=args_dict.get("array_element_list", None),
+    )
+    return array_model.export_array_elements_as_table(
+        coordinate_system=args_dict["coordinate_system"]
+    )
+
+
+def main():
+    """Get list of array elements as defined in the db (array layout)."""
+    label = Path(__file__).stem
+    args_dict, db_config = _parse(
+        label,
+        "Get list of array elements as defined in the db (array layout).",
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    if args_dict.get("list_available_layouts", False):
+        if args_dict.get("site", None) is None:
+            raise ValueError("Site must be provided to list available layouts.")
+        site_model = SiteModel(
+            mongo_db_config=db_config,
+            model_version=args_dict["model_version"],
+            site=args_dict["site"],
+        )
+        print(site_model.get_list_of_array_layouts())
+    else:
+        logger.info("Array layout: %s", args_dict["array_layout_name"])
+        layout = _layout_from_db(args_dict, db_config)
+        layout.pprint()
+
+        if not args_dict.get("output_file_from_default", False):
+            writer.ModelDataWriter.dump(
+                args_dict=args_dict,
+                output_file=args_dict["output_file"],
+                metadata=None,
+                product_data=layout,
+            )
+
+
+if __name__ == "__main__":
+    main()