gammasimtools 0.20.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

{gammasimtools-0.20.0.dist-info → gammasimtools-0.22.0.dist-info}/entry_points.txt

@@ -11,6 +11,7 @@ simtools-db-get-array-layouts-from-db = simtools.applications.db_get_array_layou
 simtools-db-get-file-from-db = simtools.applications.db_get_file_from_db:main
 simtools-db-get-parameter-from-db = simtools.applications.db_get_parameter_from_db:main
 simtools-db-inspect-databases = simtools.applications.db_inspect_databases:main
+simtools-db-upload-model-repository = simtools.applications.db_upload_model_repository:main
 simtools-derive-ctao-array-layouts = simtools.applications.derive_ctao_array_layouts:main
 simtools-derive-mirror-rnda = simtools.applications.derive_mirror_rnda:main
 simtools-derive-photon-electron-spectrum = simtools.applications.derive_photon_electron_spectrum:main
@@ -25,7 +26,7 @@ simtools-generate-corsika-histograms = simtools.applications.generate_corsika_hi
 simtools-generate-default-metadata = simtools.applications.generate_default_metadata:main
 simtools-generate-regular-arrays = simtools.applications.generate_regular_arrays:main
 simtools-generate-simtel-event-data = simtools.applications.generate_simtel_event_data:main
-simtools-maintain-simulation-model-add-production-table = simtools.applications.maintain_simulation_model_add_production_table:main
+simtools-maintain-simulation-model-add-production = simtools.applications.maintain_simulation_model_add_production:main
 simtools-maintain-simulation-model-compare-productions = simtools.applications.maintain_simulation_model_compare_productions:main
 simtools-maintain-simulation-model-verify-production-tables = simtools.applications.maintain_simulation_model_verify_production_tables:main
 simtools-merge-tables = simtools.applications.merge_tables:main
@@ -39,9 +40,9 @@ simtools-production-derive-statistics = simtools.applications.production_derive_
 simtools-production-generate-grid = simtools.applications.production_generate_grid:main
 simtools-production-merge-corsika-limits = simtools.applications.production_merge_corsika_limits:main
 simtools-run-application = simtools.applications.run_application:main
-simtools-simulate-calibration-events = simtools.applications.simulate_calibration_events:main
 simtools-simulate-flasher = simtools.applications.simulate_flasher:main
 simtools-simulate-illuminator = simtools.applications.simulate_illuminator:main
+simtools-simulate-pedestals = simtools.applications.simulate_pedestals:main
 simtools-simulate-prod = simtools.applications.simulate_prod:main
 simtools-simulate-prod-htcondor-generator = simtools.applications.simulate_prod_htcondor_generator:main
 simtools-submit-array-layouts = simtools.applications.submit_array_layouts:main

simtools/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.20.0'
-__version_tuple__ = version_tuple = (0, 20, 0)
+__version__ = version = '0.22.0'
+__version_tuple__ = version_tuple = (0, 22, 0)
 
 __commit_id__ = commit_id = None

simtools/applications/calculate_incident_angles.py

@@ -144,10 +144,7 @@ def main():
 
     logger.info("Starting calculation of incident angles")
 
-    output_base = Path(args_dict.get("output_path", "./"))
-    output_dir = (
-        output_base / label if not args_dict.get("use_plain_output_path", False) else output_base
-    )
+    output_dir = Path(args_dict.get("output_path", "./"))
     base_label = args_dict.get("label", label)
     telescope_name = args_dict["telescope"]
     label_with_telescope = f"{base_label}_{telescope_name}"

simtools/applications/convert_all_model_parameters_from_simtel.py

@@ -233,8 +233,7 @@ def read_and_export_parameters(args_dict, logger):
             parameter_version=args_dict["parameter_version"],
             output_file=io_handler.get_output_file(
                 f"{_parameter}-{args_dict['parameter_version']}.json",
-                label=f"{args_dict['telescope']}",
-                sub_dir=f"{_parameter}",
+                sub_dir=[f"{args_dict['telescope']}", f"{_parameter}"],
             ),
         )
 

simtools/applications/convert_model_parameter_from_simtel.py

@@ -109,7 +109,6 @@ def main():  # noqa: D103
         parameter_version=args_dict["parameter_version"],
         output_file=args_dict["output_file"],
         output_path=args_dict.get("output_path"),
-        use_plain_output_path=args_dict.get("use_plain_output_path"),
     )
     logger.info(f"Validated parameter: {_json_dict}")
 

simtools/applications/db_generate_compound_indexes.py

@@ -40,25 +40,12 @@ def main():  # noqa: D103
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
     db = db_handler.DatabaseHandler(mongo_db_config=db_config)
-    # databases without internal databases we don't have rights to modify
-    databases = [
-        d for d in db.db_client.list_database_names() if d not in ("config", "admin", "local")
-    ]
-    requested = db.get_db_name(
+
+    db.generate_compound_indexes_for_databases(
         db_name=args_dict["db_name"],
-        model_version=args_dict.get("db_simulation_model_version"),
-        model_name=args_dict.get("db_simulation_model"),
+        db_simulation_model=args_dict.get("db_simulation_model"),
+        db_simulation_model_version=args_dict.get("db_simulation_model_version"),
     )
-    if requested != "all" and requested not in databases:
-        raise ValueError(
-            f"Requested database '{requested}' not found. "
-            f"Following databases are available: {', '.join(databases)}"
-        )
-
-    databases = databases if requested == "all" else [requested]
-    for db_name in databases:
-        logger.info(f"Generating compound indexes for database: {db_name}")
-        db.generate_compound_indexes(db_name=db_name)
 
 
 if __name__ == "__main__":

simtools/applications/db_upload_model_repository.py

@@ -0,0 +1,122 @@
+#!/usr/bin/python3
+r"""
+Upload model parameters from simulation model repository to a local or remote database.
+
+This script clones the CTAO simulation model repository and uploads model parameters
+and production tables to a MongoDB database. It includes retry functionality for
+network operations and confirmation prompts for remote database uploads.
+
+Command line arguments
+----------------------
+db_simulation_model (str, required)
+    Name of the database simulation model.
+db_simulation_model_version (str, required)
+    Version of the database simulation model.
+branch (str, optional)
+    Repository branch to clone (if not provided, uses the version tag).
+tmp_dir (str, optional)
+    Temporary directory for cloning the repository (default: ./tmp_model_parameters).
+
+Examples
+--------
+Upload model repository to database using a version tag
+(see simulation model repository for available tags):
+
+.. code-block:: console
+
+    simtools-db-upload-model-repository \\
+        --db_simulation_model CTAO-Simulation-Model \\
+        --db_simulation_model_version v10.10.10
+
+Upload model repository using specific branch (the version tag is
+used to name the database, but no tag checkout is done):
+
+.. code-block:: console
+
+    simtools-db-upload-model-repository \\
+        --db_simulation_model CTAO-Simulation-Model \\
+        --db_simulation_model_version v10.10.10 \\
+        --branch main
+
+"""
+
+import logging
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools.configuration import configurator
+from simtools.db import db_handler, db_model_upload
+
+DEFAULT_REPOSITORY_URL = (
+    "https://gitlab.cta-observatory.org/cta-science/simulations/"
+    "simulation-model/simulation-models.git"
+)
+
+
+def _parse(label=None, description=None):
+    """
+    Parse command line configuration.
+
+    Parameters
+    ----------
+    label : str
+        Label describing application.
+    description : str
+        Description of application.
+
+    Returns
+    -------
+    tuple
+        Command line parser object and database configuration.
+    """
+    config = configurator.Configurator(label=label, description=description)
+    config.parser.add_argument(
+        "--branch",
+        help="Repository branch to clone (optional, defaults to using version tag).",
+        type=str,
+        required=False,
+    )
+    config.parser.add_argument(
+        "--tmp_dir",
+        help="Temporary directory for cloning the repository (default: ./tmp_model_parameters).",
+        type=str,
+        default="tmp_model_parameters",
+        required=False,
+    )
+
+    args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True)
+
+    if args_dict.get("db_simulation_model_version"):
+        db_config["db_simulation_model"] = args_dict.get(
+            "db_simulation_model", "CTAO-Simulation-Model"
+        )
+        db_config["db_simulation_model_version"] = args_dict["db_simulation_model_version"]
+    else:
+        raise ValueError("Setting of db_simulation_model_version is required.")
+
+    return args_dict, db_config
+
+
+def main():
+    """Application main."""
+    args_dict, db_config = _parse(
+        label=Path(__file__).stem, description="Upload model parameters from repository to database"
+    )
+    logger = logging.getLogger()
+    logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
+
+    db = db_handler.DatabaseHandler(mongo_db_config=db_config)
+    db.print_connection_info()
+
+    db_model_upload.add_complete_model(
+        tmp_dir=args_dict.get("tmp_dir"),
+        db=db,
+        db_simulation_model=args_dict.get("db_simulation_model"),
+        db_simulation_model_version=args_dict.get("db_simulation_model_version"),
+        repository_url=DEFAULT_REPOSITORY_URL,
+        repository_branch=args_dict.get("branch"),
+    )
+
+
+if __name__ == "__main__":
+    main()

simtools/applications/derive_psf_parameters.py

@@ -6,25 +6,22 @@ r"""
     This includes parameters mirror_reflection_random_angle, \
     mirror_align_random_horizontal and mirror_align_random_vertical.
 
-    The telescope zenith angle and the source distance can be set by command line arguments.
-
     The measured cumulative PSF should be provided by using the command line argument data. \
     A file name is expected, in which the file should contain 3 columns: radial distance in mm, \
     differential value of photon intensity and its integral value.
 
-    The derivation is performed through a random search. A number of random combination of the \
-    parameters are tested and the best ones are selected based on the minimum value of \
-    the Root Mean Squared Deviation between data and simulations. The range in which the \
-    parameter are drawn uniformly are defined based on the previous value on the telescope model.
+    The derivation is performed through gradient descent optimization that minimizes either the \
+    Root Mean Squared Deviation (RMSD) between measured and simulated PSF curves (default) or the \
+    Kolmogorov-Smirnov (KS) statistic when the --ks_statistic flag is used.
 
     The optimization workflow includes:
 
     * Loading and preprocessing PSF data from measurement files
-    * Generating random parameter combinations for optimization
-    * Running ray-tracing simulations for each parameter set
-    * Calculating RMSD between measured and simulated PSF curves
-    * Identifying the best-fit parameters with minimum RMSD
-    * Creating comprehensive plots and D80 vs off-axis angle analysis
+    * Running gradient descent optimization to minimize RMSD
+    * Generating cumulative PSF plots for each iteration showing optimization progression
+    * Logging parameter evolution through gradient descent steps
+    * Creating convergence plots showing RMSD and D80 evolution
+    * Automatically generating D80 vs off-axis angle analysis for best parameters
     * Optionally exporting optimized parameters as simulation model files
 
     The assumption are:
@@ -36,7 +33,7 @@ r"""
     One example of the plot generated by this applications are shown below.
 
     .. _derive_psf_parameters_plot:
-    .. image::  images/derive_psf_parameters.png
+    .. image::  images/gradient_descent.png
       :width: 49 %
 
     Command line arguments
@@ -63,47 +60,53 @@ r"""
         If activated, application will be faster by simulating fewer photons.
     write_psf_parameters (activation mode, optional)
         Write the optimized PSF parameters as simulation model parameter files.
-    random_seed (int, optional)
-        Random seed for parameter generation.
-    n_runs (int, optional)
-        Number of parameter combinations to test.
+    rmsd_threshold (float, optional)
+        RMSD threshold for gradient descent convergence (default: 0.007).
+    learning_rate (float, optional)
+        Learning rate for gradient descent optimization (default: 0.01).
+    monte_carlo_analysis (activation mode, optional)
+        Run Monte Carlo analysis to find statistical uncertainties.
 
     Example
     -------
-    LSTN-01 5.0.0
-
-    Runtime < 3 min.
+    --telescope LSTN-01 --model_version 6.0.0
 
-    Get PSF data from the DB:
-
-    .. code-block:: console
 
-        simtools-db-get-file-from-db --file_name PSFcurve_data_v2.txt
 
     Run the application:
 
     .. code-block:: console
 
         simtools-derive-psf-parameters --site North --telescope LSTN-01 \\
-            --model_version 6.0.0 --data tests/resources/PSFcurve_data_v2.txt --plot_all --test
+            --model_version 6.0.0 --data tests/resources/PSFcurve_data_v2.ecsv --plot_all --test
 
     Run with parameter export:
 
     .. code-block:: console
 
-        simtools-derive-psf-parameters --site North --telescope LSTN-01 \\
-            --model_version 6.0.0 \\
-            --data tests/resources/PSFcurve_data_v2.txt --write_psf_parameters
+        simtools-derive-psf-parameters --site North --telescope LSTN-01 --model_version 6.0.0 \\
+            --plot_all --test --rmsd_threshold 0.01 --learning_rate 0.001 \\
+            --data tests/resources/PSFcurve_data_v2.ecsv \\
+            --write_psf_parameters
+
+    Run monte carlo analysis:
 
-    The output is saved in simtools-output/tune_psf.
+    .. code-block:: console
+
+        simtools-derive-psf-parameters --site North --telescope LSTN-01 --model_version 6.0.0 \\
+            --plot_all --test --monte_carlo_analysis \\
+            --data tests/resources/PSFcurve_data_v2.ecsv \\
+            --write_psf_parameters
+
+    The output is saved in simtools-output/derive_psf_parameters.
 
     Output files include:
 
-    * Parameter optimization results in tested_psf_parameters.txt
-    * PSF comparison plots in tune_psf_[telescope].pdf
+    * Gradient descent progression log in psf_gradient_descent_[telescope].log
+    * Gradient descent convergence plots in gradient_descent_convergence_[telescope].png
+    * PSF progression plots showing evolution through iterations (if --plot_all is specified)
     * D80 vs off-axis angle plots (d80_vs_offaxis_cm.png, d80_vs_offaxis_deg.png)
     * Optimized simulation model parameter files (if --write_psf_parameters is specified)
-    * Cumulative PSF plots for all tested combinations (if --plot_all is specified)
 
 """
 
@@ -154,16 +157,38 @@ def _parse():
         required=False,
     )
     config.parser.add_argument(
-        "--random_seed",
-        help="Random seed for parameter generation.",
-        type=int,
-        default=None,
+        "--rmsd_threshold",
+        help=(
+            "RMSD threshold for gradient descent convergence "
+            "(not used with --monte_carlo_analysis)."
+        ),
+        type=float,
+        default=0.01,
     )
     config.parser.add_argument(
-        "--n_runs",
-        help="Number of parameter combinations to test.",
-        type=int,
-        default=5,
+        "--learning_rate",
+        help=(
+            "Learning rate for gradient descent optimization "
+            "(not used with --monte_carlo_analysis)."
+        ),
+        type=float,
+        default=0.01,
+    )
+    config.parser.add_argument(
+        "--monte_carlo_analysis",
+        help="Run analysis to find monte carlo uncertainties.",
+        action="store_true",
+    )
+    config.parser.add_argument(
+        "--ks_statistic",
+        help="Use KS statistic for monte carlo uncertainty analysis.",
+        action="store_true",
+    )
+    config.parser.add_argument(
+        "--fraction",
+        help="PSF containment fraction for diameter calculation (e.g., 0.8 for D80, 0.95 for D95).",
+        type=float,
+        default=0.8,
     )
     return config.initialize(
         db_config=True,
@@ -179,8 +204,7 @@ def main():  # noqa: D103
     logger.setLevel(get_log_level_from_user(args_dict["log_level"]))
 
     _io_handler = io_handler.IOHandler()
-    output_dir = _io_handler.get_output_directory(label, sub_dir="application-plots")
-    tel_model, site_model = initialize_simulation_models(
+    tel_model, site_model, _ = initialize_simulation_models(
         label=label,
         db_config=db_config,
         site=args_dict["site"],
@@ -188,7 +212,12 @@ def main():  # noqa: D103
         model_version=args_dict["model_version"],
     )
 
-    psf_opt.run_psf_optimization_workflow(tel_model, site_model, args_dict, output_dir)
+    psf_opt.run_psf_optimization_workflow(
+        tel_model,
+        site_model,
+        args_dict,
+        _io_handler.get_output_directory(),
+    )
 
 
 if __name__ == "__main__":

simtools/applications/docs_produce_array_element_report.py

@@ -56,7 +56,7 @@ def main():  # noqa: D103
     args, db_config = _parse(label_name)
 
     io_handler_instance = io_handler.IOHandler()
-    output_path = io_handler_instance.get_output_directory(label=label_name, sub_dir="productions")
+    output_path = io_handler_instance.get_output_directory()
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args["log_level"]))

simtools/applications/docs_produce_calibration_reports.py

@@ -28,7 +28,7 @@ def main():  # noqa: D103
     args, db_config = _parse(label_name)
 
     io_handler_instance = io_handler.IOHandler()
-    output_path = io_handler_instance.get_output_directory(label=label_name, sub_dir="productions")
+    output_path = io_handler_instance.get_output_directory()
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args["log_level"]))

simtools/applications/docs_produce_model_parameter_reports.py

@@ -41,7 +41,7 @@ def main():  # noqa: D103
     label_name = "reports"
     args, db_config = _parse(label_name)
     io_handler_instance = io_handler.IOHandler()
-    output_path = io_handler_instance.get_output_directory(label=label_name, sub_dir="parameters")
+    output_path = io_handler_instance.get_output_directory()
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args["log_level"]))

simtools/applications/docs_produce_simulation_configuration_report.py

@@ -29,7 +29,7 @@ def main():  # noqa: D103
     args, db_config = _parse(label_name)
 
     io_handler_instance = io_handler.IOHandler()
-    output_path = io_handler_instance.get_output_directory(label=label_name, sub_dir="productions")
+    output_path = io_handler_instance.get_output_directory()
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args["log_level"]))