gammasimtools 0.24.0__py3-none-any.whl → 0.25.0__py3-none-any.whl

simtools/applications/simulate_pedestals.py

@@ -69,7 +69,7 @@ def _parse():
         help="Calibration run mode",
         type=str,
         required=True,
-        choices=["pedestals", "dark_pedestals", "nsb_only_pedestals"],
+        choices=["pedestals", "pedestals_dark", "pedestals_nsb_only"],
     )
     config.parser.add_argument(
         "--number_of_events",

simtools/applications/simulate_prod.py

@@ -39,10 +39,6 @@ r"""
         It allows the transformation system to keep using sequential run numbers without repetition.
     run (int, required)
         Run number (actual run number will be 'start_run' + 'run').
-    data_directory (str, optional)
-        The location of the output directories corsika-data and simtel-data.
-        the label is added to the data_directory, such that the output
-        will be written to data_directory/label/simtel-data.
     pack_for_grid_register (str, optional)
         Set whether to prepare a tarball for registering the output files on the grid.
         The files are written to the specified directory.
@@ -58,13 +54,6 @@ r"""
         simtools-simulate-prod \\
         --model_version 5.0.0 --site north --primary gamma --azimuth_angle north \\
         --zenith_angle 20 --start_run 0 --run 1
-
-    By default the configuration is saved in simtools-output/test-production
-    together with the actual simulation output in corsika-data and simtel-data within.
-    The location of the latter directories can be set
-    to a different location via the option --data_directory,
-    but the label is always added to the data_directory, such that the output
-    will be written to data_directory/label/simtel-data.
 """
 
 from simtools.application_control import startup_application
@@ -76,15 +65,12 @@ def _parse():
     """Parse command line configuration."""
     config = configurator.Configurator(description="Run simulations for productions")
     config.parser.add_argument(
-        "--data_directory",
+        "--corsika_file",
         help=(
-            "The directory where to save the corsika-data and simtel-data output directories."
-            "the label is added to the data_directory, such that the output"
-            "will be written to data_directory/label/simtel-data."
+            "Path to the CORSIKA input file (only relevant for simulation software 'sim_telarray')."
         ),
-        type=str.lower,
+        type=str,
         required=False,
-        default="./simtools-output/",
     )
     config.parser.add_argument(
         "--pack_for_grid_register",
@@ -150,19 +136,18 @@ def main():
     simulator.simulate()
     simulator.validate_metadata()
 
-    app_context.logger.info(
-        f"Production run complete for primary {app_context.args['primary']} showers "
-        f"from {app_context.args['azimuth_angle']} azimuth and "
-        f"{app_context.args['zenith_angle']} zenith "
-        f"at {app_context.args['site']} site, using {app_context.args['model_version']} model."
-    )
     if app_context.args["save_reduced_event_lists"]:
         simulator.save_reduced_event_lists()
+
+    simulator.verify_simulations()
+
     if app_context.args.get("pack_for_grid_register"):
         simulator.pack_for_register(app_context.args["pack_for_grid_register"])
     if app_context.args["save_file_lists"]:
         simulator.save_file_lists()
 
+    simulator.report()
+
 
 if __name__ == "__main__":
     main()

simtools/applications/simulate_prod_htcondor_generator.py

@@ -56,6 +56,13 @@ def _parse():
         label=get_application_label(__file__),
         description="Prepare simulations production for HT Condor job submission",
     )
+    config.parser.add_argument(
+        "--number_of_runs",
+        help="Number of runs to be simulated.",
+        type=int,
+        required=True,
+        default=1,
+    )
     config.parser.add_argument(
         "--apptainer_image",
         help="Apptainer image to use for the simulation (full path).",

simtools/applications/submit_array_layouts.py

@@ -1,9 +1,11 @@
 #!/usr/bin/python3
 
 r"""
-    Submit array-layouts definition and corresponding metadata and validate list of telescopes.
+    Read array layouts from file, validate with telescopes in DB, and prepare for submission.
 
-    Includes validation that all defined telescope exists.
+    Validates that all telescope defined in the array layouts exist in the database for the
+    specified model version. Prepares both JSON-style model parameters and corresponding
+    metadata for submission.
 
     Command line arguments
     ----------------------
@@ -19,7 +21,7 @@ r"""
     Example
     -------
 
-    Submit a new array layout dictionary:
+    Submit and validate a new array layout dictionary:
 
     .. code-block:: console
 

simtools/applications/validate_file_using_schema.py

@@ -3,26 +3,25 @@ r"""
     Validate a file or files in a directory using a schema.
 
     Input files can be metadata, schema, or data files in yaml, json, or ecsv format.
+    For model parameters, the schema files are taken from the simtools model parameter
+    schema directory by default.
 
     Command line arguments
     ----------------------
     file_name (str)
       input file to be validated
-    model_parameters_directory (str)
+    file_directory (str)
         directory with json files of model parameters to be validated
     schema (str)
       schema file (jsonschema format) used for validation
     data_type (str)
-        type of input data (allowed types: metadata, schema, data)
-
-    Raises
-    ------
-    FileNotFoundError
-      if file to be validated is not found
+        type of input data (allowed types: metadata, schema, data, model_parameter)
 
     Example
     -------
 
+    Validate metadata of a file:
+
     .. code-block:: console
 
         simtools-validate-file-using-schema \\
@@ -30,16 +29,29 @@ r"""
          --schema simtools/schemas/metadata.metaschema.yml \\
          --data_type metadata
 
-"""
+    Validate schema of a file:
+
+    .. code-block:: console
+
+        simtools-validate-file-using-schema \\
+         --file_name tests/resources/model_parameters/schema-0.3.0/num_gains-1.0.0.json \\
+         --schema src/simtools/schemas/model_parameter.metaschema.yml \\
+         --data_type schema
 
-import re
-from pathlib import Path
+    Validate all model parameter files in a directory:
+
+    .. code-block:: console
+
+        simtools-validate-file-using-schema \\
+         --file_directory tests/resources/model_parameters \\
+         --data_type model_parameter
+
+"""
 
 from simtools.application_control import get_application_label, startup_application
 from simtools.configuration import configurator
 from simtools.constants import MODEL_PARAMETER_SCHEMA_PATH
 from simtools.data_model import metadata_collector, schema, validate_data
-from simtools.io import ascii_handler
 
 
 def _parse():
@@ -62,7 +74,7 @@ def _parse():
             f"{MODEL_PARAMETER_SCHEMA_PATH}."
         ),
     )
-    config.parser.add_argument("--schema", help="Json schema file", required=False)
+    config.parser.add_argument("--schema", help="Schema file", required=False)
     config.parser.add_argument(
         "--data_type",
         help="Type of input data",
@@ -70,7 +82,7 @@ def _parse():
         default="data",
     )
     config.parser.add_argument(
-        "--require_exact_data_type",
+        "--check_exact_data_type",
         help="Require exact data type for validation",
         action="store_true",
     )
@@ -82,121 +94,35 @@ def _parse():
     return config.initialize(paths=False)
 
 
-def _get_schema_file_name(args_dict, data_dict=None):
-    """
-    Get schema file name from metadata, data dict, or from command line argument.
-
-    Parameters
-    ----------
-    args_dict (dict)
-        command line arguments
-    data_dict (dict)
-        dictionary with metaschema information
-
-    Returns
-    -------
-    schema_file: str
-        schema file name
-
-    """
-    schema_file = args_dict.get("schema")
-    if schema_file is None and data_dict is not None:
-        schema_file = data_dict.get("meta_schema_url")
-    if schema_file is None:
-        metadata = metadata_collector.MetadataCollector(
-            None, metadata_file_name=args_dict["file_name"]
-        )
-        schema_file = metadata.get_data_model_schema_file_name()
-    return schema_file
-
-
-def _get_file_list(file_directory=None, file_name="*.json"):
-    """Return list of files in a directory."""
-    file_list = []
-    if file_directory is not None:
-        file_list = list(Path(file_directory).rglob(file_name))
-        if not file_list:
-            raise FileNotFoundError(f"No files found in {file_directory}")
-    elif file_name is not None:
-        file_list = [file_name]
-
-    return file_list
-
-
-def validate_dict_using_schema(args_dict, logger):
-    """
-    Validate a schema file (or several files) given in yaml or json format.
-
-    This function validate all documents in a multi-document YAML file.
-    Schema is either given as command line argument, read from the meta_schema_url or from
-    the metadata section of the data dictionary.
-
-    """
-    for file_name in _get_file_list(args_dict.get("file_directory"), args_dict.get("file_name")):
-        try:
-            data = ascii_handler.collect_data_from_file(file_name=file_name)
-        except FileNotFoundError as exc:
-            raise FileNotFoundError(f"Error reading schema file from {file_name}") from exc
-        data = data if isinstance(data, list) else [data]
-        try:
-            for data_dict in data:
-                schema.validate_dict_using_schema(
-                    data_dict,
-                    _get_schema_file_name(args_dict, data_dict),
-                    ignore_software_version=args_dict.get("ignore_software_version", False),
-                )
-        except Exception as exc:
-            raise ValueError(f"Validation of file {file_name} failed") from exc
-        logger.info(f"Successful validation of file {file_name}")
-
-
-def validate_data_files(args_dict, logger):
-    """Validate data files."""
-    if args_dict.get("file_directory") is not None:
-        tmp_args_dict = {}
-        for file_name in _get_file_list(args_dict.get("file_directory")):
-            tmp_args_dict["file_name"] = file_name
-            parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem)
-            schema_file = schema.get_model_parameter_schema_file(f"{parameter_name}")
-            tmp_args_dict["schema"] = schema_file
-            tmp_args_dict["data_type"] = "model_parameter"
-            tmp_args_dict["require_exact_data_type"] = args_dict["require_exact_data_type"]
-            validate_data_file(tmp_args_dict, logger)
-    else:
-        validate_data_file(args_dict, logger)
-
-
-def validate_data_file(args_dict, logger):
-    """Validate a data file (e.g., in ecsv, json, yaml format)."""
-    data_validator = validate_data.DataValidator(
-        schema_file=_get_schema_file_name(args_dict),
-        data_file=args_dict["file_name"],
-        check_exact_data_type=args_dict["require_exact_data_type"],
-    )
-    data_validator.validate_and_transform(
-        is_model_parameter=(args_dict["data_type"].lower() == "model_parameter")
-    )
-
-    logger.info(f"Successful validation of data file {args_dict['file_name']}")
-
-
-def validate_metadata(args_dict, logger):
-    """Validate metadata."""
-    # metadata_collector runs the metadata validation by default, no need to do anything else
-    metadata_collector.MetadataCollector(None, metadata_file_name=args_dict["file_name"])
-    logger.info(f"Successful validation of metadata {args_dict['file_name']}")
-
-
 def main():
     """Validate a file or files in a directory using a schema."""
     app_context = startup_application(_parse)
 
-    if app_context.args["data_type"].lower() == "metadata":
-        validate_metadata(app_context.args, app_context.logger)
-    elif app_context.args["data_type"].lower() == "schema":
-        validate_dict_using_schema(app_context.args, app_context.logger)
+    file_name = app_context.args.get("file_name")
+    file_directory = app_context.args.get("file_directory")
+    schema_file = app_context.args.get("schema")
+    data_type = app_context.args.get("data_type").lower()
+
+    if data_type == "metadata":
+        # metadata_collector runs the metadata validation by default, no need to do anything else
+        metadata_collector.MetadataCollector(None, metadata_file_name=file_name)
+        app_context.logger.info(f"Successful validation of metadata {file_name}")
+
+    elif data_type == "schema":
+        schema.validate_schema_from_files(
+            file_directory=file_directory,
+            file_name=file_name,
+            schema_file=schema_file,
+            ignore_software_version=app_context.args.get("ignore_software_version", False),
+        )
     else:
-        validate_data_files(app_context.args, app_context.logger)
+        validate_data.DataValidator.validate_data_files(
+            file_name=file_name,
+            file_directory=file_directory,
+            is_model_parameter=(data_type == "model_parameter"),
+            check_exact_data_type=app_context.args.get("check_exact_data_type", False),
+            schema_file=schema_file,
+        )
 
 
 if __name__ == "__main__":

simtools/configuration/commandline_parser.py

@@ -377,12 +377,6 @@ class CommandLineParser(argparse.ArgumentParser):
                 "required": True,
                 "default": 1,
             },
-            "number_of_runs": {
-                "help": "Number of runs to be simulated.",
-                "type": int,
-                "required": True,
-                "default": 1,
-            },
             "event_number_first_shower": {
                 "help": "Event number of first shower",
                 "type": int,
@@ -395,6 +389,14 @@ class CommandLineParser(argparse.ArgumentParser):
                 "required": False,
                 "default": True,
             },
+            "curved_atmosphere_min_zenith_angle": {
+                "help": (
+                    "Minimum zenith angle (deg) for using curved-atmosphere CORSIKA binaries. "
+                ),
+                "type": CommandLineParser.zenith_angle,
+                "required": False,
+                "default": 65 * u.deg,
+            },
         }
 
     @staticmethod