gammasimtools 0.6.1__py3-none-any.whl → 0.8.2__py3-none-any.whl

simtools/applications/validate_file_using_schema.py

@@ -1,15 +1,19 @@
 #!/usr/bin/python3
-"""
-    Summary
-    -------
-    Validate yaml, json, or ecsv file using a json schema file.
+r"""
+    Validate a file or files in a directory using a schema.
+
+    Input files can be metadata, schema, or data files in yaml, json, or ecsv format.
 
     Command line arguments
     ----------------------
     file_name (str)
       input file to be validated
+    model_parameters_directory (str)
+        directory with json files of model parameters to be validated
     schema (str)
       schema file (jsonschema format) used for validation
+    data_type (str)
+        type of input data (allowed types: metadata, schema, data)
 
     Raises
     ------
@@ -21,23 +25,25 @@
 
     .. code-block:: console
 
-        simtools-validate-file-using-schema \
-         --file_name tests/resources/MLTdata-preproduction.meta.yml \
-         --schema simtools/schemas/metadata.metaschema.yml
+        simtools-validate-file-using-schema \\
+         --file_name tests/resources/MLTdata-preproduction.meta.yml \\
+         --schema simtools/schemas/metadata.metaschema.yml \\
+         --data_type metadata
 
 """
 
 import logging
+from importlib.resources import files
 from pathlib import Path
 
 import simtools.utils.general as gen
 from simtools.configuration import configurator
-from simtools.data_model import data_reader, metadata_collector, validate_data
+from simtools.data_model import metadata_collector, metadata_model, validate_data
 
 
 def _parse(label, description):
     """
-    Parse command line configuration
+    Parse command line configuration.
 
     Parameters
     ----------
@@ -52,75 +58,133 @@ def _parse(label, description):
         application configuration
 
     """
-
     config = configurator.Configurator(label=label, description=description)
-    config.parser.add_argument("--file_name", help="file to be validated", required=True)
-    config.parser.add_argument("--schema", help="json schema file", required=False)
+    group = config.parser.add_mutually_exclusive_group(required=True)
+    group.add_argument("--file_name", help="File to be validated")
+    group.add_argument(
+        "--model_parameters_directory",
+        help=(
+            "Directory with json files with model parameters to be validated."
+            "All *.json files in the directory will be validated."
+            "Schema files will be taken from simtools/schemas/model_parameters/."
+            "Note that in this case the data_type argument is ignored"
+            "and data_type=model_parameter is always used."
+        ),
+    )
+    config.parser.add_argument("--schema", help="Json schema file", required=False)
+    config.parser.add_argument(
+        "--data_type",
+        help="Type of input data",
+        choices=["metadata", "schema", "data", "model_parameter"],
+        default="data",
+    )
     config.parser.add_argument(
-        "--validate_metadata",
-        help="validate metadata",
+        "--require_exact_data_type",
+        help="Require exact data type for validation",
         action="store_true",
-        default=False,
     )
     return config.initialize(paths=False)
 
 
-def _validate_yaml_or_json_file(args_dict, logger):
-    """
-    Validate a yaml or json file.
-    Schema is either given as command line argument, read from the meta_schema_url or from
-    the metadata section of the data dictionary.
-
+def _get_schema_file_name(args_dict, data_dict=None):
     """
+    Get schema file name from metadata, data dict, or from command line argument.
 
-    data_reader.read_value_from_file(
-        file_name=args_dict["file_name"],
-        schema_file=args_dict["schema"],
-        validate=True,
-    )
-    logger.debug("Successful validation of json/yaml file")
+    Parameters
+    ----------
+    args_dict (dict)
+        command line arguments
+    data_dict (dict)
+        dictionary with metaschema information
 
+    Returns
+    -------
+    schema_file: str
+        schema file name
 
-def _validate_ecsv_file(args_dict, logger):
     """
-    Validate an ecsv file
+    schema_file = args_dict.get("schema")
+    if schema_file is None and data_dict is not None:
+        schema_file = data_dict.get("meta_schema_url")
+    if schema_file is None:
+        metadata = metadata_collector.MetadataCollector(
+            None, metadata_file_name=args_dict["file_name"]
+        )
+        schema_file = metadata.get_data_model_schema_file_name()
+    return schema_file
+
+
+def validate_schema(args_dict, logger):
+    """
+    Validate a schema file given in yaml or json format.
+
+    Schema is either given as command line argument, read from the meta_schema_url or from
+    the metadata section of the data dictionary.
 
     """
+    try:
+        data = gen.collect_data_from_file(file_name=args_dict["file_name"])
+    except FileNotFoundError as exc:
+        logger.error(f"Error reading schema file from {args_dict['file_name']}")
+        raise exc
+    metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data))
+    logger.info(f"Successful validation of schema file {args_dict['file_name']}")
+
+
+def validate_data_files(args_dict, logger):
+    """Validate data files."""
+    model_parameters_directory = args_dict.get("model_parameters_directory")
+    if model_parameters_directory is not None:
+        tmp_args_dict = {}
+        for file_name in Path(model_parameters_directory).rglob("*.json"):
+            tmp_args_dict["file_name"] = file_name
+            schema_file = (
+                files("simtools") / "schemas/model_parameters" / f"{file_name.stem}.schema.yml"
+            )
+            tmp_args_dict["schema"] = schema_file
+            tmp_args_dict["data_type"] = "model_parameter"
+            tmp_args_dict["require_exact_data_type"] = args_dict["require_exact_data_type"]
+            validate_data_file(tmp_args_dict, logger)
+    else:
+        validate_data_file(args_dict, logger)
+
 
+def validate_data_file(args_dict, logger):
+    """Validate a data file (e.g., in ecsv, json, yaml format)."""
     data_validator = validate_data.DataValidator(
-        schema_file=args_dict["schema"],
+        schema_file=_get_schema_file_name(args_dict),
         data_file=args_dict["file_name"],
+        check_exact_data_type=args_dict["require_exact_data_type"],
     )
-    data_validator.validate_and_transform()
-    logger.debug("Successful validation of escv file")
-
+    data_validator.validate_and_transform(is_model_parameter=True)
+    if args_dict["data_type"].lower() == "model_parameter":
+        data_validator.validate_parameter_and_file_name()
 
-def _validate_metadata(args_dict, logger):
-    """
-    Validate metadata.
+    logger.info(f"Successful validation of data file {args_dict['file_name']}")
 
-    """
 
+def validate_metadata(args_dict, logger):
+    """Validate metadata."""
     # metadata_collector runs the metadata validation by default, no need to do anything else
     metadata_collector.MetadataCollector(None, metadata_file_name=args_dict["file_name"])
-    logger.debug("Successful validation of metadata")
+    logger.info(f"Successful validation of metadata {args_dict['file_name']}")
 
 
-def main():
+def main():  # noqa: D103
     label = Path(__file__).stem
-    args_dict, _ = _parse(label, description="Validate yaml or ecsv file using a json schema file.")
+    args_dict, _ = _parse(
+        label, description="Validate a file (metadata, schema, or data file) using a schema."
+    )
 
     logger = logging.getLogger()
     logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"]))
 
-    if args_dict["validate_metadata"]:
-        _validate_metadata(args_dict, logger)
-    elif any(args_dict["file_name"].endswith(ext) for ext in (".yml", ".yaml", ".json")):
-        _validate_yaml_or_json_file(args_dict, logger)
-    elif args_dict["file_name"].endswith(".ecsv"):
-        _validate_ecsv_file(args_dict, logger)
+    if args_dict["data_type"].lower() == "metadata":
+        validate_metadata(args_dict, logger)
+    elif args_dict["data_type"].lower() == "schema":
+        validate_schema(args_dict, logger)
     else:
-        logger.error(f"File extension not supported for {args_dict['file_name']}")
+        validate_data_files(args_dict, logger)
 
 
 if __name__ == "__main__":

simtools/applications/validate_optics.py

@@ -1,10 +1,9 @@
 #!/usr/bin/python3
 
-"""
-    Summary
-    -------
-    This application validates the optical model parameters through ray tracing simulations \
-    of the whole telescope, assuming a point-like light source. The output includes PSF (D80), \
+r"""
+    Validate the optical model parameters through ray tracing simulations of the whole telescope.
+
+    A point-like light source is assumed. The output includes PSF (D80), \
     effective mirror area and effective focal length as a function of the off-axis angle. \
 
     The telescope zenith angle and the source distance can be set by command line arguments.
@@ -45,16 +44,14 @@
         Produce a multiple pages pdf file with the image plots.
     test (activation mode, optional)
         If activated, application will be faster by simulating fewer photons.
-    verbosity (str, optional)
-        Log level to print.
 
     Example
     -------
-    LST-1 Prod5
+    LST-1 5.0.0
 
     .. code-block:: console
 
-        simtools-validate-optics --site North --telescope LST-1 --max_offset 1.0 \
+        simtools-validate-optics --site North --telescope LST-1 --max_offset 1.0 \\
         --zenith 20 --src_distance 10 --test
 
     The output is saved in simtools-output/validate_optics
@@ -80,15 +77,11 @@ import simtools.utils.general as gen
 from simtools.configuration import configurator
 from simtools.io_operations import io_handler
 from simtools.model.telescope_model import TelescopeModel
-from simtools.ray_tracing import RayTracing
+from simtools.ray_tracing.ray_tracing import RayTracing
 
 
 def _parse(label):
-    """
-    Parse command line configuratio
-
-    """
-
+    """Parse command line configuration."""
     config = configurator.Configurator(
         label=label,
         description=(
@@ -121,10 +114,10 @@ def _parse(label):
         help="Produce a multiple pages pdf file with the image plots.",
         action="store_true",
     )
-    return config.initialize(db_config=True, telescope_model=True)
+    return config.initialize(db_config=True, simulation_model=["telescope"])
 
 
-def main():
+def main():  # noqa: D103
     label = Path(__file__).stem
     args_dict, db_config = _parse(label)
 
@@ -136,7 +129,7 @@ def main():
 
     tel_model = TelescopeModel(
         site=args_dict["site"],
-        telescope_model_name=args_dict["telescope"],
+        telescope_name=args_dict["telescope"],
         model_version=args_dict["model_version"],
         label=label,
         mongo_db_config=db_config,
@@ -155,13 +148,15 @@ def main():
 
     print(f"\nValidating telescope optics with ray tracing simulations for {tel_model.name}\n")
 
-    ray = RayTracing.from_kwargs(
+    ray = RayTracing(
         telescope_model=tel_model,
-        simtel_source_path=args_dict["simtel_path"],
-        source_distance=args_dict["src_distance"] * u.km,
+        simtel_path=args_dict["simtel_path"],
         zenith_angle=args_dict["zenith"] * u.deg,
+        source_distance=args_dict["src_distance"] * u.km,
         off_axis_angle=np.linspace(
-            0, args_dict["max_offset"], int(args_dict["max_offset"] / args_dict["offset_steps"]) + 1
+            0,
+            args_dict["max_offset"],
+            int(args_dict["max_offset"] / args_dict["offset_steps"]) + 1,
         )
         * u.deg,
     )