gammasimtools 0.17.0__py3-none-any.whl → 0.19.0__py3-none-any.whl

simtools/runners/simtools_runner.py

@@ -0,0 +1,267 @@
+"""Tools for running applications in the simtools framework."""
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import simtools.utils.general as gen
+from simtools import dependencies
+from simtools.io import ascii_handler
+
+
+def run_applications(args_dict, db_config, logger):
+    """
+    Run simtools applications step-by-step as defined in a configuration file.
+
+    Parameters
+    ----------
+    args_dict : dict
+        Dictionary containing command line arguments.
+    db_config : dict
+        Database configuration
+    logger : logging.Logger
+        Logger for logging application output.
+    """
+    configurations, runtime_environment, log_file = _read_application_configuration(
+        args_dict["configuration_file"], args_dict.get("steps"), logger
+    )
+    run_time = (
+        read_runtime_environment(runtime_environment)
+        if not args_dict["ignore_runtime_environment"]
+        else []
+    )
+
+    with log_file.open("w", encoding="utf-8") as file:
+        file.write("Running simtools applications\n")
+        file.write(dependencies.get_version_string(db_config, run_time))
+
+        for config in configurations:
+            app = config.get("application")
+            if not config.get("run_application"):
+                logger.info(f"Skipping application: {app}")
+                continue
+            logger.info(f"Running application: {app}")
+            stdout, stderr = run_application(run_time, app, config.get("configuration"), logger)
+            file.write("=" * 80 + "\n")
+            file.write(f"Application: {app}\nSTDOUT:\n{stdout}\nSTDERR:\n{stderr}\n")
+
+
+def run_application(runtime_environment, application, configuration, logger):
+    """
+    Run a simtools application and return stdout and stderr.
+
+    Allow to specify a runtime environment (e.g., Docker) and a working directory.
+
+    Parameters
+    ----------
+    runtime_environment : list
+        Command to run the application in the specified runtime environment.
+    application : str
+        Name of the application to run.
+    configuration : dict
+        Configuration for the application.
+    logger : logging.Logger
+        Logger for logging application output.
+
+    Returns
+    -------
+    tuple
+        stdout and stderr from the application run.
+
+    """
+    command = [application, *_convert_dict_to_args(configuration)]
+    if runtime_environment:
+        command = runtime_environment + command
+    try:
+        result = subprocess.run(
+            command,
+            check=True,
+            capture_output=True,
+            text=True,
+        )
+    except subprocess.CalledProcessError as exc:
+        logger.error(f"Error running application {application}: {exc.stderr}")
+        raise exc
+
+    return result.stdout, result.stderr
+
+
+def _convert_dict_to_args(parameters):
+    """
+    Convert a dictionary of parameters to a list of command line arguments.
+
+    Parameters
+    ----------
+    parameters : dict
+        Dictionary containing parameters to convert.
+
+    Returns
+    -------
+    list
+        List of command line arguments.
+    """
+    args = []
+    for key, value in parameters.items():
+        if isinstance(value, bool):
+            if value:
+                args.append(f"--{key}")
+        elif isinstance(value, list):
+            args.extend([f"--{key}", *(str(item) for item in value)])
+        else:
+            args.extend([f"--{key}", str(value)])
+    return args
+
+
+def _read_application_configuration(configuration_file, steps, logger):
+    """
+    Read application configuration from file and modify for setting workflows.
+
+    Strong assumptions on the structure of input and output files:
+
+    - configuration file is expected to be in './input/<workflow directory>/<yaml file>'
+    - output files will be written out to './output/<workflow directory>/'
+
+    Replaces the placeholders in the configuration file with the actual values.
+    Sets 'USE_PLAIN_OUTPUT_PATH' to True for all applications.
+
+    Parameters
+    ----------
+    configuration_file : str
+        Configuration file name.
+    steps : list
+        List of steps to be executed (None: all steps).
+    logger : Logger
+        Logger object.
+
+    Returns
+    -------
+    dict
+        Application configuration.
+    dict:
+        Runtime environment configuration.
+    Path
+        Path to the log file.
+
+    """
+    job_configuration = ascii_handler.collect_data_from_file(configuration_file)
+    configurations = job_configuration.get("applications")
+    output_path, setting_workflow = _set_input_output_directories(configuration_file)
+    logger.info(f"Setting workflow output path to {output_path}")
+    for step_count, config in enumerate(configurations, start=1):
+        config["run_application"] = step_count in steps if steps else True
+        config = gen.change_dict_keys_case(config, True)
+        config["configuration"] = _replace_placeholders_in_configuration(
+            config.get("configuration", {}),
+            output_path,
+            setting_workflow,
+        )
+        configurations[step_count - 1] = config
+
+    return (
+        configurations,
+        job_configuration.get("runtime_environment"),
+        output_path / "simtools.log",
+    )
+
+
+def _replace_placeholders_in_configuration(
+    configuration, output_path, setting_workflow, place_holder="__SETTING_WORKFLOW__"
+):
+    """
+    Replace placeholders in the configuration dictionary.
+
+    Parameters
+    ----------
+    configuration : dict
+        Configuration dictionary.
+    output_path : Path
+        Path to the output directory.
+    setting_workflow : str
+        The setting workflow to replace the placeholder with.
+    place_holder : str
+        Placeholder to be replaced.
+
+    Returns
+    -------
+    dict
+        Configuration dictionary with placeholders replaced.
+    """
+    for key, value in configuration.items():
+        if isinstance(value, str):
+            configuration[key] = value.replace(place_holder, setting_workflow)
+        if isinstance(value, list):
+            configuration[key] = [
+                item.replace(place_holder, setting_workflow) if isinstance(item, str) else item
+                for item in value
+            ]
+    if output_path:
+        configuration["use_plain_output_path"] = True
+        configuration["output_path"] = str(output_path)
+
+    return configuration
+
+
+def _set_input_output_directories(path):
+    """
+    Set input and output directories based on the configuration file path.
+
+    Tuned to simulation models setting workflows.
+
+    Parameters
+    ----------
+    path : str or Path
+        Path to the configuration file.
+
+    Returns
+    -------
+    tuple
+        The first part is the 'input' directory, the second part is the subdirectory name
+    """
+    path = Path(path).resolve()
+    try:
+        input_index = path.parts.index("input")
+        # Get all parts after 'input', excluding the filename
+        subdirs = path.parts[input_index + 1 : -1]
+        setting_workflow = "/".join(subdirs)
+        workflow_dir = path.parts[input_index]
+    except (ValueError, IndexError) as exc:
+        raise ValueError(f"Could not find subdirectory under 'input': {exc}") from exc
+
+    output_path = Path(str(workflow_dir).replace("input", "output")) / Path(setting_workflow)
+    output_path.mkdir(parents=True, exist_ok=True)
+    return output_path, "/".join(subdirs)
+
+
+def read_runtime_environment(runtime_environment, workdir="/workdir/external/"):
+    """
+    Read the runtime environment (e.g. docker runtime) and generate the required command.
+
+    Parameters
+    ----------
+    runtime_environment : str or None
+        Path to the runtime environment configuration file.
+
+    Returns
+    -------
+    list
+        Runtime command.
+    """
+    if runtime_environment is None:
+        return []
+
+    engine = runtime_environment.get("container_engine", "docker")
+    if shutil.which(engine) is None:
+        raise RuntimeError(f"Container engine '{engine}' not found.")
+    cmd = [engine, "run", "--rm", "-v", f"{Path.cwd()}:{workdir}", "-w", workdir]
+
+    if options := runtime_environment.get("options"):
+        for opt in options:
+            cmd.extend(opt.split())
+
+    if env := runtime_environment.get("env_file"):
+        cmd += ["--env-file", env]
+    if net := runtime_environment.get("network"):
+        cmd += ["--network", net]
+
+    cmd.append(runtime_environment["image"])
+    return cmd

simtools/schemas/application_workflow.metaschema.yml

@@ -3,8 +3,8 @@ $schema: http://json-schema.org/draft-06/schema#
 $ref: '#/definitions/SimtoolsApplicationWorkflowConfiguration'
 title: SimPipe application workflow configuration metaschema
 description: YAML representation of application workflow configuration metaschema
-version: 0.3.0
-name: application_workflow.metaschema
+schema_version: 0.4.0
+schema_name: application_workflow.metaschema
 type: object
 additionalProperties: false
 
@@ -13,103 +13,136 @@ definitions:
     type: object
     additionalProperties: false
     properties:
-      CTA_SIMPIPE:
-        "$ref": "#/definitions/CTASIMPIPE"
-      SCHEMA_VERSION:
+      applications:
+        "$ref": "#/definitions/applications"
+      schema_version:
         type: string
         description: "Version of the schema."
-      SCHEMA_URL:
+      schema_url:
         type: string
         format: uri
         description: "URL of the schema."
-      SCHEMA_NAME:
+      schema_name:
         type: string
         description: "Name of the schema."
+      runtime_environment:
+        "$ref": "#/definitions/runtime_environment"
     required:
-      - CTA_SIMPIPE
-      - SCHEMA_VERSION
-      - SCHEMA_NAME
+      - applications
+      - schema_version
+      - schema_name
     title: ""
-  CTASIMPIPE:
-    type: object
+  runtime_environment:
     additionalProperties: false
-    properties:
-      LOG_PATH:
-        type: string
-        description: "Path to the log file."
-      APPLICATIONS:
-        type: array
-        items:
-          type: object
-          properties:
-            APPLICATION:
-              type: string
-              description: "Name of simtools application"
-            TEST_NAME:
-              type: string
-              description: "Test name. Should be unique for this application."
-            SKIP_FOR_PRODUCTION_DB:
-              type: boolean
-              description: "Skip test if production DB is used."
-            MODEL_VERSION_USE_CURRENT:
-              type: boolean
-              description: |
-                "Skip test if model version is changed via command line.
-                Important e.g., for cases where DB model is incomplete for some
-                model versions."
-            CONFIGURATION:
-              "$ref": "#/definitions/Configuration"
-            INTEGRATION_TESTS:
-              type: array
-              items:
-                "$ref": "#/definitions/IntegrationTest"
-            TEST_REQUIREMENT:
-              type: string
-              description: "Requirement ID of the test."
-            TEST_USE_CASE:
-              type: string
-              description: "Use case ID of the test."
-          required:
-            - APPLICATION
-            - CONFIGURATION
-    title: CTASimpipe
-  Configuration:
+    type: array
+    items:
+      type: object
+      properties:
+        image:
+          type: string
+          description: "Container image to use for the application."
+        network:
+          type: string
+          description: "Network configuration for the container."
+        environment_file:
+          type: string
+          description: "Path to the environment file for the container."
+        container_engine:
+          type: string
+          description: "Container engine to use (e.g., docker, podman)."
+          default: "docker"
+  applications:
+    additionalProperties: false
+    type: array
+    items:
+      type: object
+      properties:
+        application:
+          type: string
+          description: "Name of simtools application"
+        test_name:
+          type: string
+          description: "Test name. Should be unique for this application."
+        skip_for_production_db:
+          type: boolean
+          description: "Skip test if production DB is used."
+        model_version_use_current:
+          type: boolean
+          description: |
+            "Skip test if model version is changed via command line.
+            Important e.g., for cases where DB model is incomplete for some
+            model versions."
+        configuration:
+          "$ref": "#/definitions/configuration"
+        integration_tests:
+          type: array
+          items:
+            "$ref": "#/definitions/integration_test"
+        test_requirement:
+          type: string
+          description: "Requirement ID of the test."
+        test_use_case:
+          type: string
+          description: "Use case ID of the test."
+      required:
+        - application
+        - configuration
+    title: Applications
+  configuration:
     description: "Command line configuration of simtools application."
     type: object
     additionalProperties: true
     title: Configuration
-  IntegrationTest:
+  integration_test:
     description: "List of integration tests."
     type: object
     additionalProperties: false
     properties:
-      OUTPUT_FILE:
+      output_file:
         description: |
           "Path of output file tested to be generated by integration test."
         type: string
-      TEST_OUTPUT_FILES:
+      test_output_files:
         type: array
         items:
           type: object
           properties:
-            PATH_DESCRIPTOR:
+            path_descriptor:
               type: string
-            FILE:
+            file:
               type: string
-            EXPECTED_OUTPUT:
+            expected_output:
               description: |
                 "Expected output of integration test."
               type: object
-          required: ["PATH_DESCRIPTOR", "FILE"]
-      FILE_TYPE:
+          required: ["path_descriptor", "file"]
+      file_type:
         description: |
           "Expected file type of output file generated by integration test."
         type: string
-      REFERENCE_OUTPUT_FILE:
+      reference_output_file:
         description: |
           "Reference file used for comparison."
         type: string
-      TEST_SIMTEL_CFG_FILES:
+      model_parameter_validation:
+        description: |
+          "Validation of model parameters against reference values."
+        type: object
+        additionalProperties: false
+        properties:
+          parameter_file:
+            description: |
+              "Path to the JSON file containing model parameters."
+            type: string
+          reference_parameter_name:
+            description: |
+              "Name of the parameter to validate against the reference."
+            type: string
+          tolerance:
+            description: |
+              "Allowed tolerance for floating point comparison."
+            type: number
+      test_simtel_cfg_files:
         description: |
           "Reference file used for comparison of sim_telarray configuration files."
         type: object
@@ -119,22 +152,22 @@ definitions:
             type: string
             description: Path to the configuration file for the given version.
         minProperties: 1
-      TOLERANCE:
+      tolerance:
         description: "Allowed tolerance for floating point comparison."
         type: number
-      TEST_COLUMNS:
+      test_columns:
         description: "List of columns to be compared, each with optional cut conditions."
         type: array
         items:
           type: object
           properties:
-            TEST_COLUMN_NAME:
+            test_column_name:
               description: "Name of the column to compare."
               type: string
-            CUT_COLUMN_NAME:
+            cut_column_name:
               description: "Name of the column for applying the cut."
               type: string
-            CUT_CONDITION:
+            cut_condition:
               description: "Condition to apply to the cut column."
               type: string
     title: ApplicationWorkflow

simtools/schemas/input/MST_mirror_2f_measurements.schema.yml

@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 title: Schema for mirror panel 2F measurements
-version: 0.1.0
+schema_version: 0.1.0
 meta_schema: simpipe-schema
 meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml
 meta_schema_version: 0.1.0

simtools/schemas/input/single_pe_spectrum.schema.yml

@@ -1,7 +1,7 @@
 %YAML 1.2
 ---
 title: Schema for single photon electron spectrum
-version: 0.1.0
+schema_version: 0.1.0
 meta_schema: simpipe-schema
 meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml
 meta_schema_version: 0.1.0