dirac-cwl 1.0.2__py3-none-any.whl

dirac_cwl/transformation/__init__.py

@@ -0,0 +1,203 @@
+"""CLI interface to run a workflow as a transformation."""
+
+import glob
+import logging
+import os
+import time
+from pathlib import Path
+from typing import Dict, List, Optional
+
+import typer
+from cwl_utils.pack import pack
+from cwl_utils.parser import load_document
+from cwl_utils.parser.cwl_v1_2 import File
+from rich import print_json
+from rich.console import Console
+from schema_salad.exceptions import ValidationException
+
+from dirac_cwl.execution_hooks import (
+    TransformationExecutionHooksHint,
+)
+from dirac_cwl.job import submit_job_router
+from dirac_cwl.submission_models import (
+    JobInputModel,
+    JobSubmissionModel,
+    TransformationSubmissionModel,
+)
+
+app = typer.Typer()
+console = Console()
+
+
+# -----------------------------------------------------------------------------
+# dirac-cli commands
+# -----------------------------------------------------------------------------
+
+
+@app.command("submit")
+def submit_transformation_client(
+    task_path: str = typer.Argument(..., help="Path to the CWL file"),
+    # Specific parameter for the purpose of the prototype
+    local: Optional[bool] = typer.Option(True, help="Run the jobs locally instead of submitting them to the router"),
+):
+    """
+    Correspond to the dirac-cli command to submit transformations.
+
+    This command will:
+    - Validate the workflow
+    - Start the transformation
+    """
+    os.environ["DIRAC_PROTO_LOCAL"] = "0"
+    # Validate the workflow
+    console.print("[blue]:information_source:[/blue] [bold]CLI:[/bold] Validating the transformation...")
+    try:
+        task = load_document(pack(task_path))
+    except FileNotFoundError as ex:
+        console.print(f"[red]:heavy_multiplication_x:[/red] [bold]CLI:[/bold] Failed to load the task:\n{ex}")
+        return typer.Exit(code=1)
+    except ValidationException as ex:
+        console.print(f"[red]:heavy_multiplication_x:[/red] [bold]CLI:[/bold] Failed to validate the task:\n{ex}")
+        return typer.Exit(code=1)
+    console.print(f"\t[green]:heavy_check_mark:[/green] Task {task_path}")
+
+    transformation = TransformationSubmissionModel(task=task)
+    console.print("[green]:heavy_check_mark:[/green] [bold]CLI:[/bold] Transformation validated.")
+
+    # Submit the transformation
+    console.print("[blue]:information_source:[/blue] [bold]CLI:[/bold] Submitting the transformation...")
+    print_json(transformation.model_dump_json(indent=4))
+    if not submit_transformation_router(transformation):
+        console.print("[red]:heavy_multiplication_x:[/red] [bold]CLI:[/bold] Failed to run transformation.")
+        return typer.Exit(code=1)
+    console.print("[green]:heavy_check_mark:[/green] [bold]CLI:[/bold] Transformation done.")
+
+
+# -----------------------------------------------------------------------------
+# dirac-router commands
+# -----------------------------------------------------------------------------
+
+
+def submit_transformation_router(transformation: TransformationSubmissionModel) -> bool:
+    """Execute a transformation using the router.
+
+    If the transformation is waiting for an input from another transformation,
+    it will wait for the input to be available in the "bookkeeping".
+
+    :param transformation: The transformation to start.
+    :return: True if the transformation is executed successfully, False otherwise.
+    """
+    logger = logging.getLogger("TransformationRouter")
+
+    # Validate the transformation
+    logger.info("Validating the transformation...")
+    # Already validated by the pydantic model
+    logger.info("Transformation validated!")
+
+    # Check if the transformation is waiting for an input
+    # - if there is no execution_hooks, the transformation is not waiting for an input and can go on
+    # - if there is execution_hooks, the transformation is waiting for an input
+    job_model_params = []
+
+    try:
+        transformation_execution_hooks = TransformationExecutionHooksHint.from_cwl(transformation.task)
+    except Exception as exc:
+        raise ValueError(f"Invalid DIRAC hints:\n{exc}") from exc
+
+    if transformation_execution_hooks.configuration and transformation_execution_hooks.group_size:
+        # Get the metadata class
+        transformation_metadata = transformation_execution_hooks.to_runtime(transformation)
+
+        # Build the input cwl for the jobs to submit
+        logger.info("Getting the input data for the transformation...")
+        input_data_dict = {}
+        min_length = None
+        for input_name, group_size in transformation_execution_hooks.group_size.items():
+            # Get input query
+            logger.info("\t- Getting input query for %s...", input_name)
+            input_query = transformation_metadata.get_input_query(input_name)
+            if not input_query:
+                raise RuntimeError("Input query not found.")
+
+            # Wait for the input to be available
+            logger.info("\t- Waiting for input data for %s...", input_name)
+            logger.debug("\t\t- Query: %s", input_query)
+            logger.debug("\t\t- Group Size: %s", group_size)
+            while not (inputs := _get_inputs(input_query, group_size)):
+                logger.debug("\t\t- Result: %s", inputs)
+                time.sleep(5)
+            logger.info("\t- Input data for %s available.", input_name)
+            if not min_length or len(inputs) < min_length:
+                min_length = len(inputs)
+
+            # Update the input data in the metadata
+            # Only keep the first min_length inputs
+            input_data_dict[input_name] = inputs[:min_length]
+
+        # Get the JobModelParameter for each input
+        job_model_params = _generate_job_model_parameter(input_data_dict)
+        logger.info("Input data for the transformation retrieved!")
+
+    logger.info("Building the jobs...")
+    jobs = JobSubmissionModel(
+        task=transformation.task,
+        inputs=job_model_params,
+    )
+    logger.info("Jobs built!")
+
+    logger.info("Submitting jobs...")
+    return submit_job_router(jobs)
+
+
+# -----------------------------------------------------------------------------
+# Transformation management
+# -----------------------------------------------------------------------------
+
+
+def _get_inputs(input_query: Path | list[Path], group_size: int) -> List[List[str]]:
+    """Get the input data from the input query.
+
+    :param input_query: The input query to get the input data
+    :param group_size: The number of jobs to group together in a transformation
+    :return: A list of lists of paths to the input data, each inner list has length group_size
+    """
+    # TODO: how do we know whether a given input has already been processed?
+
+    # Retrieve all input paths matching the query
+    if isinstance(input_query, Path):
+        input_paths = glob.glob(str(input_query / "*"), root_dir="filecatalog")
+    else:
+        input_paths = []
+        for query in input_query:
+            input_paths.extend(glob.glob(str(query / "*"), root_dir="filecatalog"))
+    len_input_paths = len(input_paths)
+
+    # Ensure there are enough inputs to form at least one group
+    if len_input_paths < group_size:
+        return []
+
+    # Calculate the number of full groups
+    num_full_groups = len_input_paths // group_size
+
+    # Group the input paths into lists of size group_size
+    input_groups = [input_paths[i * group_size : (i + 1) * group_size] for i in range(num_full_groups)]
+
+    return input_groups
+
+
+def _generate_job_model_parameter(
+    input_data_dict: Dict[str, List[List[str]]],
+) -> List[JobInputModel]:
+    """Generate job model parameters from input data provided."""
+    job_model_params = []
+
+    input_names = list(input_data_dict.keys())
+    input_data_lists = [input_data_dict[input_name] for input_name in input_names]
+    grouped_input_data = [dict(zip(input_names, elements)) for elements in zip(*input_data_lists)]
+    for group in grouped_input_data:
+        cwl_inputs = {}
+        for input_name, input_data in group.items():
+            cwl_inputs[input_name] = [File(location=str(Path("lfn:") / path)) for path in input_data]
+
+        job_model_params.append(JobInputModel(sandbox=None, cwl=cwl_inputs))
+
+    return job_model_params

dirac_cwl-1.0.2.dist-info/METADATA

@@ -0,0 +1,285 @@
+Metadata-Version: 2.4
+Name: dirac-cwl
+Version: 1.0.2
+Summary: Prototype of CWL used as a production/job workflow language
+Author: DIRAC consortium
+License-Expression: GPL-3.0-only
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: System :: Distributed Computing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: cwl-utils
+Requires-Dist: cwlformat
+Requires-Dist: cwltool
+Requires-Dist: dirac>=9.0.0
+Requires-Dist: diracx-api
+Requires-Dist: lbprodrun
+Requires-Dist: mypy
+Requires-Dist: pydantic
+Requires-Dist: pyyaml
+Requires-Dist: typer
+Requires-Dist: referencing>=0.30
+Requires-Dist: rich
+Requires-Dist: ruamel.yaml
+Provides-Extra: testing
+Requires-Dist: pytest>=6; extra == "testing"
+Requires-Dist: pytest-mock; extra == "testing"
+Dynamic: license-file
+
+<p align="center">
+  <img alt="Dirac CWL Logo" src="public/CWLDiracX.png" width="300" >
+</p>
+
+# Dirac CWL Prototype
+![Workflow tests](https://github.com/DIRACGrid/dirac-cwl/actions/workflows/main.yml/badge.svg?branch=main)
+![Schema Generation](https://github.com/DIRACGrid/dirac-cwl/actions/workflows/generate-schemas.yml/badge.svg?branch=main)
+
+This Python prototype introduces a command-line interface (CLI) designed for the end-to-end execution of Common Workflow Language (CWL) workflows at different scales. It enables users to locally test CWL workflows, and then run them as jobs, transformations and/or productions.
+
+## Prototype Workflow
+
+### Local testing
+
+Initially, the user tests the CWL workflow locally using `cwltool`. This step involves validating the workflow's structure and ensuring that it executes correctly with the provided inputs.
+
+  > - CWL task: workflow structure
+  > - inputs of the task
+
+Once the workflow passes local testing, the user can choose from 3 options for submission depending on the requirements.
+
+### Submission methods
+
+1. **Submission as Dirac Jobs**: For simple workflows with a limited number of inputs, CWL tasks can be submitted as individual jobs. In this context, they are run locally as if they were run on distributed computing resources. Additionally, users can submit the same workflow with different sets of inputs in a single request, generating multiple jobs at once.
+
+  > - CWL task
+  > - [inputs1, inputs2, ...]
+  > - Dirac description (site, priority):  Dirac-specific attributes related to scheduling
+  > - Metadata (job type): Dirac-specific attributes related to scheduling + execution
+
+2. **Submission as Dirac Transformation**: For workflows requiring continuous, real-time input data or large-scale execution, CWL tasks can be submitted as transformations. As new input data becomes available, jobs are automatically generated and executed as jobs. This method is ideal for ongoing data processing and scalable operations.
+
+  > - CWL task (inputs already described within it)
+  > - Dirac description (site, priority)
+  > - Metadata (job type, group size, query parameters)
+
+3. **Submission as Dirac Productions**: For complex workflows that require multiple steps with different requirements, CWL tasks can be submitted as productions. This method allows the workflow to be split into multiple transformations, with each transformation handling a distinct step in the process. Each transformation can manage one or more jobs, enabling large-scale, multi-step execution.
+
+  > - CWL task (inputs already described within it)
+  > - Step Metadata (per step):
+  >   - Dirac description (site, priority)
+  >   - Metadata (job type, group size, query parameters)
+
+## Installation (with Pixi)
+
+This project uses [Pixi](https://pixi.sh) to manage the development environment and tasks.
+
+1) Install Pixi (see official docs for your platform)
+
+2) Create and populate the environment
+
+```bash
+pixi install
+```
+
+3) Enter the environment (optional)
+
+```bash
+pixi shell
+```
+
+That’s it. You can now run commands either inside `pixi shell` or by prefixing with `pixi run`.
+
+## Usage
+
+Inside the Pixi environment:
+
+```bash
+# Either inside a shell
+pixi shell
+
+# Submit
+dirac-cwl job submit <workflow_path> [--parameter-path <input_path>] [--metadata-path <metadata_path>]
+
+dirac-cwl transformation submit <workflow_path> [--metadata-path <metadata_path>]
+
+dirac-cwl production submit <workflow_path> [--steps-metadata-path <steps_metadata_path>]
+```
+
+Or prefix individual commands:
+
+```bash
+pixi run dirac-cwl job submit <workflow_path> --parameter-path <input_path>
+```
+
+Common tasks are defined in `pyproject.toml` and can be run with Pixi:
+
+```bash
+# Run tests
+pixi run test
+
+# Lint (mypy)
+pixi run lint
+```
+
+## Using cwltool directly
+
+To use the workflows and inputs directly with `cwltool`, you need to add the `modules` directory to the `$PATH`:
+
+```bash
+export PATH=$PATH:</path/to/dirac-cwl/src/dirac_cwl/modules>
+cwltool <workflow_path> <inputs>
+```
+
+## Contribute
+
+### Add a workflow
+
+To add a new workflow to the project, follow these steps:
+
+- Create a new directory under `workflows` (e.g. `workflows/helloworld`)
+- Add one or more variants of a workflow under different directory (e.g. `helloworld/helloworld_basic/description.cwl` and `helloworld/helloworld_with_inputs/description.cwl`)
+- In a `type_dependencies` subdirectory, add the required files to submit a job/transformation/production from a given variant.
+
+Directory Structure Example:
+
+```
+workflows/
+└── my_new_workflow/
+    |
+    ├── my_new_workflow_complete/
+    |   └── description.cwl
+    ├── my_new_workflow_step1/
+    |   └── description.cwl
+    ├── my_new_workflow_step2/
+    |   └── description.cwl
+    |
+    └── type_dependencies/
+        ├── production/
+        |   └── steps_metadata.yaml
+        ├── transformation/
+        |   └── metadata.yaml
+        └── job/
+            ├── inputs1.yaml
+            └── inputs2.yaml
+```
+
+### Add a Pre/Post-processing commamd and a Job type
+
+#### Add a Pre/Post-Command
+
+A pre/post-processing command allows the execution of code before and after the workflow.
+
+The commands should be stored at the `src/dirac_cwl/commands/` directory
+
+To add a new pre/post-processing command to the project, follow these steps:
+
+- Create a class that inherits `PreProcessCommand` if it's going to be executed before the workflow or `PostProcessCommand` if it's going to be executed after the workflow. In the rare case that the command can be executed in both stages, it should inherit both classes. These classes are located at `src/dirac_cwl/commands/core.py`.
+
+- Implement the `execute` function with the actions it's expected to do. This function recieves the `job path` as a `string` and the dictionary of keyworded arguments `**kwargs`. This function can raise exceptions if it needs to.
+
+#### Add a Job Type
+
+Job types in `dirac_cwl` have the name of "plugins". These plugins are created from the hints defined in a cwl file.
+
+The Job type should be stored at the `src/dirac_cwl/execution_hooks/plugins/` directory and should appear in the `__all__` list of the `__init__.py` file.
+
+To add a new Job type to the project, follow these steps:
+
+- Create a class that inherits `ExecutionHooksBasePlugin` from `src/dirac_cwl/execution_hooks/core.py`.
+
+- Import the pre-processing and post-processing commands that this Job type is going to execute.
+
+- Inside the `__init__` function, set the `preprocess_commands` and `postprocess_commands` lists with the commands that each step should execute. Be specially careful in the order, the commands will be executed in the same order they were specified in the lists.
+
+In the end, it should look something like this:
+
+```python
+class JobTypeExample(ExecutionHooksBasePlugin):
+  def __init__(self, **data):
+    super().__init__(**data)
+
+    # ...
+    self.preprocess_commands = [PreProcessCmd1, PreProcessCmd2, PreProcessCmd3]
+    self.postprocess_commands = [PostProcessCmd1, PostProcessCmd2, PostProcessCmd3]
+    # ...
+```
+
+In the previous example, `PreProcessCmd1` will be executed before `PreProcessCmd2`, and this will be executed before `PreProcessCmd3`.
+
+- Finally, to be able to discover this plugin from the registry, it has to appear in `pyproject.toml` entrypoints at the group `dirac_cwl.execution_hooks`. The previous example would look like:
+
+```toml
+[project.entry-points."dirac_cwl.execution_hooks"]
+# ...
+JobTypeExample = "dirac_cwl.execution_hooks.plugins:JobTypeExample"
+# ...
+```
+
+### Add a module
+
+If your workflow requires calling a script, you can add this script as a module. Follow these steps to properly integrate the module:
+
+- Add the script: Place your script in the `src/dirac_cwl/modules` directory.
+- Update `pyproject.toml`: Add the script to the `pyproject.toml` file to create a command-line interface (CLI) command.
+- Reinstall the package: Run `pixi run pip install .` to reinstall the package and make the new script available as a command.
+- Usage in CWL Workflow: Reference the command in your `description.cwl` file.
+
+**Example**
+
+Let’s say you have a script named `generic_command.py` located at `src/dirac_cwl/modules/generic_command.py`. Here's how you can integrate it:
+
+- `generic_command.py` Example Script:
+
+```python
+#!/usr/bin/env python3
+import typer
+from rich.console import Console
+
+app = typer.Typer()
+console = Console()
+
+@app.command()
+def run_example():
+    console.print("This is an example command.")
+
+if __name__ == "__main__":
+    app()
+```
+
+- Update `pyproject.toml`:
+
+```toml
+[project.scripts]
+generic-command = "dirac_cwl.modules.generic_command:app"
+```
+
+- Reinstall the package with:
+
+```bash
+pixi run pip install .
+```
+
+- Reference in `description.cwl`:
+
+```yaml
+baseCommand: [generic-command]
+```
+
+### Test your changes
+
+- Run tests via Pixi:
+
+```bash
+pixi run test
+```
+
+- Or directly:
+
+```bash
+pixi run pytest test/test_workflows.py -v
+```

dirac_cwl-1.0.2.dist-info/RECORD

@@ -0,0 +1,32 @@
+dirac_cwl/__init__.py,sha256=ta9RLzO5LQ9UzW1yiZ3OYY5fmQiWP2zjw0sbXvJnh6g,758
+dirac_cwl/submission_models.py,sha256=5jr6ScwZm9zloaRS-k29LDQiPZHDhdQAUs1xmH1BFi4,4935
+dirac_cwl/commands/__init__.py,sha256=yADJYLzbM8oYt04f0vpJ3ju1Ts2SIc5lW9M8Jle0L7Q,179
+dirac_cwl/commands/core.py,sha256=vXqAMD7KGL5MStSP-JTRWCPUEmTcmj-TwIKgjHGSGpY,1232
+dirac_cwl/commands/download_config.py,sha256=dOwhWMhyD2bVzO8i2uFm5nlgrG-Lh8xI-pYxTAwvWQs,633
+dirac_cwl/commands/group_outputs.py,sha256=E6XTan9mQA5ux50wKx_OwBx9ZVRqO45vvJmY7Uc81z8,1133
+dirac_cwl/core/__init__.py,sha256=4gQdN890Y--wTXMojmNNYg0qzPzFJSIz9Wh4cUgBVzA,30
+dirac_cwl/core/exceptions.py,sha256=x_15wckaEhPE60DHPUQK0gv6zBkhSZBr6K6kZDS-1rQ,157
+dirac_cwl/core/utility.py,sha256=4e6NcsQr_0O-mDHvJTTbfMiqkzSgSLtgyEDzXLsbDX8,1492
+dirac_cwl/data_management_mocks/data_manager.py,sha256=QUADgjcpb60mlcvyHLOjbKpQblCHGmeoba1WLkJmMYU,4271
+dirac_cwl/data_management_mocks/file_catalog.py,sha256=xpIcLcbELXuSVq-DcgAd-vbQ0SXiie9ZnkUx9FnjTGE,4544
+dirac_cwl/data_management_mocks/sandbox.py,sha256=bueqArIZR6kZVVEb258Dpx445A_Odz3X_9TqoZOmnsc,3321
+dirac_cwl/execution_hooks/__init__.py,sha256=DkZPSJ362opbfc0b6WbxmSa6bT-6s9rWFB2MjLV6dKk,1036
+dirac_cwl/execution_hooks/core.py,sha256=vn8SI_Ow16qx_yl4YmzDSfa6sooMKgkXYpx_7MmhML8,12417
+dirac_cwl/execution_hooks/registry.py,sha256=aDf0yggUvOFkOLFO--YEU3zML5TzH27_MNk6wYuHPNg,7838
+dirac_cwl/execution_hooks/plugins/__init__.py,sha256=TKtfC1FUV1vRgcG0DnhoWqlyqaj2M1pysqTxh_Vlybo,324
+dirac_cwl/execution_hooks/plugins/core.py,sha256=5oSB3aobAV9l4lMvta7wv3EejyXIX5kpB55X1uQguAo,1987
+dirac_cwl/job/__init__.py,sha256=6ENcM9E930jSIhWBmXm_QPxOk31uduYtQ3eTirEw7xo,7809
+dirac_cwl/job/job_wrapper.py,sha256=26Q5ywkmexPuxEBtOo3taZdOIwGYPRJeDT1tcunsvSM,14693
+dirac_cwl/job/job_wrapper_template.py,sha256=sQ1XuKPmo6qFzRra3qZSlk8sPTgi1VTP3Q1bq67YCBk,1458
+dirac_cwl/job/submission_clients.py,sha256=WrieGuHg-JRfu1c0QxMTXk6zS39rTagGEFTG04DOGWg,5528
+dirac_cwl/modules/crypto.py,sha256=QS4NWQYiFyWSTBvRjm0O7OS_f6hm-_K9QfBMP6ngPX8,2574
+dirac_cwl/modules/pi_gather.py,sha256=-a3SBztxi4FrMKLttpJ3JgyZ5gcRpReQSqykNpO34fs,1120
+dirac_cwl/modules/pi_simulate.py,sha256=br_c8wQKrGveGrsFEPDEQggsPbd0b2J1rSW4i27lfCs,826
+dirac_cwl/production/__init__.py,sha256=NvPI2UKLjFHL8a2TQGNOD4_yh6mHvn9ugZ4iWOTgOSw,6983
+dirac_cwl/transformation/__init__.py,sha256=SJP8HUaH-huYphE-Gxhm4I4vVyhHze69tVSvAS007wI,8151
+dirac_cwl-1.0.2.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+dirac_cwl-1.0.2.dist-info/METADATA,sha256=zzNL0gyaTMLSH-Uce7E0m-V50uO90B-C8fUA-Q4joPQ,10036
+dirac_cwl-1.0.2.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+dirac_cwl-1.0.2.dist-info/entry_points.txt,sha256=vJW7SQRh8klf-hUG2SG3IbtxLpl8HucZsk44oQxRo4g,273
+dirac_cwl-1.0.2.dist-info/top_level.txt,sha256=s23lab7e3RtDqMonc5OlaSymqnNpCjv2FIBwO4-P2wc,10
+dirac_cwl-1.0.2.dist-info/RECORD,,

dirac_cwl-1.0.2.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

dirac_cwl-1.0.2.dist-info/entry_points.txt

@@ -0,0 +1,8 @@
+[console_scripts]
+crypto = dirac_cwl.modules.crypto:app
+dirac-cwl = dirac_cwl:app
+pi-gather = dirac_cwl.modules.pi_gather:app
+pi-simulate = dirac_cwl.modules.pi_simulate:app
+
+[dirac_cwl.execution_hooks]
+QueryBasedPlugin = dirac_cwl.execution_hooks.plugins:QueryBasedPlugin