CTADIRAC 2.2.71__tar.gz → 2.2.72__tar.gz

{ctadirac-2.2.71/src/CTADIRAC.egg-info → ctadirac-2.2.72}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: CTADIRAC
-Version: 2.2.71
+Version: 2.2.72
 Summary: CTADIRAC is an extension of the DIRAC interware for dedicated CTA usage.
 Author-email: Luisa Arrabito <arrabito@in2p3.fr>
 License: BSD 3-Clause License

{ctadirac-2.2.71 → ctadirac-2.2.72}/src/CTADIRAC/Interfaces/API/CWLJob.py

@@ -3,9 +3,10 @@
 import tempfile
 from pathlib import Path
 
-from cwl_utils.parser import save
+from cwl_utils.parser import load_document, save
 from DIRAC.Interfaces.API.Dirac import Dirac
 from DIRAC.Interfaces.API.Job import Job
+from cwl_utils.parser.utils import load_inputfile
 from ruamel.yaml import YAML
 
 from CTADIRAC.Interfaces.Utilities.CWL_utils import (
@@ -33,20 +34,33 @@ class CWLJob(Job):
         apptainer_options: list | None = None,
     ) -> None:
         super().__init__()
-        self.cwl_workflow = cwl_workflow
-        self.cwl_inputs = cwl_inputs
+
+        self.cwl_workflow_path = Path(cwl_workflow)
+        self.cwl_inputs_path = Path(cwl_inputs)
+
+        self.original_cwl = load_document(self.cwl_workflow_path.read_text())
+        self.original_inputs = load_inputfile(
+            self.original_cwl.cwlVersion, self.cwl_inputs_path.read_text()
+        )
+
         self.cvmfs_base_path = cvmfs_base_path
         self.apptainer_options = apptainer_options if apptainer_options else []
 
-        self.cwl_dict = translate_cwl_workflow(
-            self.cwl_workflow, self.cvmfs_base_path, self.apptainer_options
+        cwl_dict = translate_cwl_workflow(
+            self.original_cwl,
+            self.original_inputs,
+            self.cvmfs_base_path,
+            self.apptainer_options,
         )
-        self.output_sandbox = self.cwl_dict.get("OutputSandbox", [])
-        self.output_data = self.cwl_dict.get("OutputData", [])
 
-        self.input_cwl_dict = extract_and_translate_input_files(self.cwl_inputs)
-        self.input_data = self.input_cwl_dict.get("InputData", [])
-        self.input_sandbox = self.input_cwl_dict.get("InputSandbox", [])
+        self.transformed_cwl = cwl_dict["CWLDesc"]
+        self.output_sandbox = cwl_dict.get("OutputSandbox", [])
+        self.output_data = cwl_dict.get("OutputData", [])
+
+        input_cwl_dict = extract_and_translate_input_files(self.original_inputs)
+        self.transformed_inputs = input_cwl_dict["InputDesc"]
+        self.input_data = input_cwl_dict.get("InputData", [])
+        self.input_sandbox = input_cwl_dict.get("InputSandbox", [])
 
     def submit(self):
         """Submit the CWL job to DIRAC.
@@ -60,15 +74,15 @@ class CWLJob(Job):
 
         # Create the modified Dirac compliant CWL workflow and inputs files to submit
         with tempfile.NamedTemporaryFile(
-            suffix=f"_{self.cwl_workflow.name}"
+            suffix=f"_{self.cwl_workflow_path.name}"
         ) as temp_workflow:
-            yaml.dump(save(self.cwl_dict["CWLDesc"]), temp_workflow)
+            yaml.dump(save(self.transformed_cwl), temp_workflow)
             temp_workflow.flush()
 
             with tempfile.NamedTemporaryFile(
-                suffix=f"_{self.cwl_inputs.name}"
+                suffix=f"_{self.cwl_inputs_path.name}"
             ) as temp_inputs:
-                yaml.dump(save(self.input_cwl_dict["InputDesc"]), temp_inputs)
+                yaml.dump(save(self.transformed_inputs), temp_inputs)
                 temp_inputs.flush()
 
                 self.setInputSandbox(

ctadirac-2.2.72/src/CTADIRAC/Interfaces/Utilities/CWL_utils.py

@@ -0,0 +1,248 @@
+"""Utilities to handle CWL."""
+
+from pathlib import Path
+from typing import Any
+from copy import deepcopy
+
+from cwl_utils.parser import OutputArraySchema, File
+from cwl_utils.expression import do_eval
+
+LFN_PREFIX = "lfn://"
+LFN_DIRAC_PREFIX = "LFN:"
+LOCAL_PREFIX = "file://"
+
+
+def fill_defaults(cwl, inputs):
+    """Fill in defaults into inputs.
+
+    This is needed for evaluating expressions later on.
+
+    Parameters
+    ----------
+    cwl: dict
+        The CWL definition
+    inputs: dict
+        user provided inputs.
+
+    Returns
+    -------
+    inputs: dict
+        inputs with additional values filled from CWL defaults
+    """
+    inputs_new = deepcopy(inputs)
+
+    if cwl.inputs is None:
+        return inputs_new
+
+    for inp in cwl.inputs:
+        key = inp.id.rpartition("#")[2]
+        if key not in inputs and inp.default is not None:
+            inputs_new[key] = inp.default
+    return inputs_new
+
+
+def translate_cwl_workflow(
+    cwl_obj,
+    cwl_inputs,
+    cvmfs_base_path: Path,
+    apptainer_options: list[Any],
+) -> dict[str, Any]:
+    """Translate the CWL workflow description into Dirac compliant execution."""
+    output_dict = {"CWLDesc": cwl_obj, "OutputSandbox": [], "OutputData": []}
+
+    if cwl_obj.class_ != "CommandLineTool":
+        return output_dict
+
+    if cwl_obj.hints:
+        cwl_obj = translate_docker_hints(cwl_obj, cvmfs_base_path, apptainer_options)
+
+    return extract_and_translate_output_files(cwl_obj, cwl_inputs)
+
+
+def translate_docker_hints(
+    cwl_obj, cvmfs_base_path: Path, apptainer_options: list[Any]
+):
+    """Translate CWL DockerRequirement into Dirac compliant execution."""
+    for index, hints in enumerate(cwl_obj.hints):
+        if hints.class_ == "DockerRequirement":
+            image = hints.dockerPull
+            cmd = [
+                "apptainer",
+                "run",
+                *apptainer_options,
+                str(cvmfs_base_path / f"{image}"),
+            ]
+            if isinstance(cwl_obj.baseCommand, str):
+                cmd += [cwl_obj.baseCommand]
+            else:
+                cmd += cwl_obj.baseCommand
+            cwl_obj.baseCommand = cmd
+            del cwl_obj.hints[index]
+    return cwl_obj
+
+
+def collect_outputs(cwl, inputs) -> list[str]:
+    """Collect evaluated output filenames.
+
+    Parameters
+    ----------
+    cwl: dict
+        The CWL definition
+    inputs: dict
+        user provided inputs.
+
+    Returns
+    -------
+    outputs: list[str]
+        The output filenames of this workflow given
+        the provided inputs and defaults.
+    """
+    outputs = []
+
+    if cwl.outputs is None:
+        return outputs
+
+    for output in cwl.outputs:
+        if glob := output.outputBinding.glob:
+            result = do_eval(
+                glob,
+                inputs,
+                outdir=None,
+                requirements=[],
+                tmpdir=None,
+                resources={},
+            )
+            outputs.append(result)
+
+    return outputs
+
+
+def extract_and_translate_output_files(cwl_obj, inputs) -> dict[str, Any]:
+    """Translate output files into a DIRAC compliant usage.
+
+    Extract local outputs and lfns.
+    Remove outputs path prefix.
+    """
+    output_lfns = []
+    output_sandboxes = []
+
+    inputs = fill_defaults(cwl_obj, inputs)
+
+    for outputs in cwl_obj.outputs or []:
+        if not verify_cwl_output_type(outputs.type_):
+            continue
+
+        glob = outputs.outputBinding.glob
+
+        if isinstance(glob, str):
+            glob = do_eval(
+                glob,
+                inputs,
+                outdir=None,
+                requirements=[],
+                tmpdir=None,
+                resources={},
+            )
+            if glob.startswith(LFN_PREFIX):
+                output_lfns.append(glob.replace(LFN_PREFIX, LFN_DIRAC_PREFIX))
+            else:
+                output_sandboxes.append(glob)
+
+        if isinstance(glob, list):
+            for glob_item in glob:
+                glob_item = do_eval(
+                    glob_item,
+                    inputs,
+                    outdir=None,
+                    requirements=[],
+                    tmpdir=None,
+                    resources={},
+                )
+                if glob_item.startswith(LFN_PREFIX):
+                    output_lfns.append(glob_item.replace(LFN_PREFIX, LFN_DIRAC_PREFIX))
+                else:
+                    output_sandboxes.append(glob_item)
+
+    return {
+        "CWLDesc": cwl_obj,
+        "OutputSandbox": output_sandboxes,
+        "OutputData": output_lfns,
+    }
+
+
+def verify_cwl_output_type(output_type) -> bool:
+    """Verify the cwl output type.
+
+    True if the type is File or OutputArraySchema
+    or a list of 'null' and File/OutputArraySchema
+    else False.
+    """
+    if isinstance(output_type, list):
+        for type_ in output_type:
+            if type_ == "File" or isinstance(type_, OutputArraySchema):
+                return True
+    if output_type == "File" or isinstance(output_type, OutputArraySchema):
+        return True
+    return False
+
+
+# TODO: how to deal with default values not present in the input file but in the cwl?
+def extract_and_translate_input_files(cwl_inputs) -> dict[str, Any]:
+    """Extract input files from CWL inputs, rewrite file prefix.
+
+    If the file is a Sandbox, ensure there is no absolute path,
+    and store it in the input sandbox list.
+    If the file is a LFN, remove the lfn prefix and store it in the lfns list.
+    """
+    input_sandboxes = []
+    input_lfns = []
+    cwl_inputs = deepcopy(cwl_inputs)
+
+    for key, input_value in cwl_inputs.items():
+        if isinstance(input_value, list):
+            for file in input_value:
+                if isinstance(file, File):
+                    original = file.path
+                    file.path, lfn = translate_sandboxes_and_lfns(file)
+                    if lfn:
+                        input_lfns.append(
+                            original.replace(LFN_PREFIX, LFN_DIRAC_PREFIX)
+                        )
+                    else:
+                        input_sandboxes.append(file.path)
+
+        elif isinstance(input_value, File):
+            path = input_value.path
+            input_value.path, lfn = translate_sandboxes_and_lfns(path)
+            if lfn:
+                input_lfns.append(path.replace(LFN_PREFIX, LFN_DIRAC_PREFIX))
+            else:
+                input_sandboxes.append(input_value.path)
+
+        # assume that an LFN in the inputs as string is an output filename
+        elif isinstance(input_value, str) and input_value.startswith(LFN_PREFIX):
+            cwl_inputs[key], _ = translate_sandboxes_and_lfns(input_value)
+
+    return {
+        "InputDesc": cwl_inputs,
+        "InputSandbox": input_sandboxes,
+        "InputData": input_lfns,
+    }
+
+
+def translate_sandboxes_and_lfns(file: File | str) -> tuple[str, bool]:
+    """Extract local files as sandboxes and lfns as input data."""
+    if isinstance(file, File):
+        if not file.path:
+            raise KeyError("File path is not defined.")
+        path = file.path
+    elif isinstance(file, str):
+        path = file
+
+    lfn = False
+    if path.startswith(LFN_PREFIX):
+        path = path.removeprefix(LFN_PREFIX)
+        lfn = True
+
+    path = path.removeprefix(LOCAL_PREFIX)
+    return Path(path).name, lfn

{ctadirac-2.2.71 → ctadirac-2.2.72}/src/CTADIRAC/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.2.71'
-__version_tuple__ = version_tuple = (2, 2, 71)
+__version__ = version = '2.2.72'
+__version_tuple__ = version_tuple = (2, 2, 72)

{ctadirac-2.2.71 → ctadirac-2.2.72/src/CTADIRAC.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: CTADIRAC
-Version: 2.2.71
+Version: 2.2.72
 Summary: CTADIRAC is an extension of the DIRAC interware for dedicated CTA usage.
 Author-email: Luisa Arrabito <arrabito@in2p3.fr>
 License: BSD 3-Clause License

{ctadirac-2.2.71 → ctadirac-2.2.72}/src/CTADIRAC.egg-info/SOURCES.txt

@@ -232,6 +232,8 @@ src/CTADIRAC/WorkloadManagementSystem/__init__.py
 src/CTADIRAC/WorkloadManagementSystem/Client/CTAODownloadInputData.py
 src/CTADIRAC/WorkloadManagementSystem/Client/__init__.py
 src/CTADIRAC/_dev/__init__.py
+tests/resources/cwl/inputs_process_dl0_dl1.yaml
+tests/resources/cwl/process_dl0_dl1.cwl
 tests/system/runTestSuite.py
 tests/system/testDMSDefinitions.py
 tests/system/testFileCatalogDefinitions.py

ctadirac-2.2.72/tests/resources/cwl/inputs_process_dl0_dl1.yaml

@@ -0,0 +1,9 @@
+dl0:
+  class: File
+  path: lfn:///ctao/simpipe/prod6/gamma-diffuse/010xxx/gamma_cone10_run010000.simtel.zst
+
+processing_config:
+  class: File
+  path: process_config.yaml
+
+dl1_filename: lfn:///ctao/datapipe/test.dl1.h5

ctadirac-2.2.72/tests/resources/cwl/process_dl0_dl1.cwl

@@ -0,0 +1,60 @@
+%YAML 1.1
+---
+cwlVersion: v1.2
+class: CommandLineTool
+baseCommand:
+  - ctapipe-process
+  - --DataWriter.write_dl1_parameters=True
+  - --DataWriter.write_dl2=False
+doc: |
+  Processes a single file from DL0 to DL1 using the ctapipe-process tool.
+  (DPPS-UC-130-1.2.1)
+label: process single dl0 to dl1
+inputs:
+  processing_config:
+    type: File?
+    inputBinding:
+      prefix: --config
+    doc: |
+      Sets the reconstruction parameters that apply to DL0 to DL1.
+      See `ctapipe-process --help-all` for a list of all options, or the output
+      of `ctapipe-quickstart` for sample configuration files.
+
+  dl0:
+    type: [File, string]
+    doc: |
+      path to input file, which can be at any data level transformable to DL1
+      that is supported by the installed ctapipe io plugins. I can also be a
+      URL.
+    inputBinding:
+      prefix: --input
+
+  dl1_filename:
+    type: string
+    doc: name of the DL1 output file
+    inputBinding:
+      prefix: --output
+
+  provenance_log_filename:
+    type: string
+    doc: file in which to write the local ctapipe-process provenance.
+    default: ctapipe-process_dl0_dl1.provenance.log
+    inputBinding:
+      prefix: --provenance-log
+
+outputs:
+  dl1:
+    type: File
+    doc: HDF5 format output file.
+    outputBinding:
+      glob: $(inputs.dl1_filename)
+
+  provenance_log:
+    type: File
+    doc: ctapipe format provenance log for this step.
+    outputBinding:
+      glob: $(inputs.provenance_log_filename)
+
+hints:
+  DockerRequirement:
+    dockerPull: harbor.cta-observatory.org/dpps/datapipe:dev

{ctadirac-2.2.71 → ctadirac-2.2.72}/tests/unit/api/test_cwl_job.py

@@ -198,3 +198,42 @@ def test_cwl_job_submit(
     ):
         executable = executable.text
         assert executable == "cwltool"
+
+
+def test_datapipe_cwl_job(mock_submit_job):
+    job = CWLJob(
+        "tests/resources/cwl/process_dl0_dl1.cwl",
+        "tests/resources/cwl/inputs_process_dl0_dl1.yaml",
+        cvmfs_base_path=Path("/cvmfs/ctao.dpps.test/"),
+    )
+
+    # these are set via inputs
+    assert len(job.input_data) == 1
+    assert (
+        job.input_data[0]
+        == "LFN:/ctao/simpipe/prod6/gamma-diffuse/010xxx/gamma_cone10_run010000.simtel.zst"
+    )
+
+    result = job.submit()
+    mock_submit_job.assert_called_once_with(job)
+    result_xml = ET.fromstring(result)
+
+    if input_data := result_xml.find(".//Parameter[@name='InputData']/value"):
+        input_data = input_data.text.split(";")
+        assert len(input_data) == 1
+        assert input_data[0] == "gamma_cone10_run010000.simtel.zst"
+
+    if input_sandbox := result_xml.find(".//Parameter[@name='InputSandbox']/value"):
+        input_sandbox = input_sandbox.text.split(";")
+        assert len(input_sandbox) == 1
+        assert input_sandbox[0] == "process_config.yaml"
+
+    if output_data := result_xml.find(".//Parameter[@name='OutputData']/value"):
+        output_data = output_data.text.split(";")
+        assert len(output_data) == 1
+        assert output_data[0] == "LFN:/ctao/datapipe/test.dl1.h5"
+
+    if output_sandbox := result_xml.find(".//Parameter[@name='OutputSandbox']/value"):
+        output_sandbox = output_sandbox.text.split(";")
+        assert len(job.output_sandbox) == 1
+        assert job.output_sandbox[0] == "ctapipe-process_dl0_dl1.provenance.log"

{ctadirac-2.2.71 → ctadirac-2.2.72}/tests/unit/api/test_cwl_utils.py

@@ -26,74 +26,40 @@ from CTADIRAC.Interfaces.Utilities.CWL_utils import (
 CVMFS_BASE_PATH = Path("/cvmfs/ctao.dpps.test")
 
 
-@pytest.fixture
-def mock_load_inputfile(mocker):
-    return mocker.patch("CTADIRAC.Interfaces.Utilities.CWL_utils.load_inputfile")
-
-
-@pytest.fixture
-def mock_load_document_by_uri(mocker):
-    return mocker.patch("CTADIRAC.Interfaces.Utilities.CWL_utils.load_document_by_uri")
-
-
-@pytest.fixture
-def mock_translate_docker_hints(mocker):
-    return mocker.patch(
-        "CTADIRAC.Interfaces.Utilities.CWL_utils.translate_docker_hints"
-    )
-
-
-@pytest.fixture
-def mock_extract_and_translate_output_files(mocker):
-    return mocker.patch(
-        "CTADIRAC.Interfaces.Utilities.CWL_utils.extract_and_translate_output_files"
-    )
-
-
 @pytest.mark.parametrize(
-    ("file_input", "expected_result", "expected_sandboxes", "expected_lfns"),
+    ("file_input", "expected_result", "expected_lfn"),
     [
         (
             File(path=LFN_PREFIX + "test_lfn_file.txt"),
             "test_lfn_file.txt",
-            [],
-            [LFN_DIRAC_PREFIX + "test_lfn_file.txt"],
+            True,
         ),
         (
             File(path=LOCAL_PREFIX + "test_local_file.txt"),
             "test_local_file.txt",
-            ["test_local_file.txt"],
-            [],
+            False,
         ),
         (
             LFN_PREFIX + "test_lfn_str.txt",
             "test_lfn_str.txt",
-            [],
-            [LFN_DIRAC_PREFIX + "test_lfn_str.txt"],
+            True,
         ),
         (
             LOCAL_PREFIX + "test_local_str.txt",
             "test_local_str.txt",
-            ["test_local_str.txt"],
-            [],
+            False,
         ),
-        (File(), None, [], []),  # This will raise an exception
+        (File(), None, False),  # This will raise an exception
     ],
 )
-def test_translate_sandboxes_and_lfns(
-    file_input, expected_result, expected_sandboxes, expected_lfns
-):
-    sandboxes = []
-    lfns = []
-
+def test_translate_sandboxes_and_lfns(file_input, expected_result, expected_lfn):
     if expected_result is None:
-        with pytest.raises(NotImplementedError, match="File path is not defined."):
-            translate_sandboxes_and_lfns(file_input, sandboxes, lfns)
+        with pytest.raises(KeyError, match="File path is not defined."):
+            translate_sandboxes_and_lfns(file_input)
     else:
-        result = translate_sandboxes_and_lfns(file_input, sandboxes, lfns)
+        result, is_lfn = translate_sandboxes_and_lfns(file_input)
         assert result == expected_result
-        assert sandboxes == expected_sandboxes
-        assert lfns == expected_lfns
+        assert is_lfn == expected_lfn
 
 
 @pytest.mark.parametrize(
@@ -160,11 +126,8 @@ def test_translate_sandboxes_and_lfns(
         ),
     ],
 )
-def test_extract_and_translate_input_files(
-    mock_load_inputfile, input_data, expected_result
-):
-    mock_load_inputfile.return_value = input_data
-    result = extract_and_translate_input_files(Path("dummy_path"))
+def test_extract_and_translate_input_files(input_data, expected_result):
+    result = extract_and_translate_input_files(input_data)
     assert result == expected_result
 
 
@@ -246,7 +209,8 @@ def test_verify_cwl_output_type(output_type, expected_result):
 )
 def test_extract_and_translate_output_files(outputs, expected_result):
     cwl_obj = CommandLineTool(inputs=None, outputs=outputs)
-    result = extract_and_translate_output_files(cwl_obj)
+    result = extract_and_translate_output_files(cwl_obj, {})
+
     assert result["OutputSandbox"] == expected_result["OutputSandbox"]
     assert result["OutputData"] == expected_result["OutputData"]
     assert isinstance(result["CWLDesc"], CommandLineTool)
@@ -274,32 +238,19 @@ def test_translate_docker_hints(
     assert result.baseCommand == expected_base_command
 
 
-@pytest.mark.parametrize(
-    ("hints"),
-    [
-        ([DockerRequirement(dockerPull="harbor/python:tag")],),
-    ],
-)
-def test_translate_cwl_workflow(
-    mock_load_document_by_uri,
-    mock_translate_docker_hints,
-    mock_extract_and_translate_output_files,
-    hints,
-):
+def test_translate_cwl_workflow():
     cwl_obj = CommandLineTool(
-        inputs=None, outputs=None, hints=hints, baseCommand="python"
+        inputs=None,
+        outputs=None,
+        hints=[DockerRequirement(dockerPull="harbor/python:tag")],
+        baseCommand="python",
     )
-    mock_load_document_by_uri.return_value = cwl_obj
-    mock_translate_docker_hints.return_value = cwl_obj
-    mock_extract_and_translate_output_files.return_value = {
-        "CWLDesc": cwl_obj,
-        "OutputSandbox": [],
-        "OutputData": [],
-    }
 
-    result = translate_cwl_workflow(Path("workflow.cwl"), CVMFS_BASE_PATH, [])
+    result = translate_cwl_workflow(
+        cwl_obj=cwl_obj,
+        cwl_inputs={},
+        cvmfs_base_path=CVMFS_BASE_PATH,
+        apptainer_options=[],
+    )
 
-    mock_load_document_by_uri.assert_called_once_with(Path("workflow.cwl"))
-    mock_translate_docker_hints.assert_called_once_with(cwl_obj, CVMFS_BASE_PATH, [])
-    mock_extract_and_translate_output_files.assert_called_once_with(cwl_obj)
     assert result == {"CWLDesc": cwl_obj, "OutputSandbox": [], "OutputData": []}