CTADIRAC 3.0.8.dev4__tar.gz → 3.0.9__tar.gz

{ctadirac-3.0.8.dev4/src/CTADIRAC.egg-info → ctadirac-3.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: CTADIRAC
-Version: 3.0.8.dev4
+Version: 3.0.9
 Summary: CTADIRAC is an extension of the DIRAC interware for dedicated CTA usage.
 Author-email: Luisa Arrabito <arrabito@in2p3.fr>
 License: BSD 3-Clause License

{ctadirac-3.0.8.dev4 → ctadirac-3.0.9}/src/CTADIRAC/Interfaces/API/CWLJob.py

@@ -7,7 +7,6 @@ from cwl_utils.parser import save
 from DIRAC.Interfaces.API.Dirac import Dirac
 from DIRAC.Interfaces.API.Job import Job
 from ruamel.yaml import YAML
-
 from CTADIRAC.Interfaces.Utilities.CWLTranslator import CWLTranslator
 
 
@@ -32,11 +31,10 @@ class CWLJob(Job, CWLTranslator):
     ) -> None:
         Job.__init__(self)
         CWLTranslator.__init__(self, cwl_workflow, cwl_inputs)
-
+        self.logfile_name = "cwl_job"
         self.cvmfs_base_path = cvmfs_base_path
         self.apptainer_options = apptainer_options if apptainer_options else []
         self._output_se = output_se
-
         self.translate(self.cvmfs_base_path, self.apptainer_options)
 
     def submit(self):
@@ -48,14 +46,12 @@ class CWLJob(Job, CWLTranslator):
         """
         dirac = Dirac()
         yaml = YAML()
-
         # Create the modified Dirac compliant CWL workflow and inputs files to submit
         with tempfile.NamedTemporaryFile(
             suffix=f"_{self.cwl_workflow_path.name}"
         ) as temp_workflow:
             yaml.dump(save(self.transformed_cwl), temp_workflow)
             temp_workflow.flush()
-
             with tempfile.NamedTemporaryFile(
                 suffix=f"_{self.cwl_inputs_path.name}"
             ) as temp_inputs:
@@ -76,9 +72,10 @@ class CWLJob(Job, CWLTranslator):
                 arguments_str = (
                     f"{Path(temp_workflow.name).name} {Path(temp_inputs.name).name}"
                 )
-
                 self.setExecutable(
-                    "cwltool", arguments=arguments_str, logFile=f"{self.name}.log"
+                    "cwltool",
+                    arguments=arguments_str,
+                    logFile=f"{self.logfile_name}.log",
                 )
                 res = dirac.submitJob(self)
         return res

{ctadirac-3.0.8.dev4 → ctadirac-3.0.9}/src/CTADIRAC/Interfaces/Utilities/CWLTranslator.py

@@ -95,12 +95,12 @@ class CWLTranslator:
             apptainer_options : list[Any]
                 A list of options for Apptainer.
         """
-
+        self._extract_input_files(self.original_inputs)
         if self.transformed_cwl.hints:
             self.transformed_cwl = self._translate_docker_hints(
                 self.transformed_cwl, cvmfs_base_path, apptainer_options
             )
-        self._extract_and_translate_input_files()
+
         self._extract_output_files(self.transformed_cwl, self.original_inputs)
 
     def _translate_workflow(
@@ -115,41 +115,42 @@ class CWLTranslator:
         apptainer_options : list[Any]
             A list of options for Apptainer.
         """
-        # Need to set the file basename for JSReq:
-        self.transformed_inputs = set_input_file_basename(self.transformed_inputs)
-        self.transformed_inputs = fill_defaults(
-            self.transformed_cwl, self.transformed_inputs
-        )
         # Extract the DIRAC related
-        self._extract_and_translate_input_files()
-
-        self.evaluated_steps_outputs = {}
+        self._extract_input_files(self.original_inputs)
+        # Need to set the file basename for JSReq:
         steps_inputs = deepcopy(self.transformed_inputs)
+        self.evaluated_steps_outputs = {}
         for n, wf_step in enumerate(self.transformed_cwl.steps):
-            step_name = wf_step.id.rpartition("#")[2].split("/")[0]
-            self.current_step_name = step_name
-            self.evaluated_steps_outputs.setdefault(step_name, {})
-            # replace docker hints
-            if wf_step.run.hints:
-                self.transformed_cwl.steps[n].run = self._translate_docker_hints(
-                    wf_step.run, cvmfs_base_path, apptainer_options
-                )
-            # Update the input context for next steps
-            steps_inputs = fill_defaults(wf_step.run, steps_inputs)
-            steps_inputs = set_input_file_basename(steps_inputs)
-            steps_inputs = self._evaluate_step_inputs(wf_step, steps_inputs)
-
-            # Evaluate step outputs
-            # specific evaluation if the step is an ExpressionTool
-            if isinstance(wf_step.run, ExpressionTool):
-                exptool_outputs = self._evaluate_expression_tool_outputs(
-                    self.unpacked_cwl, wf_step, steps_inputs, step_name
-                )
-                self.evaluated_steps_outputs[step_name] = exptool_outputs
-            else:
-                self._evaluate_step_outputs(wf_step, steps_inputs)
+            # TODO: handle the case where the step is also a workflow
+            if isinstance(wf_step.run, (CommandLineTool, ExpressionTool)):
+                step_name = wf_step.id.rpartition("#")[2].split("/")[0]
+                self.current_step_name = step_name
+                self.evaluated_steps_outputs.setdefault(step_name, {})
+                # replace docker hints
+                if isinstance(wf_step, CommandLineTool) and wf_step.run.hints:
+                    self.transformed_cwl.steps[n].run = self._translate_docker_hints(
+                        wf_step.run, cvmfs_base_path, apptainer_options
+                    )
+                # Update the input context for next steps
+                steps_inputs = fill_defaults(wf_step.run, steps_inputs)
+                steps_inputs = set_input_file_basename(steps_inputs)
+                steps_inputs = self._evaluate_step_inputs(wf_step, steps_inputs)
+
+                # Evaluate step outputs
+                # specific evaluation if the step is an ExpressionTool
+                if isinstance(wf_step.run, ExpressionTool):
+                    exptool_outputs = self._evaluate_expression_tool_outputs(
+                        self.unpacked_cwl, wf_step, steps_inputs, step_name
+                    )
+                    self.evaluated_steps_outputs[step_name] = exptool_outputs
+                else:
+                    self._evaluate_step_outputs(wf_step, steps_inputs)
+            elif isinstance(wf_step, Workflow):
+                raise NotImplementedError("Workflow steps are not handle yet...")
+
         # Finally extract the DIRAC related outputs
         self._extract_workflow_outputs()
+        self.remove_lfns_from_transformed_inputs()
 
     @staticmethod
     def _translate_docker_hints(
@@ -230,6 +231,7 @@ class CWLTranslator:
             list | dict[str, Any] | None
                 Evaluated CWL File or list of CWL File
             """
+            transformed_inputs = deepcopy(self.transformed_inputs)
             inp_id = inp.id.rpartition("#")[2].split("/")[-1]
             if inp.valueFrom:
                 # evaluate if it is a JS exp
@@ -300,8 +302,8 @@ class CWLTranslator:
                         ):
                             # TODO: can that be a list?
                             return self.evaluated_steps_outputs[step_ref][source]
-                    elif inp_source[0] in save(self.transformed_inputs):
-                        return save(self.transformed_inputs)[inp_source[0]]
+                    elif inp_source[0] in save(transformed_inputs):
+                        return save(transformed_inputs)[inp_source[0]]
                     elif inp_source[0] in input_context:
                         return input_context[inp_source[0]]
 
@@ -544,40 +546,64 @@ class CWLTranslator:
 
         return expr_inputs
 
-    def _extract_and_translate_input_files(self) -> None:
-        """Extract input files from CWL inputs and rewrite file paths.
-        If the file is a Sandbox, ensure there is no absolute path, and store it in the input sandbox list.
-        If the file is a LFN, remove the lfn prefix and store it in the lfns list.
-        """
+    def remove_lfns_from_transformed_inputs(self):
+        """Remove 'lfn' prefix from inputs name in tranformed inputs."""
+        for inp_name, inp_value in self.transformed_inputs.items():
+            if isinstance(inp_value, list):
+                for index, val in enumerate(inp_value):
+                    if isinstance(val, str) and self.transformed_inputs[inp_name][
+                        index
+                    ].startswith(LFN_PREFIX):
+                        # Update the transformed cwl lfn filename
+                        self.transformed_inputs[inp_name][index] = Path(
+                            val.removeprefix(LFN_PREFIX)
+                        ).name
+            elif isinstance(inp_value, str) and self.transformed_inputs[
+                inp_name
+            ].startswith(LFN_PREFIX):
+                # Update the transformed cwl lfn filename
+                self.transformed_inputs[inp_name] = Path(
+                    inp_value.removeprefix(LFN_PREFIX)
+                ).name
 
-        def rewrite_file_path(file: File | str) -> str:
-            """Rewrite file path.
+    def _extract_input_files(self, cwl_inputs: dict):
+        """Extract input sandboxes/data inputs.
 
-            Parameters
-            ----------
-                file : File | str
-                    File which path should be rewritten.
+        Parameters
+        ----------
+        inputs : dict
+            CWL inputs
+        """
+        inputs = deepcopy(cwl_inputs)
+        for inp_name, inp_value in inputs.items():
+            if isinstance(inp_value, list):
+                for index, val in enumerate(inp_value):
+                    if isinstance(val, File):
+                        self._fill_dirac_inputs(val)
+                        # Update the transformed cwl input file
+                        self.transformed_inputs[inp_name][index].path = Path(
+                            val.path
+                        ).name
+            elif isinstance(inp_value, File):
+                self._fill_dirac_inputs(inp_value)
+                # Update the transformed cwl input file
+                self.transformed_inputs[inp_name].path = Path(inp_value.path).name
+
+    def _fill_dirac_inputs(self, input_file: File):
+        """Fill DIRAC input sanbox and data lists.
 
-            Returns:
-            -----
-                str:
-                    The new file path.
-            """
-            path, is_lfn = self._translate_sandboxes_and_lfns(file)
-            (self.input_data if is_lfn else self.input_sandbox).append(path)
-            return Path(path.removeprefix(LFN_DIRAC_PREFIX)).name
-
-        for key, input_value in self.transformed_inputs.items():
-            if isinstance(input_value, list):
-                for file in input_value:
-                    if isinstance(file, File):
-                        file.path = rewrite_file_path(file)
-            elif isinstance(input_value, File):
-                input_value.path = rewrite_file_path(input_value)
-            elif isinstance(input_value, str) and input_value.startswith(LFN_PREFIX):
-                self.transformed_inputs[key] = Path(
-                    input_value.removeprefix(LFN_PREFIX)
-                ).name
+        Parameters
+        ----------
+        input_value : File
+            File input
+        """
+        if input_file.path.startswith(LFN_PREFIX):
+            self.input_data.append(
+                input_file.path.replace(LFN_PREFIX, LFN_DIRAC_PREFIX)
+            )
+        else:
+            sandbox = Path(input_file.path.replace(LOCAL_PREFIX, ""))
+            self.input_sandbox.append(str(sandbox.absolute()))
 
     def _translate_sandboxes_and_lfns(self, file: File | str) -> tuple[str, bool]:
         """Extract local files as sandboxes and lfns as input data.
@@ -619,7 +645,7 @@ class CWLTranslator:
                     if isinstance(output_value, list):
                         for out in output_value:
                             self._fill_dirac_outputs(out)
-                    else:
+                    elif output_value:
                         self._fill_dirac_outputs(output_value)
 
     def _fill_dirac_outputs(self, output_value: dict):
@@ -629,7 +655,7 @@ class CWLTranslator:
         ----------
             output_value : dict
         """
-        if output_value.get("class", None) == "File":
+        if isinstance(output_value, dict) and output_value.get("class", None) == "File":
             if output_value.get("path", "").startswith(LFN_PREFIX):
                 self.output_data.append(
                     output_value.get("path", "").replace(LFN_PREFIX, LFN_DIRAC_PREFIX)

{ctadirac-3.0.8.dev4 → ctadirac-3.0.9}/src/CTADIRAC/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.0.8.dev4'
-__version_tuple__ = version_tuple = (3, 0, 8, 'dev4')
+__version__ = version = '3.0.9'
+__version_tuple__ = version_tuple = (3, 0, 9)
 
-__commit_id__ = commit_id = 'gd0cd0765e'
+__commit_id__ = commit_id = 'g119582918'

{ctadirac-3.0.8.dev4 → ctadirac-3.0.9/src/CTADIRAC.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: CTADIRAC
-Version: 3.0.8.dev4
+Version: 3.0.9
 Summary: CTADIRAC is an extension of the DIRAC interware for dedicated CTA usage.
 Author-email: Luisa Arrabito <arrabito@in2p3.fr>
 License: BSD 3-Clause License

{ctadirac-3.0.8.dev4 → ctadirac-3.0.9}/tests/resources/cwl/complex_workflow/camera_calibration_inputs.yaml

@@ -1,13 +1,13 @@
 dl0_pedestal_data:
   - class: File
-    path: test/data/pedestals_LST_dark.simtel.gz
+    path: lfn:///ctao/test/data/pedestals_LST_dark.simtel.gz
   - class: File
-    path: test/data/pedestals_LST_half_moon.simtel.gz
+    path: lfn:///ctao/test/data/pedestals_LST_half_moon.simtel.gz
 dl0_flatfield_data:
   - class: File
-    path: test/data/flasher_LST_dark.simtel.gz
+    path: lfn:///ctao/test/data/flasher_LST_dark.simtel.gz
   - class: File
-    path: test/data/flasher_LST_half_moon.simtel.gz
+    path: lfn:///ctao/test/data/flasher_LST_half_moon.simtel.gz
 ped_process_config:
   - class: File
     path: test/config/process_pedestal.yaml

ctadirac-3.0.9/tests/resources/cwl/scatter_feat_requirement/inputs_dl0_dl1_multiple.yaml

@@ -0,0 +1,9 @@
+input_files:
+  - "class": "File"
+    "path": "lfn:///ctao.dpps.test/test/gamma_1.dl1_img.h5"
+  - "class": "File"
+    "path": "lfn:///ctao.dpps.test/test/gamma_2.dl1_img.h5"
+
+intermediate_filenames: ["gamma_1.dl1.h5", "gamma_2.dl1.h5"]
+
+output_filename: "lfn:///ctao.dpps.test/test/merged.dl1.h5"

{ctadirac-3.0.8.dev4 → ctadirac-3.0.9}/tests/unit/api/test_cwl_job.py

@@ -242,7 +242,7 @@ def test_datapipe_cwl_job(mock_submit_job):
     assert save(job.transformed_inputs) == {
         "dl0": {"class": "File", "path": "gamma_cone10_run010000.simtel.zst"},
         "processing_config": {"class": "File", "path": "process_config.yaml"},
-        "dl1_filename": "test.dl1.h5",
+        "dl1_filename": "lfn:///ctao/datapipe/test.dl1.h5",
     }
 
     result = job.submit()
@@ -278,35 +278,47 @@ def test_datapipe_cwl_workflow_job(mock_submit_job):
         "tests/resources/cwl/scatter_feat_requirement/inputs_dl0_dl1_multiple.yaml",
         cvmfs_base_path=CVMFS_BASE_PATH,
     )
-    assert len(job.input_data) == 0
 
-    assert len(job.input_sandbox) == 2
-    assert INPUT_1 in job.input_sandbox[0]
-    assert INPUT_2 in job.input_sandbox[1]
+    assert len(job.input_sandbox) == 0
 
-    assert len(job.output_data) == 0
+    assert job.input_data == [
+        f"{LFN_DIRAC_PREFIX}/ctao.dpps.test/test/gamma_1.dl1_img.h5",
+        f"{LFN_DIRAC_PREFIX}/ctao.dpps.test/test/gamma_2.dl1_img.h5",
+    ]
+
+    assert job.output_data == [f"{LFN_DIRAC_PREFIX}/ctao.dpps.test/test/merged.dl1.h5"]
     assert job.output_sandbox == [
-        MERGED_OUTPUT,
         MERGE_LOG,
         MERGE_PROVENANCE_LOG,
     ]
 
+    # Ensure the transformed inputs are correctly transformed
+    for input_file in job.transformed_inputs["input_files"]:
+        assert input_file.path in ["gamma_1.dl1_img.h5", "gamma_2.dl1_img.h5"]
+
+    assert job.transformed_inputs["output_filename"] == "merged.dl1.h5"
+
     result = job.submit()
     mock_submit_job.assert_called_once_with(job)
     result_xml = ET.fromstring(result)
 
     input_data = get_list(result_xml, "InputData")
-    assert input_data == []
+    assert input_data == [
+        f"{LFN_DIRAC_PREFIX}/ctao.dpps.test/test/gamma_1.dl1_img.h5",
+        f"{LFN_DIRAC_PREFIX}/ctao.dpps.test/test/gamma_2.dl1_img.h5",
+    ]
 
     input_sandbox = get_list(result_xml, "InputSandbox")
-    assert len(input_sandbox) == 4  # cwl + inputs + input_sandbox
+    assert len(input_sandbox) == 2  # cwl + inputs + input_sandbox
 
     output_data = get_list(result_xml, "OutputData")
-    assert len(output_data) == 0
+    assert output_data == [f"{LFN_DIRAC_PREFIX}/ctao.dpps.test/test/merged.dl1.h5"]
 
     output_sandbox = get_list(result_xml, "OutputSandbox")
-    assert len(output_sandbox) == 3
-    assert output_sandbox[0] == MERGED_OUTPUT
+    assert output_sandbox == [
+        MERGE_LOG,
+        MERGE_PROVENANCE_LOG,
+    ]
 
 
 # TODO: parametrize with the above test?
@@ -427,10 +439,10 @@ def test_expression_tool(mock_submit_job):
 
 
 CAMERA_CALIBRATION_INPUT_SANDBOXES = [
-    str(Path("test/data/pedestals_LST_dark.simtel.gz").absolute()),
-    str(Path("test/data/pedestals_LST_half_moon.simtel.gz").absolute()),
-    str(Path("test/data/flasher_LST_dark.simtel.gz").absolute()),
-    str(Path("test/data/flasher_LST_half_moon.simtel.gz").absolute()),
+    # str(Path("test/data/pedestals_LST_dark.simtel.gz").absolute()),
+    # str(Path("test/data/pedestals_LST_half_moon.simtel.gz").absolute()),
+    # str(Path("test/data/flasher_LST_dark.simtel.gz").absolute()),
+    # str(Path("test/data/flasher_LST_half_moon.simtel.gz").absolute()),
     str(Path("test/config/process_pedestal.yaml").absolute()),
     str(Path("test/config/process_flatfield.yaml").absolute()),
     str(Path("test/config/merge_config.yaml").absolute()),
@@ -446,7 +458,12 @@ CAMERA_CALIBRATION_OUTPUT_SANDBOXES = [
     "camera_calibration_lst1.mon.dl1.h5",
 ]
 CAMERA_CALIBRATION_OUTPUT_DATA = []
-CAMERA_CALIBRATION_INPUT_DATA = []
+CAMERA_CALIBRATION_INPUT_DATA = [
+    f"{LFN_DIRAC_PREFIX}/ctao/test/data/pedestals_LST_dark.simtel.gz",
+    f"{LFN_DIRAC_PREFIX}/ctao/test/data/pedestals_LST_half_moon.simtel.gz",
+    f"{LFN_DIRAC_PREFIX}/ctao/test/data/flasher_LST_dark.simtel.gz",
+    f"{LFN_DIRAC_PREFIX}/ctao/test/data/flasher_LST_half_moon.simtel.gz",
+]
 
 
 @pytest.mark.parametrize(
@@ -493,13 +510,13 @@ def test_complex_workflow(
     result_xml = ET.fromstring(result)
 
     input_data = get_list(result_xml, "InputData")
-    assert input_data == []
+    assert set(input_data) == set(expected_input_data)
 
     input_sandbox = get_list(result_xml, "InputSandbox")
-    assert len(input_sandbox) == 13  # cwl + inputs + input_sandbox
+    assert len(input_sandbox) == 9  # cwl + inputs + input_sandbox
 
     output_data = get_list(result_xml, "OutputData")
     assert len(output_data) == 0
 
     output_sandbox = get_list(result_xml, "OutputSandbox")
-    assert len(output_sandbox) == 4
+    assert output_sandbox == expected_output_sandbox

{ctadirac-3.0.8.dev4 → ctadirac-3.0.9}/tests/unit/api/test_cwl_translator.py

@@ -213,7 +213,7 @@ def test_translate_docker_hints(
             {"input1": File(path=LOCAL_PREFIX + "test_local_file.txt")},
             {
                 "transformed_inputs": {"input1": File(path="test_local_file.txt")},
-                "input_sandbox": ["test_local_file.txt"],
+                "input_sandbox": [str(Path("test_local_file.txt").absolute())],
                 "input_data": [],
             },
         ),
@@ -231,7 +231,7 @@ def test_translate_docker_hints(
                         File(path="test_local_file1.txt"),
                     ]
                 },
-                "input_sandbox": ["test_local_file1.txt"],
+                "input_sandbox": [str(Path("test_local_file1.txt").absolute())],
                 "input_data": [LFN_DIRAC_PREFIX + "/ctao/test_lfn_file1.txt"],
             },
         ),
@@ -253,7 +253,10 @@ def test_translate_docker_hints(
                         File(path="test_local_file3.txt"),
                     ],
                 },
-                "input_sandbox": ["test_local_file2.txt", "test_local_file3.txt"],
+                "input_sandbox": [
+                    str(Path("test_local_file2.txt").absolute()),
+                    str(Path("test_local_file3.txt").absolute()),
+                ],
                 "input_data": [
                     LFN_DIRAC_PREFIX + "/ctao/test_lfn_file2.txt",
                     LFN_DIRAC_PREFIX + "/ctao/test_lfn_file3.txt",
@@ -272,7 +275,9 @@ def test_translate_docker_hints(
                         File(path="test_local_file1.txt"),
                     ]
                 },
-                "input_sandbox": ["some/path/test_local_file1.txt"],
+                "input_sandbox": [
+                    str(Path("some/path/test_local_file1.txt").absolute())
+                ],
                 "input_data": [],
             },
         ),
@@ -284,7 +289,9 @@ def test_translate_docker_hints(
                 "transformed_inputs": {
                     "input1": File(path="test_local_file1.txt"),
                 },
-                "input_sandbox": ["some/path/test_local_file1.txt"],
+                "input_sandbox": [
+                    str(Path("some/path/test_local_file1.txt").absolute())
+                ],
                 "input_data": [],
             },
         ),
@@ -297,22 +304,24 @@ def test_translate_docker_hints(
             {
                 "transformed_inputs": {
                     "input1": File(path="test_local_file1.txt"),
-                    "input2": "test.h5",
+                    "input2": f"{LFN_PREFIX}/ctao/path/test.h5",
                 },
-                "input_sandbox": ["some/path/test_local_file1.txt"],
+                "input_sandbox": [
+                    str(Path("some/path/test_local_file1.txt").absolute())
+                ],
                 "input_data": [],
             },
         ),
     ],
 )
-def test_extract_and_translate_input_files(input_cwl, expected_results):
+def test_extract_input_files(input_cwl, expected_results):
     cwl_translator = CWLTranslator.__new__(CWLTranslator)
 
     cwl_translator.input_data = []
     cwl_translator.input_sandbox = []
     cwl_translator.transformed_inputs = input_cwl
 
-    cwl_translator._extract_and_translate_input_files()
+    cwl_translator._extract_input_files(input_cwl)
 
     assert save(cwl_translator.transformed_inputs) == save(
         expected_results["transformed_inputs"]

ctadirac-3.0.8.dev4/tests/resources/cwl/scatter_feat_requirement/inputs_dl0_dl1_multiple.yaml

@@ -1,9 +0,0 @@
-input_files:
-  - "class": "File"
-    "path": "gamma_1.dl1_img.h5"
-  - "class": "File"
-    "path": "gamma_2.dl1_img.h5"
-
-intermediate_filenames: ["gamma_1.dl1.h5", "gamma_2.dl1_.h5"]
-
-output_filename: "merged.dl1.h5"