CTADIRAC 3.0.8a2__tar.gz → 3.0.8.dev2__tar.gz

{ctadirac-3.0.8a2/src/CTADIRAC.egg-info → ctadirac-3.0.8.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: CTADIRAC
-Version: 3.0.8a2
+Version: 3.0.8.dev2
 Summary: CTADIRAC is an extension of the DIRAC interware for dedicated CTA usage.
 Author-email: Luisa Arrabito <arrabito@in2p3.fr>
 License: BSD 3-Clause License

{ctadirac-3.0.8a2 → ctadirac-3.0.8.dev2}/src/CTADIRAC/Interfaces/API/CWLJob.py

@@ -7,6 +7,7 @@ from cwl_utils.parser import save
 from DIRAC.Interfaces.API.Dirac import Dirac
 from DIRAC.Interfaces.API.Job import Job
 from ruamel.yaml import YAML
+
 from CTADIRAC.Interfaces.Utilities.CWLTranslator import CWLTranslator
 
 
@@ -31,10 +32,11 @@ class CWLJob(Job, CWLTranslator):
     ) -> None:
         Job.__init__(self)
         CWLTranslator.__init__(self, cwl_workflow, cwl_inputs)
-        self.logfile_name = "cwl_job"
+
         self.cvmfs_base_path = cvmfs_base_path
         self.apptainer_options = apptainer_options if apptainer_options else []
         self._output_se = output_se
+
         self.translate(self.cvmfs_base_path, self.apptainer_options)
 
     def submit(self):
@@ -46,12 +48,14 @@ class CWLJob(Job, CWLTranslator):
         """
         dirac = Dirac()
         yaml = YAML()
+
         # Create the modified Dirac compliant CWL workflow and inputs files to submit
         with tempfile.NamedTemporaryFile(
             suffix=f"_{self.cwl_workflow_path.name}"
         ) as temp_workflow:
             yaml.dump(save(self.transformed_cwl), temp_workflow)
             temp_workflow.flush()
+
             with tempfile.NamedTemporaryFile(
                 suffix=f"_{self.cwl_inputs_path.name}"
             ) as temp_inputs:
@@ -72,10 +76,9 @@ class CWLJob(Job, CWLTranslator):
                 arguments_str = (
                     f"{Path(temp_workflow.name).name} {Path(temp_inputs.name).name}"
                 )
+
                 self.setExecutable(
-                    "cwltool",
-                    arguments=arguments_str,
-                    logFile=f"{self.logfile_name}.log",
+                    "cwltool", arguments=arguments_str, logFile=f"{self.name}.log"
                 )
                 res = dirac.submitJob(self)
         return res

{ctadirac-3.0.8a2 → ctadirac-3.0.8.dev2}/src/CTADIRAC/Interfaces/Utilities/CWLTranslator.py

@@ -95,12 +95,12 @@ class CWLTranslator:
             apptainer_options : list[Any]
                 A list of options for Apptainer.
         """
-        self._extract_input_files(self.original_inputs)
+
         if self.transformed_cwl.hints:
             self.transformed_cwl = self._translate_docker_hints(
                 self.transformed_cwl, cvmfs_base_path, apptainer_options
             )
-
+        self._extract_and_translate_input_files()
         self._extract_output_files(self.transformed_cwl, self.original_inputs)
 
     def _translate_workflow(
@@ -115,39 +115,39 @@ class CWLTranslator:
         apptainer_options : list[Any]
             A list of options for Apptainer.
         """
-        # Extract the DIRAC related
-        self._extract_input_files(self.original_inputs)
         # Need to set the file basename for JSReq:
-        steps_inputs = deepcopy(self.transformed_inputs)
+        self.transformed_inputs = set_input_file_basename(self.transformed_inputs)
+        self.transformed_inputs = fill_defaults(
+            self.transformed_cwl, self.transformed_inputs
+        )
+        # Extract the DIRAC related
+        self._extract_and_translate_input_files()
+
         self.evaluated_steps_outputs = {}
+        steps_inputs = deepcopy(self.transformed_inputs)
         for n, wf_step in enumerate(self.transformed_cwl.steps):
-            # TODO: handle the case where the step is also a workflow
-            if isinstance(wf_step.run, (CommandLineTool, ExpressionTool)):
-                step_name = wf_step.id.rpartition("#")[2].split("/")[0]
-                self.current_step_name = step_name
-                self.evaluated_steps_outputs.setdefault(step_name, {})
-                # replace docker hints
-                if isinstance(wf_step, CommandLineTool) and wf_step.run.hints:
-                    self.transformed_cwl.steps[n].run = self._translate_docker_hints(
-                        wf_step.run, cvmfs_base_path, apptainer_options
-                    )
-                # Update the input context for next steps
-                steps_inputs = fill_defaults(wf_step.run, steps_inputs)
-                steps_inputs = set_input_file_basename(steps_inputs)
-                steps_inputs = self._evaluate_step_inputs(wf_step, steps_inputs)
-
-                # Evaluate step outputs
-                # specific evaluation if the step is an ExpressionTool
-                if isinstance(wf_step.run, ExpressionTool):
-                    exptool_outputs = self._evaluate_expression_tool_outputs(
-                        self.unpacked_cwl, wf_step, steps_inputs, step_name
-                    )
-                    self.evaluated_steps_outputs[step_name] = exptool_outputs
-                else:
-                    self._evaluate_step_outputs(wf_step, steps_inputs)
-            elif isinstance(wf_step, Workflow):
-                raise NotImplementedError("Workflow steps are not handle yet...")
-
+            step_name = wf_step.id.rpartition("#")[2].split("/")[0]
+            self.current_step_name = step_name
+            self.evaluated_steps_outputs.setdefault(step_name, {})
+            # replace docker hints
+            if wf_step.run.hints:
+                self.transformed_cwl.steps[n].run = self._translate_docker_hints(
+                    wf_step.run, cvmfs_base_path, apptainer_options
+                )
+            # Update the input context for next steps
+            steps_inputs = fill_defaults(wf_step.run, steps_inputs)
+            steps_inputs = set_input_file_basename(steps_inputs)
+            steps_inputs = self._evaluate_step_inputs(wf_step, steps_inputs)
+
+            # Evaluate step outputs
+            # specific evaluation if the step is an ExpressionTool
+            if isinstance(wf_step.run, ExpressionTool):
+                exptool_outputs = self._evaluate_expression_tool_outputs(
+                    self.unpacked_cwl, wf_step, steps_inputs, step_name
+                )
+                self.evaluated_steps_outputs[step_name] = exptool_outputs
+            else:
+                self._evaluate_step_outputs(wf_step, steps_inputs)
         # Finally extract the DIRAC related outputs
         self._extract_workflow_outputs()
 
@@ -230,7 +230,6 @@ class CWLTranslator:
             list | dict[str, Any] | None
                 Evaluated CWL File or list of CWL File
             """
-            transformed_inputs = deepcopy(self.transformed_inputs)
             inp_id = inp.id.rpartition("#")[2].split("/")[-1]
             if inp.valueFrom:
                 # evaluate if it is a JS exp
@@ -301,8 +300,8 @@ class CWLTranslator:
                         ):
                             # TODO: can that be a list?
                             return self.evaluated_steps_outputs[step_ref][source]
-                    elif inp_source[0] in save(transformed_inputs):
-                        return save(transformed_inputs)[inp_source[0]]
+                    elif inp_source[0] in save(self.transformed_inputs):
+                        return save(self.transformed_inputs)[inp_source[0]]
                     elif inp_source[0] in input_context:
                         return input_context[inp_source[0]]
 
@@ -379,44 +378,37 @@ class CWLTranslator:
         """
         for output in step.run.outputs:
             output_name = output.id.rpartition("#")[2].split("/")[-1]
-            glob = output.outputBinding.glob
-            if glob:
-                if isinstance(glob, str):
-                    # the glob is an expression we evaluate it
-                    glob_exp = glob if glob.startswith("$(") else None
-                    if glob_exp:
-                        eval_glob = do_eval(
-                            glob,
-                            inputs,
-                            outdir=None,
-                            requirements=[],
-                            tmpdir=None,
-                            resources={},
-                        )
-                        if step.scatter and isinstance(eval_glob, list):
-                            for out in eval_glob:
-                                self.evaluated_steps_outputs[
-                                    self.current_step_name
-                                ].setdefault(output_name, []).append(
-                                    format_output_by_type(out, output.type_)
-                                )
-                        else:
-                            self.evaluated_steps_outputs[self.current_step_name][
-                                output_name
-                            ] = format_output_by_type(eval_glob, output.type_)
+            if output.outputBinding.glob:
+                # the glob is an expression we evaluate it
+                glob_exp = (
+                    output.outputBinding.glob
+                    if output.outputBinding.glob.startswith("$(")
+                    else None
+                )
+                if glob_exp:
+                    glob = do_eval(
+                        output.outputBinding.glob,
+                        inputs,
+                        outdir=None,
+                        requirements=[],
+                        tmpdir=None,
+                        resources={},
+                    )
+                    if step.scatter and isinstance(glob, list):
+                        for out in glob:
+                            self.evaluated_steps_outputs[
+                                self.current_step_name
+                            ].setdefault(output_name, []).append(
+                                format_output_by_type(out, output.type_)
+                            )
                     else:
                         self.evaluated_steps_outputs[self.current_step_name][
                             output_name
                         ] = format_output_by_type(glob, output.type_)
-                elif isinstance(glob, list):
-                    self.evaluated_steps_outputs[self.current_step_name].setdefault(
-                        output_name, []
-                    )
-                    for g in glob:
-                        if isinstance(g, str):
-                            self.evaluated_steps_outputs[self.current_step_name][
-                                output_name
-                            ].append(format_output_by_type(g, output.type_))
+                else:
+                    self.evaluated_steps_outputs[self.current_step_name][
+                        output_name
+                    ] = format_output_by_type(output.outputBinding.glob, output.type_)
 
     def _evaluate_expression_tool_outputs(
         self,
@@ -545,44 +537,40 @@ class CWLTranslator:
 
         return expr_inputs
 
-    def _extract_input_files(self, cwl_inputs: dict):
-        """Extract input sandboxes/data inputs.
-
-        Parameters
-        ----------
-        inputs : dict
-            CWL inputs
+    def _extract_and_translate_input_files(self) -> None:
+        """Extract input files from CWL inputs and rewrite file paths.
+        If the file is a Sandbox, ensure there is no absolute path, and store it in the input sandbox list.
+        If the file is a LFN, remove the lfn prefix and store it in the lfns list.
         """
-        inputs = deepcopy(cwl_inputs)
-        for inp_name, inp_value in inputs.items():
-            if isinstance(inp_value, list):
-                for index, val in enumerate(inp_value):
-                    if isinstance(val, File):
-                        self._fill_dirac_inputs(val)
-                        # Update the transformed cwl input file
-                        self.transformed_inputs[inp_name][index].path = Path(
-                            val.path
-                        ).name
-            elif isinstance(inp_value, File):
-                self._fill_dirac_inputs(inp_value)
-                # Update the transformed cwl input file
-                self.transformed_inputs[inp_name].path = Path(inp_value.path).name
-
-    def _fill_dirac_inputs(self, input_file: File):
-        """Fill DIRAC input sanbox and data lists.
 
-        Parameters
-        ----------
-        input_value : File
-            File input
-        """
-        if input_file.path.startswith(LFN_PREFIX):
-            self.input_data.append(
-                input_file.path.replace(LFN_PREFIX, LFN_DIRAC_PREFIX)
-            )
-        else:
-            sandbox = Path(input_file.path.replace(LOCAL_PREFIX, ""))
-            self.input_sandbox.append(str(sandbox.absolute()))
+        def rewrite_file_path(file: File | str) -> str:
+            """Rewrite file path.
+
+            Parameters
+            ----------
+                file : File | str
+                    File which path should be rewritten.
+
+            Returns:
+            -----
+                str:
+                    The new file path.
+            """
+            path, is_lfn = self._translate_sandboxes_and_lfns(file)
+            (self.input_data if is_lfn else self.input_sandbox).append(path)
+            return Path(path.removeprefix(LFN_DIRAC_PREFIX)).name
+
+        for key, input_value in self.transformed_inputs.items():
+            if isinstance(input_value, list):
+                for file in input_value:
+                    if isinstance(file, File):
+                        file.path = rewrite_file_path(file)
+            elif isinstance(input_value, File):
+                input_value.path = rewrite_file_path(input_value)
+            elif isinstance(input_value, str) and input_value.startswith(LFN_PREFIX):
+                self.transformed_inputs[key] = Path(
+                    input_value.removeprefix(LFN_PREFIX)
+                ).name
 
     def _translate_sandboxes_and_lfns(self, file: File | str) -> tuple[str, bool]:
         """Extract local files as sandboxes and lfns as input data.
@@ -610,22 +598,17 @@ class CWLTranslator:
     def _extract_workflow_outputs(self):
         """Extract the DIRAC outputs (output sandbox/data) from Workflow outputs."""
         for output in self.transformed_cwl.outputs:
-            output_sources = output.outputSource
-            if isinstance(output_sources, str):
-                output_sources = [output_sources]
-            if isinstance(output_sources, list):
-                for output_src in output_sources:
-                    output_source_step = output_src.rpartition("#")[2].split("/")[0]
-                    output_source_name = output_src.rpartition("#")[2].split("/")[-1]
-                    output_value = self.evaluated_steps_outputs[output_source_step].get(
-                        output_source_name, None
-                    )
+            output_source_step = output.outputSource.rpartition("#")[2].split("/")[0]
+            output_source_name = output.outputSource.rpartition("#")[2].split("/")[-1]
+            output_value = self.evaluated_steps_outputs[output_source_step].get(
+                output_source_name, None
+            )
 
-                    if isinstance(output_value, list):
-                        for out in output_value:
-                            self._fill_dirac_outputs(out)
-                    elif output_value:
-                        self._fill_dirac_outputs(output_value)
+            if isinstance(output_value, list):
+                for out in output_value:
+                    self._fill_dirac_outputs(out)
+            else:
+                self._fill_dirac_outputs(output_value)
 
     def _fill_dirac_outputs(self, output_value: dict):
         """Fill DIRAC output sandbox and data lists.
@@ -634,7 +617,7 @@ class CWLTranslator:
         ----------
             output_value : dict
         """
-        if isinstance(output_value, dict) and output_value.get("class", None) == "File":
+        if output_value.get("class", None) == "File":
             if output_value.get("path", "").startswith(LFN_PREFIX):
                 self.output_data.append(
                     output_value.get("path", "").replace(LFN_PREFIX, LFN_DIRAC_PREFIX)

{ctadirac-3.0.8a2 → ctadirac-3.0.8.dev2}/src/CTADIRAC/Interfaces/Utilities/CWLUtilities.py

@@ -228,7 +228,7 @@ def format_output_by_type(
         if isinstance(out_type, str):
             if out_type in ["boolean", "string", "int"]:
                 return output
-            elif out_type in ["File", "File[]"]:
+            elif out_type == "File":
                 return {"class": "File", "path": output}
         elif isinstance(out_type, OutputArraySchema):
             if out_type.type_ == "File":

{ctadirac-3.0.8a2 → ctadirac-3.0.8.dev2}/src/CTADIRAC/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.0.8a2'
-__version_tuple__ = version_tuple = (3, 0, 8, 'a2')
+__version__ = version = '3.0.8.dev2'
+__version_tuple__ = version_tuple = (3, 0, 8, 'dev2')
 
-__commit_id__ = commit_id = 'gc79ca8610'
+__commit_id__ = commit_id = 'ge29143644'

{ctadirac-3.0.8a2 → ctadirac-3.0.8.dev2/src/CTADIRAC.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: CTADIRAC
-Version: 3.0.8a2
+Version: 3.0.8.dev2
 Summary: CTADIRAC is an extension of the DIRAC interware for dedicated CTA usage.
 Author-email: Luisa Arrabito <arrabito@in2p3.fr>
 License: BSD 3-Clause License

{ctadirac-3.0.8a2 → ctadirac-3.0.8.dev2}/tests/unit/api/test_cwl_job.py

@@ -242,7 +242,7 @@ def test_datapipe_cwl_job(mock_submit_job):
     assert save(job.transformed_inputs) == {
         "dl0": {"class": "File", "path": "gamma_cone10_run010000.simtel.zst"},
         "processing_config": {"class": "File", "path": "process_config.yaml"},
-        "dl1_filename": "lfn:///ctao/datapipe/test.dl1.h5",
+        "dl1_filename": "test.dl1.h5",
     }
 
     result = job.submit()

{ctadirac-3.0.8a2 → ctadirac-3.0.8.dev2}/tests/unit/api/test_cwl_translator.py

@@ -213,7 +213,7 @@ def test_translate_docker_hints(
             {"input1": File(path=LOCAL_PREFIX + "test_local_file.txt")},
             {
                 "transformed_inputs": {"input1": File(path="test_local_file.txt")},
-                "input_sandbox": [str(Path("test_local_file.txt").absolute())],
+                "input_sandbox": ["test_local_file.txt"],
                 "input_data": [],
             },
         ),
@@ -231,7 +231,7 @@ def test_translate_docker_hints(
                         File(path="test_local_file1.txt"),
                     ]
                 },
-                "input_sandbox": [str(Path("test_local_file1.txt").absolute())],
+                "input_sandbox": ["test_local_file1.txt"],
                 "input_data": [LFN_DIRAC_PREFIX + "/ctao/test_lfn_file1.txt"],
             },
         ),
@@ -253,10 +253,7 @@ def test_translate_docker_hints(
                         File(path="test_local_file3.txt"),
                     ],
                 },
-                "input_sandbox": [
-                    str(Path("test_local_file2.txt").absolute()),
-                    str(Path("test_local_file3.txt").absolute()),
-                ],
+                "input_sandbox": ["test_local_file2.txt", "test_local_file3.txt"],
                 "input_data": [
                     LFN_DIRAC_PREFIX + "/ctao/test_lfn_file2.txt",
                     LFN_DIRAC_PREFIX + "/ctao/test_lfn_file3.txt",
@@ -275,9 +272,7 @@ def test_translate_docker_hints(
                         File(path="test_local_file1.txt"),
                     ]
                 },
-                "input_sandbox": [
-                    str(Path("some/path/test_local_file1.txt").absolute())
-                ],
+                "input_sandbox": ["some/path/test_local_file1.txt"],
                 "input_data": [],
             },
         ),
@@ -289,9 +284,7 @@ def test_translate_docker_hints(
                 "transformed_inputs": {
                     "input1": File(path="test_local_file1.txt"),
                 },
-                "input_sandbox": [
-                    str(Path("some/path/test_local_file1.txt").absolute())
-                ],
+                "input_sandbox": ["some/path/test_local_file1.txt"],
                 "input_data": [],
             },
         ),
@@ -304,24 +297,22 @@ def test_translate_docker_hints(
             {
                 "transformed_inputs": {
                     "input1": File(path="test_local_file1.txt"),
-                    "input2": f"{LFN_PREFIX}/ctao/path/test.h5",
+                    "input2": "test.h5",
                 },
-                "input_sandbox": [
-                    str(Path("some/path/test_local_file1.txt").absolute())
-                ],
+                "input_sandbox": ["some/path/test_local_file1.txt"],
                 "input_data": [],
             },
         ),
     ],
 )
-def test_extract_input_files(input_cwl, expected_results):
+def test_extract_and_translate_input_files(input_cwl, expected_results):
     cwl_translator = CWLTranslator.__new__(CWLTranslator)
 
     cwl_translator.input_data = []
     cwl_translator.input_sandbox = []
     cwl_translator.transformed_inputs = input_cwl
 
-    cwl_translator._extract_input_files(input_cwl)
+    cwl_translator._extract_and_translate_input_files()
 
     assert save(cwl_translator.transformed_inputs) == save(
         expected_results["transformed_inputs"]
@@ -505,50 +496,6 @@ def test_get_scattered_inputs(step, inputs, expected_result):
                 }
             },
         ),
-        (
-            WorkflowStep(
-                id="/some/path#step_1",
-                in_=[],
-                out=[],
-                run=CommandLineTool(
-                    inputs=["input1"],
-                    outputs=[
-                        CommandOutputParameter(
-                            id="/some/path#step_1/output1",
-                            type_="File",
-                            outputBinding=CommandOutputBinding(glob="$(inputs.input1)"),
-                        ),
-                        CommandOutputParameter(
-                            id="/some/path#step_1/output2",
-                            type_="File[]",
-                            outputBinding=CommandOutputBinding(
-                                glob=["file3.txt", "file4.txt"]
-                            ),
-                        ),
-                    ],
-                    baseCommand="echo CLT1",
-                ),
-                scatter="input1",
-            ),
-            {
-                "input1": [
-                    {"class": "File", "path": "file1.txt"},
-                    {"class": "File", "path": "file2.txt"},
-                ],
-            },
-            {
-                "step_1": {
-                    "output1": [
-                        {"class": "File", "path": "file1.txt"},
-                        {"class": "File", "path": "file2.txt"},
-                    ],
-                    "output2": [
-                        {"class": "File", "path": "file3.txt"},
-                        {"class": "File", "path": "file4.txt"},
-                    ],
-                }
-            },
-        ),
     ],
 )
 def test_evaluate_step_outputs(step, inputs, expected_result):