CTADIRAC 3.0.8.dev1__tar.gz → 3.0.8.dev3__tar.gz

{ctadirac-3.0.8.dev1/src/CTADIRAC.egg-info → ctadirac-3.0.8.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: CTADIRAC
-Version: 3.0.8.dev1
+Version: 3.0.8.dev3
 Summary: CTADIRAC is an extension of the DIRAC interware for dedicated CTA usage.
 Author-email: Luisa Arrabito <arrabito@in2p3.fr>
 License: BSD 3-Clause License
@@ -15,7 +15,7 @@ Classifier: Topic :: System :: Distributed Computing
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENCE
-Requires-Dist: DIRAC==8.0.71
+Requires-Dist: DIRAC==9.0.14
 Requires-Dist: ruamel.yaml
 Requires-Dist: shellescape
 Requires-Dist: matplotlib
@@ -27,6 +27,8 @@ Requires-Dist: codecov; extra == "test"
 Requires-Dist: flake8; extra == "test"
 Requires-Dist: pytest-xdist; extra == "test"
 Requires-Dist: mock; extra == "test"
+Provides-Extra: server
+Requires-Dist: DIRAC[server]==9.0.14; extra == "server"
 Dynamic: license-file
 
 [![PyPI - Version](https://badge.fury.io/py/CTADIRAC.svg)](https://pypi.org/project/CTADIRAC/)

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/docker/dirac-client/Dockerfile

@@ -1,6 +1,5 @@
 FROM  almalinux:9
 
-ARG CTADIRAC_VERSION="2.2.72"
 ARG USERID=1000
 ARG GROUPID=1000
 

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/pyproject.toml

@@ -15,7 +15,7 @@ classifiers = [
 ]
 license = { text = "BSD 3-Clause License" }
 dependencies = [
-    "DIRAC==8.0.71",
+    "DIRAC==9.0.14",
     "ruamel.yaml",
     "shellescape",
     "matplotlib",
@@ -48,6 +48,9 @@ test = [
     "pytest-xdist",
     "mock",
 ]
+server = [
+    "DIRAC[server]==9.0.14",
+]
 
 [project.entry-points]
 dirac = {metadata = "CTADIRAC:extension_metadata"}

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/src/CTADIRAC/Interfaces/Utilities/CWLTranslator.py

@@ -378,37 +378,44 @@ class CWLTranslator:
         """
         for output in step.run.outputs:
             output_name = output.id.rpartition("#")[2].split("/")[-1]
-            if output.outputBinding.glob:
-                # the glob is an expression we evaluate it
-                glob_exp = (
-                    output.outputBinding.glob
-                    if output.outputBinding.glob.startswith("$(")
-                    else None
-                )
-                if glob_exp:
-                    glob = do_eval(
-                        output.outputBinding.glob,
-                        inputs,
-                        outdir=None,
-                        requirements=[],
-                        tmpdir=None,
-                        resources={},
-                    )
-                    if step.scatter and isinstance(glob, list):
-                        for out in glob:
-                            self.evaluated_steps_outputs[
-                                self.current_step_name
-                            ].setdefault(output_name, []).append(
-                                format_output_by_type(out, output.type_)
-                            )
+            glob = output.outputBinding.glob
+            if glob:
+                if isinstance(glob, str):
+                    # the glob is an expression we evaluate it
+                    glob_exp = glob if glob.startswith("$(") else None
+                    if glob_exp:
+                        eval_glob = do_eval(
+                            glob,
+                            inputs,
+                            outdir=None,
+                            requirements=[],
+                            tmpdir=None,
+                            resources={},
+                        )
+                        if step.scatter and isinstance(eval_glob, list):
+                            for out in eval_glob:
+                                self.evaluated_steps_outputs[
+                                    self.current_step_name
+                                ].setdefault(output_name, []).append(
+                                    format_output_by_type(out, output.type_)
+                                )
+                        else:
+                            self.evaluated_steps_outputs[self.current_step_name][
+                                output_name
+                            ] = format_output_by_type(eval_glob, output.type_)
                     else:
                         self.evaluated_steps_outputs[self.current_step_name][
                             output_name
                         ] = format_output_by_type(glob, output.type_)
-                else:
-                    self.evaluated_steps_outputs[self.current_step_name][
-                        output_name
-                    ] = format_output_by_type(output.outputBinding.glob, output.type_)
+                elif isinstance(glob, list):
+                    self.evaluated_steps_outputs[self.current_step_name].setdefault(
+                        output_name, []
+                    )
+                    for g in glob:
+                        if isinstance(g, str):
+                            self.evaluated_steps_outputs[self.current_step_name][
+                                output_name
+                            ].append(format_output_by_type(g, output.type_))
 
     def _evaluate_expression_tool_outputs(
         self,

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/src/CTADIRAC/Interfaces/Utilities/CWLUtilities.py

@@ -228,7 +228,7 @@ def format_output_by_type(
         if isinstance(out_type, str):
             if out_type in ["boolean", "string", "int"]:
                 return output
-            elif out_type == "File":
+            elif out_type in ["File", "File[]"]:
                 return {"class": "File", "path": output}
         elif isinstance(out_type, OutputArraySchema):
             if out_type.type_ == "File":

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/src/CTADIRAC/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '3.0.8.dev1'
-__version_tuple__ = version_tuple = (3, 0, 8, 'dev1')
+__version__ = version = '3.0.8.dev3'
+__version_tuple__ = version_tuple = (3, 0, 8, 'dev3')
 
-__commit_id__ = commit_id = 'g02748529b'
+__commit_id__ = commit_id = 'g5cf3cc56a'

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3/src/CTADIRAC.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: CTADIRAC
-Version: 3.0.8.dev1
+Version: 3.0.8.dev3
 Summary: CTADIRAC is an extension of the DIRAC interware for dedicated CTA usage.
 Author-email: Luisa Arrabito <arrabito@in2p3.fr>
 License: BSD 3-Clause License
@@ -15,7 +15,7 @@ Classifier: Topic :: System :: Distributed Computing
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENCE
-Requires-Dist: DIRAC==8.0.71
+Requires-Dist: DIRAC==9.0.14
 Requires-Dist: ruamel.yaml
 Requires-Dist: shellescape
 Requires-Dist: matplotlib
@@ -27,6 +27,8 @@ Requires-Dist: codecov; extra == "test"
 Requires-Dist: flake8; extra == "test"
 Requires-Dist: pytest-xdist; extra == "test"
 Requires-Dist: mock; extra == "test"
+Provides-Extra: server
+Requires-Dist: DIRAC[server]==9.0.14; extra == "server"
 Dynamic: license-file
 
 [![PyPI - Version](https://badge.fury.io/py/CTADIRAC.svg)](https://pypi.org/project/CTADIRAC/)

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/src/CTADIRAC.egg-info/requires.txt

@@ -1,9 +1,12 @@
-DIRAC==8.0.71
+DIRAC==9.0.14
 ruamel.yaml
 shellescape
 matplotlib
 typer[all]
 
+[server]
+DIRAC[server]==9.0.14
+
 [test]
 pytest
 pytest-cov

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/system/CWL/cta-prod-managedata-processing.yml

@@ -1,6 +1,6 @@
 metadata: '{"array_layout": "Prod6-Hyperarray", "thetaP": 20.0, "data_level": 2, "particle": "gamma", "sct": "False", "outputType": "Data", "site": "Paranal", "configuration_id": 15, "merged": 0, "MCCampaign": "Prod6", "phiP": 0.0}'
 file_metadata: "{}"
-base_path: /vo.cta.in2p3.fr/tests/prodsys/cwl_test/processing
+base_path: /ctao/tests/prodsys/cwl_test/processing
 output_pattern: Data/*.h5
 name_package: ctapipe
 program_category: analysis

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/system/CWL/cta-prod-managedata.yml

@@ -1,6 +1,6 @@
 metadata: '{"array_layout": "Prod6-Hyperarray", "thetaP": 20.0, "data_level": -1, "particle": "gamma", "sct": "False", "outputType": "Data", "site": "Paranal", "configuration_id": 15, "merged": 0, "MCCampaign": "Prod6", "tel_sim_prog_version": "2020-06-29b", "phiP": 0.0, "tel_sim_prog": "sim_telarray"}'
 file_metadata: "{}"
-base_path: /vo.cta.in2p3.fr/tests/prodsys/cwl_test/simulation
+base_path: /ctao/tests/prodsys/cwl_test/simulation
 output_pattern: Data/*dark*.simtel.zst
 name_package: corsika_simtelarray
 program_category: tel_sim

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/system/config/processing_merging_config_from_dataset.yml

@@ -35,4 +35,4 @@ ProdSteps:
 Common:
   MCCampaign: Prod5bTest
   configuration_id: 15
-  base_path: /vo.cta.in2p3.fr/tests/prodsys/MC
+  base_path: /ctao/tests/prodsys/MC

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/system/config/production_config.yml

@@ -20,4 +20,4 @@ ProdSteps:
 Common:
   MCCampaign: Prod5bTest
   configuration_id: 15
-  base_path: /vo.cta.in2p3.fr/tests/prodsys/MC
+  base_path: /ctao/tests/prodsys/MC

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/system/testDMSDefinitions.py

@@ -15,9 +15,9 @@ class ClientDMS:
     def __init__(self):
         self.dest_se1 = "CSCS-Disk"
         self.dest_se2 = "PIC-Disk"
-        self.test_dir = "/vo.cta.in2p3.fr/tests/dms"
+        self.test_dir = "/ctao/tests/dms"
         self.test_file = "DMS_TestFile.txt"
-        self.test_lfn = "/vo.cta.in2p3.fr/tests/dms/DMS_TestFile.txt"
+        self.test_lfn = "/ctao/tests/dms/DMS_TestFile.txt"
 
     def write_test_file(self):
         with open(self.test_file, "w") as file:

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/system/testFileCatalogDefinitions.py

@@ -7,7 +7,6 @@ import os
 
 from DIRAC.Core.Base.Script import Script
 
-Script.parseCommandLine()
 
 import DIRAC
 from DIRAC import gLogger
@@ -15,13 +14,15 @@ from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
 from DIRAC.Resources.Catalog.RucioFileCatalogClient import RucioFileCatalogClient
 from DIRAC.DataManagementSystem.Client.DataManager import DataManager
 
+Script.parseCommandLine()
+
 
 class ClientFileCatalog:
     def __init__(self, catalog):
         self.catalogs = [catalog]
         self.dest_se_1 = "PIC-Disk"
         self.dest_se_2 = "CSCS-Disk"
-        self.test_dir = "/vo.cta.in2p3.fr/tests/file_catalog"
+        self.test_dir = "/ctao/tests/file_catalog"
         self.lfn = os.path.join(self.test_dir, "testFile.txt")
         self.file_name = os.path.basename(self.lfn)
         self.file_tuple = (

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/system/testJobDefinitions.py

@@ -15,7 +15,7 @@ from testJobUtils import base_to_all_jobs, end_of_all_jobs
 
 dm = DataManager()
 dirac = Dirac()
-test_dir = "/vo.cta.in2p3.fr/tests/prodsys"
+test_dir = "/ctao/tests/prodsys"
 git_repo = "https://gitlab.cta-observatory.org/arrabito/mandel4ts.git"
 workflow_path = "DIRAC/tests/Workflow"
 exec_script = "exe-script.py"

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/system/testProdDefinitions.py

@@ -7,7 +7,6 @@ import json
 
 from DIRAC.Core.Base.Script import Script
 
-Script.parseCommandLine()
 
 # from DIRAC
 from DIRAC import gLogger
@@ -21,12 +20,14 @@ from testJobDefinitions import create_workflow_body_step
 from testJobDefinitions import create_workflow_body_step1
 from testJobDefinitions import create_workflow_body_step2
 
+Script.parseCommandLine()
+
 dm = DataManager()
 prod_client = ProductionClient()
 prod_desc = prod_client.prodDescription
 dirac = Dirac()
 
-testDir = "/vo.cta.in2p3.fr/tests/prodsys"
+testDir = "/ctao/tests/prodsys"
 
 
 def create_production_step(name, type, input_query=None, output_query=None):

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/system/testSuiteDefinition.py

@@ -9,7 +9,6 @@ import DIRAC
 from DIRAC.Interfaces.API.Dirac import Dirac
 from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
 
-DIRAC.initialize()  # Initialize configuration
 
 from testJobDefinitions import (
     hello_world,
@@ -37,6 +36,8 @@ from testDMSDefinitions import (
 )
 from testFileCatalogDefinitions import ClientFileCatalog
 
+DIRAC.initialize()  # Initialize configuration
+
 dirac = Dirac()
 
 """To add parser and running or skipping test using unittest
@@ -55,7 +56,7 @@ class WMSTestCase(unittest.TestCase):
         self,
         sites=None,
         s_elements=None,
-        test_dir="/vo.cta.in2p3.fr/tests/wms",
+        test_dir="/ctao/tests/wms",
         input_file="WMS_TestInputFile.txt",
         output_file="WMS_TestOutputFile.txt",
         exe_script="exe_script.sh",
@@ -250,7 +251,7 @@ class ProdSystemFullTests(WMSTestCase):
     def test_submit_full_prod(self):
         print("\n")
         print(self.separation)
-        test_dir = "/vo.cta.in2p3.fr/tests/prodsys/MC"
+        test_dir = "/ctao/tests/prodsys/MC"
         prod_name = "prodTestFullProduction"
         prod_test_dir = os.path.join(test_dir, "Prod5bTest/LaPalma/gamma-diffuse")
 

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/unit/api/test_cwl_translator.py

@@ -496,6 +496,50 @@ def test_get_scattered_inputs(step, inputs, expected_result):
                 }
             },
         ),
+        (
+            WorkflowStep(
+                id="/some/path#step_1",
+                in_=[],
+                out=[],
+                run=CommandLineTool(
+                    inputs=["input1"],
+                    outputs=[
+                        CommandOutputParameter(
+                            id="/some/path#step_1/output1",
+                            type_="File",
+                            outputBinding=CommandOutputBinding(glob="$(inputs.input1)"),
+                        ),
+                        CommandOutputParameter(
+                            id="/some/path#step_1/output2",
+                            type_="File[]",
+                            outputBinding=CommandOutputBinding(
+                                glob=["file3.txt", "file4.txt"]
+                            ),
+                        ),
+                    ],
+                    baseCommand="echo CLT1",
+                ),
+                scatter="input1",
+            ),
+            {
+                "input1": [
+                    {"class": "File", "path": "file1.txt"},
+                    {"class": "File", "path": "file2.txt"},
+                ],
+            },
+            {
+                "step_1": {
+                    "output1": [
+                        {"class": "File", "path": "file1.txt"},
+                        {"class": "File", "path": "file2.txt"},
+                    ],
+                    "output2": [
+                        {"class": "File", "path": "file3.txt"},
+                        {"class": "File", "path": "file4.txt"},
+                    ],
+                }
+            },
+        ),
     ],
 )
 def test_evaluate_step_outputs(step, inputs, expected_result):

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/unit/production/__init__.py

@@ -85,7 +85,7 @@ MERGING_CONFIG_2 = {
 COMMON_CONFIG = {
     "MCCampaign": "Prod5bTest",
     "configuration_id": 1,
-    "base_path": "/vo.cta.in2p3.fr/tests/",
+    "base_path": "/ctao/tests/",
 }
 
 WORKFLOW_CONFIG = {

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/unit/scripts/production_system/test_production_scripts.py

@@ -84,15 +84,14 @@ def run_cli(cmd):
 @pytest.mark.xfail(reason="Needs to be fixed.")
 def test_cta_prod_submit(tmp_path):
     production_config = tmp_path / "production_config.yml"
-    with open(production_config, "w") as f:
-        f.write(PRODUCTION_CONFIG)
+    production_config.write_text(PRODUCTION_CONFIG)
     script_name = "cta-prod-submit"
     prod_name = "cta_prod_submit_test"
     mode = "dry-run"
 
     cmd = f"{script_name} {prod_name} {production_config} {mode}"
     result = run_cli(cmd)
-    output, err = result
+    _, err = result
     assert err.decode("utf-8") == ""
 
 
@@ -111,12 +110,11 @@ pointing: North"""
 @pytest.mark.xfail(reason="Needs to be fixed.")
 def test_cta_prod_create_workflow_config(tmp_path):
     workflow_config = tmp_path / "workflow_config.yml"
-    with open(workflow_config, "w") as f:
-        f.write(WORKFLOW_CONFIG)
+    workflow_config.write_text(WORKFLOW_CONFIG)
 
     script_name = "cta-prod-create-workflow-config"
 
     cmd = f"{script_name} {workflow_config}"
     result = run_cli(cmd)
-    output, err = result
+    _, err = result
     assert err.decode("utf-8") == ""

{ctadirac-3.0.8.dev1 → ctadirac-3.0.8.dev3}/tests/unit/transformation/agent/test_transformation_failover_utils.py

@@ -181,7 +181,7 @@ get_transformation_files_result = {
     "OK": True,
     "Value": [
         {
-            "LFN": "/vo.cta.in2p3.fr/tests/prodsys/MC/Prod5bTest/LaPalma/gamma-diffuse/sim_telarray/448/Data/000xxx/gamma_20deg_0deg_run8___cta-prod5b-lapalma_desert-2158m-LaPalma-dark_cone10.simtel.zst",
+            "LFN": "/ctao/tests/prodsys/MC/Prod5bTest/LaPalma/gamma-diffuse/sim_telarray/448/Data/000xxx/gamma_20deg_0deg_run8___cta-prod5b-lapalma_desert-2158m-LaPalma-dark_cone10.simtel.zst",
             "TransformationID": 449,
             "FileID": 2742,
             "Status": "Processed",