DIRAC 9.0.7__py3-none-any.whl → 9.0.8__py3-none-any.whl

DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -3,9 +3,13 @@
 import datetime
 import errno
 import os
+import requests
 from packaging.version import Version
 
-from cachetools import cachedmethod, LRUCache
+from cachetools import cachedmethod, LRUCache, TTLCache, cached
+from threading import Lock
+from typing import Optional
+
 
 # Requires at least version 3.3.3
 from fts3 import __version__ as fts3_version
@@ -44,6 +48,50 @@ BRING_ONLINE_TIMEOUT = 259200
 IDP_CACHE_SIZE = 8
 
 
+_scitag_cache = TTLCache(maxsize=10, ttl=3600)
+_scitag_lock = Lock()
+_scitag_json_cache = TTLCache(maxsize=1, ttl=86400)
+_scitag_json_lock = Lock()
+
+
+@cached(_scitag_cache, lock=_scitag_lock)
+def get_scitag(vo: str, activity: Optional[str] = None) -> int:
+    """
+    Get the scitag based on the VO and activity.
+    If the VO is not found in the scitag.json, it defaults to 1.
+    If no specific activity is provided, it defaults to the "default" activityName.
+
+    :param vo: The VO for which to get the scitag
+    :param activity: The activity for which to get the scitag
+    :return: The scitag value
+    """
+
+    @cached(_scitag_json_cache, lock=_scitag_json_lock)
+    def get_remote_json():
+        gLogger.verbose("Fetching https://scitags.org/api.json from the network")
+        response = requests.get("https://scitags.org/api.json")
+        response.raise_for_status()
+        return response.json()
+
+    vo_id = 1  # Default VO ID
+    activity_id = 1  # Default activity ID
+
+    try:
+        # Load the JSON data from the cache or network
+        sj = get_remote_json()
+
+        for experiment in sj.get("experiments", []):
+            if experiment.get("expName") == vo.lower():
+                vo_id = experiment.get("expId")
+                for act in experiment.get("activities", []):
+                    if act.get("activityName") == activity:
+                        activity_id = act.get("activityId")
+    except Exception as e:
+        gLogger.error(f"Error fetching or parsing scitag.json. Using default scitag values.", repr(e))
+    # Logic to determine the scitag based on vo and activity (this is what FTS wants)
+    return vo_id << 6 | activity_id  # Example logic, replace with actual implementation
+
+
 class FTS3Job(JSerializable):
     """Abstract class to represent a job to be executed by FTS. It belongs
     to an FTS3Operation
@@ -470,6 +518,9 @@ class FTS3Job(JSerializable):
 
                 ftsFileID = getattr(ftsFile, "fileID")
 
+                # scitag 65 is 1 << 6 | 1 (default experiment, default activity)
+                scitag = get_scitag(vo=self.vo, activity=self.activity)
+
                 # Under normal circumstances, we simply submit an fts transfer as such:
                 # * srcProto://myFile -> destProto://myFile
                 #
@@ -499,6 +550,7 @@ class FTS3Job(JSerializable):
                         filesize=ftsFile.size,
                         metadata=stageTrans_metadata,
                         activity=self.activity,
+                        scitag=scitag,
                     )
                     transfers.append(stageTrans)
 
@@ -572,6 +624,7 @@ class FTS3Job(JSerializable):
                     activity=self.activity,
                     source_token=srcToken,
                     destination_token=dstToken,
+                    scitag=scitag,
                 )
 
                 transfers.append(trans)

DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py

@@ -204,6 +204,7 @@ def generateFTS3Job(sourceSE, targetSE, lfns, multiHopSE=None):
     newJob.sourceSE = sourceSE
     newJob.targetSE = targetSE
     newJob.multiHopSE = multiHopSE
+    newJob.vo = "lhcb"
     filesToSubmit = []
 
     for i, lfn in enumerate(lfns, start=1):

DIRAC/DataManagementSystem/Client/test/Test_scitag.py

@@ -0,0 +1,69 @@
+from unittest.mock import Mock, patch
+
+import pytest
+
+from DIRAC.DataManagementSystem.Client.FTS3Job import get_scitag
+
+
+class TestGetScitag:
+    def test_valid_vo_and_activity(self):
+        """Test get_scitag with valid VO and activity."""
+        result = get_scitag("atlas", "Analysis Input")
+        expected = 2 << 6 | 17  # atlas expId=2, analysis activityId=17
+        assert result == expected
+
+    def test_valid_vo_no_activity(self):
+        """Test get_scitag with valid VO but no specific activity (should use default)."""
+        result = get_scitag("cms")
+        expected = 3 << 6 | 1  # cms expId=200, default activityId=1
+        assert result == expected
+
+    def test_invalid_vo(self):
+        """Test get_scitag with invalid VO (should use default vo_id=1)."""
+        result = get_scitag("nonexistent")
+        expected = 1 << 6 | 1  # default vo_id=1, default activity_id=1
+        assert result == expected
+
+    def test_valid_vo_invalid_activity(self):
+        """Test get_scitag with valid VO but invalid activity."""
+        result = get_scitag("atlas", "nonexistent_activity")
+        expected = 2 << 6 | 1  # atlas expId=2, default activity_id=1
+        assert result == expected
+
+    def test_case_insensitive_vo(self):
+        """Test that VO matching is case insensitive."""
+        result = get_scitag("ATLAS", "Data Brokering")
+        expected = 2 << 6 | 3  # atlas expId=2, production activityId=3
+        assert result == expected
+
+
+@pytest.mark.parametrize(
+    "vo,activity,expected_vo_id,expected_activity_id",
+    [
+        ("atlas", "Analysis Output", 2, 18),
+        ("atlas", "Debug", 2, 9),
+        ("cms", "Cache", 3, 3),
+        ("cms", "default", 3, 1),
+        ("nonexistent", "any", 1, 1),  # defaults
+        ("atlas", "nonexistent", 2, 1),  # valid vo, invalid activity
+    ],
+)
+def test_parametrized_scenarios(vo, activity, expected_vo_id, expected_activity_id):
+    """Parametrized test for various VO and activity combinations."""
+    result = get_scitag(vo, activity)
+    expected = expected_vo_id << 6 | expected_activity_id
+    assert result == expected
+
+
+@pytest.mark.parametrize(
+    "vo,expected_result",
+    [
+        ("atlas", 2 << 6 | 1),  # Should use default activity
+        ("cms", 3 << 6 | 1),  # Should use default activity
+        ("unknown", 1 << 6 | 1),  # Should use all defaults
+    ],
+)
+def test_no_activity_parameter(vo, expected_result):
+    """Test behavior when no activity parameter is provided."""
+    result = get_scitag(vo)
+    assert result == expected_result

DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -148,10 +148,10 @@ class SiteDirector(AgentModule):
             self.sendSubmissionAccounting = True
 
         # Get the site description dictionary
-        siteNames = self.am_getOption("Site")
-        ceTypes = self.am_getOption("CETypes")
-        ces = self.am_getOption("CEs")
-        tags = self.am_getOption("Tags")
+        siteNames = self.am_getOption("Site", []) or None
+        ceTypes = self.am_getOption("CETypes", []) or None
+        ces = self.am_getOption("CEs", []) or None
+        tags = self.am_getOption("Tags", []) or None
 
         # Display options used
         self.log.always("VO:", self.vo)

DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg

@@ -263,10 +263,14 @@ Agents
     # the DN of the certificate proxy used to submit pilots. If not found here, what is in Operations/Pilot section of the CS will be used
     PilotDN =
 
-    # Site = # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
-    # CEs =  # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
-    # CETypes = # List of Tags that are required to be present in the CE/Queue definition
-    # Tags =
+    # List of Sites that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
+    Site =
+    # List of CEs that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
+    CEs =
+    # List of CETypes that are required to be present in the CE/Queue definition
+    CETypes =
+    # List of Tags that are required to be present in the CE/Queue definition
+    Tags =
 
     # How many cycles to skip if queue is not working
     FailedQueueCycleFactor = 10

{dirac-9.0.7.dist-info → dirac-9.0.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.7
+Version: 9.0.8
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.7
+Requires-Dist: DIRACCommon==v9.0.8
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1

{dirac-9.0.7.dist-info → dirac-9.0.8.dist-info}/RECORD

@@ -365,7 +365,7 @@ DIRAC/DataManagementSystem/Client/DataManager.py,sha256=edLmRUHJtZEYAMyen3y9qwdG
 DIRAC/DataManagementSystem/Client/DirectoryListing.py,sha256=CvQZ5Da5WhgwdDL8iLk0xVBjKZrBHTdwMKWwSJwzwEI,7582
 DIRAC/DataManagementSystem/Client/FTS3Client.py,sha256=Kk1wy4YnkYuOI6IltyDYZqkzxV7Zsf7ZuoKT12_VbTw,3531
 DIRAC/DataManagementSystem/Client/FTS3File.py,sha256=gZFN_Uxc2tblUCETb4qWPJ2M1tSmKUvTMiZUL-yJ6M0,3122
-DIRAC/DataManagementSystem/Client/FTS3Job.py,sha256=ZrQEMRFEfCTTkbcaio6DLV1dijp4e8gJgO8kvphW9p4,40045
+DIRAC/DataManagementSystem/Client/FTS3Job.py,sha256=5MAtSZoWb_rwglNHdYJAuoXzoW5_TAX96E5jUWEYds8,42090
 DIRAC/DataManagementSystem/Client/FTS3Operation.py,sha256=L3yqc0X0ziluMPegoQU6o3aYDnTYwLtujShgbGFJjsw,24455
 DIRAC/DataManagementSystem/Client/FailoverTransfer.py,sha256=2V-dKtgAndMuw9iZHYV019V0hLwlezUNP33wvClpsaA,13373
 DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py,sha256=sc_6mT0BCbXgj5wOEFtCxNc0AvyQ186A_yLi2lnduvw,80094
@@ -376,7 +376,8 @@ DIRAC/DataManagementSystem/Client/CmdDirCompletion/AbstractFileSystem.py,sha256=
 DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py,sha256=TdOCEfVQ3QppTl0iHrf472rT3YkDw2EX2sUviKIYL2s,2394
 DIRAC/DataManagementSystem/Client/CmdDirCompletion/__init__.py,sha256=IU_DNOfkPtg8iBqQatUKTNBMj2LldGrhf6KseBgx14M,236
 DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py,sha256=qb0mERTlHOysiSfNPDsbLbadWgbuQLulT9Py8RqTIYc,3956
-DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py,sha256=PwdHIHEJpP4I7plHRhrIBHrftIW0aRHCasYx7AUQfd4,15432
+DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py,sha256=DZHRZPfiTJrqinXolneRE06Wf51sVtRw0Fx0bq85AAA,15455
+DIRAC/DataManagementSystem/Client/test/Test_scitag.py,sha256=llvY98ODaBZhyhDquP9IMzCB-uXJz8xL-TO2-sbNf68,2545
 DIRAC/DataManagementSystem/Client/test/__init__.py,sha256=IL2xDyzb5nr2dkH2hpT4Zx6vvc9m4jDryuUmfyIY_RY,58
 DIRAC/DataManagementSystem/Client/test/mock_DM.py,sha256=S7qWHjN3ZyjqKIjHkXwuUWJqp75waEchc1IsNB9y-28,1167
 DIRAC/DataManagementSystem/Client/test/new_dir_completion.py,sha256=_Km7jE4IRe-pY8BJ62hK6IbwRROoj0wxvL7WMClEJ0E,1212
@@ -1118,7 +1119,7 @@ DIRAC/Workflow/Modules/test/Test_Modules.py,sha256=d92bA2SpF7Wwg9gy3roEvPt_7QlnB
 DIRAC/Workflow/Utilities/Utils.py,sha256=9f395ylWaLRaaC6EExPAyvOhPK3FXD7I_XGevqueRkM,2545
 DIRAC/Workflow/Utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 DIRAC/Workflow/Utilities/test/Test_Utilities.py,sha256=DNjFPpBmpojwCWhZSUSoG3AIhMQvqyiZdSuVkrHkOtk,2595
-DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg,sha256=lCelD92HtTXp46CCwCb8S58BjITG1fBgtFs116vehww,8949
+DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg,sha256=uHa3ZiSC7cyhbJUNvWwqiL0E-m9UmBinrLIQdWmre3k,9030
 DIRAC/WorkloadManagementSystem/__init__.py,sha256=9-_-HOT_8S3i-TMmTR_gFVVlNyktBRk-S2qSuOBKoIc,50
 DIRAC/WorkloadManagementSystem/Agent/JobAgent.py,sha256=LzLypd3m4a6M-J5_nmUMsMqgflb_1Xm77eONDE6G1Vg,40843
 DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py,sha256=TR8nDO43DuJxOQVIdOUrqpWjRjJp52dVzOKUUewL9U4,14708
@@ -1126,7 +1127,7 @@ DIRAC/WorkloadManagementSystem/Agent/PilotLoggingAgent.py,sha256=ZIgvFpasGTh76GW
 DIRAC/WorkloadManagementSystem/Agent/PilotStatusAgent.py,sha256=qY6TbYCPOFFXhHffmRJLNEbWvZPyg5Lc5B_8BbyQ7zc,9711
 DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py,sha256=qzDFCGZ8EtjxDUaPgyFDlSmJfyF2KLuyXTC7au3-p2Q,4860
 DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py,sha256=IvHshnw2xN0AZrruEu86C47GDez8enBD6jjNIZd6QcA,38594
-DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py,sha256=7b0CVkSgJ9b_lX-6CT6exzaWgEHH6pLjj5T1RJTnYJU,45114
+DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py,sha256=ZSGWVKO64ztnv_R19I6TT7660jf7LMuYcff_aPq2xCM,45162
 DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py,sha256=foEbmRotEmfeQG6nyIsJv1kSJkm4flkQsPYbSylS3SM,24572
 DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py,sha256=iNIlWQEDBk6R1S8oHOIusZUwxOwLtgwuzR_4s32-o5w,8707
 DIRAC/WorkloadManagementSystem/Agent/TaskQueuesAgent.py,sha256=ypsmo233TFXq9IC5uz6pum7_joOh2gFPUYNmmCpukAY,943
@@ -1295,9 +1296,9 @@ DIRAC/tests/Workflow/Integration/exe-script.py,sha256=B_slYdTocEzqfQLRhwuPiLyYUn
 DIRAC/tests/Workflow/Integration/helloWorld.py,sha256=tBgEHH3ZF7ZiTS57gtmm3DW-Qxgm_57HWHpM-Y8XSws,205
 DIRAC/tests/Workflow/Regression/helloWorld.py,sha256=69eCgFuVSYo-mK3Dj2dw1c6g86sF5FksKCf8V2aGVoM,509
 DIRAC/tests/Workflow/Regression/helloWorld.xml,sha256=xwydIcFTAHIX-YPfQfyxuQ7hzvIO3IhR3UAF7ORgkGg,5310
-dirac-9.0.7.dist-info/licenses/LICENSE,sha256=uyr4oV6jmjUeepXZPPjkJRwa5q5MrI7jqJz5sVXNblQ,32452
-dirac-9.0.7.dist-info/METADATA,sha256=0Y2bxOpUIA4SmXbeMd2Lp1_MLU8_2m83KFlge8LwsLs,10016
-dirac-9.0.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dirac-9.0.7.dist-info/entry_points.txt,sha256=hupzIL8aVmjK3nn7RLKdhcaiPmLOiD3Kulh3CSDHKmw,16492
-dirac-9.0.7.dist-info/top_level.txt,sha256=RISrnN9kb_mPqmVu8_o4jF-DSX8-h6AcgfkO9cgfkHA,6
-dirac-9.0.7.dist-info/RECORD,,
+dirac-9.0.8.dist-info/licenses/LICENSE,sha256=uyr4oV6jmjUeepXZPPjkJRwa5q5MrI7jqJz5sVXNblQ,32452
+dirac-9.0.8.dist-info/METADATA,sha256=gpKRR8YAjDBxrxogcUUGPNXojPku4FhNUVqBVSPRNWg,10016
+dirac-9.0.8.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dirac-9.0.8.dist-info/entry_points.txt,sha256=hupzIL8aVmjK3nn7RLKdhcaiPmLOiD3Kulh3CSDHKmw,16492
+dirac-9.0.8.dist-info/top_level.txt,sha256=RISrnN9kb_mPqmVu8_o4jF-DSX8-h6AcgfkO9cgfkHA,6
+dirac-9.0.8.dist-info/RECORD,,