DIRAC 9.0.5__py3-none-any.whl → 9.0.6__py3-none-any.whl

DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -1,9 +1,11 @@
-""" FTS3Job module containing only the FTS3Job class """
+"""FTS3Job module containing only the FTS3Job class"""
 
 import datetime
 import errno
+import os
 from packaging.version import Version
 
+from cachetools import cachedmethod, LRUCache
 
 # Requires at least version 3.3.3
 from fts3 import __version__ as fts3_version
@@ -26,8 +28,9 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement
 
 from DIRAC.FrameworkSystem.Client.Logger import gLogger
 from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager
+from DIRAC.FrameworkSystem.Utilities.TokenManagementUtilities import getIdProviderClient
 
-from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, returnValueOrRaise
 from DIRAC.Core.Utilities.DErrno import cmpError
 
 from DIRAC.Core.Utilities.JEncode import JSerializable
@@ -36,6 +39,10 @@ from DIRAC.DataManagementSystem.Client.FTS3File import FTS3File
 # 3 days in seconds
 BRING_ONLINE_TIMEOUT = 259200
 
+# Number of IdP to keep in cache. Should correspond roughly
+# to the number of groups performing transfers
+IDP_CACHE_SIZE = 8
+
 
 class FTS3Job(JSerializable):
     """Abstract class to represent a job to be executed by FTS. It belongs
@@ -78,6 +85,8 @@ class FTS3Job(JSerializable):
         "userGroup",
     ]
 
+    _idp_cache = LRUCache(maxsize=IDP_CACHE_SIZE)
+
     def __init__(self):
         self.submitTime = None
         self.lastUpdate = None
@@ -113,6 +122,11 @@ class FTS3Job(JSerializable):
         # when a job is in a final state
         self.accountingDict = None
 
+    @classmethod
+    @cachedmethod(lambda cls: cls._idp_cache)
+    def _getIdpClient(cls, group_name: str):
+        return returnValueOrRaise(getIdProviderClient(group_name, None, client_name_prefix="fts"))
+
     def monitor(self, context=None, ftsServer=None, ucert=None):
         """Queries the fts server to monitor the job.
         The internal state of the object is updated depending on the
@@ -509,11 +523,10 @@ class FTS3Job(JSerializable):
                     if not res["OK"]:
                         return res
                     srcTokenPath = res["Value"]
-                    res = gTokenManager.getToken(
-                        userGroup=self.userGroup,
-                        requiredTimeLeft=3600,
+                    res = self._getIdpClient(self.userGroup).fetchToken(
+                        grant_type="client_credentials",
                         scope=[f"storage.read:/{srcTokenPath}", "offline_access"],
-                        useCache=False,
+                        # TODO: add a specific audience
                     )
                     if not res["OK"]:
                         return res
@@ -528,11 +541,17 @@ class FTS3Job(JSerializable):
                     if not res["OK"]:
                         return res
                     dstTokenPath = res["Value"]
-                    res = gTokenManager.getToken(
-                        userGroup=self.userGroup,
-                        requiredTimeLeft=3600,
-                        scope=[f"storage.modify:/{dstTokenPath}", f"storage.read:/{dstTokenPath}", "offline_access"],
-                        useCache=False,
+                    res = self._getIdpClient(self.userGroup).fetchToken(
+                        grant_type="client_credentials",
+                        scope=[
+                            f"storage.modify:/{dstTokenPath}",
+                            f"storage.read:/{dstTokenPath}",
+                            # Needed because CNAF
+                            # https://ggus.eu/index.php?mode=ticket_info&ticket_id=165048
+                            f"storage.read:/{os.path.dirname(dstTokenPath)}",
+                            "offline_access",
+                        ],
+                        # TODO: add a specific audience
                     )
                     if not res["OK"]:
                         return res
@@ -728,6 +747,7 @@ class FTS3Job(JSerializable):
             retry=3,
             metadata=job_metadata,
             priority=self.priority,
+            unmanaged_tokens=True,
             **dest_spacetoken,
         )
 

DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py

@@ -10,11 +10,12 @@ DEFAULT_RT_EXPIRATION_TIME = 24 * 3600
 DEFAULT_AT_EXPIRATION_TIME = 1200
 
 
-def getIdProviderClient(userGroup: str, idProviderClientName: str = None):
+def getIdProviderClient(userGroup: str, idProviderClientName: str = None, client_name_prefix: str = ""):
     """Get an IdProvider client
 
     :param userGroup: group name
     :param idProviderClientName: name of an identity provider in the DIRAC CS
+    :param client_name_prefix: prefix of the client in the CS options
     """
     # Get IdProvider credentials from CS
     if not idProviderClientName and userGroup:
@@ -23,7 +24,7 @@ def getIdProviderClient(userGroup: str, idProviderClientName: str = None):
         return S_ERROR(f"The {userGroup} group belongs to the VO that is not tied to any Identity Provider.")
 
     # Prepare the client instance of the appropriate IdP
-    return IdProviderFactory().getIdProvider(idProviderClientName)
+    return IdProviderFactory().getIdProvider(idProviderClientName, client_name_prefix=client_name_prefix)
 
 
 def getCachedKey(

DIRAC/Resources/IdProvider/IdProviderFactory.py

@@ -1,6 +1,7 @@
-"""  The Identity Provider Factory instantiates IdProvider objects
-     according to their configuration
+"""The Identity Provider Factory instantiates IdProvider objects
+according to their configuration
 """
+
 import jwt
 
 from DIRAC import S_OK, S_ERROR, gLogger, gConfig
@@ -40,11 +41,12 @@ class IdProviderFactory:
             return result
         return self.getIdProvider(result["Value"])
 
-    def getIdProvider(self, name, **kwargs):
+    def getIdProvider(self, name, client_name_prefix="", **kwargs):
         """This method returns a IdProvider instance corresponding to the supplied
         name.
 
         :param str name: the name of the Identity Provider client
+        :param str client_name_prefix: name of the client of the IdP
 
         :return: S_OK(IdProvider)/S_ERROR()
         """
@@ -68,8 +70,14 @@ class IdProviderFactory:
             if not result["OK"]:
                 self.log.error("Failed to read configuration", f"{name}: {result['Message']}")
                 return result
+
             pDict = result["Value"]
 
+            if client_name_prefix:
+                client_name_prefix = client_name_prefix + "_"
+            pDict["client_id"] = pDict[f"{client_name_prefix}client_id"]
+            pDict["client_secret"] = pDict[f"{client_name_prefix}client_secret"]
+
         pDict.update(kwargs)
         pDict["ProviderName"] = name
 

{dirac-9.0.5.dist-info → dirac-9.0.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.5
+Version: 9.0.6
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.5
+Requires-Dist: DIRACCommon==v9.0.6
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1

{dirac-9.0.5.dist-info → dirac-9.0.6.dist-info}/RECORD

@@ -365,7 +365,7 @@ DIRAC/DataManagementSystem/Client/DataManager.py,sha256=edLmRUHJtZEYAMyen3y9qwdG
 DIRAC/DataManagementSystem/Client/DirectoryListing.py,sha256=CvQZ5Da5WhgwdDL8iLk0xVBjKZrBHTdwMKWwSJwzwEI,7582
 DIRAC/DataManagementSystem/Client/FTS3Client.py,sha256=Kk1wy4YnkYuOI6IltyDYZqkzxV7Zsf7ZuoKT12_VbTw,3531
 DIRAC/DataManagementSystem/Client/FTS3File.py,sha256=gZFN_Uxc2tblUCETb4qWPJ2M1tSmKUvTMiZUL-yJ6M0,3122
-DIRAC/DataManagementSystem/Client/FTS3Job.py,sha256=X0BCd35SSGjGBr3tL0TmQiPDxMT8H5gHugBK86x0I04,38482
+DIRAC/DataManagementSystem/Client/FTS3Job.py,sha256=uiiTDowNrv91OeFHWPCrygZR--JqIdBOgZOv0AS5tEg,39393
 DIRAC/DataManagementSystem/Client/FTS3Operation.py,sha256=L3yqc0X0ziluMPegoQU6o3aYDnTYwLtujShgbGFJjsw,24455
 DIRAC/DataManagementSystem/Client/FailoverTransfer.py,sha256=2V-dKtgAndMuw9iZHYV019V0hLwlezUNP33wvClpsaA,13373
 DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py,sha256=sc_6mT0BCbXgj5wOEFtCxNc0AvyQ186A_yLi2lnduvw,80094
@@ -531,7 +531,7 @@ DIRAC/FrameworkSystem/Service/TornadoUserProfileManagerHandler.py,sha256=9z8Vky8
 DIRAC/FrameworkSystem/Service/UserProfileManagerHandler.py,sha256=DyZFvqFeKjxVYNXqGYLkTlYgj3raUsfsb68GE4FX5x4,5707
 DIRAC/FrameworkSystem/Service/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 DIRAC/FrameworkSystem/Utilities/MonitoringUtilities.py,sha256=wt6QhJ6A_ifRP8kAgRWo-vtS_jJa8e5Xi9GdrRuA-i8,2556
-DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py,sha256=qbh6WQMvbczBjiP8Dig3PCQWODXSgPXeYpgXLTOaIiU,2895
+DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py,sha256=eEeKs9JMJOiokjDtDcRlyQiRWIM1TNYwcVy1C4oozuY,3034
 DIRAC/FrameworkSystem/Utilities/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 DIRAC/FrameworkSystem/Utilities/diracx.py,sha256=cyHtKuRY-maOUmY96uhj5sEKzb6nTEhNvOd2LlDhp18,3647
 DIRAC/FrameworkSystem/Utilities/test/Test_TokenManagementUtilities.py,sha256=XIaNe18QjuBUmPvDFBmKO0UNSEiYJCzQP-XuH-JMY3s,6551
@@ -948,7 +948,7 @@ DIRAC/Resources/Computing/test/Test_SSHComputingElement.py,sha256=ahYUZ9hySaEXik
 DIRAC/Resources/Computing/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 DIRAC/Resources/IdProvider/CheckInIdProvider.py,sha256=Tq8ENK6yhC7Tl6JkTF1q8mCOrFRfsN3ediF251iU4oM,1163
 DIRAC/Resources/IdProvider/IAMIdProvider.py,sha256=kk2ty2mLevxGmJRBpLYlQXce_vy4cxp-dO2w7b1rCz0,822
-DIRAC/Resources/IdProvider/IdProviderFactory.py,sha256=-MbDuQ7bIGcBhaHDAlA9NUzgC6VwMlujfHUOwf8uzEE,3899
+DIRAC/Resources/IdProvider/IdProviderFactory.py,sha256=bAUYnGQQUhCCXTq1o68lRlOPG--77oR9JoOAX6t0TNs,4238
 DIRAC/Resources/IdProvider/OAuth2IdProvider.py,sha256=smoGhSD1JsyjJDot9XmMUmT7fw8kcfS3w392s2Lg9P8,23650
 DIRAC/Resources/IdProvider/Utilities.py,sha256=bHF-aNtOapEN7Z18XKOFfAcXCHceLbEWU9dOrKeHxDo,1785
 DIRAC/Resources/IdProvider/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -1295,9 +1295,9 @@ DIRAC/tests/Workflow/Integration/exe-script.py,sha256=B_slYdTocEzqfQLRhwuPiLyYUn
 DIRAC/tests/Workflow/Integration/helloWorld.py,sha256=tBgEHH3ZF7ZiTS57gtmm3DW-Qxgm_57HWHpM-Y8XSws,205
 DIRAC/tests/Workflow/Regression/helloWorld.py,sha256=69eCgFuVSYo-mK3Dj2dw1c6g86sF5FksKCf8V2aGVoM,509
 DIRAC/tests/Workflow/Regression/helloWorld.xml,sha256=xwydIcFTAHIX-YPfQfyxuQ7hzvIO3IhR3UAF7ORgkGg,5310
-dirac-9.0.5.dist-info/licenses/LICENSE,sha256=uyr4oV6jmjUeepXZPPjkJRwa5q5MrI7jqJz5sVXNblQ,32452
-dirac-9.0.5.dist-info/METADATA,sha256=WgBoEsoG3B1PTa2YPy4NsEzaG4q01d5BOVhMYEIZbTs,10016
-dirac-9.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-dirac-9.0.5.dist-info/entry_points.txt,sha256=hupzIL8aVmjK3nn7RLKdhcaiPmLOiD3Kulh3CSDHKmw,16492
-dirac-9.0.5.dist-info/top_level.txt,sha256=RISrnN9kb_mPqmVu8_o4jF-DSX8-h6AcgfkO9cgfkHA,6
-dirac-9.0.5.dist-info/RECORD,,
+dirac-9.0.6.dist-info/licenses/LICENSE,sha256=uyr4oV6jmjUeepXZPPjkJRwa5q5MrI7jqJz5sVXNblQ,32452
+dirac-9.0.6.dist-info/METADATA,sha256=tbETxonTh5kgVSZJNZeZgQNue961cdGH4IXCZEgmWN8,10016
+dirac-9.0.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+dirac-9.0.6.dist-info/entry_points.txt,sha256=hupzIL8aVmjK3nn7RLKdhcaiPmLOiD3Kulh3CSDHKmw,16492
+dirac-9.0.6.dist-info/top_level.txt,sha256=RISrnN9kb_mPqmVu8_o4jF-DSX8-h6AcgfkO9cgfkHA,6
+dirac-9.0.6.dist-info/RECORD,,