DIRAC 9.0.4__tar.gz → 9.0.6__tar.gz

{dirac-9.0.4/src/DIRAC.egg-info → dirac-9.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.4
+Version: 9.0.6
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.4
+Requires-Dist: DIRACCommon==v9.0.6
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1

{dirac-9.0.4 → dirac-9.0.6}/release.notes

@@ -1,3 +1,19 @@
+[v9.0.6]
+
+*FrameworkSystem
+
+NEW: (#8328) allow for multiple client per provider
+
+*DataManagementSystem
+
+NEW: (#8328) FTS3 gets tokens directly
+
+[v9.0.5]
+
+*Subsystem
+
+CHANGE: (#8353) add PreferredURLPatterns to pilot.json
+
 [v9.0.4]
 [v9.0.3]
 

{dirac-9.0.4 → dirac-9.0.6}/setup.cfg

@@ -26,7 +26,7 @@ install_requires =
 	certifi
 	cwltool
 	diraccfg
-	DIRACCommon==v9.0.4
+	DIRACCommon==v9.0.6
 	diracx-client >=v0.0.1
 	diracx-core >=v0.0.1
 	diracx-cli >=v0.0.1

{dirac-9.0.4 → dirac-9.0.6}/src/DIRAC/Core/scripts/dirac_apptainer_exec.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
-""" Starts a DIRAC command inside an apptainer container.
-"""
+"""Starts a DIRAC command inside an apptainer container."""
 
 import os
 import sys
@@ -52,8 +51,9 @@ def main():
         if switch[0].lower() == "i" or switch[0].lower() == "image":
             user_image = switch[1]
 
-    dirac_env_var = os.environ.get("DIRAC", os.getcwd())
-    diracos_env_var = os.environ.get("DIRACOS", os.getcwd())
+    cwd = os.path.realpath(os.getcwd())
+    dirac_env_var = os.environ.get("DIRAC", cwd)
+    diracos_env_var = os.environ.get("DIRACOS", cwd)
     etc_dir = os.path.join(DIRAC.rootPath, "etc")
     rc_script = os.path.join(os.path.realpath(sys.base_prefix), "diracosrc")
 
@@ -74,7 +74,7 @@ def main():
     cmd.extend(["--contain"])  # use minimal /dev and empty other directories (e.g. /tmp and $HOME)
     cmd.extend(["--ipc"])  # run container in a new IPC namespace
     cmd.extend(["--pid"])  # run container in a new PID namespace
-    cmd.extend(["--bind", f"{os.getcwd()}"])  # bind current directory for dirac_container.sh
+    cmd.extend(["--bind", cwd])  # bind current directory for dirac_container.sh
     if proxy_location:
         cmd.extend(["--bind", f"{proxy_location}:/etc/proxy"])  # bind proxy file
     cmd.extend(["--bind", f"{getCAsLocation()}:/etc/grid-security/certificates"])  # X509_CERT_DIR
@@ -89,13 +89,13 @@ def main():
         if safe_listdir(bind_path):
             cmd.extend(["--bind", f"{bind_path}:{bind_path}"])
         else:
-            gLogger.warning(f"Bind path {bind_path} does not exist, skipping")
-    cmd.extend(["--cwd", f"{os.getcwd()}"])  # set working directory
+            gLogger.warn(f"Bind path {bind_path} does not exist, skipping")
+    cmd.extend(["--cwd", cwd])  # set working directory
 
     rootImage = user_image or gConfig.getValue("/Resources/Computing/Singularity/ContainerRoot") or CONTAINER_DEFROOT
 
     if os.path.isdir(rootImage) or os.path.isfile(rootImage):
-        cmd.extend([rootImage, f"{os.getcwd()}/dirac_container.sh"])
+        cmd.extend([rootImage, f"{cwd}/dirac_container.sh"])
     else:
         # if we are here is because there's no image, or it is not accessible (e.g. not on CVMFS)
         gLogger.error("Apptainer image to exec not found: ", rootImage)

{dirac-9.0.4 → dirac-9.0.6}/src/DIRAC/DataManagementSystem/Client/FTS3Job.py

@@ -1,9 +1,11 @@
-""" FTS3Job module containing only the FTS3Job class """
+"""FTS3Job module containing only the FTS3Job class"""
 
 import datetime
 import errno
+import os
 from packaging.version import Version
 
+from cachetools import cachedmethod, LRUCache
 
 # Requires at least version 3.3.3
 from fts3 import __version__ as fts3_version
@@ -26,8 +28,9 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement
 
 from DIRAC.FrameworkSystem.Client.Logger import gLogger
 from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager
+from DIRAC.FrameworkSystem.Utilities.TokenManagementUtilities import getIdProviderClient
 
-from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
+from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, returnValueOrRaise
 from DIRAC.Core.Utilities.DErrno import cmpError
 
 from DIRAC.Core.Utilities.JEncode import JSerializable
@@ -36,6 +39,10 @@ from DIRAC.DataManagementSystem.Client.FTS3File import FTS3File
 # 3 days in seconds
 BRING_ONLINE_TIMEOUT = 259200
 
+# Number of IdP to keep in cache. Should correspond roughly
+# to the number of groups performing transfers
+IDP_CACHE_SIZE = 8
+
 
 class FTS3Job(JSerializable):
     """Abstract class to represent a job to be executed by FTS. It belongs
@@ -78,6 +85,8 @@ class FTS3Job(JSerializable):
         "userGroup",
     ]
 
+    _idp_cache = LRUCache(maxsize=IDP_CACHE_SIZE)
+
     def __init__(self):
         self.submitTime = None
         self.lastUpdate = None
@@ -113,6 +122,11 @@ class FTS3Job(JSerializable):
         # when a job is in a final state
         self.accountingDict = None
 
+    @classmethod
+    @cachedmethod(lambda cls: cls._idp_cache)
+    def _getIdpClient(cls, group_name: str):
+        return returnValueOrRaise(getIdProviderClient(group_name, None, client_name_prefix="fts"))
+
     def monitor(self, context=None, ftsServer=None, ucert=None):
         """Queries the fts server to monitor the job.
         The internal state of the object is updated depending on the
@@ -509,11 +523,10 @@ class FTS3Job(JSerializable):
                     if not res["OK"]:
                         return res
                     srcTokenPath = res["Value"]
-                    res = gTokenManager.getToken(
-                        userGroup=self.userGroup,
-                        requiredTimeLeft=3600,
+                    res = self._getIdpClient(self.userGroup).fetchToken(
+                        grant_type="client_credentials",
                         scope=[f"storage.read:/{srcTokenPath}", "offline_access"],
-                        useCache=False,
+                        # TODO: add a specific audience
                     )
                     if not res["OK"]:
                         return res
@@ -528,11 +541,17 @@ class FTS3Job(JSerializable):
                     if not res["OK"]:
                         return res
                     dstTokenPath = res["Value"]
-                    res = gTokenManager.getToken(
-                        userGroup=self.userGroup,
-                        requiredTimeLeft=3600,
-                        scope=[f"storage.modify:/{dstTokenPath}", f"storage.read:/{dstTokenPath}", "offline_access"],
-                        useCache=False,
+                    res = self._getIdpClient(self.userGroup).fetchToken(
+                        grant_type="client_credentials",
+                        scope=[
+                            f"storage.modify:/{dstTokenPath}",
+                            f"storage.read:/{dstTokenPath}",
+                            # Needed because CNAF
+                            # https://ggus.eu/index.php?mode=ticket_info&ticket_id=165048
+                            f"storage.read:/{os.path.dirname(dstTokenPath)}",
+                            "offline_access",
+                        ],
+                        # TODO: add a specific audience
                     )
                     if not res["OK"]:
                         return res
@@ -728,6 +747,7 @@ class FTS3Job(JSerializable):
             retry=3,
             metadata=job_metadata,
             priority=self.priority,
+            unmanaged_tokens=True,
             **dest_spacetoken,
         )
 

{dirac-9.0.4 → dirac-9.0.6}/src/DIRAC/FrameworkSystem/Utilities/TokenManagementUtilities.py

@@ -10,11 +10,12 @@ DEFAULT_RT_EXPIRATION_TIME = 24 * 3600
 DEFAULT_AT_EXPIRATION_TIME = 1200
 
 
-def getIdProviderClient(userGroup: str, idProviderClientName: str = None):
+def getIdProviderClient(userGroup: str, idProviderClientName: str = None, client_name_prefix: str = ""):
     """Get an IdProvider client
 
     :param userGroup: group name
     :param idProviderClientName: name of an identity provider in the DIRAC CS
+    :param client_name_prefix: prefix of the client in the CS options
     """
     # Get IdProvider credentials from CS
     if not idProviderClientName and userGroup:
@@ -23,7 +24,7 @@ def getIdProviderClient(userGroup: str, idProviderClientName: str = None):
         return S_ERROR(f"The {userGroup} group belongs to the VO that is not tied to any Identity Provider.")
 
     # Prepare the client instance of the appropriate IdP
-    return IdProviderFactory().getIdProvider(idProviderClientName)
+    return IdProviderFactory().getIdProvider(idProviderClientName, client_name_prefix=client_name_prefix)
 
 
 def getCachedKey(

{dirac-9.0.4 → dirac-9.0.6}/src/DIRAC/Resources/IdProvider/IdProviderFactory.py

@@ -1,6 +1,7 @@
-"""  The Identity Provider Factory instantiates IdProvider objects
-     according to their configuration
+"""The Identity Provider Factory instantiates IdProvider objects
+according to their configuration
 """
+
 import jwt
 
 from DIRAC import S_OK, S_ERROR, gLogger, gConfig
@@ -40,11 +41,12 @@ class IdProviderFactory:
             return result
         return self.getIdProvider(result["Value"])
 
-    def getIdProvider(self, name, **kwargs):
+    def getIdProvider(self, name, client_name_prefix="", **kwargs):
         """This method returns a IdProvider instance corresponding to the supplied
         name.
 
         :param str name: the name of the Identity Provider client
+        :param str client_name_prefix: name of the client of the IdP
 
         :return: S_OK(IdProvider)/S_ERROR()
         """
@@ -68,8 +70,14 @@ class IdProviderFactory:
             if not result["OK"]:
                 self.log.error("Failed to read configuration", f"{name}: {result['Message']}")
                 return result
+
             pDict = result["Value"]
 
+            if client_name_prefix:
+                client_name_prefix = client_name_prefix + "_"
+            pDict["client_id"] = pDict[f"{client_name_prefix}client_id"]
+            pDict["client_secret"] = pDict[f"{client_name_prefix}client_secret"]
+
         pDict.update(kwargs)
         pDict["ProviderName"] = name
 

{dirac-9.0.4 → dirac-9.0.6}/src/DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py

@@ -11,7 +11,6 @@ import os
 import shutil
 import tarfile
 from typing import Any
-
 from git import Repo
 
 from DIRAC import S_OK, gConfig, gLogger
@@ -217,6 +216,10 @@ class PilotCStoJSONSynchronizer:
 
         pilotDict["ConfigurationServers"] = configurationServers
 
+        preferredURLPatterns = gConfigurationData.extractOptionFromCFG("/DIRAC/PreferredURLPatterns")
+        if preferredURLPatterns:
+            pilotDict["PreferredURLPatterns"] = preferredURLPatterns
+
         self.log.debug("Got pilotDict", str(pilotDict))
 
         return S_OK(pilotDict)

{dirac-9.0.4 → dirac-9.0.6/src/DIRAC.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.4
+Version: 9.0.6
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.4
+Requires-Dist: DIRACCommon==v9.0.6
 Requires-Dist: diracx-client>=v0.0.1
 Requires-Dist: diracx-core>=v0.0.1
 Requires-Dist: diracx-cli>=v0.0.1

{dirac-9.0.4 → dirac-9.0.6}/src/DIRAC.egg-info/requires.txt

@@ -4,7 +4,7 @@ cachetools
 certifi
 cwltool
 diraccfg
-DIRACCommon==v9.0.4
+DIRACCommon==v9.0.6
 diracx-client>=v0.0.1
 diracx-core>=v0.0.1
 diracx-cli>=v0.0.1