DIRAC 9.0.0a68__tar.gz → 9.0.0a70__tar.gz

{dirac-9.0.0a68/src/DIRAC.egg-info → dirac-9.0.0a70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.0a68
+Version: 9.0.0a70
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,9 +19,10 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.0a68
-Requires-Dist: diracx-client>=v0.0.1a18
-Requires-Dist: diracx-core>=v0.0.1a18
+Requires-Dist: DIRACCommon==v9.0.0a70
+Requires-Dist: diracx-client>=v0.0.1
+Requires-Dist: diracx-core>=v0.0.1
+Requires-Dist: diracx-cli>=v0.0.1
 Requires-Dist: db12
 Requires-Dist: fts3
 Requires-Dist: gfal2-python
@@ -40,13 +41,13 @@ Requires-Dist: python-dateutil
 Requires-Dist: pytz
 Requires-Dist: requests
 Requires-Dist: rucio-clients>=34.4.2
-Requires-Dist: setuptools
 Requires-Dist: sqlalchemy
 Requires-Dist: typing_extensions>=4.3.0
 Requires-Dist: Authlib>=1.0.0.a2
 Requires-Dist: pyjwt
 Requires-Dist: dominate
 Requires-Dist: zstandard
+Requires-Dist: xattr
 Provides-Extra: server
 Requires-Dist: CMRESHandler; extra == "server"
 Requires-Dist: opensearch-py; extra == "server"

{dirac-9.0.0a68 → dirac-9.0.0a70}/dirac.cfg

@@ -219,7 +219,7 @@ Registry
       IdPRole = some_special_scope
 
       # Virtual organization associated with the group
-      VOMSVO = lhcb
+      VO = lhcb
 
       # Just for normal users:
       JobShare = 200

{dirac-9.0.0a68 → dirac-9.0.0a70}/release.notes

@@ -1,3 +1,186 @@
+[v9.0.0a70]
+
+
+[v9.0.13]
+
+*DataManagementSystem
+
+FIX: (#8396) submit unmanaged token for FTS3 replication
+
+*Resources
+
+NEW: (#8382) Attempt to set the xrd checksum attribute when copying files localy
+
+[v9.0.12]
+
+*Resources
+
+FIX: (#8393) make client_id and secret optional to avoid KeyError exceptions.
+
+*ConfigurationSystem
+
+NEW: (#8392) handle adding user AffiliationEnds section in the CS when modifying the user from CSAPI
+
+*MonitoringSystem
+
+FIX: (#8391) Addapt webApp to new getTransformations API
+
+[v9.0.11]
+
+*WorkloadManagement
+
+FIX: (#8389) Use ObjectLoader for DB instantiation in WMS components
+FIX: (#8389) Reuse DB objects when killing/deleting jobs
+
+*Resources
+
+CHANGE: (#8362) SSHCE export inputs/import outputs as a JSON file instead of using the stdout/stderr
+
+[v9.0.10]
+
+*DataManagementSystem
+
+FIX: (#8381) make sure the DataManager knows the VO
+
+*Interfaces
+
+FIX: (#8380) DConfigCache - reset Registry caches after loading the configuration from the local disk cache
+
+*TransformationSystem
+
+CHANGE: (#8373) This contains various speed of for the TS, mostly by using temporary tables for the queries, only requesting what's needed in the DB, and bulking things more efficiently
+
+[v9.0.9]
+
+CHANGE: (#8359) Subprocess: use psutil for killing processes
+
+[v9.0.8]
+
+*DataManagementSystem
+
+NEW: (#8263) add scitag to FTS transfers
+
+[v9.0.7]
+
+*DataManagementSystem
+
+FIX: (#8367) FTS reports correct accounting for failed transfers
+
+*WorkloadManagementSystem
+
+FIX: (#8365) SiteDirector: siteNames, ceTypes, ces and tags are None by default
+
+*FrameworkSystem
+
+FIX: (#8361) use cachedmethod instead of cached in ProxyDB
+
+*TransformationSystem
+
+FIX: (#8361) use cachedmethod instead of cached in Utilities
+
+*WorkloadManagement
+
+CHANGE: (#8360) use PilotManager to get the pilot output(s) from a job ID in WMSAdministrator service
+
+*ResourceStatusSystem
+
+FIX: (#8357) Avoid negative values in the Space Accounting
+
+*Resources
+
+FIX: (#8355) AREX CE: fix for understanding slightly different formats
+
+*Core
+
+NEW: (#8352) Report errors for every failed attempt in M2SSLTransport initAsClient
+FIX: (#8352) Use resolved socket address for SSL connections in M2SSLTransport
+
+[v9.0.6]
+
+*FrameworkSystem
+
+NEW: (#8328) allow for multiple client per provider
+
+*DataManagementSystem
+
+NEW: (#8328) FTS3 gets tokens directly
+
+[v9.0.5]
+
+*Subsystem
+
+CHANGE: (#8353) add PreferredURLPatterns to pilot.json
+
+[v9.0.4]
+[v9.0.3]
+
+*DiracX
+
+NEW: (#8347) Depend on released versions of DiracX
+NEW: (#8347) Depend on diracx-cli to ease management of DiracX CS conversion
+
+*FrameworkSystem
+
+CHANGE: (#8324) Remove Registry/group option VOMSVO (simply use "VO")
+
+[v9.0.2]
+
+*MonitoringSystem
+
+FIX: (#8343) remove Setup remnants
+
+*CERNLDAPSyncPlugin
+
+FIX: (#8341) cernAccountType is a list -- ensure CERNPersonId filled correctly
+NEW: (#8334) sync CCID / CERN Person ID to CS
+
+*WorkloadManagement
+
+FIX: (#8340) Site Director stops trying to leverage banned CEs
+FIX: (#8338) TokenManager - pass audience parameter as resource to fetchToken() to satisfy Check-In
+
+*General
+
+Remove setuptools from runtime dependencies, no longer required.
+
+*Core
+
+CHANGE: (#8323) Agressively close connections when serivces are overloaded
+For examples look into release.notes
+CHANGE: (#8312) allow one and only one diracx token in the proxy file
+
+[v9.0.1]
+
+*WorkloadManagementSystem
+
+FIX: (#8321) Restore uploading local outputfiles from the job's directory to a specific LFN.
+FIX: (#8317) use correct interface to _filterJobStateTransition from DIRAC APIs
+
+*WorkloadManagement
+
+FIX: (#8318) Killing jobs
+
+*FileCatalog
+
+FIX: (#8316) Fix non-existent charset in mysql schema definition
+
+*RequestManagementSystem
+
+CHANGE: (#8303) REA defaults changed for higher perfs
+
+[v9.0.0]
+
+*ConfigurationSystem
+
+CHANGE: (#8298) Apply PreferredURLPatterns to Configuration refresh server list
+
+[v9.0.0a69]
+
+*WorkloadManagementSystem
+
+CHANGE: (#8297) Remove the master CS aliases from the list of servers when generating the pilot.json
+FIX: (#8296) removed MatcherServiceTime JobParameters
+
 [v9.0.0a68]
 
 *ConfigurationSystem

{dirac-9.0.0a68 → dirac-9.0.0a70}/setup.cfg

@@ -26,9 +26,10 @@ install_requires =
 	certifi
 	cwltool
 	diraccfg
-	DIRACCommon==v9.0.0a68
-	diracx-client >=v0.0.1a18
-	diracx-core >=v0.0.1a18
+	DIRACCommon==v9.0.0a70
+	diracx-client >=v0.0.1
+	diracx-core >=v0.0.1
+	diracx-cli >=v0.0.1
 	db12
 	fts3
 	gfal2-python
@@ -47,13 +48,13 @@ install_requires =
 	pytz
 	requests
 	rucio-clients >=34.4.2
-	setuptools
 	sqlalchemy
 	typing_extensions >=4.3.0
 	Authlib >=1.0.0.a2
 	pyjwt
 	dominate
 	zstandard
+	xattr
 zip_safe = False
 include_package_data = True
 

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/AccountingSystem/Client/Types/Network.py

@@ -1,6 +1,6 @@
-""" Accounting class to stores network metrics gathered by perfSONARs.
+"""Accounting class to stores network metrics gathered by perfSONARs.
 
-    Filled by "Accounting/Network" agent
+Filled by "Accounting/Network" agent
 """
 
 from DIRAC.AccountingSystem.Client.Types.BaseAccountingType import BaseAccountingType
@@ -16,12 +16,12 @@ class Network(BaseAccountingType):
 
         # IPv6 address has up to 45 chars
         self.definitionKeyFields = [
-            ("SourceIP", "VARCHAR(50)"),
-            ("DestinationIP", "VARCHAR(50)"),
-            ("SourceHostName", "VARCHAR(50)"),
-            ("DestinationHostName", "VARCHAR(50)"),
-            ("Source", "VARCHAR(50)"),
-            ("Destination", "VARCHAR(50)"),
+            ("SourceIP", "VARCHAR(64)"),
+            ("DestinationIP", "VARCHAR(64)"),
+            ("SourceHostName", "VARCHAR(255)"),
+            ("DestinationHostName", "VARCHAR(255)"),
+            ("Source", "VARCHAR(255)"),
+            ("Destination", "VARCHAR(255)"),
         ]
 
         self.definitionAccountingFields = [

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/AccountingSystem/Client/Types/PilotSubmission.py

@@ -1,6 +1,6 @@
-""" Accounting Type for Pilot Submission
+"""Accounting Type for Pilot Submission
 
-    Filled by the "WorkloadManagement/SiteDirector" agent(s)
+Filled by the "WorkloadManagement/SiteDirector" agent(s)
 """
 
 from DIRAC.AccountingSystem.Client.Types.BaseAccountingType import BaseAccountingType
@@ -13,7 +13,7 @@ class PilotSubmission(BaseAccountingType):
         super().__init__()
 
         self.definitionKeyFields = [
-            ("HostName", "VARCHAR(100)"),
+            ("HostName", "VARCHAR(255)"),
             ("SiteDirector", "VARCHAR(100)"),
             ("Site", "VARCHAR(100)"),
             ("CE", "VARCHAR(100)"),

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/ConfigurationSystem/Client/CSAPI.py

@@ -456,13 +456,23 @@ class CSAPI:
             gLogger.error("User is not registered: ", repr(username))
             return S_OK(False)
         for prop in properties:
-            if prop == "Groups":
+            if prop in ["Groups", "AffiliationEnds"]:
                 continue
             prevVal = self.__csMod.getValue(f"{self.__baseSecurity}/Users/{username}/{prop}")
             if not prevVal or prevVal != properties[prop]:
                 gLogger.info(f"Setting {prop} property for user {username} to {properties[prop]}")
                 self.__csMod.setOptionValue(f"{self.__baseSecurity}/Users/{username}/{prop}", properties[prop])
                 modifiedUser = True
+        if properties.get("AffiliationEnds", None):
+            user_affiliationends_section = f"{self.__baseSecurity}/Users/{username}/AffiliationEnds"
+            # add the section for AffiliationEnds
+            res = gConfig.getSections(user_affiliationends_section)
+            if not res["OK"]:
+                self.__csMod.createSection(user_affiliationends_section)
+            # now set value VO = end date
+            for vo, end_date in properties["AffiliationEnds"].items():
+                self.__csMod.setOptionValue(f"{user_affiliationends_section}/{vo}", end_date)
+                modifiedUser = True
         if "Groups" in properties:
             result = self.listGroups()
             if not result["OK"]:

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py

@@ -4,15 +4,6 @@ Some Helper functions to retrieve common location from the CS
 from DIRAC.Core.Utilities.Extensions import extensionsByPriority
 
 
-def getSetup() -> str:
-    """
-    Return setup name
-    """
-    from DIRAC import gConfig
-
-    return gConfig.getValue("/DIRAC/Setup", "")
-
-
 def getVO(defaultVO: str = "") -> str:
     """
     Return VO from configuration

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py

@@ -3,18 +3,14 @@
 import errno
 import inspect
 import sys
-
-from threading import Lock
 from collections.abc import Iterable
+from threading import Lock
+from typing import Optional
 
 from cachetools import TTLCache, cached
 from cachetools.keys import hashkey
 
-
-from typing import Optional
-from collections.abc import Iterable
-
-from DIRAC import S_OK, S_ERROR
+from DIRAC import S_ERROR, S_OK
 from DIRAC.ConfigurationSystem.Client.Config import gConfig
 from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
 
@@ -488,28 +484,6 @@ def getVOMSAttributeForGroup(group):
     return gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VOMSRole", getDefaultVOMSAttribute())
 
 
-def getDefaultVOMSVO():
-    """Get default VOMS VO
-
-    :return: str
-    """
-    return gConfig.getValue(f"{gBaseRegistrySection}/DefaultVOMSVO", "") or getVO()
-
-
-def getVOMSVOForGroup(group):
-    """Search VOMS VO for group
-
-    :param str group: group name
-
-    :return: str
-    """
-    vomsVO = gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VOMSVO", getDefaultVOMSVO())
-    if not vomsVO:
-        vo = getVOForGroup(group)
-        vomsVO = getVOOption(vo, "VOMSName", "")
-    return vomsVO
-
-
 def getGroupsWithVOMSAttribute(vomsAttr):
     """Search groups with VOMS attribute
 

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py

@@ -32,6 +32,9 @@ class CERNLDAPSyncPlugin:
         else:
             userDict["PrimaryCERNAccount"] = self._findOwnerAccountName(username, attributes)
 
+        if userDict["CERNAccountType"] in ["Primary", "Secondary"]:
+            userDict["CERNPersonId"] = attributes.get("employeeId", [None])[0]
+
     def _findOwnerAccountName(self, username, attributes):
         """Find the owner account from a CERN LDAP entry.
 
@@ -64,7 +67,7 @@ class CERNLDAPSyncPlugin:
         status, result, response, _ = self._connection.search(
             "OU=Users,OU=Organic Units,DC=cern,DC=ch",
             f"(CN={commonName})",
-            attributes=["cernAccountOwner", "cernAccountType"],
+            attributes=["cernAccountOwner", "cernAccountType", "employeeId"],
         )
         if not status:
             raise ValueError(f"Bad status from LDAP search: {result}")

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg

@@ -6,6 +6,9 @@ Services
   {
     HandlerPath = DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py
     Port = 9135
+    # Set to True to make a DiracX model validation on commits
+    # If validation fails, commit will also failed
+    VerifyDiracXSyncOnCommit = False
     # Subsection to configure authorization over the service
     Authorization
     {

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/ConfigurationSystem/private/Modificator.py

@@ -1,13 +1,16 @@
 """ This is the guy that actually modifies the content of the CS
 """
-import zlib
-import difflib
 import datetime
+import difflib
+import zlib
 
 from diraccfg import CFG
-from DIRAC.Core.Utilities import List
+
+from DIRAC import S_ERROR
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
+from DIRAC.Core.Utilities import List
+from DIRAC.FrameworkSystem.Utilities.diracx import diracxVerifyConfig
 
 
 class Modificator:
@@ -239,6 +242,11 @@ class Modificator:
         return str(self.cfgData)
 
     def commit(self):
+        retOpt = self.getValue("/Systems/Configuration/Services/Server/VerifyDiracXSyncOnCommit")
+        if retOpt == "True":
+            resVerif = diracxVerifyConfig(self.cfgData)
+            if not resVerif["OK"]:
+                return S_ERROR(resVerif["Message"])
         compressedData = zlib.compress(str(self.cfgData).encode(), 9)
         return self.rpcClient.commitNewData(compressedData)
 

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/ConfigurationSystem/private/RefresherBase.py

@@ -1,7 +1,7 @@
 import time
 
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
-from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs
+from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs, groupURLsByPriority
 from DIRAC.Core.Utilities import List
 from DIRAC.Core.Utilities.EventDispatcher import gEventDispatcher
 from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK
@@ -138,7 +138,9 @@ class RefresherBase:
         if not initialServerList:
             return S_OK()
 
-        randomServerList = List.randomize(initialServerList)
+        randomServerList = []
+        for urlGroup in groupURLsByPriority(initialServerList):
+            randomServerList.extend(List.randomize(urlGroup))
         gLogger.debug(f"Randomized server list is {', '.join(randomServerList)}")
 
         for sServer in randomServerList:

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/Core/DISET/ServiceReactor.py

@@ -200,6 +200,7 @@ class ServiceReactor:
                                   services at the same time
         """
         sel = self.__getListeningSelector(svcName)
+        throttleExpires = None
         while self.__alive:
             clientTransport = None
             try:
@@ -223,12 +224,19 @@ class ServiceReactor:
                 gLogger.warn(f"Client connected from banned ip {clientIP}")
                 clientTransport.close()
                 continue
+            # Handle throttling
+            if self.__services[svcName].wantsThrottle and throttleExpires is None:
+                throttleExpires = time.time() + THROTTLE_SERVICE_SLEEP_SECONDS
+            if throttleExpires:
+                if time.time() > throttleExpires:
+                    throttleExpires = None
+                else:
+                    gLogger.warn("Rejecting client due to throttling", str(clientTransport.getRemoteAddress()))
+                    clientTransport.close()
+                    continue
             # Handle connection
             self.__stats.connectionStablished()
             self.__services[svcName].handleConnection(clientTransport)
-            while self.__services[svcName].wantsThrottle:
-                gLogger.warn("Sleeping as service requested throttling", svcName)
-                time.sleep(THROTTLE_SERVICE_SLEEP_SECONDS)
             # Renew context?
             now = time.time()
             renewed = False

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py

@@ -110,19 +110,18 @@ class SSLTransport(BaseTransport):
         if self.serverMode():
             raise RuntimeError("SSLTransport is in server mode.")
 
-        error = None
+        errors = []
         host, port = self.stServerAddress
 
         # The following piece of code was inspired by the python socket documentation
         # as well as the implementation of M2Crypto.httpslib.HTTPSConnection
 
-        # We ignore the returned sockaddr because SSL.Connection.connect needs
-        # a host name.
+        # Get all available addresses (IPv6 and IPv4) and try them in order
         try:
             addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
         except OSError as e:
             return S_ERROR(f"DNS lookup failed {e!r}")
-        for family, _socketType, _proto, _canonname, _socketAddress in addrInfoList:
+        for family, _socketType, _proto, _canonname, socketAddress in addrInfoList:
             try:
                 self.oSocket = SSL.Connection(self.__ctx, family=family)
 
@@ -138,7 +137,10 @@ class SSLTransport(BaseTransport):
                 # set SNI server name since we know it at this point
                 self.oSocket.set_tlsext_host_name(host)
 
-                self.oSocket.connect((host, port))
+                # tell the connection which host we are connecting to so we can
+                # use the address we obtained from DNS
+                self.oSocket.set1_host(host)
+                self.oSocket.connect(socketAddress)
 
                 # Once the connection is established, we can use the timeout
                 # asked for RPC
@@ -151,12 +153,12 @@ class SSLTransport(BaseTransport):
             # They should be propagated upwards and caught by the BaseClient
             # not to enter the retry loop
             except OSError as e:
-                error = f"{e}:{repr(e)}"
+                errors.append(f"{socketAddress} {e}:{repr(e)}")
 
                 if self.oSocket is not None:
                     self.close()
 
-        return S_ERROR(error)
+        return S_ERROR("; ".join(errors))
 
     def initAsServer(self):
         """Prepare this server socket for use."""

{dirac-9.0.0a68 → dirac-9.0.0a70}/src/DIRAC/Core/Security/DiracX.py

@@ -40,7 +40,7 @@ from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueO
 
 PEM_BEGIN = "-----BEGIN DIRACX-----"
 PEM_END = "-----END DIRACX-----"
-RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(.*)\n{PEM_END}", re.MULTILINE | re.DOTALL)
+RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(.*?)\n{PEM_END}", re.DOTALL)
 
 
 @convertToReturnValue
@@ -62,21 +62,26 @@ def addTokenToPEM(pemPath, group):
             token_type=token_content.get("token_type"),
             refresh_token=token_content.get("refresh_token"),
         )
-
         token_pem = f"{PEM_BEGIN}\n"
         data = base64.b64encode(serialize_credentials(token).encode("utf-8")).decode()
         token_pem += textwrap.fill(data, width=64)
         token_pem += f"\n{PEM_END}\n"
 
-        with open(pemPath, "a") as f:
-            f.write(token_pem)
+        pem = Path(pemPath).read_text()
+        # Remove any existing DiracX token there would be
+        new_pem = re.sub(RE_DIRACX_PEM, "", pem)
+        new_pem += token_pem
+
+        Path(pemPath).write_text(new_pem)
 
 
 def diracxTokenFromPEM(pemPath) -> dict[str, Any] | None:
     """Extract the DiracX token from the proxy PEM file"""
     pem = Path(pemPath).read_text()
-    if match := RE_DIRACX_PEM.search(pem):
-        match = match.group(1)
+    if match := RE_DIRACX_PEM.findall(pem):
+        if len(match) > 1:
+            raise ValueError("Found multiple DiracX tokens, this should never happen")
+        match = match[0]
         return json.loads(base64.b64decode(match).decode("utf-8"))