DIRAC 9.0.0a68__py3-none-any.whl → 9.0.0a70__py3-none-any.whl

DIRAC/AccountingSystem/Client/Types/Network.py

@@ -1,6 +1,6 @@
-""" Accounting class to stores network metrics gathered by perfSONARs.
+"""Accounting class to stores network metrics gathered by perfSONARs.
 
-    Filled by "Accounting/Network" agent
+Filled by "Accounting/Network" agent
 """
 
 from DIRAC.AccountingSystem.Client.Types.BaseAccountingType import BaseAccountingType
@@ -16,12 +16,12 @@ class Network(BaseAccountingType):
 
         # IPv6 address has up to 45 chars
         self.definitionKeyFields = [
-            ("SourceIP", "VARCHAR(50)"),
-            ("DestinationIP", "VARCHAR(50)"),
-            ("SourceHostName", "VARCHAR(50)"),
-            ("DestinationHostName", "VARCHAR(50)"),
-            ("Source", "VARCHAR(50)"),
-            ("Destination", "VARCHAR(50)"),
+            ("SourceIP", "VARCHAR(64)"),
+            ("DestinationIP", "VARCHAR(64)"),
+            ("SourceHostName", "VARCHAR(255)"),
+            ("DestinationHostName", "VARCHAR(255)"),
+            ("Source", "VARCHAR(255)"),
+            ("Destination", "VARCHAR(255)"),
         ]
 
         self.definitionAccountingFields = [

DIRAC/AccountingSystem/Client/Types/PilotSubmission.py

@@ -1,6 +1,6 @@
-""" Accounting Type for Pilot Submission
+"""Accounting Type for Pilot Submission
 
-    Filled by the "WorkloadManagement/SiteDirector" agent(s)
+Filled by the "WorkloadManagement/SiteDirector" agent(s)
 """
 
 from DIRAC.AccountingSystem.Client.Types.BaseAccountingType import BaseAccountingType
@@ -13,7 +13,7 @@ class PilotSubmission(BaseAccountingType):
         super().__init__()
 
         self.definitionKeyFields = [
-            ("HostName", "VARCHAR(100)"),
+            ("HostName", "VARCHAR(255)"),
             ("SiteDirector", "VARCHAR(100)"),
             ("Site", "VARCHAR(100)"),
             ("CE", "VARCHAR(100)"),

DIRAC/ConfigurationSystem/Client/CSAPI.py

@@ -456,13 +456,23 @@ class CSAPI:
             gLogger.error("User is not registered: ", repr(username))
             return S_OK(False)
         for prop in properties:
-            if prop == "Groups":
+            if prop in ["Groups", "AffiliationEnds"]:
                 continue
             prevVal = self.__csMod.getValue(f"{self.__baseSecurity}/Users/{username}/{prop}")
             if not prevVal or prevVal != properties[prop]:
                 gLogger.info(f"Setting {prop} property for user {username} to {properties[prop]}")
                 self.__csMod.setOptionValue(f"{self.__baseSecurity}/Users/{username}/{prop}", properties[prop])
                 modifiedUser = True
+        if properties.get("AffiliationEnds", None):
+            user_affiliationends_section = f"{self.__baseSecurity}/Users/{username}/AffiliationEnds"
+            # add the section for AffiliationEnds
+            res = gConfig.getSections(user_affiliationends_section)
+            if not res["OK"]:
+                self.__csMod.createSection(user_affiliationends_section)
+            # now set value VO = end date
+            for vo, end_date in properties["AffiliationEnds"].items():
+                self.__csMod.setOptionValue(f"{user_affiliationends_section}/{vo}", end_date)
+                modifiedUser = True
         if "Groups" in properties:
             result = self.listGroups()
             if not result["OK"]:

DIRAC/ConfigurationSystem/Client/Helpers/CSGlobals.py

@@ -4,15 +4,6 @@ Some Helper functions to retrieve common location from the CS
 from DIRAC.Core.Utilities.Extensions import extensionsByPriority
 
 
-def getSetup() -> str:
-    """
-    Return setup name
-    """
-    from DIRAC import gConfig
-
-    return gConfig.getValue("/DIRAC/Setup", "")
-
-
 def getVO(defaultVO: str = "") -> str:
     """
     Return VO from configuration

DIRAC/ConfigurationSystem/Client/Helpers/Registry.py

@@ -3,18 +3,14 @@
 import errno
 import inspect
 import sys
-
-from threading import Lock
 from collections.abc import Iterable
+from threading import Lock
+from typing import Optional
 
 from cachetools import TTLCache, cached
 from cachetools.keys import hashkey
 
-
-from typing import Optional
-from collections.abc import Iterable
-
-from DIRAC import S_OK, S_ERROR
+from DIRAC import S_ERROR, S_OK
 from DIRAC.ConfigurationSystem.Client.Config import gConfig
 from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
 
@@ -488,28 +484,6 @@ def getVOMSAttributeForGroup(group):
     return gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VOMSRole", getDefaultVOMSAttribute())
 
 
-def getDefaultVOMSVO():
-    """Get default VOMS VO
-
-    :return: str
-    """
-    return gConfig.getValue(f"{gBaseRegistrySection}/DefaultVOMSVO", "") or getVO()
-
-
-def getVOMSVOForGroup(group):
-    """Search VOMS VO for group
-
-    :param str group: group name
-
-    :return: str
-    """
-    vomsVO = gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VOMSVO", getDefaultVOMSVO())
-    if not vomsVO:
-        vo = getVOForGroup(group)
-        vomsVO = getVOOption(vo, "VOMSName", "")
-    return vomsVO
-
-
 def getGroupsWithVOMSAttribute(vomsAttr):
     """Search groups with VOMS attribute
 

DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py

@@ -32,6 +32,9 @@ class CERNLDAPSyncPlugin:
         else:
             userDict["PrimaryCERNAccount"] = self._findOwnerAccountName(username, attributes)
 
+        if userDict["CERNAccountType"] in ["Primary", "Secondary"]:
+            userDict["CERNPersonId"] = attributes.get("employeeId", [None])[0]
+
     def _findOwnerAccountName(self, username, attributes):
         """Find the owner account from a CERN LDAP entry.
 
@@ -64,7 +67,7 @@ class CERNLDAPSyncPlugin:
         status, result, response, _ = self._connection.search(
             "OU=Users,OU=Organic Units,DC=cern,DC=ch",
             f"(CN={commonName})",
-            attributes=["cernAccountOwner", "cernAccountType"],
+            attributes=["cernAccountOwner", "cernAccountType", "employeeId"],
         )
         if not status:
             raise ValueError(f"Bad status from LDAP search: {result}")

DIRAC/ConfigurationSystem/ConfigTemplate.cfg

@@ -6,6 +6,9 @@ Services
   {
     HandlerPath = DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py
     Port = 9135
+    # Set to True to make a DiracX model validation on commits
+    # If validation fails, commit will also failed
+    VerifyDiracXSyncOnCommit = False
     # Subsection to configure authorization over the service
     Authorization
     {

DIRAC/ConfigurationSystem/private/Modificator.py

@@ -1,13 +1,16 @@
 """ This is the guy that actually modifies the content of the CS
 """
-import zlib
-import difflib
 import datetime
+import difflib
+import zlib
 
 from diraccfg import CFG
-from DIRAC.Core.Utilities import List
+
+from DIRAC import S_ERROR
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
+from DIRAC.Core.Utilities import List
+from DIRAC.FrameworkSystem.Utilities.diracx import diracxVerifyConfig
 
 
 class Modificator:
@@ -239,6 +242,11 @@ class Modificator:
         return str(self.cfgData)
 
     def commit(self):
+        retOpt = self.getValue("/Systems/Configuration/Services/Server/VerifyDiracXSyncOnCommit")
+        if retOpt == "True":
+            resVerif = diracxVerifyConfig(self.cfgData)
+            if not resVerif["OK"]:
+                return S_ERROR(resVerif["Message"])
         compressedData = zlib.compress(str(self.cfgData).encode(), 9)
         return self.rpcClient.commitNewData(compressedData)
 

DIRAC/ConfigurationSystem/private/RefresherBase.py

@@ -1,7 +1,7 @@
 import time
 
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
-from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs
+from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs, groupURLsByPriority
 from DIRAC.Core.Utilities import List
 from DIRAC.Core.Utilities.EventDispatcher import gEventDispatcher
 from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK
@@ -138,7 +138,9 @@ class RefresherBase:
         if not initialServerList:
             return S_OK()
 
-        randomServerList = List.randomize(initialServerList)
+        randomServerList = []
+        for urlGroup in groupURLsByPriority(initialServerList):
+            randomServerList.extend(List.randomize(urlGroup))
         gLogger.debug(f"Randomized server list is {', '.join(randomServerList)}")
 
         for sServer in randomServerList:

DIRAC/Core/DISET/ServiceReactor.py

@@ -200,6 +200,7 @@ class ServiceReactor:
                                   services at the same time
         """
         sel = self.__getListeningSelector(svcName)
+        throttleExpires = None
         while self.__alive:
             clientTransport = None
             try:
@@ -223,12 +224,19 @@ class ServiceReactor:
                 gLogger.warn(f"Client connected from banned ip {clientIP}")
                 clientTransport.close()
                 continue
+            # Handle throttling
+            if self.__services[svcName].wantsThrottle and throttleExpires is None:
+                throttleExpires = time.time() + THROTTLE_SERVICE_SLEEP_SECONDS
+            if throttleExpires:
+                if time.time() > throttleExpires:
+                    throttleExpires = None
+                else:
+                    gLogger.warn("Rejecting client due to throttling", str(clientTransport.getRemoteAddress()))
+                    clientTransport.close()
+                    continue
             # Handle connection
             self.__stats.connectionStablished()
             self.__services[svcName].handleConnection(clientTransport)
-            while self.__services[svcName].wantsThrottle:
-                gLogger.warn("Sleeping as service requested throttling", svcName)
-                time.sleep(THROTTLE_SERVICE_SLEEP_SECONDS)
             # Renew context?
             now = time.time()
             renewed = False

DIRAC/Core/DISET/private/Transports/M2SSLTransport.py

@@ -110,19 +110,18 @@ class SSLTransport(BaseTransport):
         if self.serverMode():
             raise RuntimeError("SSLTransport is in server mode.")
 
-        error = None
+        errors = []
         host, port = self.stServerAddress
 
         # The following piece of code was inspired by the python socket documentation
         # as well as the implementation of M2Crypto.httpslib.HTTPSConnection
 
-        # We ignore the returned sockaddr because SSL.Connection.connect needs
-        # a host name.
+        # Get all available addresses (IPv6 and IPv4) and try them in order
         try:
             addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
         except OSError as e:
             return S_ERROR(f"DNS lookup failed {e!r}")
-        for family, _socketType, _proto, _canonname, _socketAddress in addrInfoList:
+        for family, _socketType, _proto, _canonname, socketAddress in addrInfoList:
             try:
                 self.oSocket = SSL.Connection(self.__ctx, family=family)
 
@@ -138,7 +137,10 @@ class SSLTransport(BaseTransport):
                 # set SNI server name since we know it at this point
                 self.oSocket.set_tlsext_host_name(host)
 
-                self.oSocket.connect((host, port))
+                # tell the connection which host we are connecting to so we can
+                # use the address we obtained from DNS
+                self.oSocket.set1_host(host)
+                self.oSocket.connect(socketAddress)
 
                 # Once the connection is established, we can use the timeout
                 # asked for RPC
@@ -151,12 +153,12 @@ class SSLTransport(BaseTransport):
             # They should be propagated upwards and caught by the BaseClient
             # not to enter the retry loop
             except OSError as e:
-                error = f"{e}:{repr(e)}"
+                errors.append(f"{socketAddress} {e}:{repr(e)}")
 
                 if self.oSocket is not None:
                     self.close()
 
-        return S_ERROR(error)
+        return S_ERROR("; ".join(errors))
 
     def initAsServer(self):
         """Prepare this server socket for use."""

DIRAC/Core/Security/DiracX.py

@@ -40,7 +40,7 @@ from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueO
 
 PEM_BEGIN = "-----BEGIN DIRACX-----"
 PEM_END = "-----END DIRACX-----"
-RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(.*)\n{PEM_END}", re.MULTILINE | re.DOTALL)
+RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(.*?)\n{PEM_END}", re.DOTALL)
 
 
 @convertToReturnValue
@@ -62,21 +62,26 @@ def addTokenToPEM(pemPath, group):
             token_type=token_content.get("token_type"),
             refresh_token=token_content.get("refresh_token"),
         )
-
         token_pem = f"{PEM_BEGIN}\n"
         data = base64.b64encode(serialize_credentials(token).encode("utf-8")).decode()
         token_pem += textwrap.fill(data, width=64)
         token_pem += f"\n{PEM_END}\n"
 
-        with open(pemPath, "a") as f:
-            f.write(token_pem)
+        pem = Path(pemPath).read_text()
+        # Remove any existing DiracX token there would be
+        new_pem = re.sub(RE_DIRACX_PEM, "", pem)
+        new_pem += token_pem
+
+        Path(pemPath).write_text(new_pem)
 
 
 def diracxTokenFromPEM(pemPath) -> dict[str, Any] | None:
     """Extract the DiracX token from the proxy PEM file"""
     pem = Path(pemPath).read_text()
-    if match := RE_DIRACX_PEM.search(pem):
-        match = match.group(1)
+    if match := RE_DIRACX_PEM.findall(pem):
+        if len(match) > 1:
+            raise ValueError("Found multiple DiracX tokens, this should never happen")
+        match = match[0]
         return json.loads(base64.b64decode(match).decode("utf-8"))
 
 

DIRAC/Core/Security/test/test_diracx_token_from_pem.py

@@ -0,0 +1,161 @@
+import base64
+import json
+import pytest
+import tempfile
+from pathlib import Path
+from unittest.mock import patch, mock_open
+
+from DIRAC.Core.Security.DiracX import diracxTokenFromPEM, PEM_BEGIN, PEM_END, RE_DIRACX_PEM
+
+
+class TestDiracxTokenFromPEM:
+    """Test cases for diracxTokenFromPEM function"""
+
+    def create_valid_token_data(self):
+        """Create valid token data for testing"""
+        return {
+            "access_token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.test",
+            "refresh_token": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.refresh",
+            "expires_in": 3600,
+            "token_type": "Bearer",
+        }
+
+    def create_pem_content(self, token_data=None, include_other_content=True):
+        """Create PEM content with embedded DiracX token"""
+        if token_data is None:
+            token_data = self.create_valid_token_data()
+
+        # Encode token data
+        token_json = json.dumps(token_data)
+        encoded_token = base64.b64encode(token_json.encode("utf-8")).decode()
+
+        # Create PEM content
+        pem_content = ""
+        if include_other_content:
+            pem_content += "-----BEGIN CERTIFICATE-----\n"
+            pem_content += "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...\n"
+            pem_content += "-----END CERTIFICATE-----\n"
+
+        pem_content += f"{PEM_BEGIN}\n"
+        pem_content += encoded_token + "\n"
+        pem_content += f"{PEM_END}\n"
+
+        return pem_content
+
+    def test_valid_token_extraction(self):
+        """Test successful extraction of valid token from PEM file"""
+        token_data = self.create_valid_token_data()
+        pem_content = self.create_pem_content(token_data)
+
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
+            f.write(pem_content)
+            temp_path = f.name
+
+        try:
+            result = diracxTokenFromPEM(temp_path)
+            assert result == token_data
+        finally:
+            Path(temp_path).unlink()
+
+    def test_no_token_in_pem(self):
+        """Test behavior when no DiracX token is present in PEM file"""
+        pem_content = """-----BEGIN CERTIFICATE-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA...
+-----END CERTIFICATE-----"""
+
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
+            f.write(pem_content)
+            temp_path = f.name
+
+        try:
+            result = diracxTokenFromPEM(temp_path)
+            assert result is None
+        finally:
+            Path(temp_path).unlink()
+
+    def test_multiple_tokens_error(self):
+        """Test that multiple tokens raise ValueError"""
+        token_data = self.create_valid_token_data()
+
+        # Create PEM with two tokens
+        pem_content = self.create_pem_content(token_data)
+        pem_content += "\n" + self.create_pem_content(token_data, include_other_content=False)
+
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
+            f.write(pem_content)
+            temp_path = f.name
+
+        try:
+            with pytest.raises(ValueError, match="Found multiple DiracX tokens"):
+                diracxTokenFromPEM(temp_path)
+        finally:
+            Path(temp_path).unlink()
+
+    def test_malformed_base64(self):
+        """Test behavior with malformed base64 data"""
+        pem_content = f"""{PEM_BEGIN}
+invalid_base64_data_that_will_cause_error!
+{PEM_END}"""
+
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
+            f.write(pem_content)
+            temp_path = f.name
+
+        try:
+            with pytest.raises(Exception):  # base64.b64decode will raise an exception
+                diracxTokenFromPEM(temp_path)
+        finally:
+            Path(temp_path).unlink()
+
+    def test_invalid_json_in_token(self):
+        """Test behavior with invalid JSON in token data"""
+        invalid_json = "this is not valid json"
+        encoded_invalid = base64.b64encode(invalid_json.encode("utf-8")).decode()
+
+        pem_content = f"""{PEM_BEGIN}
+{encoded_invalid}
+{PEM_END}"""
+
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
+            f.write(pem_content)
+            temp_path = f.name
+
+        try:
+            with pytest.raises(json.JSONDecodeError):
+                diracxTokenFromPEM(temp_path)
+        finally:
+            Path(temp_path).unlink()
+
+    def test_token_with_unicode_characters(self):
+        """Test token with unicode characters"""
+        unicode_token = {
+            "access_token": "token_with_unicode_ñ_é_ü",
+            "refresh_token": "refresh_with_emoji_🚀_🎉",
+            "expires_in": 3600,
+            "token_type": "Bearer",
+        }
+
+        pem_content = self.create_pem_content(unicode_token)
+
+        with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".pem") as f:
+            f.write(pem_content)
+            temp_path = f.name
+
+        try:
+            result = diracxTokenFromPEM(temp_path)
+            assert result == unicode_token
+        finally:
+            Path(temp_path).unlink()
+
+    def test_regex_pattern_validation(self):
+        """Test that the regex pattern correctly identifies DiracX tokens"""
+        # Test that the regex matches the expected pattern
+        token_data = self.create_valid_token_data()
+        token_json = json.dumps(token_data)
+        encoded_token = base64.b64encode(token_json.encode("utf-8")).decode()
+
+        test_content = f"{PEM_BEGIN}\n{encoded_token}\n{PEM_END}"
+        matches = RE_DIRACX_PEM.findall(test_content)
+
+        assert len(matches) == 1
+        assert matches[0] == encoded_token

DIRAC/Core/Tornado/Server/TornadoService.py

@@ -73,7 +73,7 @@ class TornadoService(BaseRequestHandler):  # pylint: disable=abstract-method
     :py:class:`BaseRequestHandler <DIRAC.Core.Tornado.Server.private.BaseRequestHandler.BaseRequestHandler>` for more details.
 
     In order to pass information around and keep some states, we use instance attributes.
-    These are initialized in the :py:meth:`.initialize` method.
+    These are initialized in the ``initialize`` methods.
 
     The handler only define the ``post`` verb. Please refer to :py:meth:`.post` for the details.
 

DIRAC/Core/Utilities/ElasticSearchDB.py

@@ -82,7 +82,6 @@ def generateDocs(data, withTimeStamp=True):
 
 
 class ElasticSearchDB:
-
     """
     .. class:: ElasticSearchDB
 
@@ -506,7 +505,7 @@ class ElasticSearchDB:
             indexName = self.generateFullIndexName(indexPrefix, period)
         else:
             indexName = indexPrefix
-        sLog.debug(f"Bulk indexing into {indexName} of {data}")
+        sLog.debug(f"Bulk indexing into {indexName} of {len(data)}")
 
         res = self.existingIndex(indexName)
         if not res["OK"]: