DIRAC 9.0.0a67__tar.gz → 9.0.0a69__tar.gz

{dirac-9.0.0a67/src/DIRAC.egg-info → dirac-9.0.0a69}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.0a67
+Version: 9.0.0a69
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.0a67
+Requires-Dist: DIRACCommon==v9.0.0a69
 Requires-Dist: diracx-client>=v0.0.1a18
 Requires-Dist: diracx-core>=v0.0.1a18
 Requires-Dist: db12

{dirac-9.0.0a67 → dirac-9.0.0a69}/release.notes

@@ -1,3 +1,16 @@
+[v9.0.0a69]
+
+*WorkloadManagementSystem
+
+CHANGE: (#8297) Remove the master CS aliases from the list of servers when generating the pilot.json
+FIX: (#8296) removed MatcherServiceTime JobParameters
+
+[v9.0.0a68]
+
+*ConfigurationSystem
+
+CHANGE: (#8295) Move getDIRACPlatform to DIRACCommon
+
 [v9.0.0a67]
 
 *DataManagementSystem

{dirac-9.0.0a67 → dirac-9.0.0a69}/setup.cfg

@@ -26,7 +26,7 @@ install_requires =
 	certifi
 	cwltool
 	diraccfg
-	DIRACCommon==v9.0.0a67
+	DIRACCommon==v9.0.0a69
 	diracx-client >=v0.0.1a18
 	diracx-core >=v0.0.1a18
 	db12

{dirac-9.0.0a67 → dirac-9.0.0a69}/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py

@@ -1,11 +1,14 @@
 """ Helper for the CS Resources section
 """
-import re
 from urllib import parse
 
 from DIRAC import S_ERROR, S_OK, gConfig, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
 from DIRAC.Core.Utilities.List import fromChar, uniqueElements
+from DIRACCommon.ConfigurationSystem.Client.Helpers.Resources import (
+    getDIRACPlatform as _getDIRACPlatform,
+    _platformSortKey,
+)
 
 gBaseResourcesSection = "/Resources"
 
@@ -328,8 +331,8 @@ def getCompatiblePlatforms(originalPlatforms):
     if not (result["OK"] and result["Value"]):
         return S_ERROR("OS compatibility info not found")
 
-    platformsDict = {k: v.replace(" ", "").split(",") for k, v in result["Value"].items()}  # can be an iterator
-    for k, v in platformsDict.items():  # can be an iterator
+    platformsDict = {k: v.replace(" ", "").split(",") for k, v in result["Value"].items()}
+    for k, v in platformsDict.items():
         if k not in v:
             v.append(k)
 
@@ -355,7 +358,6 @@ def getDIRACPlatform(OSList):
     :param list OSList: list of platforms defined by resource providers
     :return: a list of DIRAC platforms that can be specified in job descriptions
     """
-
     # For backward compatibility allow a single string argument
     osList = OSList
     if isinstance(OSList, str):
@@ -365,31 +367,12 @@ def getDIRACPlatform(OSList):
     if not (result["OK"] and result["Value"]):
         return S_ERROR("OS compatibility info not found")
 
-    platformsDict = {k: v.replace(" ", "").split(",") for k, v in result["Value"].items()}  # can be an iterator
-    for k, v in platformsDict.items():  # can be an iterator
+    platformsDict = {k: set(v.replace(" ", "").split(",")) for k, v in result["Value"].items()}
+    for k, v in platformsDict.items():
         if k not in v:
-            v.append(k)
-
-    # making an OS -> platforms dict
-    os2PlatformDict = dict()
-    for platform, osItems in platformsDict.items():  # can be an iterator
-        for osItem in osItems:
-            if os2PlatformDict.get(osItem):
-                os2PlatformDict[osItem].append(platform)
-            else:
-                os2PlatformDict[osItem] = [platform]
-
-    platforms = []
-    for os in osList:
-        if os in os2PlatformDict:
-            platforms += os2PlatformDict[os]
-
-    if not platforms:
-        return S_ERROR(f"No compatible DIRAC platform found for {','.join(OSList)}")
+            v.add(k)
 
-    platforms.sort(key=_platformSortKey, reverse=True)
-
-    return S_OK(platforms)
+    return _getDIRACPlatform(osList, platformsDict)
 
 
 def getDIRACPlatforms():
@@ -451,7 +434,7 @@ def getInfoAboutProviders(of=None, providerName=None, option="", section=""):
             result = gConfig.getConfigurationTree(relPath)
             if not result["OK"]:
                 return result
-            for key, value in result["Value"].items():  # can be an iterator
+            for key, value in result["Value"].items():
                 if value:
                     resDict[key.replace(relPath, "")] = value
             return S_OK(resDict)
@@ -459,18 +442,3 @@ def getInfoAboutProviders(of=None, providerName=None, option="", section=""):
             return gConfig.getSections(f"{gBaseResourcesSection}/{of}Providers/{providerName}/{section}/")
     else:
         return S_OK(gConfig.getValue(f"{gBaseResourcesSection}/{of}Providers/{providerName}/{section}/{option}"))
-
-
-def _platformSortKey(version: str) -> list[str]:
-    # Loosely based on distutils.version.LooseVersion
-    parts = []
-    for part in re.split(r"(\d+|[a-z]+|\.| -)", version.lower()):
-        if not part or part == ".":
-            continue
-        if part[:1] in "0123456789":
-            part = part.zfill(8)
-        else:
-            while parts and parts[-1] == "00000000":
-                parts.pop()
-        parts.append(part)
-    return parts

{dirac-9.0.0a67 → dirac-9.0.0a69}/src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py

@@ -9,7 +9,6 @@ from DIRAC.ConfigurationSystem.Client import ConfigurationData
 from DIRAC.ConfigurationSystem.Client.Helpers.Resources import (
     getDIRACPlatform,
     getCompatiblePlatforms,
-    _platformSortKey,
     getQueue,
 )
 
@@ -77,21 +76,6 @@ def test_getDIRACPlatform(mocker, mockGCReplyInput, requested, expectedRes, expe
         assert set(res["Value"]) == set(expectedValue), res["Value"]
 
 
-@pytest.mark.parametrize(
-    "string,expected",
-    [
-        ("Darwin_arm64_12.4", ["darwin", "_", "arm", "64", "_", "12", "4"]),
-        ("Linux_x86_64_glibc-2.17", ["linux", "_", "x", "86", "_", "64", "_", "glibc", "-", "2", "17"]),
-        ("Linux_aarch64_glibc-2.28", ["linux", "_", "aarch", "64", "_", "glibc", "-", "2", "28"]),
-    ],
-)
-def test_platformSortKey(string, expected):
-    actual = _platformSortKey(string)
-    for a, e in zip_longest(actual, expected):
-        # Numbers are padded with zeros so string comparison works
-        assert a.lstrip("0") == e
-
-
 @pytest.mark.parametrize(
     "mockGCReplyInput, requested, expectedRes, expectedValue",
     [

{dirac-9.0.0a67 → dirac-9.0.0a69}/src/DIRAC/Interfaces/scripts/dirac_wms_job_parameters.py

@@ -17,7 +17,6 @@ Example:
    'LocalAccount': 'prod006',
    'LocalBatchID': '',
    'LocalJobID': '277821.ce.labmc.inf.utfsm.cl',
-   'MatcherServiceTime': '2.27646398544',
    'Memory(kB)': '858540kB',
    'ModelName': 'Intel(R)Xeon(R)CPU5110@1.60GHz',
    'NormCPUTime(s)': '1.02',

{dirac-9.0.0a67 → dirac-9.0.0a69}/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py

@@ -236,7 +236,6 @@ class JobAgent(AgentModule):
         jobGroup = matcherInfo["Group"]
         owner = matcherInfo["Owner"]
         ceDict = matcherInfo["CEDict"]
-        matchTime = matcherInfo["matchTime"]
 
         optimizerParams = {}
         for key in matcherInfo:
@@ -263,9 +262,6 @@ class JobAgent(AgentModule):
         self.log.verbose("Job request successful: \n", jobRequest["Value"])
         self.log.info("Received", f"JobID={jobID}, JobType={jobType}, Owner={owner}, JobGroup={jobGroup}")
         self.jobCount += 1
-        self.jobs[jobID]["JobReport"].setJobParameter(
-            par_name="MatcherServiceTime", par_value=str(matchTime), sendFlag=False
-        )
 
         self.jobs[jobID]["JobReport"].setJobStatus(minorStatus="Job Received by Agent", sendFlag=False)
         result_setupProxy = self._setupProxy(owner, jobGroup)
@@ -547,7 +543,7 @@ class JobAgent(AgentModule):
             jobRequest = MatcherClient().requestJob(ceDict)
             matchTime = time.time() - start
 
-            self.log.info("MatcherTime", f"= {matchTime:.2f} (s)")
+            self.log.verbose("MatcherTime", f"= {matchTime:.2f} (s)")
             if jobRequest["OK"]:
                 jobRequest["Value"]["matchTime"] = matchTime
                 jobRequest["Value"]["CEDict"] = ceDict

{dirac-9.0.0a67 → dirac-9.0.0a69}/src/DIRAC/WorkloadManagementSystem/Agent/PilotSyncAgent.py

@@ -1,4 +1,4 @@
-""" This agent syncs CS and pilot files to a web server of your choice
+"""This agent syncs CS and pilot files to a web server of your choice
 
 .. literalinclude:: ../ConfigTemplate.cfg
   :start-after: ##BEGIN PilotSyncAgent
@@ -7,6 +7,7 @@
   :caption: PilotsSyncAgent options
 
 """
+
 import os
 import json
 import shutil
@@ -38,8 +39,8 @@ class PilotSyncAgent(AgentModule):
         self.workingDirectory = self.am_getOption("WorkDirectory")
         self.saveDir = self.am_getOption("SaveDirectory", self.saveDir)
         self.uploadLocations = self.am_getOption("UploadLocations", self.uploadLocations)
-        includeMasterCS = self.am_getOption("IncludeMasterCS", self.includeMasterCS)
-        if isinstance(includeMasterCS, str) and includeMasterCS.lower() in ["n", "no", "false"]:
+        self.includeMasterCS = self.am_getOption("IncludeMasterCS", self.includeMasterCS)
+        if isinstance(self.includeMasterCS, str) and self.includeMasterCS.lower() in ["n", "no", "false"]:
             self.includeMasterCS = False
 
         self.certAndKeyLocation = getHostCertificateAndKeyLocation()

{dirac-9.0.0a67 → dirac-9.0.0a69}/src/DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py

@@ -285,7 +285,6 @@ class PushJobAgent(JobAgent):
                 jobGroup = matcherInfo["Group"]
                 owner = matcherInfo["Owner"]
                 ceDict = matcherInfo["CEDict"]
-                matchTime = matcherInfo["matchTime"]
 
                 optimizerParams = {}
                 for key in matcherInfo:
@@ -313,9 +312,6 @@ class PushJobAgent(JobAgent):
                 self.log.verbose("Job request successful: \n", jobRequest["Value"])
                 self.log.info("Received", f"JobID={jobID}, JobType={jobType}, Owner={owner}, JobGroup={jobGroup}")
 
-                self.jobs[jobID]["JobReport"].setJobParameter(
-                    par_name="MatcherServiceTime", par_value=str(matchTime), sendFlag=False
-                )
                 self.jobs[jobID]["JobReport"].setJobStatus(
                     status=JobStatus.MATCHED, minorStatus="Job Received by Agent", sendFlag=False
                 )

{dirac-9.0.0a67 → dirac-9.0.0a69}/src/DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py

@@ -1,9 +1,10 @@
-""" CStoJSONSynchronizer
-  Module that keeps the pilot parameters file synchronized with the information
-  in the Operations/Pilot section of the CS. If there are additions in the CS,
-  these are incorporated to the file.
-  The module uploads to a web server the latest version of the pilot scripts.
+"""CStoJSONSynchronizer
+Module that keeps the pilot parameters file synchronized with the information
+in the Operations/Pilot section of the CS. If there are additions in the CS,
+these are incorporated to the file.
+The module uploads to a web server the latest version of the pilot scripts.
 """
+
 import datetime
 import glob
 import os
@@ -19,6 +20,67 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
 from DIRAC.Core.Utilities.ReturnValues import DOKReturnType, DReturnType
 
+import socket
+from urllib.parse import urlparse
+
+
+def exclude_master_cs_aliases(urls: list[str], master_cs_url: str) -> list[str]:
+    """
+    Excludes URLs that are DNS aliases of the given MasterCS server URL.
+
+    This function resolves the IP addresses of the MasterCS server and each URL in the input list.
+    It returns a new list containing only those URLs whose hostnames do not resolve to any of the
+    MasterCS server's IP addresses, effectively excluding all DNS aliases of the MasterCS server.
+
+    Args:
+        urls (list[str]): A list of URLs to filter. Each URL should be a string in a valid URL format.
+        master_cs_url (str): The reference URL (e.g., MasterCS server URL) whose DNS aliases are to be excluded.
+
+    Returns:
+        list[str]: A new list of URLs with all aliases of the MasterCS server removed.
+                  If the MasterCS hostname cannot be resolved, the original list is returned unchanged.
+
+    Example:
+        >>> urls = [
+        ...     'dips://lbvobox303.cern.ch:9135/Configuration/Server',
+        ...     'dips://ccwlcglhcb02.in2p3.fr:9135/Configuration/Server',
+        ...     'dips://lbvobox302.cern.ch:9135/Configuration/Server',
+        ... ]
+        >>> master_cs_url = "dips://mastercs.cern.ch:9135/Configuration/Server"
+        >>> exclude_master_cs_aliases(urls, master_cs_url)
+        ['dips://ccwlcglhcb02.in2p3.fr:9135/Configuration/Server']
+
+    Notes:
+        - If the MasterCS hostname cannot be resolved, the function returns the original list.
+        - If a hostname in the input list cannot be resolved, it is included in the result.
+        - The comparison is based on IP addresses, not hostnames.
+    """
+    master_cs_hostname = urlparse(master_cs_url).hostname
+    if not master_cs_hostname:
+        return urls
+
+    # Resolve IP addresses for the MasterCS hostname
+    try:
+        master_cs_ips = set(socket.gethostbyname_ex(master_cs_hostname)[2])
+    except socket.gaierror:
+        return urls
+
+    # Function to get IPs for a hostname
+    def get_ips(hostname):
+        try:
+            return set(socket.gethostbyname_ex(hostname)[2])
+        except socket.gaierror:
+            return set()
+
+    filtered_urls = []
+    for url in urls:
+        hostname = urlparse(url).hostname
+        ips = get_ips(hostname)
+        if not ips & master_cs_ips:
+            filtered_urls.append(url)
+
+    return filtered_urls
+
 
 class PilotCStoJSONSynchronizer:
     """
@@ -151,7 +213,8 @@ class PilotCStoJSONSynchronizer:
         configurationServers = gConfig.getServersList()
         if not includeMasterCS:
             masterCS = gConfigurationData.getMasterServer()
-            configurationServers = list(set(configurationServers) - {masterCS})
+            configurationServers = exclude_master_cs_aliases(configurationServers, masterCS)
+
         pilotDict["ConfigurationServers"] = configurationServers
 
         self.log.debug("Got pilotDict", str(pilotDict))

{dirac-9.0.0a67 → dirac-9.0.0a69/src/DIRAC.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.0a67
+Version: 9.0.0a69
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,7 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.0a67
+Requires-Dist: DIRACCommon==v9.0.0a69
 Requires-Dist: diracx-client>=v0.0.1a18
 Requires-Dist: diracx-core>=v0.0.1a18
 Requires-Dist: db12

{dirac-9.0.0a67 → dirac-9.0.0a69}/src/DIRAC.egg-info/requires.txt

@@ -4,7 +4,7 @@ cachetools
 certifi
 cwltool
 diraccfg
-DIRACCommon==v9.0.0a67
+DIRACCommon==v9.0.0a69
 diracx-client>=v0.0.1a18
 diracx-core>=v0.0.1a18
 db12