DIRAC 9.0.0a68__py3-none-any.whl → 9.0.0a70__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py

@@ -65,6 +65,11 @@ class JobManagerHandlerMixin:
                 return result
             cls.pilotAgentsDB = result["Value"](parentLogger=cls.log)
 
+            result = ObjectLoader().loadObject("StorageManagementSystem.DB.StorageManagementDB", "StorageManagementDB")
+            if not result["OK"]:
+                return result
+            cls.storageManagementDB = result["Value"](parentLogger=cls.log)
+
         except RuntimeError as excp:
             return S_ERROR(f"Can't connect to DB: {excp!r}")
 
@@ -449,7 +454,16 @@ class JobManagerHandlerMixin:
             jobList, RIGHT_DELETE
         )
 
-        result = kill_delete_jobs(RIGHT_DELETE, validJobList, nonauthJobList, force=force)
+        result = kill_delete_jobs(
+            RIGHT_DELETE,
+            validJobList,
+            nonauthJobList,
+            force=force,
+            jobdb=self.jobDB,
+            taskqueuedb=self.taskQueueDB,
+            pilotagentsdb=self.pilotAgentsDB,
+            storagemanagementdb=self.storageManagementDB,
+        )
 
         result["requireProxyUpload"] = len(ownerJobList) > 0 and self.__checkIfProxyUploadIsRequired()
 
@@ -478,7 +492,16 @@ class JobManagerHandlerMixin:
             jobList, RIGHT_KILL
         )
 
-        result = kill_delete_jobs(RIGHT_KILL, validJobList, nonauthJobList, force=force)
+        result = kill_delete_jobs(
+            RIGHT_KILL,
+            validJobList,
+            nonauthJobList,
+            force=force,
+            jobdb=self.jobDB,
+            taskqueuedb=self.taskQueueDB,
+            pilotagentsdb=self.pilotAgentsDB,
+            storagemanagementdb=self.storageManagementDB,
+        )
 
         result["requireProxyUpload"] = len(ownerJobList) > 0 and self.__checkIfProxyUploadIsRequired()
 

DIRAC/WorkloadManagementSystem/Service/WMSAdministratorHandler.py

@@ -150,37 +150,24 @@ class WMSAdministratorHandlerMixin:
         :param str jobID: job ID
         :return: S_OK(dict)/S_ERROR()
         """
-        pilotReference = ""
-        # Get the pilot grid reference first from the job parameters
-
-        credDict = self.getRemoteCredentials()
-        vo = credDict.get("VO", Registry.getVOForGroup(credDict["group"]))
-        res = self.elasticJobParametersDB.getJobParameters(int(jobID), vo=vo, paramList=["Pilot_Reference"])
-        if not res["OK"]:
-            return res
-        if res["Value"].get(int(jobID)):
-            pilotReference = res["Value"][int(jobID)]["Pilot_Reference"]
-
-        if not pilotReference:
-            res = self.jobDB.getJobParameter(int(jobID), "Pilot_Reference")
-            if not res["OK"]:
-                return res
-            pilotReference = res["Value"]
-
-        if not pilotReference:
-            # Failed to get the pilot reference, try to look in the attic parameters
-            res = self.jobDB.getAtticJobParameters(int(jobID), ["Pilot_Reference"])
-            if res["OK"]:
-                c = -1
-                # Get the pilot reference for the last rescheduling cycle
-                for cycle in res["Value"]:
-                    if cycle > c:
-                        pilotReference = res["Value"][cycle]["Pilot_Reference"]
-                        c = cycle
-
-        if pilotReference:
-            return self.pilotManager.getPilotOutput(pilotReference)
-        return S_ERROR("No pilot job reference found")
+        result = self.pilotManager.getPilots(jobID)
+
+        if not result["OK"]:
+            return result
+        pilotJobReferences = result["Value"].keys()
+
+        outputs = {"StdOut": "", "StdErr": ""}
+        for pilotRef in pilotJobReferences:
+            result = self.pilotManager.getPilotOutput(pilotRef)
+            if not result["OK"]:
+                stdout = f"Could not retrieve output: {result['Message']}"
+                error = f"Could not retrieve error: {result['Message']}"
+            else:
+                stdout, error = result["Value"]["StdOut"], result["Value"]["StdErr"]
+            outputs["StdOut"] += f"# PilotJobReference: {pilotRef}\n\n{stdout}\n"
+            outputs["StdErr"] += f"# PilotJobReference: {pilotRef}\n\n{error}\n"
+
+        return S_OK(outputs)
 
 
 class WMSAdministratorHandler(WMSAdministratorHandlerMixin, RequestHandler):

DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py

@@ -1,16 +1,16 @@
-""" CStoJSONSynchronizer
-  Module that keeps the pilot parameters file synchronized with the information
-  in the Operations/Pilot section of the CS. If there are additions in the CS,
-  these are incorporated to the file.
-  The module uploads to a web server the latest version of the pilot scripts.
+"""CStoJSONSynchronizer
+Module that keeps the pilot parameters file synchronized with the information
+in the Operations/Pilot section of the CS. If there are additions in the CS,
+these are incorporated to the file.
+The module uploads to a web server the latest version of the pilot scripts.
 """
+
 import datetime
 import glob
 import os
 import shutil
 import tarfile
 from typing import Any
-
 from git import Repo
 
 from DIRAC import S_OK, gConfig, gLogger
@@ -19,6 +19,67 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
 from DIRAC.Core.Utilities.ReturnValues import DOKReturnType, DReturnType
 
+import socket
+from urllib.parse import urlparse
+
+
+def exclude_master_cs_aliases(urls: list[str], master_cs_url: str) -> list[str]:
+    """
+    Excludes URLs that are DNS aliases of the given MasterCS server URL.
+
+    This function resolves the IP addresses of the MasterCS server and each URL in the input list.
+    It returns a new list containing only those URLs whose hostnames do not resolve to any of the
+    MasterCS server's IP addresses, effectively excluding all DNS aliases of the MasterCS server.
+
+    Args:
+        urls (list[str]): A list of URLs to filter. Each URL should be a string in a valid URL format.
+        master_cs_url (str): The reference URL (e.g., MasterCS server URL) whose DNS aliases are to be excluded.
+
+    Returns:
+        list[str]: A new list of URLs with all aliases of the MasterCS server removed.
+                  If the MasterCS hostname cannot be resolved, the original list is returned unchanged.
+
+    Example:
+        >>> urls = [
+        ...     'dips://lbvobox303.cern.ch:9135/Configuration/Server',
+        ...     'dips://ccwlcglhcb02.in2p3.fr:9135/Configuration/Server',
+        ...     'dips://lbvobox302.cern.ch:9135/Configuration/Server',
+        ... ]
+        >>> master_cs_url = "dips://mastercs.cern.ch:9135/Configuration/Server"
+        >>> exclude_master_cs_aliases(urls, master_cs_url)
+        ['dips://ccwlcglhcb02.in2p3.fr:9135/Configuration/Server']
+
+    Notes:
+        - If the MasterCS hostname cannot be resolved, the function returns the original list.
+        - If a hostname in the input list cannot be resolved, it is included in the result.
+        - The comparison is based on IP addresses, not hostnames.
+    """
+    master_cs_hostname = urlparse(master_cs_url).hostname
+    if not master_cs_hostname:
+        return urls
+
+    # Resolve IP addresses for the MasterCS hostname
+    try:
+        master_cs_ips = set(socket.gethostbyname_ex(master_cs_hostname)[2])
+    except socket.gaierror:
+        return urls
+
+    # Function to get IPs for a hostname
+    def get_ips(hostname):
+        try:
+            return set(socket.gethostbyname_ex(hostname)[2])
+        except socket.gaierror:
+            return set()
+
+    filtered_urls = []
+    for url in urls:
+        hostname = urlparse(url).hostname
+        ips = get_ips(hostname)
+        if not ips & master_cs_ips:
+            filtered_urls.append(url)
+
+    return filtered_urls
+
 
 class PilotCStoJSONSynchronizer:
     """
@@ -151,9 +212,14 @@ class PilotCStoJSONSynchronizer:
         configurationServers = gConfig.getServersList()
         if not includeMasterCS:
             masterCS = gConfigurationData.getMasterServer()
-            configurationServers = list(set(configurationServers) - {masterCS})
+            configurationServers = exclude_master_cs_aliases(configurationServers, masterCS)
+
         pilotDict["ConfigurationServers"] = configurationServers
 
+        preferredURLPatterns = gConfigurationData.extractOptionFromCFG("/DIRAC/PreferredURLPatterns")
+        if preferredURLPatterns:
+            pilotDict["PreferredURLPatterns"] = preferredURLPatterns.replace(" ", "").split(",")
+
         self.log.debug("Got pilotDict", str(pilotDict))
 
         return S_OK(pilotDict)

{dirac-9.0.0a68.dist-info → dirac-9.0.0a70.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.0a68
+Version: 9.0.0a70
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,9 +19,10 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
-Requires-Dist: DIRACCommon==v9.0.0a68
-Requires-Dist: diracx-client>=v0.0.1a18
-Requires-Dist: diracx-core>=v0.0.1a18
+Requires-Dist: DIRACCommon==v9.0.0a70
+Requires-Dist: diracx-client>=v0.0.1
+Requires-Dist: diracx-core>=v0.0.1
+Requires-Dist: diracx-cli>=v0.0.1
 Requires-Dist: db12
 Requires-Dist: fts3
 Requires-Dist: gfal2-python
@@ -40,13 +41,13 @@ Requires-Dist: python-dateutil
 Requires-Dist: pytz
 Requires-Dist: requests
 Requires-Dist: rucio-clients>=34.4.2
-Requires-Dist: setuptools
 Requires-Dist: sqlalchemy
 Requires-Dist: typing_extensions>=4.3.0
 Requires-Dist: Authlib>=1.0.0.a2
 Requires-Dist: pyjwt
 Requires-Dist: dominate
 Requires-Dist: zstandard
+Requires-Dist: xattr
 Provides-Extra: server
 Requires-Dist: CMRESHandler; extra == "server"
 Requires-Dist: opensearch-py; extra == "server"