DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py

@@ -4,176 +4,12 @@
     getParameterVectorLength() - to get the total size of the bunch of parametric jobs
     generateParametricJobs() - to get a list of expanded descriptions of all the jobs
 """
-import re
 
-from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
-from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
-from DIRAC.Core.Utilities.DErrno import EWMSJDL
+# Import from DIRACCommon for backward compatibility
+from DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob import (
+    getParameterVectorLength,
+    generateParametricJobs,
+)
 
-
-def __getParameterSequence(nPar, parList=[], parStart=1, parStep=0, parFactor=1):
-    if parList:
-        if nPar != len(parList):
-            return []
-        else:
-            parameterList = list(parList)
-    else:
-        # The first parameter must have the same type as the other ones even if not defined explicitly
-        parameterList = [parStart * type(parFactor)(1) + type(parStep)(0)]
-        for np in range(1, nPar):
-            parameterList.append(parameterList[np - 1] * parFactor + parStep)
-
-    return parameterList
-
-
-def getParameterVectorLength(jobClassAd):
-    """Get the length of parameter vector in the parametric job description
-
-    :param jobClassAd: ClassAd job description object
-    :return: result structure with the Value: int number of parameter values, None if not a parametric job
-    """
-
-    nParValues = None
-    attributes = jobClassAd.getAttributes()
-    for attribute in attributes:
-        if attribute.startswith("Parameters"):
-            if jobClassAd.isAttributeList(attribute):
-                parameterList = jobClassAd.getListFromExpression(attribute)
-                nThisParValues = len(parameterList)
-            else:
-                nThisParValues = jobClassAd.getAttributeInt(attribute)
-            if nParValues is not None and nParValues != nThisParValues:
-                return S_ERROR(
-                    EWMSJDL,
-                    "Different length of parameter vectors: for %s, %s != %d" % (attribute, nParValues, nThisParValues),
-                )
-            nParValues = nThisParValues
-    if nParValues is not None and nParValues <= 0:
-        return S_ERROR(EWMSJDL, "Illegal number of job parameters %d" % (nParValues))
-    return S_OK(nParValues)
-
-
-def __updateAttribute(classAd, attribute, parName, parValue):
-    # If there is something to do:
-    pattern = r"%%\(%s\)s" % parName
-    if parName == "0":
-        pattern = "%s"
-    expr = classAd.get_expression(attribute)
-    if not re.search(pattern, expr):
-        return False
-
-    pattern = "%%(%s)s" % parName
-    if parName == "0":
-        pattern = "%s"
-
-    parValue = parValue.strip()
-    if classAd.isAttributeList(attribute):
-        parValue = parValue.strip()
-        if parValue.startswith("{"):
-            parValue = parValue.lstrip("{").rstrip("}").strip()
-
-    expr = classAd.get_expression(attribute)
-    newexpr = expr.replace(pattern, str(parValue))
-    classAd.set_expression(attribute, newexpr)
-    return True
-
-
-def generateParametricJobs(jobClassAd):
-    """Generate a series of ClassAd job descriptions expanding
-        job parameters
-
-    :param jobClassAd: ClassAd job description object
-    :return: list of ClassAd job description objects
-    """
-    if not jobClassAd.lookupAttribute("Parameters"):
-        return S_OK([jobClassAd.asJDL()])
-
-    result = getParameterVectorLength(jobClassAd)
-    if not result["OK"]:
-        return result
-    nParValues = result["Value"]
-    if nParValues is None:
-        return S_ERROR(EWMSJDL, "Can not determine the number of job parameters")
-
-    parameterDict = {}
-    attributes = jobClassAd.getAttributes()
-    for attribute in attributes:
-        for key in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
-            if attribute.startswith(key):
-                seqID = "0" if "." not in attribute else attribute.split(".")[1]
-                parameterDict.setdefault(seqID, {})
-                if key == "Parameters":
-                    if jobClassAd.isAttributeList(attribute):
-                        parList = jobClassAd.getListFromExpression(attribute)
-                        if len(parList) != nParValues:
-                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
-                        parameterDict[seqID]["ParameterList"] = parList
-                    else:
-                        if attribute != "Parameters":
-                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
-                        nPar = jobClassAd.getAttributeInt(attribute)
-                        if nPar is None:
-                            value = jobClassAd.get_expression(attribute)
-                            return S_ERROR(EWMSJDL, f"Inconsistent parametric job description: {attribute}={value}")
-                        parameterDict[seqID]["Parameters"] = nPar
-                else:
-                    value = jobClassAd.getAttributeInt(attribute)
-                    if value is None:
-                        value = jobClassAd.getAttributeFloat(attribute)
-                        if value is None:
-                            value = jobClassAd.get_expression(attribute)
-                            return S_ERROR(f"Illegal value for {attribute} JDL field: {value}")
-                    parameterDict[seqID][key] = value
-
-    if "0" in parameterDict and not parameterDict.get("0"):
-        parameterDict.pop("0")
-
-    parameterLists = {}
-    for seqID in parameterDict:
-        parList = __getParameterSequence(
-            nParValues,
-            parList=parameterDict[seqID].get("ParameterList", []),
-            parStart=parameterDict[seqID].get("ParameterStart", 1),
-            parStep=parameterDict[seqID].get("ParameterStep", 0),
-            parFactor=parameterDict[seqID].get("ParameterFactor", 1),
-        )
-        if not parList:
-            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
-
-        parameterLists[seqID] = parList
-
-    jobDescList = []
-    jobDesc = jobClassAd.asJDL()
-    # Width of the sequential parameter number
-    zLength = len(str(nParValues - 1))
-    for n in range(nParValues):
-        newJobDesc = jobDesc
-        newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength))
-        newClassAd = ClassAd(newJobDesc)
-        for seqID in parameterLists:
-            parameter = parameterLists[seqID][n]
-            for attribute in newClassAd.getAttributes():
-                __updateAttribute(newClassAd, attribute, seqID, str(parameter))
-
-        for seqID in parameterLists:
-            for attribute in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
-                if seqID == "0":
-                    newClassAd.deleteAttribute(attribute)
-                else:
-                    newClassAd.deleteAttribute(f"{attribute}.{seqID}")
-
-            parameter = parameterLists[seqID][n]
-            if seqID == "0":
-                attribute = "Parameter"
-            else:
-                attribute = f"Parameter.{seqID}"
-            if isinstance(parameter, str) and parameter.startswith("{"):
-                newClassAd.insertAttributeInt(attribute, str(parameter))
-            else:
-                newClassAd.insertAttributeString(attribute, str(parameter))
-
-        newClassAd.insertAttributeInt("ParameterNumber", n)
-        newJDL = newClassAd.asJDL()
-        jobDescList.append(newJDL)
-
-    return S_OK(jobDescList)
+# Re-export for backward compatibility
+__all__ = ["getParameterVectorLength", "generateParametricJobs"]

DIRAC/WorkloadManagementSystem/Utilities/PilotCStoJSONSynchronizer.py

@@ -1,16 +1,16 @@
-""" CStoJSONSynchronizer
-  Module that keeps the pilot parameters file synchronized with the information
-  in the Operations/Pilot section of the CS. If there are additions in the CS,
-  these are incorporated to the file.
-  The module uploads to a web server the latest version of the pilot scripts.
+"""CStoJSONSynchronizer
+Module that keeps the pilot parameters file synchronized with the information
+in the Operations/Pilot section of the CS. If there are additions in the CS,
+these are incorporated to the file.
+The module uploads to a web server the latest version of the pilot scripts.
 """
+
 import datetime
 import glob
 import os
 import shutil
 import tarfile
 from typing import Any
-
 from git import Repo
 
 from DIRAC import S_OK, gConfig, gLogger
@@ -19,6 +19,67 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
 from DIRAC.Core.Utilities.ReturnValues import DOKReturnType, DReturnType
 
+import socket
+from urllib.parse import urlparse
+
+
+def exclude_master_cs_aliases(urls: list[str], master_cs_url: str) -> list[str]:
+    """
+    Excludes URLs that are DNS aliases of the given MasterCS server URL.
+
+    This function resolves the IP addresses of the MasterCS server and each URL in the input list.
+    It returns a new list containing only those URLs whose hostnames do not resolve to any of the
+    MasterCS server's IP addresses, effectively excluding all DNS aliases of the MasterCS server.
+
+    Args:
+        urls (list[str]): A list of URLs to filter. Each URL should be a string in a valid URL format.
+        master_cs_url (str): The reference URL (e.g., MasterCS server URL) whose DNS aliases are to be excluded.
+
+    Returns:
+        list[str]: A new list of URLs with all aliases of the MasterCS server removed.
+                  If the MasterCS hostname cannot be resolved, the original list is returned unchanged.
+
+    Example:
+        >>> urls = [
+        ...     'dips://lbvobox303.cern.ch:9135/Configuration/Server',
+        ...     'dips://ccwlcglhcb02.in2p3.fr:9135/Configuration/Server',
+        ...     'dips://lbvobox302.cern.ch:9135/Configuration/Server',
+        ... ]
+        >>> master_cs_url = "dips://mastercs.cern.ch:9135/Configuration/Server"
+        >>> exclude_master_cs_aliases(urls, master_cs_url)
+        ['dips://ccwlcglhcb02.in2p3.fr:9135/Configuration/Server']
+
+    Notes:
+        - If the MasterCS hostname cannot be resolved, the function returns the original list.
+        - If a hostname in the input list cannot be resolved, it is included in the result.
+        - The comparison is based on IP addresses, not hostnames.
+    """
+    master_cs_hostname = urlparse(master_cs_url).hostname
+    if not master_cs_hostname:
+        return urls
+
+    # Resolve IP addresses for the MasterCS hostname
+    try:
+        master_cs_ips = set(socket.gethostbyname_ex(master_cs_hostname)[2])
+    except socket.gaierror:
+        return urls
+
+    # Function to get IPs for a hostname
+    def get_ips(hostname):
+        try:
+            return set(socket.gethostbyname_ex(hostname)[2])
+        except socket.gaierror:
+            return set()
+
+    filtered_urls = []
+    for url in urls:
+        hostname = urlparse(url).hostname
+        ips = get_ips(hostname)
+        if not ips & master_cs_ips:
+            filtered_urls.append(url)
+
+    return filtered_urls
+
 
 class PilotCStoJSONSynchronizer:
     """
@@ -151,9 +212,14 @@ class PilotCStoJSONSynchronizer:
         configurationServers = gConfig.getServersList()
         if not includeMasterCS:
             masterCS = gConfigurationData.getMasterServer()
-            configurationServers = list(set(configurationServers) - {masterCS})
+            configurationServers = exclude_master_cs_aliases(configurationServers, masterCS)
+
         pilotDict["ConfigurationServers"] = configurationServers
 
+        preferredURLPatterns = gConfigurationData.extractOptionFromCFG("/DIRAC/PreferredURLPatterns")
+        if preferredURLPatterns:
+            pilotDict["PreferredURLPatterns"] = preferredURLPatterns
+
         self.log.debug("Got pilotDict", str(pilotDict))
 
         return S_OK(pilotDict)

DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py

@@ -1,16 +1,17 @@
-""" Module holding function(s) creating the pilot wrapper.
+"""Module holding function(s) creating the pilot wrapper.
 
-    This is a DIRAC-free module, so it could possibly be used also outside of DIRAC installations.
+This is a DIRAC-free module, so it could possibly be used also outside of DIRAC installations.
 
-    The main client of this module is the SiteDirector, that invokes the functions here more or less like this::
+The main client of this module is the SiteDirector, that invokes the functions here more or less like this::
 
-        pilotFilesCompressedEncodedDict = getPilotFilesCompressedEncodedDict(pilotFiles)
-        localPilot = pilotWrapperScript(pilotFilesCompressedEncodedDict,
-                                        pilotOptions,
-                                        pilotExecDir)
-       _writePilotWrapperFile(localPilot=localPilot)
+    pilotFilesCompressedEncodedDict = getPilotFilesCompressedEncodedDict(pilotFiles)
+    localPilot = pilotWrapperScript(pilotFilesCompressedEncodedDict,
+                                    pilotOptions,
+                                    pilotExecDir)
+   _writePilotWrapperFile(localPilot=localPilot)
 
 """
+
 from __future__ import absolute_import, division, print_function
 
 import base64
@@ -18,6 +19,7 @@ import bz2
 import os
 import tempfile
 
+
 pilotWrapperContent = """#!/bin/bash
 # Reduce the maximum allowed number of open file descriptors as micromamba
 # gets stuck due to https://github.com/DaanDeMeyer/reproc/pull/103
@@ -110,8 +112,26 @@ logger.addHandler(buffer)
 # just logging the environment as first thing
 logger.debug('===========================================================')
 logger.debug('Environment of execution host\\n')
-for key, val in os.environ.items():
-  logger.debug(key + '=' + val)
+# Clear any non-UTF encodable environment variables as they cause the pilot to fail
+# Use os.environb if it exists (gives raw bytes), else fall back to os.environ.
+environb = getattr(os, "environb", os.environ)
+for key, val in environb.items():
+    try:
+        # Decode if these are bytes; if they're already text, leave them as-is.
+        if isinstance(key, bytes):
+            key_decoded = key.decode("utf-8")
+        else:
+            key_decoded = key
+        if isinstance(val, bytes):
+            val_decoded = val.decode("utf-8")
+        else:
+            val_decoded = val
+    except UnicodeDecodeError as e:
+        logger.error("Dropping %%s=%%s due to decode error: %%s", key, val, e)
+        del environb[key]
+        continue
+
+    logger.debug(key_decoded + '=' + val_decoded)
 logger.debug('===========================================================\\n')
 
 logger.debug(sys.version)
@@ -393,7 +413,17 @@ def getPilotFilesCompressedEncodedDict(pilotFiles, proxy=None):
         pilotFilesCompressedEncodedDict[os.path.basename(pf)] = pfContentEncoded
 
     if proxy is not None:
-        compressedAndEncodedProxy = base64.b64encode(bz2.compress(proxy.dumpAllToString()["Value"].encode()))
+        from pathlib import Path  # pylint: disable=import-error
+        from DIRAC.Core.Security.ProxyFile import writeChainToTemporaryFile  # pylint: disable=import-error
+
+        retVal = writeChainToTemporaryFile(proxy)
+        if not retVal["OK"]:
+            raise RuntimeError("Failed to write proxy to temporary file: %s" % retVal["Message"])
+        proxyLocation = Path(retVal["Value"])
+        proxy_string = proxyLocation.read_text()
+        proxyLocation.unlink()
+
+        compressedAndEncodedProxy = base64.b64encode(bz2.compress(proxy_string.encode()))
         pilotFilesCompressedEncodedDict["proxy"] = compressedAndEncodedProxy
 
     return pilotFilesCompressedEncodedDict

DIRAC/WorkloadManagementSystem/Utilities/RemoteRunner.py

@@ -49,6 +49,11 @@ class RemoteRunner:
             f"site {self._workloadSite}, CE {self._workloadCE}, queue {self._workloadQueue}",
         )
 
+        # The CE interface needs to drop the token section from the proxy file to interact with the CE
+        # So we save the current proxy file location (which likely contains the DiracX token)
+        # and we will restore it at the end of the job
+        originalProxyLocation = os.environ.get("X509_USER_PROXY")
+
         # Set up Application Queue
         if not (result := self._setUpWorkloadCE(numberOfProcessors))["OK"]:
             result["Errno"] = DErrno.ERESUNA
@@ -87,6 +92,8 @@ class RemoteRunner:
             time.sleep(timeBetweenRetries)
         else:
             result["Errno"] = DErrno.EWMSSUBM
+            # Restore the original proxy location
+            os.environ["X509_USER_PROXY"] = originalProxyLocation
             return result
 
         jobID = result["Value"][0]
@@ -107,6 +114,8 @@ class RemoteRunner:
                 time.sleep(timeBetweenRetries)
             else:
                 result["Errno"] = DErrno.EWMSSTATUS
+                # Restore the original proxy location
+                os.environ["X509_USER_PROXY"] = originalProxyLocation
                 return result
 
             jobStatus = result["Value"][jobID]
@@ -123,6 +132,8 @@ class RemoteRunner:
             time.sleep(timeBetweenRetries)
         else:
             result["Errno"] = DErrno.EWMSJMAN
+            # Restore the original proxy location
+            os.environ["X509_USER_PROXY"] = originalProxyLocation
             return result
 
         output, error = result["Value"]
@@ -131,6 +142,8 @@ class RemoteRunner:
         self.log.info("Checking the integrity of the outputs...")
         if not (result := self._checkOutputIntegrity("."))["OK"]:
             result["Errno"] = DErrno.EWMSJMAN
+            # Restore the original proxy location
+            os.environ["X509_USER_PROXY"] = originalProxyLocation
             return result
         self.log.info("The output has been retrieved and declared complete")
 
@@ -146,6 +159,9 @@ class RemoteRunner:
                 self.log.warn("Failed to clean the output remotely", result["Message"])
             self.log.info("The job has been remotely removed")
 
+        # Restore the original proxy location
+        os.environ["X509_USER_PROXY"] = originalProxyLocation
+
         commandStatus = {"Done": 0, "Failed": -1, "Killed": -2}
         return S_OK((commandStatus[jobStatus], output, error))
 

DIRAC/WorkloadManagementSystem/Utilities/Utils.py

@@ -4,9 +4,12 @@ import os
 import sys
 import json
 
-from DIRAC import gLogger, S_OK
+from DIRAC import gLogger, S_OK, S_ERROR
 from DIRAC.Core.Utilities.File import mkDir
 from DIRAC.FrameworkSystem.private.standardLogging.Logging import Logging
+from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
+from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
+from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB
 
 
 def createJobWrapper(
@@ -113,3 +116,35 @@ def createJobWrapper(
     if rootLocation != wrapperPath:
         generatedFiles["JobExecutableRelocatedPath"] = os.path.join(rootLocation, os.path.basename(jobExeFile))
     return S_OK(generatedFiles)
+
+
+def rescheduleJobs(jobIDs: list[int], source: str = "") -> dict:
+    """Utility to reschedule jobs (not atomic, nor bulk)
+    Requires direct access to the JobDB and TaskQueueDB
+
+    :param jobIDs: list of jobIDs
+    :param source: source of the reschedule
+    :return: S_OK/S_ERROR
+    :rtype: dict
+
+    """
+
+    failedJobs = []
+
+    for jobID in jobIDs:
+        result = JobDB().rescheduleJob(jobID)
+        if not result["OK"]:
+            failedJobs.append(jobID)
+            continue
+        TaskQueueDB().deleteJob(jobID)
+        JobLoggingDB().addLoggingRecord(
+            result["JobID"],
+            status=result["Status"],
+            minorStatus=result["MinorStatus"],
+            applicationStatus="Unknown",
+            source=source,
+        )
+
+    if failedJobs:
+        return S_ERROR(f"Failed to reschedule jobs {failedJobs}")
+    return S_OK()

DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py

@@ -0,0 +1,15 @@
+from DIRAC.WorkloadManagementSystem.Client import JobStatus
+
+
+def _filterJobStateTransition(jobStates, candidateState):
+    """Given a dictionary of jobs states,
+    return a list of jobs that are allowed to transition to the given candidate state.
+    """
+    allowedJobs = []
+
+    for js in jobStates.items():
+        stateRes = JobStatus.JobsStateMachine(js[1]["Status"]).getNextState(candidateState)
+        if stateRes["OK"]:
+            if stateRes["Value"] == candidateState:
+                allowedJobs.append(js[0])
+    return allowedJobs

DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py

@@ -175,24 +175,10 @@ def test_logLevelValidator_invalid():
 
 def test_platformValidator_valid():
     """Test the platform validator with valid input."""
-    with patch(
-        "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms",
-        return_value=S_OK(["x86_64-slc6-gcc62-opt"]),
-    ):
-        job = BaseJobDescriptionModel(executable=EXECUTABLE, platform="x86_64-slc6-gcc62-opt")
+    job = BaseJobDescriptionModel(executable=EXECUTABLE, platform="x86_64-slc6-gcc62-opt")
     assert job.platform == "x86_64-slc6-gcc62-opt"
 
 
-def test_platformValidator_invalid():
-    """Test the platform validator with invalid input."""
-    with patch(
-        "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms",
-        return_value=S_OK(["x86_64-slc6-gcc62-opt"]),
-    ):
-        with pytest.raises(ValidationError):
-            BaseJobDescriptionModel(executable=EXECUTABLE, platform="x86_64-slc6-gcc62-opt2")
-
-
 @pytest.mark.parametrize(
     "validSites, selectedSites",
     [