DIRAC 9.0.0a64__py3-none-any.whl → 9.0.0a67__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Utilities/ParametricJob.py

@@ -4,176 +4,12 @@
     getParameterVectorLength() - to get the total size of the bunch of parametric jobs
     generateParametricJobs() - to get a list of expanded descriptions of all the jobs
 """
-import re
 
-from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
-from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR
-from DIRAC.Core.Utilities.DErrno import EWMSJDL
+# Import from DIRACCommon for backward compatibility
+from DIRACCommon.WorkloadManagementSystem.Utilities.ParametricJob import (
+    getParameterVectorLength,
+    generateParametricJobs,
+)
 
-
-def __getParameterSequence(nPar, parList=[], parStart=1, parStep=0, parFactor=1):
-    if parList:
-        if nPar != len(parList):
-            return []
-        else:
-            parameterList = list(parList)
-    else:
-        # The first parameter must have the same type as the other ones even if not defined explicitly
-        parameterList = [parStart * type(parFactor)(1) + type(parStep)(0)]
-        for np in range(1, nPar):
-            parameterList.append(parameterList[np - 1] * parFactor + parStep)
-
-    return parameterList
-
-
-def getParameterVectorLength(jobClassAd):
-    """Get the length of parameter vector in the parametric job description
-
-    :param jobClassAd: ClassAd job description object
-    :return: result structure with the Value: int number of parameter values, None if not a parametric job
-    """
-
-    nParValues = None
-    attributes = jobClassAd.getAttributes()
-    for attribute in attributes:
-        if attribute.startswith("Parameters"):
-            if jobClassAd.isAttributeList(attribute):
-                parameterList = jobClassAd.getListFromExpression(attribute)
-                nThisParValues = len(parameterList)
-            else:
-                nThisParValues = jobClassAd.getAttributeInt(attribute)
-            if nParValues is not None and nParValues != nThisParValues:
-                return S_ERROR(
-                    EWMSJDL,
-                    "Different length of parameter vectors: for %s, %s != %d" % (attribute, nParValues, nThisParValues),
-                )
-            nParValues = nThisParValues
-    if nParValues is not None and nParValues <= 0:
-        return S_ERROR(EWMSJDL, "Illegal number of job parameters %d" % (nParValues))
-    return S_OK(nParValues)
-
-
-def __updateAttribute(classAd, attribute, parName, parValue):
-    # If there is something to do:
-    pattern = r"%%\(%s\)s" % parName
-    if parName == "0":
-        pattern = "%s"
-    expr = classAd.get_expression(attribute)
-    if not re.search(pattern, expr):
-        return False
-
-    pattern = "%%(%s)s" % parName
-    if parName == "0":
-        pattern = "%s"
-
-    parValue = parValue.strip()
-    if classAd.isAttributeList(attribute):
-        parValue = parValue.strip()
-        if parValue.startswith("{"):
-            parValue = parValue.lstrip("{").rstrip("}").strip()
-
-    expr = classAd.get_expression(attribute)
-    newexpr = expr.replace(pattern, str(parValue))
-    classAd.set_expression(attribute, newexpr)
-    return True
-
-
-def generateParametricJobs(jobClassAd):
-    """Generate a series of ClassAd job descriptions expanding
-        job parameters
-
-    :param jobClassAd: ClassAd job description object
-    :return: list of ClassAd job description objects
-    """
-    if not jobClassAd.lookupAttribute("Parameters"):
-        return S_OK([jobClassAd.asJDL()])
-
-    result = getParameterVectorLength(jobClassAd)
-    if not result["OK"]:
-        return result
-    nParValues = result["Value"]
-    if nParValues is None:
-        return S_ERROR(EWMSJDL, "Can not determine the number of job parameters")
-
-    parameterDict = {}
-    attributes = jobClassAd.getAttributes()
-    for attribute in attributes:
-        for key in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
-            if attribute.startswith(key):
-                seqID = "0" if "." not in attribute else attribute.split(".")[1]
-                parameterDict.setdefault(seqID, {})
-                if key == "Parameters":
-                    if jobClassAd.isAttributeList(attribute):
-                        parList = jobClassAd.getListFromExpression(attribute)
-                        if len(parList) != nParValues:
-                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
-                        parameterDict[seqID]["ParameterList"] = parList
-                    else:
-                        if attribute != "Parameters":
-                            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
-                        nPar = jobClassAd.getAttributeInt(attribute)
-                        if nPar is None:
-                            value = jobClassAd.get_expression(attribute)
-                            return S_ERROR(EWMSJDL, f"Inconsistent parametric job description: {attribute}={value}")
-                        parameterDict[seqID]["Parameters"] = nPar
-                else:
-                    value = jobClassAd.getAttributeInt(attribute)
-                    if value is None:
-                        value = jobClassAd.getAttributeFloat(attribute)
-                        if value is None:
-                            value = jobClassAd.get_expression(attribute)
-                            return S_ERROR(f"Illegal value for {attribute} JDL field: {value}")
-                    parameterDict[seqID][key] = value
-
-    if "0" in parameterDict and not parameterDict.get("0"):
-        parameterDict.pop("0")
-
-    parameterLists = {}
-    for seqID in parameterDict:
-        parList = __getParameterSequence(
-            nParValues,
-            parList=parameterDict[seqID].get("ParameterList", []),
-            parStart=parameterDict[seqID].get("ParameterStart", 1),
-            parStep=parameterDict[seqID].get("ParameterStep", 0),
-            parFactor=parameterDict[seqID].get("ParameterFactor", 1),
-        )
-        if not parList:
-            return S_ERROR(EWMSJDL, "Inconsistent parametric job description")
-
-        parameterLists[seqID] = parList
-
-    jobDescList = []
-    jobDesc = jobClassAd.asJDL()
-    # Width of the sequential parameter number
-    zLength = len(str(nParValues - 1))
-    for n in range(nParValues):
-        newJobDesc = jobDesc
-        newJobDesc = newJobDesc.replace("%n", str(n).zfill(zLength))
-        newClassAd = ClassAd(newJobDesc)
-        for seqID in parameterLists:
-            parameter = parameterLists[seqID][n]
-            for attribute in newClassAd.getAttributes():
-                __updateAttribute(newClassAd, attribute, seqID, str(parameter))
-
-        for seqID in parameterLists:
-            for attribute in ["Parameters", "ParameterStart", "ParameterStep", "ParameterFactor"]:
-                if seqID == "0":
-                    newClassAd.deleteAttribute(attribute)
-                else:
-                    newClassAd.deleteAttribute(f"{attribute}.{seqID}")
-
-            parameter = parameterLists[seqID][n]
-            if seqID == "0":
-                attribute = "Parameter"
-            else:
-                attribute = f"Parameter.{seqID}"
-            if isinstance(parameter, str) and parameter.startswith("{"):
-                newClassAd.insertAttributeInt(attribute, str(parameter))
-            else:
-                newClassAd.insertAttributeString(attribute, str(parameter))
-
-        newClassAd.insertAttributeInt("ParameterNumber", n)
-        newJDL = newClassAd.asJDL()
-        jobDescList.append(newJDL)
-
-    return S_OK(jobDescList)
+# Re-export for backward compatibility
+__all__ = ["getParameterVectorLength", "generateParametricJobs"]

DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py

@@ -1,127 +1,4 @@
-from DIRAC import S_ERROR, S_OK, gLogger
-from DIRAC.StorageManagementSystem.DB.StorageManagementDB import StorageManagementDB
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
-from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
-from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB
-from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB
-from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_DELETE, RIGHT_KILL
-
-
-def _deleteJob(jobID, force=False):
-    """Set the job status to "Deleted"
-    and remove the pilot that ran and its logging info if the pilot is finished.
-
-    :param int jobID: job ID
-    :return: S_OK()/S_ERROR()
-    """
-    if not (result := JobDB().setJobStatus(jobID, JobStatus.DELETED, "Checking accounting", force=force))["OK"]:
-        gLogger.warn("Failed to set job Deleted status", result["Message"])
-        return result
-
-    if not (result := TaskQueueDB().deleteJob(jobID))["OK"]:
-        gLogger.warn("Failed to delete job from the TaskQueue")
-
-    # if it was the last job for the pilot
-    result = PilotAgentsDB().getPilotsForJobID(jobID)
-    if not result["OK"]:
-        gLogger.error("Failed to get Pilots for JobID", result["Message"])
-        return result
-    for pilot in result["Value"]:
-        res = PilotAgentsDB().getJobsForPilot(pilot)
-        if not res["OK"]:
-            gLogger.error("Failed to get jobs for pilot", res["Message"])
-            return res
-        if not res["Value"]:  # if list of jobs for pilot is empty, delete pilot
-            result = PilotAgentsDB().getPilotInfo(pilotID=pilot)
-            if not result["OK"]:
-                gLogger.error("Failed to get pilot info", result["Message"])
-                return result
-            ret = PilotAgentsDB().deletePilot(result["Value"]["PilotJobReference"])
-            if not ret["OK"]:
-                gLogger.error("Failed to delete pilot from PilotAgentsDB", ret["Message"])
-                return ret
-
-    return S_OK()
-
-
-def _killJob(jobID, sendKillCommand=True, force=False):
-    """Kill one job
-
-    :param int jobID: job ID
-    :param bool sendKillCommand: send kill command
-
-    :return: S_OK()/S_ERROR()
-    """
-    if sendKillCommand:
-        if not (result := JobDB().setJobCommand(jobID, "Kill"))["OK"]:
-            gLogger.warn("Failed to set job Kill command", result["Message"])
-            return result
-
-    gLogger.info("Job marked for termination", jobID)
-    if not (result := JobDB().setJobStatus(jobID, JobStatus.KILLED, "Marked for termination", force=force))["OK"]:
-        gLogger.warn("Failed to set job Killed status", result["Message"])
-    if not (result := TaskQueueDB().deleteJob(jobID))["OK"]:
-        gLogger.warn("Failed to delete job from the TaskQueue", result["Message"])
-
-    return S_OK()
-
-
-def kill_delete_jobs(right, validJobList, nonauthJobList=[], force=False):
-    """Kill (== set the status to "KILLED") or delete (== set the status to "DELETED") jobs as necessary
-
-    :param str right: RIGHT_KILL or RIGHT_DELETE
-
-    :return: S_OK()/S_ERROR()
-    """
-    badIDs = []
-
-    killJobList = []
-    deleteJobList = []
-    if validJobList:
-        result = JobDB().getJobsAttributes(killJobList, ["Status"])
-        if not result["OK"]:
-            return result
-        jobStates = result["Value"]
-
-        # Get the jobs allowed to transition to the Killed state
-        killJobList.extend(_filterJobStateTransition(jobStates, JobStatus.KILLED))
-
-        if right == RIGHT_DELETE:
-            # Get the jobs allowed to transition to the Deleted state
-            deleteJobList.extend(_filterJobStateTransition(jobStates, JobStatus.DELETED))
-
-        for jobID in killJobList:
-            result = _killJob(jobID, force=force)
-            if not result["OK"]:
-                badIDs.append(jobID)
-
-        for jobID in deleteJobList:
-            result = _deleteJob(jobID, force=force)
-            if not result["OK"]:
-                badIDs.append(jobID)
-
-        # Look for jobs that are in the Staging state to send kill signal to the stager
-        stagingJobList = [jobID for jobID, sDict in jobStates.items() if sDict["Status"] == JobStatus.STAGING]
-
-        if stagingJobList:
-            stagerDB = StorageManagementDB()
-            gLogger.info("Going to send killing signal to stager as well!")
-            result = stagerDB.killTasksBySourceTaskID(stagingJobList)
-            if not result["OK"]:
-                gLogger.warn("Failed to kill some Stager tasks", result["Message"])
-
-    if nonauthJobList or badIDs:
-        result = S_ERROR("Some jobs failed deletion")
-        if nonauthJobList:
-            gLogger.warn("Non-authorized JobIDs won't be deleted", str(nonauthJobList))
-            result["NonauthorizedJobIDs"] = nonauthJobList
-        if badIDs:
-            gLogger.warn("JobIDs failed to be deleted", str(badIDs))
-            result["FailedJobIDs"] = badIDs
-        return result
-
-    jobsList = killJobList if right == RIGHT_KILL else deleteJobList
-    return S_OK(jobsList)
 
 
 def _filterJobStateTransition(jobStates, candidateState):

DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobModel.py

@@ -175,11 +175,7 @@ def test_logLevelValidator_invalid():
 
 def test_platformValidator_valid():
     """Test the platform validator with valid input."""
-    with patch(
-        "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms",
-        return_value=S_OK(["x86_64-slc6-gcc62-opt"]),
-    ):
-        job = BaseJobDescriptionModel(executable=EXECUTABLE, platform="x86_64-slc6-gcc62-opt")
+    job = BaseJobDescriptionModel(executable=EXECUTABLE, platform="x86_64-slc6-gcc62-opt")
     assert job.platform == "x86_64-slc6-gcc62-opt"
 
 

DIRAC/WorkloadManagementSystem/Utilities/test/Test_ParametricJob.py

@@ -1,138 +1,55 @@
-""" This is a test of the parametric job generation tools
-"""
+""" This is a test of the parametric job generation tools"""
 # pylint: disable= missing-docstring
 
 import pytest
 
+# Test imports from DIRAC to verify backward compatibility
 from DIRAC.WorkloadManagementSystem.Utilities.ParametricJob import generateParametricJobs, getParameterVectorLength
 from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 
-TEST_JDL_NO_PARAMETERS = """
-[
-    Executable = "my_executable";
-    Arguments = "%s";
-    JobName = "Test_%n";
-]
-"""
 
-TEST_JDL_SIMPLE = """
-[
-    Executable = "my_executable";
-    Arguments = "%s";
-    JobName = "Test_%n";
-    Parameters = { "a", "b", "c" }
-]
-"""
-
-TEST_JDL_SIMPLE_BUNCH = """
-[
-    Executable = "my_executable";
-    Arguments = "%s";
-    JobName = "Test_%n";
-    Parameters = 3;
-    ParameterStart = 5;
-]
-"""
-
-TEST_JDL_SIMPLE_PROGRESSION = """
-[
-    Executable = "my_executable";
-    Arguments = "%s";
-    JobName = "Test_%n";
-    Parameters = 3;
-    ParameterStart = 1;
-    ParameterStep = 1;
-    ParameterFactor = 2;
-]
-"""
-
-TEST_JDL_MULTI = """
-[
-    Executable = "my_executable";
-    Arguments = "%(A)s %(B)s";
-    JobName = "Test_%n";
-    Parameters = 3;
-    ParameterStart.A = 1;
-    ParameterStep.A = 1;
-    ParameterFactor.A = 2;
-    Parameters.B = { "a","b","c" };
-]
-"""
-
-TEST_JDL_MULTI_BAD = """
-[
-    Executable = "my_executable";
-    Arguments = "%(A)s %(B)s";
-    JobName = "Test_%n";
-    Parameters = 3;
-    ParameterStart.A = 1;
-    ParameterStep.A = 1;
-    ParameterFactor.A = 2;
-    Parameters.B = { "a","b","c","d" };
-]
-"""
-
-
-@pytest.mark.parametrize(
-    "jdl, expectedArguments",
-    [
-        (TEST_JDL_SIMPLE, ["a", "b", "c"]),
-        (TEST_JDL_SIMPLE_BUNCH, ["5", "5", "5"]),
-        (TEST_JDL_SIMPLE_PROGRESSION, ["1", "3", "7"]),
-        (TEST_JDL_MULTI, ["1 a", "3 b", "7 c"]),
-        (TEST_JDL_NO_PARAMETERS, []),
-    ],
-)
-def test_getParameterVectorLength_successful(jdl: str, expectedArguments: list[str]):
+def test_backward_compatibility_import():
+    """Test that imports from DIRAC still work (backward compatibility)"""
     # Arrange
-    jobDescription = ClassAd(jdl)
-
-    # Act
-    result = getParameterVectorLength(jobDescription)
-
-    # Assert
-    assert result["OK"], result["Message"]
-    if expectedArguments:
-        assert result["Value"] == len(expectedArguments)
-    else:
-        assert result["Value"] == None
-
-
-@pytest.mark.parametrize("jdl", [TEST_JDL_MULTI_BAD])
-def test_getParameterVectorLength_unsuccessful(jdl: str):
-    # Arrange
-    jobDescription = ClassAd(jdl)
-
-    # Act
-    result = getParameterVectorLength(jobDescription)
-
-    # Assert
-    assert not result["OK"], result["Value"]
-
-
-@pytest.mark.parametrize(
-    "jdl, expectedArguments",
+    jdl = """
     [
-        (TEST_JDL_SIMPLE, ["a", "b", "c"]),
-        (TEST_JDL_SIMPLE_BUNCH, ["5", "5", "5"]),
-        (TEST_JDL_SIMPLE_PROGRESSION, ["1", "3", "7"]),
-        (TEST_JDL_MULTI, ["1 a", "3 b", "7 c"]),
-    ],
-)
-def test_generateParametricJobs(jdl: str, expectedArguments: list[str]):
-    # Arrange
-    parametricJobDescription = ClassAd(jdl)
-
-    # Act
-    result = generateParametricJobs(parametricJobDescription)
-
-    # Assert
-    assert result["OK"], result["Message"]
-    assert result["Value"]
-    jobDescList = result["Value"]
-    assert len(jobDescList) == len(expectedArguments)
-
-    for i in range(len(jobDescList)):
-        jobDescription = ClassAd(jobDescList[i])
-        assert jobDescription.getAttributeString("JobName") == f"Test_{i}"
-        assert jobDescription.getAttributeString("Arguments") == expectedArguments[i]
+        Executable = "my_executable";
+        Arguments = "%s";
+        JobName = "Test_%n";
+        Parameters = { "a", "b", "c" }
+    ]
+    """
+
+    # Act - Test that we can import and use the functions from DIRAC
+    jobDescription = ClassAd(jdl)
+    vector_result = getParameterVectorLength(jobDescription)
+    generate_result = generateParametricJobs(jobDescription)
+
+    # Assert - Verify functions work correctly
+    assert vector_result["OK"]
+    assert vector_result["Value"] == 3
+    assert generate_result["OK"]
+    assert len(generate_result["Value"]) == 3
+
+
+# Import and run the comprehensive tests from DIRACCommon to avoid duplication
+# This ensures the DIRAC re-exports work with the full test suite
+try:
+    from DIRACCommon.tests.WorkloadManagementSystem.Utilities.test_ParametricJob import (
+        test_getParameterVectorLength_successful,
+        test_getParameterVectorLength_unsuccessful,
+        test_generateParametricJobs,
+    )
+
+    # Re-export the DIRACCommon tests so they run as part of DIRAC test suite
+    # This validates that the backward compatibility imports work correctly
+    __all__ = [
+        "test_backward_compatibility_import",
+        "test_getParameterVectorLength_successful",
+        "test_getParameterVectorLength_unsuccessful",
+        "test_generateParametricJobs",
+    ]
+
+except ImportError:
+    # If DIRACCommon tests can't be imported, just run the backward compatibility test
+    __all__ = ["test_backward_compatibility_import"]

DIRAC/__init__.py

@@ -1,59 +1,60 @@
 """
-   DIRAC - Distributed Infrastructure with Remote Agent Control
-
-   The distributed data production and analysis system of LHCb and other VOs.
-
-   DIRAC is a software framework for distributed computing which
-   allows to integrate various computing resources in a single
-   system. At the same time it integrates all kinds of computing
-   activities like Monte Carlo simulations, data processing, or
-   final user analysis.
-
-   It is build as number of cooperating systems:
-    - Accounting
-    - Configuration
-    - Core
-      - Base
-      - Security
-      - Utilities
-      - Workflow
-    - Framework
-    - RequestManagement
-    - Resources
-    - Transformation
-
-    Which are used by other system providing functionality to
-    the end user:
-    - DataManagement
-    - Interfaces
-    - ResourceStatus
-    - StorageManagement
-    - WorkloadManagement
-
-    It defines the following data members:
-    - version:       DIRAC version string
-
-    - errorMail:     mail address for important errors
-    - alarmMail:     mail address for important alarms
-
-    It loads Modules from :
-    - DIRAC.Core.Utililies
-
-    It loads:
-    - S_OK:           OK return structure
-    - S_ERROR:        ERROR return structure
-    - gLogger:        global Logger object
-    - gConfig:        global Config object
-
-    It defines the following functions:
-    - abort:          aborts execution
-    - exit:           finish execution using callbacks
-    - siteName:       returns DIRAC name for current site
-
-    - getPlatform():      DIRAC platform string for current host
-    - getPlatformTuple(): DIRAC platform tuple for current host
+DIRAC - Distributed Infrastructure with Remote Agent Control
+
+The distributed data production and analysis system of LHCb and other VOs.
+
+DIRAC is a software framework for distributed computing which
+allows to integrate various computing resources in a single
+system. At the same time it integrates all kinds of computing
+activities like Monte Carlo simulations, data processing, or
+final user analysis.
+
+It is build as number of cooperating systems:
+ - Accounting
+ - Configuration
+ - Core
+   - Base
+   - Security
+   - Utilities
+   - Workflow
+ - Framework
+ - RequestManagement
+ - Resources
+ - Transformation
+
+ Which are used by other system providing functionality to
+ the end user:
+ - DataManagement
+ - Interfaces
+ - ResourceStatus
+ - StorageManagement
+ - WorkloadManagement
+
+ It defines the following data members:
+ - version:       DIRAC version string
+
+ - errorMail:     mail address for important errors
+ - alarmMail:     mail address for important alarms
+
+ It loads Modules from :
+ - DIRAC.Core.Utililies
+
+ It loads:
+ - S_OK:           OK return structure
+ - S_ERROR:        ERROR return structure
+ - gLogger:        global Logger object
+ - gConfig:        global Config object
+
+ It defines the following functions:
+ - abort:          aborts execution
+ - exit:           finish execution using callbacks
+ - siteName:       returns DIRAC name for current site
+
+ - getPlatform():      DIRAC platform string for current host
+ - getPlatformTuple(): DIRAC platform tuple for current host
 
 """
+
 import importlib.metadata
 import os
 import re
@@ -237,7 +238,7 @@ def initialize(
         log_level = getattr(LogLevel, gLogger.getLevel())
         gLogger.setLevel(LogLevel.ALWAYS)
     try:
-        returnValueOrRaise(localCfg.initialize())
+        returnValueOrRaise(localCfg.initialize(requireSuccessfulSync=require_auth))
     finally:
         # Restore the pre-existing log level
         gLogger.setLevel(log_level)

{dirac-9.0.0a64.dist-info → dirac-9.0.0a67.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: DIRAC
-Version: 9.0.0a64
+Version: 9.0.0a67
 Summary: DIRAC is an interware, meaning a software framework for distributed computing.
 Home-page: https://github.com/DIRACGrid/DIRAC/
 License: GPL-3.0-only
@@ -19,6 +19,7 @@ Requires-Dist: cachetools
 Requires-Dist: certifi
 Requires-Dist: cwltool
 Requires-Dist: diraccfg
+Requires-Dist: DIRACCommon==v9.0.0a67
 Requires-Dist: diracx-client>=v0.0.1a18
 Requires-Dist: diracx-core>=v0.0.1a18
 Requires-Dist: db12