DIRAC 9.0.0a42__py3-none-any.whl → 9.0.7__py3-none-any.whl

DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py

@@ -11,6 +11,7 @@ from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
     RIGHT_GET_INFO,
     RIGHT_RESCHEDULE,
     RIGHT_RESET,
+    RIGHT_SUBMIT,
 )
 from DIRAC.WorkloadManagementSystem.Utilities.JobStatusUtility import JobStatusUtility
 
@@ -317,6 +318,11 @@ class JobState:
     def getInputData(self):
         return JobState.__db.jobDB.getInputData(self.__jid)
 
+    right_setInputData = RIGHT_SUBMIT
+
+    def setInputData(self, inputData):
+        return JobState.__db.jobDB.setInputData(self.__jid, inputData)
+
     right_insertIntoTQ = RIGHT_CHANGE_STATUS
 
     def insertIntoTQ(self, manifest=None):

DIRAC/WorkloadManagementSystem/Client/JobStateUpdateClient.py

@@ -28,3 +28,6 @@ class JobStateUpdateClient(Client):
 
         else:
             self.serverURL = url
+
+    def setJobStatus(self, jobID, status="", minorStatus="", source="Unknown", datetime_=None, force=False, **kwargs):
+        return self._getRPC(**kwargs).setJobStatus(jobID, status, minorStatus, source, datetime_, force)

DIRAC/WorkloadManagementSystem/Client/JobStatus.py

@@ -1,154 +1,10 @@
-"""
-This module contains constants and lists for the possible job states.
-"""
-
-from DIRAC import gLogger, S_OK, S_ERROR
-from DIRAC.Core.Utilities.StateMachine import State, StateMachine
-from DIRAC.Core.Utilities.Decorators import deprecated
-
-from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
-
-
-#:
-SUBMITTING = "Submitting"
-#:
-RECEIVED = "Received"
-#:
-CHECKING = "Checking"
-#:
-STAGING = "Staging"
-#:
-SCOUTING = "Scouting"
-#:
-WAITING = "Waiting"
-#:
-MATCHED = "Matched"
-#: The Rescheduled status is effectively never stored in the DB.
-#: It could be considered a "virtual" status, and might even be dropped.
-RESCHEDULED = "Rescheduled"
-#:
-RUNNING = "Running"
-#:
-STALLED = "Stalled"
-#:
-COMPLETING = "Completing"
-#:
-DONE = "Done"
-#:
-COMPLETED = "Completed"
-#:
-FAILED = "Failed"
-#:
-DELETED = "Deleted"
-#:
-KILLED = "Killed"
-
-#: Possible job states
-JOB_STATES = [
-    SUBMITTING,
-    RECEIVED,
-    CHECKING,
-    SCOUTING,
-    STAGING,
-    WAITING,
-    MATCHED,
-    RESCHEDULED,
-    RUNNING,
-    STALLED,
-    COMPLETING,
-    DONE,
-    COMPLETED,
-    FAILED,
-    DELETED,
-    KILLED,
-]
-
-# Job States when the payload work has finished
-JOB_FINAL_STATES = [DONE, COMPLETED, FAILED, KILLED]
-
-# WMS internal job States indicating the job object won't be updated
-JOB_REALLY_FINAL_STATES = [DELETED]
-
-
-class JobsStateMachine(StateMachine):
-    """Jobs state machine"""
-
-    def __init__(self, state):
-        """c'tor
-        Defines the state machine transactions
-        """
-        super().__init__(state)
+"""Backward compatibility wrapper - moved to DIRACCommon
 
-        # States transitions
-        self.states = {
-            DELETED: State(15),  # final state
-            KILLED: State(14, [DELETED], defState=KILLED),
-            FAILED: State(13, [RESCHEDULED, DELETED], defState=FAILED),
-            DONE: State(12, [DELETED], defState=DONE),
-            COMPLETED: State(11, [DONE, FAILED], defState=COMPLETED),
-            COMPLETING: State(10, [DONE, FAILED, COMPLETED, STALLED, KILLED], defState=COMPLETING),
-            STALLED: State(9, [RUNNING, FAILED, KILLED], defState=STALLED),
-            RUNNING: State(8, [STALLED, DONE, FAILED, RESCHEDULED, COMPLETING, KILLED, RECEIVED], defState=RUNNING),
-            RESCHEDULED: State(7, [WAITING, RECEIVED, DELETED, FAILED, KILLED], defState=RESCHEDULED),
-            MATCHED: State(6, [RUNNING, FAILED, RESCHEDULED, KILLED], defState=MATCHED),
-            WAITING: State(5, [MATCHED, RESCHEDULED, DELETED, KILLED], defState=WAITING),
-            STAGING: State(4, [CHECKING, WAITING, FAILED, KILLED], defState=STAGING),
-            SCOUTING: State(3, [CHECKING, FAILED, STALLED, KILLED], defState=SCOUTING),
-            CHECKING: State(2, [SCOUTING, STAGING, WAITING, RESCHEDULED, FAILED, DELETED, KILLED], defState=CHECKING),
-            RECEIVED: State(1, [SCOUTING, CHECKING, WAITING, FAILED, DELETED, KILLED], defState=RECEIVED),
-            SUBMITTING: State(0, [RECEIVED, CHECKING, DELETED, KILLED], defState=SUBMITTING),  # initial state
-        }
+This module has been moved to DIRACCommon.WorkloadManagementSystem.Client.JobStatus to avoid
+circular dependencies and allow DiracX to use these utilities without
+triggering DIRAC's global state initialization.
 
-
-@deprecated("Use filterJobStateTransition instead")
-def checkJobStateTransition(jobID, candidateState, currentStatus=None, jobMonitoringClient=None):
-    """Utility to check if a job state transition is allowed"""
-    if not currentStatus:
-        if not jobMonitoringClient:
-            from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
-
-            jobMonitoringClient = JobMonitoringClient()
-
-        res = jobMonitoringClient.getJobsStatus(jobID)
-        if not res["OK"]:
-            return res
-        try:
-            currentStatus = res["Value"][jobID]["Status"]
-        except KeyError:
-            return S_ERROR("Job does not exist")
-
-    res = JobsStateMachine(currentStatus).getNextState(candidateState)
-    if not res["OK"]:
-        return res
-
-    # If the JobsStateMachine does not accept the candidate, return an ERROR
-    if candidateState != res["Value"]:
-        gLogger.error(
-            "Job Status Error",
-            f"{jobID} can't move from {currentStatus} to {candidateState}",
-        )
-        return S_ERROR("Job state transition not allowed")
-    return S_OK()
-
-
-def filterJobStateTransition(jobIDs, candidateState):
-    """Given a list of jobIDs, return a list that are allowed to transition
-    to the given candidate state.
-    """
-    allowedJobs = []
-
-    if not isinstance(jobIDs, list):
-        jobIDs = [jobIDs]
-
-    res = JobMonitoringClient().getJobsStatus(jobIDs)
-    if not res["OK"]:
-        return res
-
-    for jobID in jobIDs:
-        if jobID in res["Value"]:
-            curState = res["Value"][jobID]["Status"]
-            stateRes = JobsStateMachine(curState).getNextState(candidateState)
-            if stateRes["OK"]:
-                if stateRes["Value"] == candidateState:
-                    allowedJobs.append(jobID)
-    return S_OK(allowedJobs)
+All exports are maintained for backward compatibility.
+"""
+# Re-export everything from DIRACCommon for backward compatibility
+from DIRACCommon.WorkloadManagementSystem.Client.JobStatus import *  # noqa: F401, F403

DIRAC/WorkloadManagementSystem/Client/PoolXMLSlice.py

@@ -5,24 +5,19 @@
 """
 import os
 
-from DIRAC import S_OK, S_ERROR, gLogger
+from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.Resources.Catalog.PoolXMLCatalog import PoolXMLCatalog
 
-COMPONENT_NAME = "PoolXMLSlice"
-
 
 class PoolXMLSlice:
-    #############################################################################
-    def __init__(self, catalogName):
+    def __init__(self, catalogName, jobID_path: os.PathLike):
         """Standard constructor"""
         self.fileName = catalogName
-        self.name = COMPONENT_NAME
-        self.log = gLogger.getSubLogger(self.name)
+        self.jobID_path = jobID_path
+        self.log = gLogger.getSubLogger(self.__class__.__name__)
 
-    #############################################################################
     def execute(self, dataDict):
         """Given a dictionary of resolved input data, this will creates a POOL XML slice."""
-        poolXMLCatName = self.fileName
         try:
             poolXMLCat = PoolXMLCatalog()
             self.log.verbose("Creating POOL XML slice")
@@ -46,21 +41,19 @@ class PoolXMLSlice:
             xmlSlice = poolXMLCat.toXML()
             self.log.verbose("POOL XML Slice is: ")
             self.log.verbose(xmlSlice)
-            with open(poolXMLCatName, "w") as poolSlice:
+            with open(self.jobID_path / self.fileName, "w") as poolSlice:
                 poolSlice.write(xmlSlice)
-            self.log.info(f"POOL XML Catalogue slice written to {poolXMLCatName}")
+            self.log.info(f"POOL XML Catalogue slice written to {self.jobID_path / self.fileName}")
             try:
                 # Temporary solution to the problem of storing the SE in the Pool XML slice
-                with open(f"{poolXMLCatName}.temp", "w") as poolSlice_temp:
+                with open(self.jobID_path / (self.fileName + ".temp"), "w") as poolSlice_temp:
                     xmlSlice = poolXMLCat.toXML(True)
                     poolSlice_temp.write(xmlSlice)
-            except Exception as x:
-                self.log.warn(f"Attempted to write catalog also to {poolXMLCatName}.temp but this failed")
-        except Exception as x:
-            self.log.error(str(x))
+            except Exception:
+                self.log.warn(f"Attempted to write catalog also to {self.fileName}.temp but this failed")
+                self.log.exception()
+        except Exception:
+            self.log.exception()
             return S_ERROR("Exception during construction of POOL XML slice")
 
         return S_OK("POOL XML Slice created")
-
-
-# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#

DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py

@@ -1,13 +1,18 @@
 """ Client for the SandboxStore.
     Will connect to the WorkloadManagement/SandboxStore service.
 """
+from __future__ import annotations
 
 import hashlib
 import os
 import re
 import tarfile
 import tempfile
+from contextlib import contextmanager
 from io import BytesIO, StringIO
+from typing import Literal
+
+import zstandard
 
 from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
@@ -18,9 +23,27 @@ from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 
 
+@contextmanager
+def ZstdCompatibleTarFile(tarFileName: os.PathLike, *, mode: Literal["r"] = "r"):
+    """Context manager to extend tarfile.open to support zstd compressed files.
+
+    This is only needed for Python <=3.13.
+    """
+    with open(tarFileName, "rb") as f:
+        magic = f.read(4)
+    # Read magic bytes to determine compression format
+    if magic.startswith(b"\x28\xb5\x2f\xfd"):  # zstd magic number
+        dctx = zstandard.ZstdDecompressor()
+        with open(tarFileName, "rb") as f, dctx.stream_reader(f) as decompressor:
+            with tarfile.open(fileobj=decompressor, mode=f"{mode}|") as tf:
+                yield tf
+    else:
+        with tarfile.open(name=tarFileName, mode=mode) as tf:
+            yield tf
+
+
 class SandboxStoreClient:
     __validSandboxTypes = ("Input", "Output")
-    __smdb = None
 
     def __init__(self, rpcClient=None, transferClient=None, smdb=False, **kwargs):
         """Constructor
@@ -37,21 +60,8 @@ class SandboxStoreClient:
         self.__transferClient = transferClient
         self.__kwargs = kwargs
         self.__vo = None
-        SandboxStoreClient.__smdb = smdb
         if "delegatedGroup" in kwargs:
             self.__vo = getVOForGroup(kwargs["delegatedGroup"])
-        if SandboxStoreClient.__smdb is True:
-            try:
-                from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
-
-                SandboxStoreClient.__smdb = SandboxMetadataDB()
-                result = SandboxStoreClient.__smdb._getConnection()  # pylint: disable=protected-access
-                if not result["OK"]:
-                    SandboxStoreClient.__smdb = False
-                else:
-                    result["Value"].close()
-            except (ImportError, RuntimeError, AttributeError):
-                SandboxStoreClient.__smdb = False
 
     def __getRPCClient(self):
         """Get an RPC client for SB service"""
@@ -206,7 +216,7 @@ class SandboxStoreClient:
 
         try:
             sandboxSize = 0
-            with tarfile.open(name=tarFileName, mode="r") as tf:
+            with ZstdCompatibleTarFile(tarFileName, mode="r") as tf:
                 for tarinfo in tf:
                     tf.extract(tarinfo, path=destinationDir)
                     sandboxSize += tarinfo.size
@@ -227,29 +237,6 @@ class SandboxStoreClient:
     ##############
     # Jobs
 
-    def assignSandboxesToJob(self, jobId, sbList, ownerName="", ownerGroup=""):
-        """
-        Assign sandboxes to a job.
-        sbList must be a list of sandboxes and relation types
-        sbList = [ ( "SB:SEName|SEPFN", "Input" ), ( "SB:SEName|SEPFN", "Output" ) ]
-        """
-        eId = f"Job:{jobId}"
-        for sbT in sbList:
-            if sbT[1] not in self.__validSandboxTypes:
-                return S_ERROR(f"Invalid Sandbox type {sbT[1]}")
-        if SandboxStoreClient.__smdb and ownerName and ownerGroup:
-            return SandboxStoreClient.__smdb.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup)
-        return self.__getRPCClient().assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup)
-
-    def unassignJobs(self, jobIdList):
-        """Unassign SB to a job"""
-        if isinstance(jobIdList, int):
-            jobIdList = [jobIdList]
-        entitiesList = []
-        for jobId in jobIdList:
-            entitiesList.append(f"Job:{jobId}")
-        return self.__getRPCClient().unassignEntities(entitiesList)
-
     def downloadSandboxForJob(self, jobId, sbType, destinationPath="", inMemory=False, unpack=True):
         """Download SB for a job"""
         result = self.__getRPCClient().getSandboxesAssignedToEntity(f"Job:{jobId}")

DIRAC/WorkloadManagementSystem/Client/WMSClient.py

@@ -2,11 +2,10 @@
     methods necessary to communicate with the Workload Management System
 """
 import os
-from io import StringIO
 import time
+from io import StringIO
 
-from DIRAC import S_OK, S_ERROR, gLogger
-
+from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.Utilities import File
 from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd

DIRAC/WorkloadManagementSystem/Client/test/Test_Client_DownloadInputData.py

@@ -1,6 +1,8 @@
 """Test for WMS clients."""
 # pylint: disable=protected-access, missing-docstring, invalid-name
 
+from pathlib import Path
+import shutil
 import pytest
 
 from unittest.mock import MagicMock
@@ -280,3 +282,30 @@ def test_DLI_execute_NoLocal(mocker, dli, mockSE):
     assert res["Value"]["Failed"]
     assert "/a/lfn/1.txt" in res["Value"]["Failed"], res
     assert res["Value"]["Failed"][0] == "/a/lfn/1.txt", res
+
+
+def test_DLI_execute_jobIDPath(mocker, dli, mockSE):
+    """Specify a jobIDPath. Output should be in the jobIDPath directory."""
+    mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2)
+
+    # Create the jobIDPath directory
+    jobIDPath = Path().cwd() / "job" / "12345"
+    jobIDPath.mkdir(parents=True, exist_ok=True)
+
+    dli.configuration["JobIDPath"] = str(jobIDPath)
+
+    mocker.patch("DIRAC.WorkloadManagementSystem.Client.DownloadInputData.gConfig.getValue", return_value=2)
+    mockObjectSE = mockSE.return_value
+    mockObjectSE.getFileMetadata.return_value = S_OK(
+        {"Successful": {"/a/lfn/1.txt": {"Cached": 1, "Accessible": 0}}, "Failed": {}}
+    )
+    dli._downloadFromSE = MagicMock(side_effect=[S_ERROR("Failed to down"), S_OK({"path": jobIDPath / "1.txt"})])
+    dli._isCache = MagicMock(return_value=True)
+    res = dli.execute(dataToResolve=["/a/lfn/1.txt"])
+
+    assert res["OK"]
+    assert not res["Value"]["Failed"]
+    assert "/a/lfn/1.txt" in res["Value"]["Successful"], res
+
+    # Check that the output path is in the jobIDPath directory
+    assert res["Value"]["Successful"]["/a/lfn/1.txt"]["path"] == jobIDPath / "1.txt", res

DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg

@@ -263,14 +263,10 @@ Agents
     # the DN of the certificate proxy used to submit pilots. If not found here, what is in Operations/Pilot section of the CS will be used
     PilotDN =
 
-    # List of sites that will be treated by this SiteDirector (No value can refer to any Site defined in the CS)
-    Site =
-    # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
-    CEs =
-    # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
-    CETypes =
-    # List of Tags that are required to be present in the CE/Queue definition
-    Tags =
+    # Site = # List of CEs that will be treated by this SiteDirector (No value can refer to any CE defined in the CS)
+    # CEs =  # List of CE types that will be treated by this SiteDirector (No value can refer to any type of CE defined in the CS)
+    # CETypes = # List of Tags that are required to be present in the CE/Queue definition
+    # Tags =
 
     # How many cycles to skip if queue is not working
     FailedQueueCycleFactor = 10