DIRAC 9.0.0a62__py3-none-any.whl → 9.0.0a64__py3-none-any.whl

DIRAC/ConfigurationSystem/Client/Helpers/Registry.py

@@ -457,7 +457,7 @@ def getVOForGroup(group):
 
     :return: str
     """
-    return getVO() or gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VO", "")
+    return gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VO", "") or getVO()
 
 
 def getIdPForGroup(group):

DIRAC/Core/Security/DiracX.py

@@ -47,7 +47,7 @@ RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(.*)\n{PEM_END}", re.MULTILINE | re.D
 def addTokenToPEM(pemPath, group):
     from DIRAC.Core.Base.Client import Client
 
-    vo = Registry.getVOMSVOForGroup(group)
+    vo = Registry.getVOForGroup(group)
     if not vo:
         gLogger.error(f"ERROR: Could not find VO for group {group}, DiracX will not work!")
     disabledVOs = gConfig.getValue("/DiracX/DisabledVOs", [])

DIRAC/Core/Tornado/Client/ClientSelector.py

@@ -17,7 +17,6 @@ from DIRAC.Core.DISET.RPCClient import RPCClient
 from DIRAC.Core.DISET.TransferClient import TransferClient
 from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient
 
-
 sLog = gLogger.getSubLogger(__name__)
 
 
@@ -82,6 +81,10 @@ def ClientSelector(disetClient, *args, **kwargs):  # We use same interface as RP
             rpc = tornadoClient(*args, **kwargs)
         else:
             rpc = disetClient(*args, **kwargs)
+    except NotImplementedError as e:
+        # We catch explicitly NotImplementedError to avoid just printing "there's an error"
+        # If we mis-configured the CS for legacy adapted services, we MUST have an error.
+        raise e
     except Exception as e:  # pylint: disable=broad-except
         # If anything went wrong in the resolution, we return default RPCClient
         # So the behaviour is exactly the same as before implementation of Tornado

DIRAC/Core/Utilities/Extensions.py

@@ -73,6 +73,15 @@ def findServices(modules):
     return findModules(modules, "Service", "*Handler")
 
 
+def findFutureServices(modules):
+    """Find the legacy adapted services for one or more DIRAC extension(s)
+
+    :param list/str/module module: One or more Python modules or Python module names
+    :returns: list of tuples of the form (SystemName, ServiceName)
+    """
+    return findModules(modules, "FutureClient")
+
+
 @iterateThenSort
 def findDatabases(module):
     """Find the DB SQL schema defintions for one or more DIRAC extension(s)
@@ -182,7 +191,7 @@ def parseArgs():
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers(required=True, dest="function")
     defaultExtensions = extensionsByPriority()
-    for func in [findSystems, findAgents, findExecutors, findServices, findDatabases]:
+    for func in [findSystems, findAgents, findExecutors, findServices, findDatabases, findFutureServices]:
         subparser = subparsers.add_parser(func.__name__)
         subparser.add_argument("--extensions", nargs="+", default=defaultExtensions)
         subparser.set_defaults(func=func)

DIRAC/Interfaces/API/Dirac.py

@@ -46,6 +46,7 @@ from DIRAC.WorkloadManagementSystem.Client import JobStatus
 from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
 from DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient import SandboxStoreClient
 from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient
+from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import _filterJobStateTransition
 
 
 def parseArguments(args):
@@ -421,6 +422,14 @@ class Dirac(API):
                 sandbox = [isFile.strip() for isFile in sandbox.split(",")]
             for isFile in sandbox:
                 self.log.debug(f"Resolving Input Sandbox {isFile}")
+                # If the sandbox is already in the sandbox store, download it
+                if isFile.startswith("SB:"):
+                    result = SandboxStoreClient(useCertificates=self.useCertificates).downloadSandbox(
+                        isFile, destinationDir=os.getcwd()
+                    )
+                    if not result["OK"]:
+                        return S_ERROR(f"Cannot download Input sandbox {isFile}: {result['Message']}")
+                    continue
                 if isFile.lower().startswith("lfn:"):  # isFile is an LFN
                     isFile = isFile[4:]
                 # Attempt to copy into job working directory, unless it is already there
@@ -1450,10 +1459,13 @@ class Dirac(API):
         # Remove any job IDs that can't change to the Killed or Deleted states
         filteredJobs = set()
         for filterState in (JobStatus.KILLED, JobStatus.DELETED):
-            filterRes = JobStatus.filterJobStateTransition(jobIDs, filterState)
-            if not filterRes["OK"]:
-                return filterRes
-            filteredJobs.update(filterRes["Value"])
+            # get a dictionary of jobID:status
+            res = JobMonitoringClient().getJobsStatus(jobIDs)
+            if not res["OK"]:
+                return res
+            js = {k: v["Status"] for k, v in res["Value"].items()}
+            # then filter
+            filteredJobs.update(_filterJobStateTransition(js, filterState))
 
         return WMSClient(useCertificates=self.useCertificates).deleteJob(list(filteredJobs))
 
@@ -1480,11 +1492,13 @@ class Dirac(API):
             return ret
         jobIDs = ret["Value"]
 
-        # Remove any job IDs that can't change to the rescheduled state
-        filterRes = JobStatus.filterJobStateTransition(jobIDs, JobStatus.RESCHEDULED)
-        if not filterRes["OK"]:
-            return filterRes
-        jobIDsToReschedule = filterRes["Value"]
+        # get a dictionary of jobID:status
+        res = JobMonitoringClient().getJobsStatus(jobIDs)
+        if not res["OK"]:
+            return res
+        js = {k: v["Status"] for k, v in res["Value"].items()}
+        # then filter
+        jobIDsToReschedule = _filterJobStateTransition(js, JobStatus.RESCHEDULED)
 
         return WMSClient(useCertificates=self.useCertificates).rescheduleJob(jobIDsToReschedule)
 
@@ -1510,10 +1524,13 @@ class Dirac(API):
         # Remove any job IDs that can't change to the Killed or Deleted states
         filteredJobs = set()
         for filterState in (JobStatus.KILLED, JobStatus.DELETED):
-            filterRes = JobStatus.filterJobStateTransition(jobIDs, filterState)
-            if not filterRes["OK"]:
-                return filterRes
-            filteredJobs.update(filterRes["Value"])
+            # get a dictionary of jobID:status
+            res = JobMonitoringClient().getJobsStatus(jobIDs)
+            if not res["OK"]:
+                return res
+            js = {k: v["Status"] for k, v in res["Value"].items()}
+            # then filter
+            filteredJobs.update(_filterJobStateTransition(js, filterState))
 
         return WMSClient(useCertificates=self.useCertificates).killJob(list(filteredJobs))
 

DIRAC/Interfaces/API/Job.py

@@ -1105,6 +1105,7 @@ class Job(API):
                 uniqueInputSandbox = uniqueElements(finalInputSandbox.split(";"))
                 paramsDict["InputSandbox"]["value"] = ";".join(uniqueInputSandbox)
                 self.log.verbose(f"Final unique Input Sandbox {';'.join(uniqueInputSandbox)}")
+                paramsDict["InputSandbox"]["type"] = "JDL"
             else:
                 paramsDict["InputSandbox"] = {}
                 paramsDict["InputSandbox"]["value"] = extraFiles

DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py

@@ -32,9 +32,12 @@ from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.TransformationSystem.Client import TransformationStatus
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
-from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
-from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
+from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
+    RIGHT_DELETE,
+    RIGHT_KILL,
+)
+from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs
 
 # # agent's name
 AGENT_NAME = "Transformation/TransformationCleaningAgent"
@@ -58,8 +61,6 @@ class TransformationCleaningAgent(AgentModule):
 
         # # transformation client
         self.transClient = None
-        # # wms client
-        self.wmsClient = None
         # # request client
         self.reqClient = None
         # # file catalog client
@@ -120,14 +121,10 @@ class TransformationCleaningAgent(AgentModule):
 
         # # transformation client
         self.transClient = TransformationClient()
-        # # wms client
-        self.wmsClient = WMSClient()
         # # request client
         self.reqClient = ReqClient()
         # # file catalog client
         self.metadataClient = FileCatalogClient()
-        # # job monitoring client
-        self.jobMonitoringClient = JobMonitoringClient()
         # # job DB
         self.jobDB = JobDB()
 
@@ -271,7 +268,7 @@ class TransformationCleaningAgent(AgentModule):
 
             # Remove JobIDs that were unknown to the TransformationSystem
             jobGroupsToCheck = [str(transDict["TransformationID"]).zfill(8) for transDict in toClean + toArchive]
-            res = self.jobMonitoringClient.getJobs({"JobGroup": jobGroupsToCheck})
+            res = self.jobDB.selectJobs({"JobGroup": jobGroupsToCheck})
             if not res["OK"]:
                 return res
             jobIDsToRemove = [int(jobID) for jobID in res["Value"]]
@@ -613,8 +610,8 @@ class TransformationCleaningAgent(AgentModule):
         # Prevent 0 job IDs
         jobIDs = [int(j) for j in transJobIDs if int(j)]
         allRemove = True
-        for jobList in breakListIntoChunks(jobIDs, 500):
-            res = self.wmsClient.killJob(jobList, force=True)
+        for jobList in breakListIntoChunks(jobIDs, 1000):
+            res = kill_delete_jobs(RIGHT_KILL, jobList, force=True)
             if res["OK"]:
                 self.log.info(f"Successfully killed {len(jobList)} jobs from WMS")
             elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res):
@@ -626,7 +623,7 @@ class TransformationCleaningAgent(AgentModule):
                 self.log.error("Failed to kill jobs", f"(n={len(res['FailedJobIDs'])})")
                 allRemove = False
 
-            res = self.wmsClient.deleteJob(jobList)
+            res = kill_delete_jobs(RIGHT_DELETE, jobList, force=True)
             if res["OK"]:
                 self.log.info("Successfully deleted jobs from WMS", f"(n={len(jobList)})")
             elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res):

DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py

@@ -35,10 +35,12 @@ from DIRAC.RequestManagementSystem.Client.Operation import Operation
 from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
 from DIRAC.RequestManagementSystem.Client.Request import Request
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
-from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
 from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB
+from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_DELETE
+from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs
+from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters
 
 
 class JobCleaningAgent(AgentModule):
@@ -230,11 +232,11 @@ class JobCleaningAgent(AgentModule):
             if not res["OK"]:
                 self.log.error("No DN found", f"for {user}")
                 return res
-            wmsClient = WMSClient(useCertificates=True, delegatedDN=res["Value"][0], delegatedGroup=ownerGroup)
             if remove:
+                wmsClient = WMSClient(useCertificates=True, delegatedDN=res["Value"][0], delegatedGroup=ownerGroup)
                 result = wmsClient.removeJob(jobsList)
             else:
-                result = wmsClient.deleteJob(jobsList)
+                result = kill_delete_jobs(RIGHT_DELETE, jobsList)
             if not result["OK"]:
                 self.log.error(
                     f"Could not {'remove' if remove else 'delete'} jobs",
@@ -294,7 +296,8 @@ class JobCleaningAgent(AgentModule):
         failed = {}
         successful = {}
 
-        result = JobMonitoringClient().getJobParameters(jobIDList, ["OutputSandboxLFN"])
+        jobIDs = [int(jobID) for jobID in jobIDList]
+        result = getJobParameters(jobIDs, "OutputSandboxLFN")
         if not result["OK"]:
             return result
         osLFNDict = result["Value"]

DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py

@@ -14,16 +14,16 @@ import datetime
 from DIRAC import S_ERROR, S_OK, gConfig
 from DIRAC.AccountingSystem.Client.Types.Job import Job
 from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath
-from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.Core.Utilities import DErrno
 from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd
 from DIRAC.Core.Utilities.TimeUtilities import fromString, second, toEpoch
 from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus, JobStatus
-from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB
 from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB
+from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_KILL
+from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs
 from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters
 from DIRAC.WorkloadManagementSystem.Utilities.Utils import rescheduleJobs
 
@@ -235,7 +235,7 @@ class StalledJobAgent(AgentModule):
         # Set the jobs Failed, send them a kill signal in case they are not really dead
         # and send accounting info
         if setFailed:
-            res = self._sendKillCommand(jobID)
+            res = kill_delete_jobs(RIGHT_KILL, [jobID], nonauthJobList=[], force=True)
             if not res["OK"]:
                 self.log.error("Failed to kill job", jobID)
 
@@ -574,26 +574,3 @@ class StalledJobAgent(AgentModule):
                 continue
 
         return S_OK()
-
-    def _sendKillCommand(self, job):
-        """Send a kill signal to the job such that it cannot continue running.
-
-        :param int job: ID of job to send kill command
-        """
-
-        res = self.jobDB.getJobAttribute(job, "Owner")
-        if not res["OK"]:
-            return res
-        owner = res["Value"]
-
-        res = self.jobDB.getJobAttribute(job, "OwnerGroup")
-        if not res["OK"]:
-            return res
-        ownerGroup = res["Value"]
-
-        wmsClient = WMSClient(
-            useCertificates=True,
-            delegatedDN=getDNForUsername(owner)["Value"][0] if owner else None,
-            delegatedGroup=ownerGroup,
-        )
-        return wmsClient.killJob(job)

DIRAC/WorkloadManagementSystem/Agent/StatesAccountingAgent.py

@@ -135,8 +135,15 @@ class StatesAccountingAgent(AgentModule):
             for backend in self.datastores:
                 if backend.lower() == "monitoring":
                     site_name = rD["Site"]
-                    rD["Tier"] = site_metadata[site_name]["Tier"]
-                    rD["Type"] = site_metadata[site_name]["Type"]
+                    if site_name not in site_metadata:
+                        self.log.warn(
+                            f"Site {site_name} not found in site metadata, using default values",
+                        )
+                        rD["Tier"] = "4"
+                        rD["Type"] = site_name.split(".")[0]
+                    else:
+                        rD["Tier"] = site_metadata[site_name]["Tier"]
+                        rD["Type"] = site_metadata[site_name]["Type"]
                     rD["timestamp"] = int(TimeUtilities.toEpochMilliSeconds(now))
                     self.datastores["Monitoring"].addRecord(rD)
 

DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py

@@ -1,10 +1,11 @@
 """ Test class for Job Cleaning Agent
 """
-import pytest
 from unittest.mock import MagicMock
 
+import pytest
+
 # DIRAC Components
-from DIRAC import gLogger, S_OK
+from DIRAC import S_OK, gLogger
 from DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent import JobCleaningAgent
 
 gLogger.setLevel("DEBUG")
@@ -32,7 +33,6 @@ def jca(mocker):
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.selectJobs", side_effect=mockReply)
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.__init__", side_effect=mockNone)
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone)
-    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC)
 
     jca = JobCleaningAgent()
     jca.log = gLogger
@@ -98,7 +98,7 @@ def test_deleteJobsByStatus(jca, conditions, mockReplyInput, expected):
     "inputs, params, expected",
     [
         ([], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
-        (["a", "b"], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
+        (["123", "456"], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}),
         (
             [],
             {"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}},
@@ -113,11 +113,11 @@ def test_deleteJobsByStatus(jca, conditions, mockReplyInput, expected):
             {"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt", 2: "/some/other/lfn/2.txt"}}},
         ),
         (
-            ["a", "b"],
+            ["123", "456"],
             {"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}},
             {"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt"}}},
         ),
-        (["a", "b"], {"OK": False}, {"OK": False}),
+        (["123", "456"], {"OK": False}, {"OK": False}),
     ],
 )
 def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
@@ -127,10 +127,10 @@ def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.am_getOption", return_value=mockAM)
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB", return_value=mockNone)
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone)
-    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC)
     mocker.patch(
         "DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.getDNForUsername", return_value=S_OK(["/bih/boh/DN"])
     )
+    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.getJobParameters", return_value=params)
 
     jobCleaningAgent = JobCleaningAgent()
     jobCleaningAgent.log = gLogger
@@ -138,8 +138,6 @@ def test_deleteJobOversizedSandbox(mocker, inputs, params, expected):
     jobCleaningAgent._AgentModule__configDefaults = mockAM
     jobCleaningAgent.initialize()
 
-    mockJMC.getJobParameters.return_value = params
-
     result = jobCleaningAgent.deleteJobOversizedSandbox(inputs)
 
     assert result == expected

DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py

@@ -28,8 +28,7 @@ def sja(mocker):
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.rescheduleJobs", return_value=MagicMock())
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.PilotAgentsDB", return_value=MagicMock())
     mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getJobParameters", return_value=MagicMock())
-    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.WMSClient", return_value=MagicMock())
-    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getDNForUsername", return_value=MagicMock())
+    mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.kill_delete_jobs", return_value=MagicMock())
 
     stalledJobAgent = StalledJobAgent()
     stalledJobAgent._AgentModule__configDefaults = mockAM

DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py

@@ -14,12 +14,14 @@ except ImportError:
 
 @createClient("WorkloadManagement/JobMonitoring")
 class JobMonitoringClient(Client):
+    # Set to None to raise an error if this service is set as "legacy adapted"
+    # See ClientSelector
+    diracxClient = None
+
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.setServer("WorkloadManagement/JobMonitoring")
 
-    diracxClient = futureJobMonitoringClient
-
     @ignoreEncodeWarning
     def getJobsStatus(self, jobIDs):
         res = self._getRPC().getJobsStatus(jobIDs)

DIRAC/WorkloadManagementSystem/Client/JobStatus.py

@@ -2,12 +2,7 @@
 This module contains constants and lists for the possible job states.
 """
 
-from DIRAC import gLogger, S_OK, S_ERROR
 from DIRAC.Core.Utilities.StateMachine import State, StateMachine
-from DIRAC.Core.Utilities.Decorators import deprecated
-
-from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
-
 
 #:
 SUBMITTING = "Submitting"
@@ -98,57 +93,3 @@ class JobsStateMachine(StateMachine):
             RECEIVED: State(1, [SCOUTING, CHECKING, STAGING, WAITING, FAILED, DELETED, KILLED], defState=RECEIVED),
             SUBMITTING: State(0, [RECEIVED, CHECKING, DELETED, KILLED], defState=SUBMITTING),  # initial state
         }
-
-
-@deprecated("Use filterJobStateTransition instead")
-def checkJobStateTransition(jobID, candidateState, currentStatus=None, jobMonitoringClient=None):
-    """Utility to check if a job state transition is allowed"""
-    if not currentStatus:
-        if not jobMonitoringClient:
-            from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
-
-            jobMonitoringClient = JobMonitoringClient()
-
-        res = jobMonitoringClient.getJobsStatus(jobID)
-        if not res["OK"]:
-            return res
-        try:
-            currentStatus = res["Value"][jobID]["Status"]
-        except KeyError:
-            return S_ERROR("Job does not exist")
-
-    res = JobsStateMachine(currentStatus).getNextState(candidateState)
-    if not res["OK"]:
-        return res
-
-    # If the JobsStateMachine does not accept the candidate, return an ERROR
-    if candidateState != res["Value"]:
-        gLogger.error(
-            "Job Status Error",
-            f"{jobID} can't move from {currentStatus} to {candidateState}",
-        )
-        return S_ERROR("Job state transition not allowed")
-    return S_OK()
-
-
-def filterJobStateTransition(jobIDs, candidateState):
-    """Given a list of jobIDs, return a list that are allowed to transition
-    to the given candidate state.
-    """
-    allowedJobs = []
-
-    if not isinstance(jobIDs, list):
-        jobIDs = [jobIDs]
-
-    res = JobMonitoringClient().getJobsStatus(jobIDs)
-    if not res["OK"]:
-        return res
-
-    for jobID in jobIDs:
-        if jobID in res["Value"]:
-            curState = res["Value"][jobID]["Status"]
-            stateRes = JobsStateMachine(curState).getNextState(candidateState)
-            if stateRes["OK"]:
-                if stateRes["Value"] == candidateState:
-                    allowedJobs.append(jobID)
-    return S_OK(allowedJobs)

DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py

@@ -1,13 +1,18 @@
 """ Client for the SandboxStore.
     Will connect to the WorkloadManagement/SandboxStore service.
 """
+from __future__ import annotations
 
 import hashlib
 import os
 import re
 import tarfile
 import tempfile
+from contextlib import contextmanager
 from io import BytesIO, StringIO
+from typing import Literal
+
+import zstandard
 
 from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
@@ -18,6 +23,25 @@ from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 
 
+@contextmanager
+def ZstdCompatibleTarFile(tarFileName: os.PathLike, *, mode: Literal["r"] = "r"):
+    """Context manager to extend tarfile.open to support zstd compressed files.
+
+    This is only needed for Python <=3.13.
+    """
+    with open(tarFileName, "rb") as f:
+        magic = f.read(4)
+    # Read magic bytes to determine compression format
+    if magic.startswith(b"\x28\xb5\x2f\xfd"):  # zstd magic number
+        dctx = zstandard.ZstdDecompressor()
+        with open(tarFileName, "rb") as f, dctx.stream_reader(f) as decompressor:
+            with tarfile.open(fileobj=decompressor, mode=f"{mode}|") as tf:
+                yield tf
+    else:
+        with tarfile.open(name=tarFileName, mode=mode) as tf:
+            yield tf
+
+
 class SandboxStoreClient:
     __validSandboxTypes = ("Input", "Output")
 
@@ -192,7 +216,7 @@ class SandboxStoreClient:
 
         try:
             sandboxSize = 0
-            with tarfile.open(name=tarFileName, mode="r") as tf:
+            with ZstdCompatibleTarFile(tarFileName, mode="r") as tf:
                 for tarinfo in tf:
                     tf.extract(tarinfo, path=destinationDir)
                     sandboxSize += tarinfo.size

DIRAC/WorkloadManagementSystem/Client/WMSClient.py

@@ -2,11 +2,10 @@
     methods necessary to communicate with the Workload Management System
 """
 import os
-from io import StringIO
 import time
+from io import StringIO
 
-from DIRAC import S_OK, S_ERROR, gLogger
-
+from DIRAC import S_ERROR, S_OK, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.Utilities import File
 from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd

DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py

@@ -77,17 +77,11 @@ class JobStateUpdateClient(FutureClient):
     def setJobAttribute(self, jobID: str | int, attribute: str, value: str):
         with DiracXClient() as api:
             if attribute == "Status":
-                api.jobs.set_job_statuses(
+                return api.jobs.set_job_statuses(
                     {jobID: {datetime.now(tz=timezone.utc): {"Status": value}}},
                 )
             else:
-                api.jobs.patch_metadata({jobID: {attribute: value}})
-
-    @stripValueIfOK
-    @convertToReturnValue
-    def setJobFlag(self, jobID: str | int, flag: str):
-        with DiracXClient() as api:
-            api.jobs.patch_metadata({jobID: {flag: True}})
+                return api.jobs.patch_metadata({jobID: {attribute: value}})
 
     @stripValueIfOK
     @convertToReturnValue
@@ -151,12 +145,6 @@ class JobStateUpdateClient(FutureClient):
             updates = {job_id: {k: v} for job_id, (k, v) in jobsParameterDict.items()}
             api.jobs.patch_metadata(updates)
 
-    @stripValueIfOK
-    @convertToReturnValue
-    def unsetJobFlag(self, jobID: str | int, flag: str):
-        with DiracXClient() as api:
-            api.jobs.patch_metadata({jobID: {flag: False}})
-
     @stripValueIfOK
     @convertToReturnValue
     def updateJobFromStager(self, jobID: str | int, status: str):