DIRAC 9.0.0a69__py3-none-any.whl → 9.0.0a70__py3-none-any.whl

DIRAC/TransformationSystem/Agent/InputDataAgent.py

@@ -74,7 +74,10 @@ class InputDataAgent(AgentModule):
         """Main execution method"""
 
         # Get all the transformations
-        result = self.transClient.getTransformations({"Status": "Active", "Type": self.transformationTypes})
+        result = self.transClient.getTransformations(
+            {"Status": "Active", "Type": self.transformationTypes},
+            columns=["TransformationID", "AuthorDN", "AuthorGroup"],
+        )
         if not result["OK"]:
             self.log.error("InputDataAgent.execute: Failed to get transformations.", result["Message"])
             return S_OK()

DIRAC/TransformationSystem/Agent/MCExtensionAgent.py

@@ -1,4 +1,4 @@
-""" Agent to extend the number of tasks given the Transformation definition
+"""Agent to extend the number of tasks given the Transformation definition
 
 The following options can be set for the MCExtensionAgent.
 
@@ -8,6 +8,7 @@ The following options can be set for the MCExtensionAgent.
   :dedent: 2
   :caption: MCExtensionAgent options
 """
+
 from DIRAC import S_OK, gLogger
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
@@ -54,7 +55,9 @@ class MCExtensionAgent(AgentModule):
             return S_OK("Disabled via CS flag")
 
         # Obtain the transformations in Cleaning status and remove any mention of the jobs/files
-        res = self.transClient.getTransformations({"Status": "Active", "Type": self.transformationTypes})
+        res = self.transClient.getTransformations(
+            {"Status": "Active", "Type": self.transformationTypes}, columns=["TransformationID", "MaxNumberOfTasks"]
+        )
         if res["OK"]:
             for transDict in res["Value"]:
                 transID = transDict["TransformationID"]

DIRAC/TransformationSystem/Agent/TaskManagerAgentBase.py

@@ -22,7 +22,6 @@ from DIRAC.TransformationSystem.Client.FileReport import FileReport
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
 from DIRAC.TransformationSystem.Client.WorkflowTasks import WorkflowTasks
 from DIRAC.WorkloadManagementSystem.Client import JobStatus
-from DIRAC.WorkloadManagementSystem.Client.JobManagerClient import JobManagerClient
 
 AGENT_NAME = "Transformation/TaskManagerAgentBase"
 
@@ -39,7 +38,6 @@ class TaskManagerAgentBase(AgentModule, TransformationAgentsUtilities):
         TransformationAgentsUtilities.__init__(self)
 
         self.transClient = None
-        self.jobManagerClient = None
         self.transType = []
 
         self.tasksPerLoop = 50
@@ -68,7 +66,6 @@ class TaskManagerAgentBase(AgentModule, TransformationAgentsUtilities):
 
         # Default clients
         self.transClient = TransformationClient()
-        self.jobManagerClient = JobManagerClient()
 
         # Bulk submission flag
         self.bulkSubmissionFlag = self.am_getOption("BulkSubmission", self.bulkSubmissionFlag)
@@ -234,7 +231,9 @@ class TaskManagerAgentBase(AgentModule, TransformationAgentsUtilities):
             selectCond["Type"] = transType
         if agentType:
             selectCond["AgentType"] = agentType
-        res = self.transClient.getTransformations(condDict=selectCond)
+        res = self.transClient.getTransformations(
+            condDict=selectCond, columns=["TransformationID", "Body", "Author", "AuthorGroup"]
+        )
         if not res["OK"]:
             self.log.error("Failed to get transformations:", res["Message"])
         elif not res["Value"]:

DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py

@@ -20,6 +20,7 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
 from DIRAC.Core.Base.AgentModule import AgentModule
 from DIRAC.Core.Utilities.DErrno import cmpError
 from DIRAC.Core.Utilities.List import breakListIntoChunks
+from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.Core.Utilities.Proxy import executeWithUserProxy
 from DIRAC.Core.Utilities.ReturnValues import returnSingleResult
 from DIRAC.RequestManagementSystem.Client.File import File
@@ -32,7 +33,6 @@ from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.TransformationSystem.Client import TransformationStatus
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
-from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 from DIRAC.WorkloadManagementSystem.Service.JobPolicy import (
     RIGHT_DELETE,
     RIGHT_KILL,
@@ -65,8 +65,11 @@ class TransformationCleaningAgent(AgentModule):
         self.reqClient = None
         # # file catalog client
         self.metadataClient = None
-        # # JobDB
+        # # databases
         self.jobDB = None
+        self.pilotAgentsDB = None
+        self.taskQueueDB = None
+        self.storageManagementDB = None
 
         # # transformations types
         self.transformationTypes = None
@@ -125,8 +128,26 @@ class TransformationCleaningAgent(AgentModule):
         self.reqClient = ReqClient()
         # # file catalog client
         self.metadataClient = FileCatalogClient()
-        # # job DB
-        self.jobDB = JobDB()
+        # # databases
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobDB", "JobDB")
+        if not result["OK"]:
+            return result
+        self.jobDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
+        if not result["OK"]:
+            return result
+        self.pilotAgentsDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
+        if not result["OK"]:
+            return result
+        self.taskQueueDB = result["Value"]()
+
+        result = ObjectLoader().loadObject("StorageManagementSystem.DB.StorageManagementDB", "StorageManagementDB")
+        if not result["OK"]:
+            return result
+        self.storageManagementDB = result["Value"]()
 
         return S_OK()
 
@@ -144,7 +165,8 @@ class TransformationCleaningAgent(AgentModule):
 
         # Obtain the transformations in Cleaning status and remove any mention of the jobs/files
         res = self.transClient.getTransformations(
-            {"Status": TransformationStatus.CLEANING, "Type": self.transformationTypes}
+            {"Status": TransformationStatus.CLEANING, "Type": self.transformationTypes},
+            columns=["TransformationID", "Author", "AuthorGroup", "Type"],
         )
         if res["OK"]:
             for transDict in res["Value"]:
@@ -161,7 +183,10 @@ class TransformationCleaningAgent(AgentModule):
             self.log.error("Failed to get transformations", res["Message"])
 
         # Obtain the transformations in RemovingFiles status and removes the output files
-        res = self.transClient.getTransformations({"Status": "RemovingFiles", "Type": self.transformationTypes})
+        res = self.transClient.getTransformations(
+            {"Status": "RemovingFiles", "Type": self.transformationTypes},
+            columns=["TransformationID", "Author", "AuthorGroup"],
+        )
         if res["OK"]:
             for transDict in res["Value"]:
                 if self.shifterProxy:
@@ -183,6 +208,7 @@ class TransformationCleaningAgent(AgentModule):
             {"Status": TransformationStatus.COMPLETED, "Type": self.transformationTypes},
             older=olderThanTime,
             timeStamp="LastUpdate",
+            columns=["TransformationID", "Author", "AuthorGroup"],
         )
         if res["OK"]:
             for transDict in res["Value"]:
@@ -230,7 +256,10 @@ class TransformationCleaningAgent(AgentModule):
             return res
         transformationIDs = res["Value"]
         if transformationIDs:
-            res = self.transClient.getTransformations({"TransformationID": transformationIDs})
+            res = self.transClient.getTransformations(
+                {"TransformationID": transformationIDs},
+                columns=["TransformationID", "Status", "Author", "AuthorGroup", "Type"],
+            )
             if not res["OK"]:
                 self.log.error("Failed to get transformations", res["Message"])
                 return res
@@ -607,11 +636,17 @@ class TransformationCleaningAgent(AgentModule):
         :param self: self reference
         :param list trasnJobIDs: job IDs
         """
+        db_kwargs = dict(
+            jobdb=self.jobDB,
+            taskqueuedb=self.taskQueueDB,
+            pilotagentsdb=self.pilotAgentsDB,
+            storagemanagementdb=self.storageManagementDB,
+        )
         # Prevent 0 job IDs
         jobIDs = [int(j) for j in transJobIDs if int(j)]
         allRemove = True
         for jobList in breakListIntoChunks(jobIDs, 1000):
-            res = kill_delete_jobs(RIGHT_KILL, jobList, force=True)
+            res = kill_delete_jobs(RIGHT_KILL, jobList, force=True, **db_kwargs)
             if res["OK"]:
                 self.log.info(f"Successfully killed {len(jobList)} jobs from WMS")
             elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res):
@@ -623,7 +658,7 @@ class TransformationCleaningAgent(AgentModule):
                 self.log.error("Failed to kill jobs", f"(n={len(res['FailedJobIDs'])})")
                 allRemove = False
 
-            res = kill_delete_jobs(RIGHT_DELETE, jobList, force=True)
+            res = kill_delete_jobs(RIGHT_DELETE, jobList, force=True, **db_kwargs)
             if res["OK"]:
                 self.log.info("Successfully deleted jobs from WMS", f"(n={len(jobList)})")
             elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res):

DIRAC/TransformationSystem/Agent/ValidateOutputDataAgent.py

@@ -76,7 +76,9 @@ class ValidateOutputDataAgent(AgentModule):
         self.updateWaitingIntegrity()
         gLogger.info("-" * 40)
 
-        res = self.transClient.getTransformations({"Status": "ValidatingOutput", "Type": self.transformationTypes})
+        res = self.transClient.getTransformations(
+            {"Status": "ValidatingOutput", "Type": self.transformationTypes}, columns=["TransformationID"]
+        )
         if not res["OK"]:
             gLogger.error("Failed to get ValidatingOutput transformations", res["Message"])
             return res
@@ -98,7 +100,7 @@ class ValidateOutputDataAgent(AgentModule):
     def updateWaitingIntegrity(self):
         """Get 'WaitingIntegrity' transformations, update to 'ValidatedOutput'"""
         gLogger.info("Looking for transformations in the WaitingIntegrity status to update")
-        res = self.transClient.getTransformations({"Status": "WaitingIntegrity"})
+        res = self.transClient.getTransformations({"Status": "WaitingIntegrity"}, columns=["TransformationID"])
         if not res["OK"]:
             gLogger.error("Failed to get WaitingIntegrity transformations", res["Message"])
             return res

DIRAC/TransformationSystem/Client/TransformationClient.py

@@ -114,13 +114,21 @@ class TransformationClient(Client):
         newer=None,
         timeStamp=None,
         orderAttribute=None,
-        limit=100,
+        limit=None,
         extraParams=False,
         columns=None,
     ):
         """gets all the transformations in the system, incrementally. "limit" here is just used to determine the offset."""
         rpcClient = self._getRPC()
 
+        # If the body is requested (or is served by default)
+        # we take smaller chunk, not to take too much memory
+        # on the server
+        if columns and "Body" not in columns:
+            limit = 100_000
+        else:
+            limit = 1_000
+
         transformations = []
         if condDict is None:
             condDict = {}

DIRAC/TransformationSystem/Client/Utilities.py

@@ -9,7 +9,7 @@ Utilities for Transformation system
 import ast
 import random
 
-from cachetools import LRUCache, cached
+from cachetools import LRUCache, cachedmethod
 from cachetools.keys import hashkey
 from DIRAC import S_OK, S_ERROR, gLogger
 
@@ -24,6 +24,9 @@ from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
 
+# Module-level cache for isSameSEInList method (shared across PluginUtilities instances)
+_is_same_se_in_list_cache = LRUCache(maxsize=1024)
+
 
 class PluginUtilities:
     """
@@ -402,8 +405,8 @@ class PluginUtilities:
 
         return StorageElement(se1).isSameSE(StorageElement(se2))
 
-    @cached(
-        LRUCache(maxsize=1024),
+    @cachedmethod(
+        lambda self: _is_same_se_in_list_cache,
         key=lambda _, a, b: hashkey(a, *sorted(b)),
     )
     def isSameSEInList(self, se1, seList):

DIRAC/TransformationSystem/DB/TransformationDB.py

@@ -6,6 +6,9 @@ This class is typically used as a base class for more specific data processing
 databases
 """
 
+# Disable it because pylint does not understand decorator (convertToReturnValue)
+
+# pylint: disable=invalid-sequence-index
 import re
 import time
 import threading
@@ -15,6 +18,7 @@ from errno import ENOENT
 from DIRAC import gLogger, S_OK, S_ERROR
 from DIRAC.Core.Base.DB import DB
 from DIRAC.Core.Utilities.DErrno import cmpError
+from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueOrRaise
 from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
 from DIRAC.Core.Utilities.List import stringListToString, intListToString, breakListIntoChunks
@@ -25,6 +29,7 @@ from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery
 
 MAX_ERROR_COUNT = 10
 
+TMP_TABLE_JOIN_LIMIT = 100
 #############################################################################
 
 
@@ -270,6 +275,7 @@ class TransformationDB(DB):
         self.__updateTransformationLogging(transID, message, author, connection=connection)
         return S_OK(transID)
 
+    @convertToReturnValue
     def getTransformations(
         self,
         condDict=None,
@@ -289,32 +295,54 @@ class TransformationDB(DB):
             columns = self.TRANSPARAMS
         else:
             columns = [c for c in columns if c in self.TRANSPARAMS]
-        req = "SELECT {} FROM Transformations {}".format(
-            intListToString(columns),
-            self.buildCondition(condDict, older, newer, timeStamp, orderAttribute, limit, offset=offset),
-        )
-        res = self._query(req, conn=connection)
-        if not res["OK"]:
-            return res
-        if condDict is None:
-            condDict = {}
-        webList = []
+
+        join_query = ""
+
+        try:
+            # If we request multiple TransformationIDs, and they are more than TMP_TABLE_JOIN_LIMIT,
+            # we create a temporary table to speed up the query
+            if (
+                "TransformationID" in condDict
+                and isinstance(condDict["TransformationID"], list)
+                and len(condDict["TransformationID"]) > TMP_TABLE_JOIN_LIMIT
+            ):
+                # Create temporary table for TransformationIDs
+                transIDs = condDict.pop("TransformationID")
+                sqlCmd = "CREATE TEMPORARY TABLE to_query_TransformationIDs (TransID INTEGER NOT NULL, PRIMARY KEY (TransID)) ENGINE=MEMORY;"
+                returnValueOrRaise(self._update(sqlCmd, conn=connection))
+                join_query = " JOIN to_query_TransformationIDs t ON TransformationID = t.TransID"
+
+                # Insert TransformationIDs into temporary table
+                sqlCmd = "INSERT INTO to_query_TransformationIDs (TransID) VALUES ( %s )"
+                returnValueOrRaise(self._updatemany(sqlCmd, [(transID,) for transID in transIDs], conn=connection))
+
+            req = "SELECT {} FROM Transformations {} {}".format(
+                intListToString(columns),
+                join_query,
+                self.buildCondition(condDict, older, newer, timeStamp, orderAttribute, limit, offset=offset),
+            )
+            matching_transformations = returnValueOrRaise(self._query(req, conn=connection))
+
+        finally:
+            # Clean up temporary table
+            if join_query:
+                sqlCmd = "DROP TEMPORARY TABLE to_query_TransformationIDs"
+                self._update(sqlCmd, conn=connection)
+
+        # TODO: optimize by getting all the extra params at once
         resultList = []
-        for row in res["Value"]:
+        for row in matching_transformations:
             # Prepare the structure for the web
-            rList = [str(item) if not isinstance(item, int) else item for item in row]
             transDict = dict(zip(columns, row))
-            webList.append(rList)
             if extraParams:
-                res = self.__getAdditionalParameters(transDict["TransformationID"], connection=connection)
-                if not res["OK"]:
-                    return res
-                transDict.update(res["Value"])
+                trans_extra_param = returnValueOrRaise(
+                    self.__getAdditionalParameters(transDict["TransformationID"], connection=connection)
+                )
+
+                transDict.update(trans_extra_param)
             resultList.append(transDict)
-        result = S_OK(resultList)
-        result["Records"] = webList
-        result["ParameterNames"] = columns
-        return result
+
+        return resultList
 
     def getTransformation(self, transName, extraParams=False, connection=False):
         """Get Transformation definition and parameters of Transformation identified by TransformationID"""
@@ -710,21 +738,38 @@ class TransformationDB(DB):
         countDict["Total"] = sum(countDict.values())
         return S_OK(countDict)
 
+    @convertToReturnValue
     def __addFilesToTransformation(self, transID, fileIDs, connection=False):
-        req = "SELECT FileID from TransformationFiles"
-        req = req + " WHERE TransformationID = %d AND FileID IN (%s);" % (transID, intListToString(fileIDs))
-        res = self._query(req, conn=connection)
-        if not res["OK"]:
-            return res
-        for tupleIn in res["Value"]:
-            fileIDs.remove(tupleIn[0])
-        if not fileIDs:
-            return S_OK([])
-        values = [(transID, fileID) for fileID in fileIDs]
-        req = "INSERT INTO TransformationFiles (TransformationID,FileID,LastUpdate,InsertedTime) VALUES (%s, %s, UTC_TIMESTAMP(), UTC_TIMESTAMP())"
-        if not (res := self._updatemany(req, values, conn=connection))["OK"]:
-            return res
-        return S_OK(fileIDs)
+        # Create temporary table for FileIDs
+        sqlCmd = "CREATE TEMPORARY TABLE to_query_FileIDs (FileID INT(11) UNSIGNED NOT NULL, PRIMARY KEY (FileID)) ENGINE=MEMORY;"
+        returnValueOrRaise(self._update(sqlCmd, conn=connection))
+
+        try:
+            # Insert FileIDs into temporary table
+            sqlCmd = "INSERT INTO to_query_FileIDs (FileID) VALUES ( %s )"
+            returnValueOrRaise(self._updatemany(sqlCmd, [(fileID,) for fileID in fileIDs], conn=connection))
+
+            # Query existing files using JOIN
+            req = (
+                "SELECT tf.FileID FROM TransformationFiles tf JOIN to_query_FileIDs t ON tf.FileID = t.FileID WHERE tf.TransformationID = %d;"
+                % transID
+            )
+            res = returnValueOrRaise(self._query(req, conn=connection))
+
+            # Remove already existing fileIDs using set difference for efficiency
+            existingFileIDs = {tupleIn[0] for tupleIn in res}
+            fileIDs = list(set(fileIDs) - existingFileIDs)
+            if not fileIDs:
+                return []
+
+            values = [(transID, fileID) for fileID in fileIDs]
+            req = "INSERT INTO TransformationFiles (TransformationID,FileID,LastUpdate,InsertedTime) VALUES (%s, %s, UTC_TIMESTAMP(), UTC_TIMESTAMP())"
+            returnValueOrRaise(self._updatemany(req, values, conn=connection))
+            return fileIDs
+        finally:
+            # Clean up temporary table
+            sqlCmd = "DROP TEMPORARY TABLE to_query_FileIDs"
+            returnValueOrRaise(self._update(sqlCmd, conn=connection))
 
     def __insertExistingTransformationFiles(self, transID, fileTuplesList, connection=False):
         """Inserting already transformation files in TransformationFiles table (e.g. for deriving transformations)"""
@@ -1271,18 +1316,35 @@ class TransformationDB(DB):
     # These methods manipulate the DataFiles table
     #
 
+    @convertToReturnValue
     def __getFileIDsForLfns(self, lfns, connection=False):
         """Get file IDs for the given list of lfns
         warning: if the file is not present, we'll see no errors
         """
-        req = f"SELECT LFN,FileID FROM DataFiles WHERE LFN in ({stringListToString(lfns)});"
-        res = self._query(req, conn=connection)
-        if not res["OK"]:
-            return res
-        lfns = dict(res["Value"])
-        # Reverse dictionary
-        fids = {fileID: lfn for lfn, fileID in lfns.items()}
-        return S_OK((fids, lfns))
+
+        if not lfns:
+            return ({}, {})
+        # Create temporary table for LFNs
+        sqlCmd = "CREATE TEMPORARY TABLE to_query_LFNs (LFN VARCHAR(255) NOT NULL, PRIMARY KEY (LFN)) ENGINE=MEMORY;"
+        returnValueOrRaise(self._update(sqlCmd, conn=connection))
+
+        try:
+            # Insert LFNs into temporary table
+            sqlCmd = "INSERT INTO to_query_LFNs (LFN) VALUES ( %s )"
+            returnValueOrRaise(self._updatemany(sqlCmd, [(lfn,) for lfn in lfns], conn=connection))
+
+            # Query using JOIN with temporary table
+            req = "SELECT df.LFN, df.FileID FROM DataFiles df JOIN to_query_LFNs t ON df.LFN = t.LFN;"
+            res = returnValueOrRaise(self._query(req, conn=connection))
+
+            lfns = dict(res)
+            # Reverse dictionary
+            fids = {fileID: lfn for lfn, fileID in lfns.items()}
+            return (fids, lfns)
+        finally:
+            # Clean up temporary table
+            sqlCmd = "DROP TEMPORARY TABLE to_query_LFNs"
+            self._update(sqlCmd, conn=connection)
 
     def __getLfnsForFileIDs(self, fileIDs, connection=False):
         """Get lfns for the given list of fileIDs"""

DIRAC/TransformationSystem/Utilities/ReplicationCLIParameters.py

@@ -1,10 +1,10 @@
 """
 Command Line Parameters for creating the Replication transformations Script
 """
-from DIRAC import S_OK, S_ERROR, gLogger
+from DIRAC import S_ERROR, S_OK, gLogger
+from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
 from DIRAC.Core.Security.Properties import SecurityProperty
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
-from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOMSVOForGroup
 
 
 class Params:
@@ -144,7 +144,7 @@ class Params:
             return False
         proxyValues = proxyInfo.get("Value", {})
         group = proxyValues.get("group", "")
-        vomsvo = getVOMSVOForGroup(group)
+        vomsvo = getVOForGroup(group)
         if not vomsvo:
             self.errorMessages.append("ERROR: ProxyGroup not associated to VOMS VO, get a different proxy")
             return False

DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py

@@ -16,7 +16,7 @@ from urllib.request import urlopen
 from DIRAC.Interfaces.API.Dirac import Dirac
 from DIRAC.Core.Utilities.File import mkDir
 from DIRAC.Core.Base.Script import Script
-from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO, getSetup
+from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
 
 
@@ -79,14 +79,12 @@ def __configurePilot(basepath, vo):
     This method was created specifically for LHCb pilots, more info
     about othe VOs is needed to make it more general.
     """
-    currentSetup = getSetup()
     masterCS = gConfigurationData.getMasterServer()
 
     os.system(
         "python "
         + basepath
-        + "dirac-pilot.py -S %s -l %s -C %s -N ce.debug.ch -Q default -n DIRAC.JobDebugger.ch -dd"
-        % (currentSetup, vo, masterCS)
+        + f"dirac-pilot.py -l {vo} -C {masterCS} -N ce.debug.ch -Q default -n DIRAC.JobDebugger.ch -dd"
     )
 
     diracdir = os.path.expanduser("~") + os.path.sep

DIRAC/TransformationSystem/test/Test_replicationTransformation.py

@@ -1,14 +1,13 @@
 """Test the dirac-transformation-replication script and helper"""
 import unittest
+from unittest.mock import MagicMock as Mock
+from unittest.mock import patch
 
-from unittest.mock import MagicMock as Mock, patch
-
-from DIRAC import S_OK, S_ERROR
-
-from DIRAC.TransformationSystem.Utilities.ReplicationTransformation import createDataTransformation
+from DIRAC import S_ERROR, S_OK
 from DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters import Params
+from DIRAC.TransformationSystem.Utilities.ReplicationTransformation import createDataTransformation
 
-GET_VOMS = "DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters.getVOMSVOForGroup"
+GET_VOMS = "DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters.getVOForGroup"
 GET_PROXY = "DIRAC.TransformationSystem.Utilities.ReplicationCLIParameters.getProxyInfo"
 
 

DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py

@@ -1,4 +1,4 @@
-"""  The Site Director is an agent performing pilot job submission to particular sites/Computing Elements.
+"""The Site Director is an agent performing pilot job submission to particular sites/Computing Elements.
 
 .. literalinclude:: ../ConfigTemplate.cfg
   :start-after: ##BEGIN SiteDirector
@@ -7,6 +7,7 @@
   :caption: SiteDirector options
 
 """
+
 import datetime
 import os
 from collections import defaultdict
@@ -147,10 +148,10 @@ class SiteDirector(AgentModule):
             self.sendSubmissionAccounting = True
 
         # Get the site description dictionary
-        siteNames = self.am_getOption("Site", [])
-        ceTypes = self.am_getOption("CETypes", [])
-        ces = self.am_getOption("CEs", [])
-        tags = self.am_getOption("Tags", [])
+        siteNames = self.am_getOption("Site", []) or None
+        ceTypes = self.am_getOption("CETypes", []) or None
+        ces = self.am_getOption("CEs", []) or None
+        tags = self.am_getOption("Tags", []) or None
 
         # Display options used
         self.log.always("VO:", self.vo)
@@ -229,12 +230,8 @@ class SiteDirector(AgentModule):
             site = self.queueDict[queueName]["Site"]
             ce = self.queueDict[queueName]["CEName"]
 
-            # Check the status of the Site
-            if site in siteMaskList:
-                continue
-
-            # Check the status of the CE (only for RSS=Active)
-            if ce not in ceMaskList:
+            # Check the status of the Site and CE
+            if site in siteMaskList and ce in ceMaskList:
                 continue
 
             self.log.warn("Queue not considered because not usable:", queueName)