wmglobalqueue 2.3.10__py3-none-any.whl

WMCore/WorkQueue/WorkQueue.py

@@ -0,0 +1,1237 @@
+#!/usr/bin/env python
+
+"""
+WorkQueue provides functionality to queue large chunks of work,
+thus acting as a buffer for the next steps in job processing
+
+WMSpec objects are fed into the queue, split into coarse grained work units
+and released when a suitable resource is found to execute them.
+
+https://twiki.cern.ch/twiki/bin/view/CMS/WMCoreJobPool
+"""
+
+from __future__ import division, print_function
+
+from builtins import str as newstr, bytes
+from future.utils import viewitems, listvalues
+
+import os
+import threading
+import time
+from collections import defaultdict
+
+from WMCore import Lexicon
+from WMCore.ACDC.DataCollectionService import DataCollectionService
+from WMCore.Database.CMSCouch import CouchInternalServerError, CouchNotFoundError
+from WMCore.Services.CRIC.CRIC import CRIC
+from WMCore.Services.DBS.DBSReader import DBSReader
+from WMCore.Services.LogDB.LogDB import LogDB
+from WMCore.Services.ReqMgr.ReqMgr import ReqMgr
+from WMCore.Services.RequestDB.RequestDBReader import RequestDBReader
+from WMCore.Services.Rucio.Rucio import Rucio
+from WMCore.Services.WorkQueue.WorkQueue import WorkQueue as WorkQueueDS
+from WMCore.WMSpec.WMWorkload import WMWorkloadHelper, getWorkloadFromTask
+from WMCore.WorkQueue.DataLocationMapper import WorkQueueDataLocationMapper
+from WMCore.WorkQueue.DataStructs.ACDCBlock import ACDCBlock
+from WMCore.WorkQueue.DataStructs.WorkQueueElement import possibleSites
+from WMCore.WorkQueue.DataStructs.WorkQueueElementsSummary import getGlobalSiteStatusSummary
+from WMCore.WorkQueue.Policy.End import endPolicy
+from WMCore.WorkQueue.Policy.Start import startPolicy
+from WMCore.WorkQueue.WorkQueueBackend import WorkQueueBackend
+from WMCore.WorkQueue.WorkQueueBase import WorkQueueBase
+from WMCore.WorkQueue.WorkQueueExceptions import (TERMINAL_EXCEPTIONS, WorkQueueError, WorkQueueNoMatchingElements,
+                                                  WorkQueueWMSpecError)
+from WMCore.WorkQueue.WorkQueueUtils import cmsSiteNames
+
+
+# Convenience constructor functions
+
+def globalQueue(logger=None, dbi=None, **kwargs):
+    """Convenience method to create a WorkQueue suitable for use globally
+    """
+    defaults = {'PopulateFilesets': False,
+                'LocalQueueFlag': False,
+                'TrackLocationOrSubscription': 'location'
+                }
+    defaults.update(kwargs)
+    return WorkQueue(logger, dbi, **defaults)
+
+
+def localQueue(logger=None, dbi=None, **kwargs):
+    """Convenience method to create a WorkQueue suitable for use locally
+    """
+    defaults = {'TrackLocationOrSubscription': 'location'}
+    defaults.update(kwargs)
+    return WorkQueue(logger, dbi, **defaults)
+
+
+class WorkQueue(WorkQueueBase):
+    """
+    _WorkQueue_
+
+    WorkQueue object - interface to WorkQueue functionality.
+    """
+
+    def __init__(self, logger=None, dbi=None, **params):
+
+        WorkQueueBase.__init__(self, logger, dbi)
+        self.parent_queue = None
+        self.params = params
+
+        # config argument (within params) shall be reference to
+        # Configuration instance
+        self.config = params.get("Config", None)
+        self.params.setdefault('CouchUrl', os.environ.get('COUCHURL'))
+        if not self.params.get('CouchUrl'):
+            raise RuntimeError('CouchUrl config value mandatory')
+        self.params.setdefault('DbName', 'workqueue')
+        self.params.setdefault('InboxDbName', self.params['DbName'] + '_inbox')
+        self.params.setdefault('ParentQueueCouchUrl', None)  # We get work from here
+
+        self.backend = WorkQueueBackend(self.params['CouchUrl'], self.params['DbName'],
+                                        self.params['InboxDbName'],
+                                        self.params['ParentQueueCouchUrl'], self.params.get('QueueURL'),
+                                        logger=self.logger)
+        self.workqueueDS = WorkQueueDS(self.params['CouchUrl'], self.params['DbName'],
+                                       self.params['InboxDbName'])
+        if self.params.get('ParentQueueCouchUrl'):
+            try:
+                if self.params.get('ParentQueueInboxCouchDBName'):
+                    self.parent_queue = WorkQueueBackend(self.params['ParentQueueCouchUrl'].rsplit('/', 1)[0],
+                                                         self.params['ParentQueueCouchUrl'].rsplit('/', 1)[1],
+                                                         self.params['ParentQueueInboxCouchDBName'])
+                else:
+                    self.parent_queue = WorkQueueBackend(self.params['ParentQueueCouchUrl'].rsplit('/', 1)[0],
+                                                         self.params['ParentQueueCouchUrl'].rsplit('/', 1)[1])
+            except IndexError as ex:
+                # Probable cause: Someone didn't put the global WorkQueue name in
+                # the ParentCouchUrl
+                msg = "Parsing failure for ParentQueueCouchUrl - probably missing dbname in input\n"
+                msg += "Exception: %s\n" % str(ex)
+                msg += str("ParentQueueCouchUrl: %s\n" % self.params['ParentQueueCouchUrl'])
+                self.logger.error(msg)
+                raise WorkQueueError(msg)
+            self.params['ParentQueueCouchUrl'] = self.parent_queue.queueUrl
+
+        # save each DBSReader instance in the class object, such that
+        # the same object is not shared amongst multiple threads
+        self.dbses = {}
+
+        self.params.setdefault('QueueDepth', 1)  # when less than this locally
+        self.params.setdefault('WorkPerCycle', 100)
+        self.params.setdefault('RowsPerSlice', 2500)
+        self.params.setdefault('MaxRowsPerCycle', 50000)
+        self.params.setdefault('LocationRefreshInterval', 600)
+        self.params.setdefault('FullLocationRefreshInterval', 7200)
+        self.params.setdefault('TrackLocationOrSubscription', 'location')
+        self.params.setdefault('ReleaseIncompleteBlocks', False)
+        self.params.setdefault('ReleaseRequireSubscribed', True)
+        self.params.setdefault('PopulateFilesets', True)
+        self.params.setdefault('LocalQueueFlag', True)
+        self.params.setdefault('QueueRetryTime', 86400)
+        self.params.setdefault('stuckElementAlertTime', 172800)
+        self.params.setdefault('reqmgrCompleteGraceTime', 604800)
+        self.params.setdefault('cancelGraceTime', 86400)
+
+        self.params.setdefault('JobDumpConfig', None)
+        self.params.setdefault('BossAirConfig', None)
+
+        self.params['QueueURL'] = self.backend.queueUrl  # url this queue is visible on
+        # backend took previous QueueURL and sanitized it
+        self.params.setdefault('WMBSUrl', None)  # this will only be set on local Queue
+        if self.params.get('WMBSUrl'):
+            self.params['WMBSUrl'] = Lexicon.sanitizeURL(self.params['WMBSUrl'])['url']
+        self.params.setdefault('Team', "")
+
+        if self.params.get('CacheDir'):
+            try:
+                os.makedirs(self.params['CacheDir'])
+            except OSError:
+                pass
+        elif self.params.get('PopulateFilesets'):
+            raise RuntimeError('CacheDir mandatory for local queue')
+
+        if self.params.get('CRIC'):
+            self.cric = self.params['CRIC']
+        else:
+            self.cric = CRIC()
+
+        self.params.setdefault('SplittingMapping', {})
+        self.params['SplittingMapping'].setdefault('DatasetBlock',
+                                                   {'name': 'Block',
+                                                    'args': {}}
+                                                   )
+        self.params['SplittingMapping'].setdefault('MonteCarlo',
+                                                   {'name': 'MonteCarlo',
+                                                    'args': {}}
+                                                   )
+        self.params['SplittingMapping'].setdefault('Dataset',
+                                                   {'name': 'Dataset',
+                                                    'args': {}}
+                                                   )
+        self.params['SplittingMapping'].setdefault('Block',
+                                                   {'name': 'Block',
+                                                    'args': {}}
+                                                   )
+        self.params['SplittingMapping'].setdefault('ResubmitBlock',
+                                                   {'name': 'ResubmitBlock',
+                                                    'args': {}}
+                                                   )
+
+        self.params.setdefault('EndPolicySettings', {})
+
+        assert (self.params['TrackLocationOrSubscription'] in ('subscription',
+                                                               'location'))
+        # Can only release blocks on location
+        if self.params['TrackLocationOrSubscription'] == 'location':
+            if self.params['SplittingMapping']['DatasetBlock']['name'] != 'Block':
+                raise RuntimeError('Only blocks can be released on location')
+
+        self.params.setdefault('rucioAccount', "wmcore_transferor")
+
+        self.rucio = Rucio(self.params['rucioAccount'],
+                           self.params['rucioUrl'], self.params['rucioAuthUrl'],
+                           configDict=dict(logger=self.logger))
+
+
+        self.dataLocationMapper = WorkQueueDataLocationMapper(self.logger, self.backend,
+                                                              rucio=self.rucio,
+                                                              cric=self.cric,
+                                                              locationFrom=self.params['TrackLocationOrSubscription'],
+                                                              incompleteBlocks=self.params['ReleaseIncompleteBlocks'],
+                                                              requireBlocksSubscribed=not self.params[
+                                                                  'ReleaseIncompleteBlocks'],
+                                                              fullRefreshInterval=self.params[
+                                                                  'FullLocationRefreshInterval'],
+                                                              updateIntervalCoarseness=self.params[
+                                                                  'LocationRefreshInterval'])
+
+        # used for only global WQ
+        if self.params.get('ReqMgrServiceURL'):
+            self.reqmgrSvc = ReqMgr(self.params['ReqMgrServiceURL'])
+
+        if self.params.get('RequestDBURL'):
+            # This is need for getting post call
+            # TODO: Change ReqMgr api to accept post for for retrieving the data and remove this
+            self.requestDB = RequestDBReader(self.params['RequestDBURL'])
+
+        # set the thread name before create the log db.
+        # only sets that when it is not set already
+        # setLogDB
+
+        myThread = threading.currentThread()
+        if myThread.getName() == "MainThread":  # this should be only GQ case other cases thread name should be set
+            myThread.setName(self.__class__.__name__)
+
+        centralurl = self.params.get("central_logdb_url")
+        identifier = self.params.get("log_reporter")
+        self.logdb = LogDB(centralurl, identifier, logger=self.logger)
+
+        self.logger.debug("WorkQueue created successfully")
+
+    def __len__(self):
+        """Returns number of Available elements in queue"""
+        return self.backend.queueLength()
+
+    def setStatus(self, status, elementIDs=None, SubscriptionId=None, WorkflowName=None):
+        """
+        _setStatus_, throws an exception if no elements are updated
+
+        """
+        try:
+            if not elementIDs:
+                elementIDs = []
+            iter(elementIDs)
+            if isinstance(elementIDs, (newstr, bytes)):
+                raise TypeError
+        except TypeError:
+            elementIDs = [elementIDs]
+
+        if status == 'Canceled':  # Cancel needs special actions
+            return self.cancelWork(elementIDs, SubscriptionId, WorkflowName)
+
+        args = {}
+        if SubscriptionId:
+            args['SubscriptionId'] = SubscriptionId
+        if WorkflowName:
+            args['RequestName'] = WorkflowName
+
+        affected = self.backend.getElements(elementIDs=elementIDs, **args)
+        if not affected:
+            raise WorkQueueNoMatchingElements("No matching elements")
+
+        for x in affected:
+            x['Status'] = status
+        elements = self.backend.saveElements(*affected)
+        if len(affected) != len(elements):
+            raise RuntimeError("Some elements not updated, see log for details")
+
+        return elements
+
+    def setPriority(self, newpriority, *workflowNames):
+        """
+        Update priority for a workflow, throw exception if no elements affected
+        """
+        self.logger.info("Priority change request to %s for %s", newpriority, str(workflowNames))
+        affected = []
+        for wf in workflowNames:
+            affected.extend(self.backend.getElements(returnIdOnly=True, RequestName=wf))
+
+        self.backend.updateElements(*affected, Priority=newpriority)
+
+        if not affected:
+            raise RuntimeError("Priority not changed: No matching elements")
+
+    def resetWork(self, ids):
+        """Put work back in Available state, from here either another queue
+         or wmbs can pick it up.
+
+         If work was Acquired by a child queue, the next status update will
+         cancel the work in the child.
+
+         Note: That the same child queue is free to pick the work up again,
+          there is no permanent blacklist of queues.
+        """
+        self.logger.info("Resetting elements %s", str(ids))
+        try:
+            iter(ids)
+        except TypeError:
+            ids = [ids]
+
+        return self.backend.updateElements(*ids, Status='Available',
+                                           ChildQueueUrl=None, WMBSUrl=None)
+
+    def getWork(self, jobSlots, siteJobCounts, excludeWorkflows=None):
+        """
+        Get available work from the queue, inject into wmbs & mark as running
+
+        jobSlots is dict format of {site: estimateJobSlot}
+        of the resources to get work for.
+
+        siteJobCounts is a dict format of {site: {prio: jobs}}
+        """
+        excludeWorkflows = excludeWorkflows or []
+        results = []
+        if not self.backend.isAvailable():
+            self.logger.warning('Backend busy or down: skipping fetching of work')
+            return results
+
+        # TODO AMR: perhaps numElems limit should be removed for LQ -> WMBS acquisition
+        matches, _ = self.backend.availableWork(jobSlots, siteJobCounts,
+                                                excludeWorkflows=excludeWorkflows,
+                                                numElems=self.params['WorkPerCycle'],
+                                                rowsPerSlice=self.params['RowsPerSlice'],
+                                                maxRows=self.params['MaxRowsPerCycle'])
+
+        self.logger.info('Got %i elements matching the constraints', len(matches))
+        if not matches:
+            return results
+
+        myThread = threading.currentThread()
+        # cache wmspecs for lifetime of function call, likely we will have multiple elements for same spec.
+        # TODO: Check to see if we can skip spec loading - need to persist some more details to element
+        wmspecCache = {}
+        for match in matches:
+            blockName, dbsBlock = None, None
+            if self.params['PopulateFilesets']:
+                if match['RequestName'] not in wmspecCache:
+                    wmspec = self.backend.getWMSpec(match['RequestName'])
+                    wmspecCache[match['RequestName']] = wmspec
+                else:
+                    wmspec = wmspecCache[match['RequestName']]
+
+                try:
+                    if match['StartPolicy'] == 'Dataset':
+                        # actually returns dataset name and dataset info
+                        blockName, dbsBlock = self._getDBSDataset(match)
+                    elif match['Inputs']:
+                        blockName, dbsBlock = self._getDBSBlock(match, wmspec)
+                except Exception as ex:
+                    msg = "%s, %s: \n" % (wmspec.name(), list(match['Inputs']))
+                    msg += "failed to retrieve data from DBS/Rucio in LQ: \n%s" % str(ex)
+                    self.logger.exception(msg)
+                    self.logdb.post(wmspec.name(), msg, 'error')
+                    continue
+
+                try:
+                    match['Subscription'] = self._wmbsPreparation(match,
+                                                                  wmspec,
+                                                                  blockName,
+                                                                  dbsBlock)
+                    self.logdb.delete(wmspec.name(), "error", this_thread=True)
+                except Exception as ex:
+                    if getattr(myThread, 'transaction', None) is not None:
+                        myThread.transaction.rollback()
+                    msg = "Failed to create subscription for %s with block name %s" % (wmspec.name(), blockName)
+                    msg += "\nError: %s" % str(ex)
+                    self.logger.exception(msg)
+                    self.logdb.post(wmspec.name(), msg, 'error')
+                    continue
+
+            results.append(match)
+
+        del wmspecCache  # remove cache explicitly
+        self.logger.info('Injected %s out of %s units into WMBS', len(results), len(matches))
+        return results
+
+    def _getDbs(self, dbsUrl):
+        """
+        If we have already construct a DBSReader object pointing to
+        the DBS URL provided, return it. Otherwise, create and return
+        a new instance.
+        :param dbsUrl: string with the DBS url
+        :return: an instance of DBSReader
+        """
+        if dbsUrl in self.dbses:
+            return self.dbses[dbsUrl]
+        return DBSReader(dbsUrl)
+
+    def _getDBSDataset(self, match):
+        """
+        Given a workqueue element with Dataset start policy, find all blocks
+        with valid files and resolve their location in Rucio.
+        :param match: workqueue element dictionary
+        :return: a tuple of the dataset name and its files and RSEs
+        """
+        dbsDatasetDict = {'Files': [], 'PhEDExNodeNames': []}
+        dbs = self._getDbs(match['Dbs'])
+        datasetName = list(match['Inputs'])[0]
+
+        blocks = dbs.listFileBlocks(datasetName)
+        for blockName in blocks:
+            blockSummary = dbs.getFileBlock(blockName)
+            if not blockSummary['Files']:
+                self.logger.warning("Block name %s has no valid files. Skipping it.", blockName)
+                continue
+            blockSummary['PhEDExNodeNames'] = self.rucio.getDataLockedAndAvailable(name=blockName,
+                                                                                   account=self.params['rucioAccount'])
+            dbsDatasetDict['Files'].extend(blockSummary['Files'])
+            dbsDatasetDict['PhEDExNodeNames'].extend(blockSummary['PhEDExNodeNames'])
+
+        dbsDatasetDict['PhEDExNodeNames'] = list(set(dbsDatasetDict['PhEDExNodeNames']))
+
+        return datasetName, dbsDatasetDict
+
+    def _getDBSBlock(self, match, wmspec):
+        """Get DBS info for this block"""
+        blockName = list(match['Inputs'])[0]  # TODO: Allow more than one
+
+        if match['ACDC']:
+            acdcInfo = match['ACDC']
+            acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
+            splitedBlockName = ACDCBlock.splitBlockName(blockName)
+            fileLists = acdc.getChunkFiles(acdcInfo['collection'],
+                                           acdcInfo['fileset'],
+                                           splitedBlockName['Offset'],
+                                           splitedBlockName['NumOfFiles'])
+
+            block = {}
+            block["Files"] = fileLists
+            return blockName, block
+        else:
+            dbs = self._getDbs(match['Dbs'])
+            if wmspec.getTask(match['TaskName']).parentProcessingFlag():
+                dbsBlockDict = dbs.getFileBlockWithParents(blockName)
+                dbsBlockDict['PhEDExNodeNames'] = self.rucio.getDataLockedAndAvailable(name=blockName,
+                                                                                       account=self.params['rucioAccount'])
+            elif wmspec.getRequestType() == 'StoreResults':
+                dbsBlockDict = dbs.getFileBlock(blockName)
+                dbsBlockDict['PhEDExNodeNames'] = dbs.listFileBlockLocation(blockName)
+            else:
+                dbsBlockDict = dbs.getFileBlock(blockName)
+                dbsBlockDict['PhEDExNodeNames'] = self.rucio.getDataLockedAndAvailable(name=blockName,
+                                                                                       account=self.params['rucioAccount'])
+
+        return blockName, dbsBlockDict
+
+    def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock):
+        """Inject data into wmbs and create subscription. """
+        from WMCore.WorkQueue.WMBSHelper import WMBSHelper
+        # the parent element (from local couch) can be fetch via:
+        # curl -ks -X GET 'http://localhost:5984/workqueue/<ParentQueueId>'
+
+        # Keep in mind that WQE contains sites, wmbs location contains pnns
+        commonSites = possibleSites(match)
+        commonLocation = self.cric.PSNstoPNNs(commonSites, allowPNNLess=True)
+        msg = "Running WMBS preparation for %s with ParentQueueId %s,\n  with common location %s"
+        self.logger.info(msg, match['RequestName'], match['ParentQueueId'], commonLocation)
+
+        mask = match['Mask']
+        wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask,
+                                self.params['CacheDir'], commonLocation)
+
+        sub, match['NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles(block=dbsBlock)
+        self.logger.info("Created top level subscription %s for %s with %s files",
+                         sub['id'], match['RequestName'], match['NumOfFilesAdded'])
+
+        # update couch with wmbs subscription info
+        match['SubscriptionId'] = sub['id']
+        match['Status'] = 'Running'
+        # do update rather than save to avoid conflicts from other thread writes
+        self.backend.updateElements(match.id, Status='Running', SubscriptionId=sub['id'],
+                                    NumOfFilesAdded=match['NumOfFilesAdded'])
+        self.logger.info("LQE %s set to 'Running' for request %s", match.id, match['RequestName'])
+
+        return sub
+
+    def _assignToChildQueue(self, queue, *elements):
+        """Assign work from parent to queue"""
+        workByRequest = {}
+        for ele in elements:
+            ele['Status'] = 'Negotiating'
+            ele['ChildQueueUrl'] = queue
+            ele['ParentQueueUrl'] = self.params['ParentQueueCouchUrl']
+            ele['WMBSUrl'] = self.params["WMBSUrl"]
+            workByRequest.setdefault(ele['RequestName'], 0)
+            workByRequest[ele['RequestName']] += 1
+        self.logger.info("Setting GQE status to 'Negotiating' and assigning to this child queue for:")
+        for reqName, numElem in workByRequest.items():
+            self.logger.info("    %d elements for: %s", numElem, reqName)
+
+        work = self.parent_queue.saveElements(*elements)
+        self.logger.info("GQE successfully saved for:")
+        for ele in work:
+            self.logger.info("    %s under GQE id: %s", ele['RequestName'], ele.id)
+        return work
+
+    def doneWork(self, elementIDs=None, SubscriptionId=None, WorkflowName=None):
+        """Mark work as done
+        """
+        return self.setStatus('Done', elementIDs=elementIDs,
+                              SubscriptionId=SubscriptionId,
+                              WorkflowName=WorkflowName)
+
+    def killWMBSWorkflows(self, reqNames):
+        """
+        Kill/cancel workflows in WMBS and CouchDB.
+        Also update job state transition in three data sources: local couch,
+        local WMBS and dashboard.
+        :param reqNames: list of request names
+        :return: a list of workflows that failed to be cancelled
+        """
+        failedWfs = []
+        if not reqNames:
+            return failedWfs
+
+        # import inside function since GQ doesn't need this.
+        from WMCore.WorkQueue.WMBSHelper import killWorkflow
+        myThread = threading.currentThread()
+        myThread.dbi = self.conn.dbi
+        myThread.logger = self.logger
+
+        for workflow in reqNames:
+            try:
+                self.logger.info("Killing workflow in WMBS: %s", workflow)
+                killWorkflow(workflow, self.params["JobDumpConfig"], self.params["BossAirConfig"])
+            except Exception as ex:
+                failedWfs.append(workflow)
+                msg = "Failed to kill workflow '%s' in WMBS. Error: %s" % (workflow, str(ex))
+                msg += "\nIt will be retried in the next loop"
+                self.logger.error(msg)
+        return failedWfs
+
+    def cancelWork(self, elementIDs=None, SubscriptionId=None, WorkflowName=None, elements=None):
+        """Cancel work - delete in wmbs, delete from workqueue db, set canceled in inbox
+           Elements may be directly provided or determined from series of filter arguments
+        """
+        if not elements:
+            args = {}
+            if SubscriptionId:
+                args['SubscriptionId'] = SubscriptionId
+            if WorkflowName:
+                args['RequestName'] = WorkflowName
+            elements = self.backend.getElements(elementIDs=elementIDs, **args)
+
+        # take wf from args in case no elements exist for workflow (i.e. work was negotiating)
+        requestNames = set([x['RequestName'] for x in elements]) | set([wf for wf in [WorkflowName] if wf])
+        if not requestNames:
+            return []
+        inbox_elements = []
+        for wf in requestNames:
+            inbox_elements.extend(self.backend.getInboxElements(WorkflowName=wf))
+
+        # if local queue, kill jobs, update parent to Canceled and delete elements
+        if self.params['LocalQueueFlag']:
+            # if we can talk to wmbs kill the jobs
+            badWfsCancel = []
+            if self.params['PopulateFilesets']:
+                self.logger.info("Canceling work for workflow(s): %s", requestNames)
+                badWfsCancel = self.killWMBSWorkflows(requestNames)
+            # now we remove any wf that failed to be cancelled (and its inbox elements)
+            requestNames -= set(badWfsCancel)
+            for wf in badWfsCancel:
+                elementsToRemove = self.backend.getInboxElements(WorkflowName=wf)
+                inbox_elements = list(set(inbox_elements) - set(elementsToRemove))
+            self.logger.info("New list of cancelled requests: %s", requestNames)
+
+            # Don't update as fails sometimes due to conflicts (#3856)
+            for x in inbox_elements:
+                if x['Status'] != 'Canceled':
+                    x.load().__setitem__('Status', 'Canceled')
+
+            self.backend.saveElements(*inbox_elements)
+
+        # if global queue, update non-acquired to Canceled, update parent to CancelRequested
+        else:
+            # Cancel in global if work has not been passed to a child queue
+            elements_to_cancel = [x for x in elements if not x['ChildQueueUrl'] and x['Status'] != 'Canceled']
+            # ensure all elements receive cancel request, covers case where initial cancel request missed some elements
+            # without this elements may avoid the cancel and not be cleared up till they finish
+            elements_not_requested = [x for x in elements if
+                                      x['ChildQueueUrl'] and (x['Status'] != 'CancelRequested' and not x.inEndState())]
+
+            self.logger.info("Canceling work for workflow(s): %s", requestNames)
+            if elements_to_cancel:
+                self.backend.updateElements(*[x.id for x in elements_to_cancel], Status='Canceled')
+                self.logger.info("Cancel-ed element(s) %s", str([x.id for x in elements_to_cancel]))
+
+            if elements_not_requested:
+                # Don't update as fails sometimes due to conflicts (#3856)
+                for x in elements_not_requested:
+                    x.load().__setitem__('Status', 'CancelRequested')
+                self.backend.saveElements(*elements_not_requested)
+                self.logger.info("CancelRequest-ed element(s) %s", str([x.id for x in elements_not_requested]))
+
+            inboxElemIds = [x.id for x in inbox_elements if x['Status'] != 'CancelRequested' and not x.inEndState()]
+            self.backend.updateInboxElements(*inboxElemIds, Status='CancelRequested')
+            # if we haven't had any updates for a while assume agent is dead and move to canceled
+            if self.params.get('cancelGraceTime', -1) > 0 and elements:
+                last_update = max([float(x.updatetime) for x in elements])
+                if (time.time() - last_update) > self.params['cancelGraceTime']:
+                    self.logger.info("%s cancellation has stalled, mark as finished", elements[0]['RequestName'])
+                    # Don't update as fails sometimes due to conflicts (#3856)
+                    for x in elements:
+                        if not x.inEndState():
+                            x.load().__setitem__('Status', 'Canceled')
+                    self.backend.saveElements(*[x for x in elements if not x.inEndState()])
+
+        return [x.id for x in elements]
+
+    def deleteWorkflows(self, *requests):
+        """Delete requests if finished"""
+        for request in requests:
+            request = self.backend.getInboxElements(elementIDs=[request])
+            if len(request) != 1:
+                raise RuntimeError('Invalid number of requests for %s' % request[0]['RequestName'])
+            request = request[0]
+
+            if request.inEndState():
+                self.logger.info('Deleting request "%s" as it is %s', request.id, request['Status'])
+                self.backend.deleteElements(request)
+            else:
+                self.logger.debug('Not deleting "%s" as it is %s', request.id, request['Status'])
+
+    # NOTE: this function is not executed by local workqueue
+    def queueWork(self, wmspecUrl, request=None, team=None):
+        """
+        Take and queue work from a WMSpec.
+
+        If request name is provided but doesn't match WMSpec name
+        an error is raised.
+
+        If team is provided work will only be available to queue's
+        belonging to that team.
+
+        Duplicate specs will be ignored.
+        """
+        self.logger.info('queueWork() begin queueing "%s"', wmspecUrl)
+        wmspec = WMWorkloadHelper()
+        wmspec.load(wmspecUrl)
+
+        if request:  # validate request name
+            if request != wmspec.name():
+                raise WorkQueueWMSpecError(wmspec,
+                                           'Request & workflow name mismatch %s vs %s' % (request, wmspec.name()))
+
+        # Either pull the existing inbox element or create a new one.
+        try:
+            inbound = self.backend.getInboxElements(elementIDs=[wmspec.name()], loadSpec=True)
+            self.logger.info('Resume splitting of "%s"', wmspec.name())
+        except CouchNotFoundError:
+            inbound = [self.backend.createWork(wmspec, Status='Negotiating',
+                                               TeamName=team, WMBSUrl=self.params["WMBSUrl"])]
+            self.backend.insertElements(inbound)
+
+        work = self.processInboundWork(inbound, throw=True)
+        return len(work)
+
+    def addWork(self, inboundElem, rucioObj=None):
+        """
+        Check and add new elements to an existing running request,
+        if supported by the start policy.
+
+        :param inboundElem: dict representation for a WorkQueueElement object,
+            including the WMSpec file.
+        :param rucioObj: object to the Rucio class
+        :return: amount of new work units added to the request
+        """
+        result = []
+        self.logger.info('Trying to add more work for: %s', inboundElem['RequestName'])
+
+        try:
+            # Check we haven't already split the work, unless it's continuous processing
+            work, rejectedWork, badWork = self._splitWork(inboundElem['WMSpec'], data=inboundElem['Inputs'],
+                                                          mask=inboundElem['Mask'], inbound=inboundElem,
+                                                          continuous=True, rucioObj=rucioObj)
+
+            # if there is new work, then insert it into the database
+            newWork = self.backend.insertElements(work, parent=inboundElem)
+
+            # store the inputs in the global queue inbox workflow element
+            processedInputs = []
+            for unit in newWork:
+                processedInputs.extend(list(unit['Inputs']))
+
+            # update the list of processed and rejected inputs with what is already
+            # defined in the workqueue inbox
+            processedInputs.extend(inboundElem['ProcessedInputs'])
+            rejectedWork.extend(inboundElem['RejectedInputs'])
+            if newWork:
+                # then also update the timestamp for when new data was found
+                self.backend.updateInboxElements(inboundElem.id,
+                                                 ProcessedInputs=processedInputs,
+                                                 RejectedInputs=rejectedWork,
+                                                 TimestampFoundNewData=int(time.time()))
+            # if global queue, then update workflow stats to request mgr couch doc
+            # remove the "UnittestFlag" - need to create the reqmgrSvc emulator
+            if not self.params.get("UnittestFlag", False):
+                # get statistics for the new work. It's already validated on the server side
+                totalStats = self._getTotalStats(newWork)
+                self.reqmgrSvc.updateRequestStats(inboundElem['WMSpec'].name(), totalStats)
+
+            if badWork:
+                msg = "Request with the following unprocessable input data: %s" % badWork
+                self.logdb.post(inboundElem['RequestName'], msg, 'warning')
+        except Exception as exc:
+            self.logger.error('Generic exception adding work to WQE inbox: %s. Error: %s',
+                              inboundElem, str(exc))
+        else:
+            result.extend(newWork)
+
+        self.logger.info('Added %d new elements for request: %s', len(result), inboundElem['RequestName'])
+        return len(result)
+
+    def status(self, status=None, elementIDs=None,
+               dictKey=None, wmbsInfo=None, loadSpec=False,
+               **filters):
+        """
+        Return elements in the queue.
+
+        status, elementIDs & filters are 'AND'ed together to filter elements.
+        dictKey returns the output as a dict with the dictKey as the key.
+        wmbsInfo causes elements to be synced with their status in WMBS.
+        loadSpec causes the workflow for each spec to be loaded.
+        """
+        items = self.backend.getElements(status=status,
+                                         elementIDs=elementIDs,
+                                         loadSpec=loadSpec,
+                                         **filters)
+
+        if wmbsInfo:
+            self.logger.debug("Syncing element statuses with WMBS for workflow: %s", filters.get("RequestName"))
+            for item in items:
+                for wmbs in wmbsInfo:
+                    if item['SubscriptionId'] == wmbs['subscription_id']:
+                        item.updateFromSubscription(wmbs)
+                        break
+
+        # if dictKey, format as a dict with the appropriate key
+        if dictKey:
+            tmp = defaultdict(list)
+            for item in items:
+                tmp[item[dictKey]].append(item)
+            items = dict(tmp)
+        return items
+
+    def getWMBSSubscriptionStatus(self):
+        """
+        Fetches all the subscriptions in this agent and make a summary of
+        every single one of them, to be used to update WQEs
+        :return: a list of dictionaries
+        """
+        from WMCore.WorkQueue.WMBSHelper import wmbsSubscriptionStatus
+        self.logger.info("Fetching WMBS subscription status information")
+        wmbsStatus = wmbsSubscriptionStatus(logger=self.logger,
+                                            dbi=self.conn.dbi,
+                                            conn=self.conn.getDBConn(),
+                                            transaction=self.conn.existingTransaction())
+        return wmbsStatus
+
+    def statusInbox(self, status=None, elementIDs=None, dictKey=None, **filters):
+        """
+        Return elements in the inbox.
+
+        status, elementIDs & filters are 'AND'ed together to filter elements.
+        dictKey returns the output as a dict with the dictKey as the key.
+        """
+        items = self.backend.getInboxElements(status, elementIDs, **filters)
+
+        # if dictKey, given format as a dict with the appropriate key
+        if dictKey:
+            tmp = defaultdict(list)
+            for item in items:
+                tmp[item[dictKey]].append(item)
+            items = dict(tmp)
+
+        return items
+
+    def updateLocationInfo(self):
+        """
+        Update locations info for elements.
+        """
+        self.logger.info('Executing data location update...')
+        if not self.backend.isAvailable():
+            self.logger.warning('Backend busy or down: skipping location update')
+            return 0
+        result = self.dataLocationMapper()
+        self.backend.recordTaskActivity('location_refresh')
+        return result
+
+    def _printLog(self, msg, printFlag, logLevel):
+        if printFlag:
+            print(msg)
+        else:
+            getattr(self.logger, logLevel)(msg)
+
+    def pullWorkConditionCheck(self, printFlag=False):
+
+        if not self.params['ParentQueueCouchUrl']:
+            msg = 'Unable to pull work from parent, ParentQueueCouchUrl not provided'
+            self._printLog(msg, printFlag, "warning")
+            return False
+        if not self.backend.isAvailable() or not self.parent_queue.isAvailable():
+            msg = 'Backend busy or down: skipping work pull'
+            self._printLog(msg, printFlag, "warning")
+            return False
+
+        left_over = self.parent_queue.getElements('Negotiating', returnIdOnly=True,
+                                                  ChildQueueUrl=self.params['QueueURL'])
+        if left_over:
+            msg = 'Not pulling more work. Still replicating %d previous units, ids:\n%s' % (len(left_over), left_over)
+            self._printLog(msg, printFlag, "warning")
+            return False
+
+        still_processing = self.backend.getInboxElements('Negotiating', returnIdOnly=True)
+        if still_processing:
+            msg = 'Not pulling more work. Still processing %d previous units' % len(still_processing)
+            self._printLog(msg, printFlag, "warning")
+            return False
+
+        return True
+
+    def freeResouceCheck(self):
+        """
+        This method looks into the WMBS and BossAir tables and collect
+        two types of information:
+         1) sites and the total slots available for job creation
+         2) sites and the number of pending jobs grouped by priority
+        With that information in hands, it looks at the local workqueue elements
+        sitting in Available status and update the 2nd data structure (thus it
+        updates number of jobs pending by priority according to the LQEs), which
+        is then used to know which work can be acquired from the parent queue or not.
+        :return: a tuple of dictionaries (or empty lists)
+        """
+        from WMCore.WorkQueue.WMBSHelper import freeSlots
+        resources, jobCounts = freeSlots(self.params['QueueDepth'], knownCmsSites=cmsSiteNames())
+        # now update jobCounts with work that is already available in the local queue
+        _, jobCounts = self.backend.calculateAvailableWork(resources, jobCounts)
+
+        return (resources, jobCounts)
+
+    def getAvailableWorkfromParent(self, resources, jobCounts, printFlag=False):
+        self.logger.info("Going to fetch work from the parent queue: %s", self.parent_queue.queueUrl)
+        work, _ = self.parent_queue.availableWork(resources, jobCounts, self.params['Team'],
+                                                  numElems=self.params['WorkPerCycle'],
+                                                  rowsPerSlice=self.params['RowsPerSlice'],
+                                                  maxRows=self.params['MaxRowsPerCycle'])
+        if not work:
+            self._printLog('No available work in parent queue.', printFlag, "warning")
+        return work
+
+    def pullWork(self, resources=None):
+        """
+        Pull work from another WorkQueue to be processed:
+        :param resources: optional dictionary with sites and the amount
+        of slots free
+        """
+        jobCounts = {}
+        if self.pullWorkConditionCheck() is False:
+            return 0
+
+        # NOTE: resources parameter is only used by unit tests, which do
+        # not use WMBS and BossAir tables
+        if not resources:
+            (resources, jobCounts) = self.freeResouceCheck()
+        if not resources and not jobCounts:
+            return 0
+
+        work = self.getAvailableWorkfromParent(resources, jobCounts)
+        if not work:
+            return 0
+
+        work = self._assignToChildQueue(self.params['QueueURL'], *work)
+
+        return len(work)
+
+    def closeWork(self):
+        """
+        Global queue service that looks for the inbox elements that are still active
+        and checks whether they should be closed for new data or not.
+        An element is closed automatically when one of the following conditions holds true:
+        - The StartPolicy doesn't define a OpenRunningTimeout or this delay is set to 0
+        - A period longer than OpenRunningTimeout has passed since the last child element
+           was created or an open block was found and the StartPolicy newDataAvailable
+           function returns False.
+
+        :return: list of workqueue_inbox elements that have been closed
+        """
+        workflowsToClose = []
+        if self.params['LocalQueueFlag']:
+            # this is a Global WorkQueue only functionality
+            return workflowsToClose
+        if not self.backend.isAvailable():
+            self.logger.warning('Backend busy or down: Can not close work at this time')
+            return workflowsToClose
+
+        workflowsToCheck = self.backend.getInboxElements(OpenForNewData=True)
+        self.logger.info("Retrieved a list of %d open workflows", len(workflowsToCheck))
+        currentTime = time.time()
+        for element in workflowsToCheck:
+            # fetch attributes from the inbox workqueue element
+            startPol = element.get('StartPolicy', {})
+            openRunningTimeout = startPol.get('OpenRunningTimeout', 0)
+            foundNewDataTime = element.get('TimestampFoundNewData', 0)
+            if not openRunningTimeout:
+                self.logger.info("Workflow %s has no OpenRunningTimeout. Queuing to be closed.",
+                                 element['RequestName'])
+                workflowsToClose.append(element.id)
+            elif (currentTime - foundNewDataTime) > openRunningTimeout:
+                # then it's been too long since the last element has been found
+                self.logger.info("Workflow %s has expired OpenRunningTimeout. Queuing to be closed.",
+                                 element['RequestName'])
+                workflowsToClose.append(element.id)
+
+        if workflowsToClose:
+            try:
+                self.logger.info('Closing workflows in workqueue_inbox for: %s', workflowsToClose)
+                self.backend.updateInboxElements(*workflowsToClose, OpenForNewData=False)
+                msg = 'Closed inbox elements for: %s.\n' % ', '.join(workflowsToClose)
+            except CouchInternalServerError as ex:
+                msg = 'Failed to close workflows with a CouchInternalServerError exception. '
+                msg += 'Details: {}'.format(str(ex))
+                self.logger.error(msg)
+            except Exception as ex:
+                msg = 'Failed to close workflows with a generic exception. '
+                msg += 'Details: {}'.format(str(ex))
+                self.logger.exception(msg)
+        else:
+            msg = 'No workflows to close.\n'
+
+        self.backend.recordTaskActivity('workclosing', msg)
+
+        return workflowsToClose
+
+    def deleteCompletedWFElements(self):
+        """
+        deletes Workflow when workflow is in finished status
+        """
+        deletableStates = ["completed", "closed-out", "failed",
+                           "announced", "aborted-completed", "rejected",
+                           "normal-archived", "aborted-archived", "rejected-archived"]
+
+        # fetch workflows known to workqueue + workqueue_inbox and with spec attachments
+        reqNames = self.backend.getWorkflows(includeInbox=True, includeSpecs=True)
+        self.logger.info("Retrieved %d workflows known by WorkQueue", len(reqNames))
+        requestsInfo = self.requestDB.getRequestByNames(reqNames)
+        deleteRequests = []
+        for key, value in viewitems(requestsInfo):
+            if (value["RequestStatus"] is None) or (value["RequestStatus"] in deletableStates):
+                deleteRequests.append(key)
+        self.logger.info("Found %d out of %d workflows in a deletable state",
+                         len(deleteRequests), len(reqNames))
+        return self.backend.deleteWQElementsByWorkflow(deleteRequests)
+
+    def performSyncAndCancelAction(self, skipWMBS):
+        """
+        Apply end policies to determine work status & cleanup finished work
+        """
+        if not self.backend.isAvailable():
+            self.logger.warning('Backend busy or down: skipping cleanup tasks')
+            return
+
+        if self.params['LocalQueueFlag']:
+            self.backend.fixConflicts()  # before doing anything fix any conflicts
+
+        wf_to_cancel = []  # record what we did for task_activity
+        finished_elements = []
+
+        useWMBS = not skipWMBS and self.params['LocalQueueFlag']
+        if useWMBS:
+            wmbsWflowSummary = self.getWMBSSubscriptionStatus()
+        else:
+            wmbsWflowSummary = []
+        # Get queue elements grouped by their workflow with updated wmbs progress
+        # Cancel if requested, update locally and remove obsolete elements
+        self.logger.info('Fetching workflow information (including inbox and specs)')
+        workflowsList = self.backend.getWorkflows(includeInbox=True, includeSpecs=True)
+        for wf in workflowsList:
+            parentQueueDeleted = True
+            try:
+                elements = self.status(RequestName=wf, wmbsInfo=wmbsWflowSummary)
+                parents = self.backend.getInboxElements(RequestName=wf)
+
+                self.logger.debug("Queue %s status follows:", self.backend.queueUrl)
+                results = endPolicy(elements, parents, self.params['EndPolicySettings'])
+                for result in results:
+                    self.logger.debug("Request %s, Status %s, Full info: %s",
+                                      result['RequestName'], result['Status'], result)
+
+                    # check for cancellation requests (affects entire workflow)
+                    if result['Status'] == 'CancelRequested':
+                        self.logger.info('Canceling work for workflow: %s', wf)
+                        canceled = self.cancelWork(WorkflowName=wf)
+                        if canceled:  # global wont cancel if work in child queue
+                            wf_to_cancel.append(wf)
+                            break
+                    elif result['Status'] == 'Negotiating':
+                        self.logger.debug("Waiting for %s to finish splitting", wf)
+                        continue
+
+                    parent = result['ParentQueueElement']
+                    if parent.modified:
+                        self.backend.saveElements(parent)
+
+                    if result.inEndState():
+                        if elements:
+                            self.logger.debug("Request %s finished (%s)",
+                                              result['RequestName'], parent.statusMetrics())
+                            finished_elements.extend(result['Elements'])
+                        else:
+                            parentQueueDeleted = False
+                        continue
+
+                    updated_elements = [x for x in result['Elements'] if x.modified]
+                    for x in updated_elements:
+                        self.logger.debug("Updating progress %s (%s): %s", x['RequestName'], x.id, x.statusMetrics())
+                        self.backend.updateElements(x.id, **x.statusMetrics())
+
+                if not parentQueueDeleted:
+                    self.logger.info('Waiting for parent queue to delete "%s"', wf)
+
+            except Exception as ex:
+                self.logger.error('Error processing workflow "%s": %s', wf, str(ex))
+
+        msg = 'Finished elements: %s\nCanceled workflows: %s' % (', '.join(["%s (%s)" % (x.id, x['RequestName']) \
+                                                                            for x in finished_elements]),
+                                                                 ', '.join(wf_to_cancel))
+
+        self.logger.debug(msg)
+        self.backend.recordTaskActivity('housekeeping', msg)
+
+    def performQueueCleanupActions(self, skipWMBS=False):
+
+        try:
+            self.logger.info("Deleting completed workflow WQ elements ...")
+            res = self.deleteCompletedWFElements()
+            self.logger.info("Deleted %d elements from workqueue/inbox database", res)
+        except Exception as ex:
+            self.logger.exception('Error deleting WQ elements. Details: %s', str(ex))
+
+        try:
+            self.logger.info("Syncing and cancelling work ...")
+            self.performSyncAndCancelAction(skipWMBS)
+        except Exception as ex:
+            self.logger.error('Error syncing and canceling WQ elements. Details: %s', str(ex))
+
+    def _splitWork(self, wmspec, data=None, mask=None, inbound=None, continuous=False, rucioObj=None):
+        """
+        Split work from a parent into WorkQeueueElements.
+
+        If data param supplied use that rather than getting input data from
+        wmspec. Used for instance when global splits by Block (avoids having to
+        modify wmspec block whitelist - thus all appear as same wf in wmbs)
+
+        mask can be used to specify i.e. event range.
+
+        The inbound and continuous parameters are used to split
+        and already split inbox element.
+        """
+        # give preference to rucio object created by the CherryPy threads
+        if not rucioObj:
+            rucioObj = self.rucio
+
+        totalUnits, rejectedWork, badWork = [], [], []
+        # split each top level task into constituent work elements
+        # get the acdc server and db name
+        for topLevelTask in wmspec.taskIterator():
+            spec = getWorkloadFromTask(topLevelTask)
+            policyName = spec.startPolicy()
+            if not policyName:
+                raise RuntimeError("WMSpec doesn't define policyName, current value: '%s'" % policyName)
+
+            policy = startPolicy(policyName, self.params['SplittingMapping'],
+                                 rucioObj=rucioObj, logger=self.logger)
+            if not policy.supportsWorkAddition() and continuous:
+                # Can't split further with a policy that doesn't allow it
+                continue
+            if continuous:
+                policy.modifyPolicyForWorkAddition(inbound)
+            self.logger.info('Splitting %s with policy name %s and policy params %s',
+                             topLevelTask.getPathName(), policyName,
+                             self.params['SplittingMapping'].get(policyName))
+            units, rejectedWork, badWork = policy(spec, topLevelTask, data, mask, continuous=continuous)
+            self.logger.info('Work splitting completed with %d units, %d rejectedWork and %d badWork',
+                             len(units), len(rejectedWork), len(badWork))
+            for unit in units:
+                msg = 'Queuing element {} for {} with policy {}, '.format(unit.id, unit['Task'].getPathName(),
+                                                                          unit['StartPolicy'])
+                msg += 'with {} job(s) and {} lumis'.format(unit['Jobs'], unit['NumberOfLumis'])
+                if unit['Inputs']:
+                    msg += ' on %s' % list(unit['Inputs'])[0]
+                if unit['Mask']:
+                    msg += ' on events %d-%d' % (unit['Mask']['FirstEvent'], unit['Mask']['LastEvent'])
+                self.logger.info(msg)
+            totalUnits.extend(units)
+
+        return (totalUnits, rejectedWork, badWork)
+
+    def _getTotalStats(self, units):
+        totalToplevelJobs = 0
+        totalEvents = 0
+        totalLumis = 0
+        totalFiles = 0
+
+        for unit in units:
+            totalToplevelJobs += unit['Jobs']
+            totalEvents += unit['NumberOfEvents']
+            totalLumis += unit['NumberOfLumis']
+            totalFiles += unit['NumberOfFiles']
+
+        return {'total_jobs': totalToplevelJobs,
+                'input_events': totalEvents,
+                'input_lumis': totalLumis,
+                'input_num_files': totalFiles}
+
+    def processInboundWork(self, inbound_work=None, throw=False, continuous=False, rucioObj=None):
+        """Retrieve work from inbox, split and store
+        If request passed then only process that request
+        """
+        inbound_work = inbound_work or []
+        msg = "Executing processInboundWork with {} inbound_work, ".format(len(inbound_work))
+        msg += "throw: {} and continuous: {}".format(throw, continuous)
+        self.logger.info(msg)
+        if self.params['LocalQueueFlag']:
+            self.logger.info("fixing conflict...")
+            self.backend.fixConflicts()  # db should be consistent
+
+        result = []
+        if not inbound_work and continuous:
+            # This is not supported
+            return result
+        if not inbound_work:
+            inbound_work = self.backend.getElementsForSplitting()
+            self.logger.info('Retrieved %d elements for splitting with continuous flag: %s',
+                             len(inbound_work), continuous)
+        for inbound in inbound_work:
+            try:
+                # Check we haven't already split the work, unless it's continuous processing
+                work = not continuous and self.backend.getElementsForParent(inbound)
+                if work:
+                    self.logger.info('Request "%s" already split - Resuming', inbound['RequestName'])
+                else:
+                    work, rejectedWork, badWork = self._splitWork(inbound['WMSpec'], data=inbound['Inputs'],
+                                                                  mask=inbound['Mask'], inbound=inbound,
+                                                                  continuous=continuous, rucioObj=rucioObj)
+
+                    # save inbound work to signal we have completed queueing
+                    # if this fails, rerunning will pick up here
+                    newWork = self.backend.insertElements(work, parent=inbound)
+                    # get statistics for the new work
+                    totalStats = self._getTotalStats(newWork)
+
+                    if not continuous:
+                        # Update to Acquired when it's the first processing of inbound work
+                        self.backend.updateInboxElements(inbound.id, Status='Acquired')
+
+                    # store the inputs in the global queue inbox workflow element
+                    if not self.params.get('LocalQueueFlag'):
+                        processedInputs = []
+                        for unit in work:
+                            processedInputs.extend(list(unit['Inputs']))
+                        self.backend.updateInboxElements(inbound.id, ProcessedInputs=processedInputs,
+                                                         RejectedInputs=rejectedWork)
+                        # if global queue, then update workflow stats to request mgr couch doc
+                        # remove the "UnittestFlag" - need to create the reqmgrSvc emulator
+                        if not self.params.get("UnittestFlag", False):
+                            self.reqmgrSvc.updateRequestStats(inbound['WMSpec'].name(), totalStats)
+
+                    if badWork:
+                        msg = "Request with the following unprocessable input data: %s" % badWork
+                        self.logdb.post(inbound['RequestName'], msg, 'warning')
+            except TERMINAL_EXCEPTIONS as ex:
+                msg = 'Terminal exception splitting WQE: %s' % inbound
+                self.logger.error(msg)
+                self.logdb.post(inbound['RequestName'], msg, 'error')
+                if not continuous:
+                    # Only fail on first splitting
+                    self.logger.error('Failing workflow "%s": %s', inbound['RequestName'], str(ex))
+                    self.backend.updateInboxElements(inbound.id, Status='Failed')
+                    if throw:
+                        raise
+            except Exception as ex:
+                if continuous:
+                    continue
+                msg = 'Exception splitting wqe %s for %s: %s' % (inbound.id, inbound['RequestName'], str(ex))
+                self.logger.exception(msg)
+                self.logdb.post(inbound['RequestName'], msg, 'error')
+
+                if throw:
+                    raise
+                continue
+            else:
+                result.extend(work)
+
+        requests = ', '.join(list(set(['"%s"' % x['RequestName'] for x in result])))
+        if requests:
+            self.logger.info('Split work for request(s): %s', requests)
+
+        return result
+
+    def getWMBSInjectionStatus(self, workflowName=None, drainMode=False):
+        """
+        if the parent queue exist return the result from parent queue.
+        other wise return the result from the current queue.
+        (In general parent queue always exist when it is called from local queue
+        except T1 skim case)
+        returns list of [{workflowName: injection status (True or False)}]
+        if the workflow is not exist return []
+        """
+        if self.parent_queue and not drainMode:
+            return self.parent_queue.getWMBSInjectStatus(workflowName)
+        return self.backend.getWMBSInjectStatus(workflowName)
+
+    def monitorWorkQueue(self, status=None):
+        """
+        Uses the workqueue data-service to retrieve a few basic information
+        regarding all the elements in the queue.
+        """
+        status = status or []
+        results = {}
+        start = int(time.time())
+        results['workByStatus'] = self.workqueueDS.getJobsByStatus()
+        results['workByStatusAndPriority'] = self.workqueueDS.getJobsByStatusAndPriority()
+        results['workByAgentAndStatus'] = self.workqueueDS.getChildQueuesAndStatus()
+        results['workByAgentAndPriority'] = self.workqueueDS.getChildQueuesAndPriority()
+
+        # now the heavy procesing for the site information
+        elements = self.workqueueDS.getElementsByStatus(status)
+        uniSites, posSites = getGlobalSiteStatusSummary(elements, status=status)
+        results['uniqueJobsPerSiteAAA'] = uniSites
+        results['possibleJobsPerSiteAAA'] = posSites
+        uniSites, posSites = getGlobalSiteStatusSummary(elements, status=status, dataLocality=True)
+        results['uniqueJobsPerSite'] = uniSites
+        results['possibleJobsPerSite'] = posSites
+
+        end = int(time.time())
+        results["total_query_time"] = end - start
+        return results