reqmgr2ms-output 2.3.8rc8__tar.gz → 2.3.8rc10__tar.gz

{reqmgr2ms-output-2.3.8rc8/src/python/reqmgr2ms_output.egg-info → reqmgr2ms-output-2.3.8rc10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.8rc8
+Version: 2.3.8rc10
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-output-2.3.8rc8 → reqmgr2ms-output-2.3.8rc10}/src/python/WMCore/MicroService/MSOutput/MSOutput.py

@@ -519,6 +519,7 @@ class MSOutput(MSCore):
                            (msPipelineNonRelVal, self.msOutNonRelValColl)]
         for pipeColl in pipeCollections:
             wfCounters = 0
+            wfCountersOk = 0
             pipeLine = pipeColl[0]
             dbColl = pipeColl[1]
             pipeLineName = pipeLine.getPipelineName()
@@ -526,8 +527,10 @@ class MSOutput(MSCore):
                 # FIXME:
                 #    To redefine those exceptions as MSoutputExceptions and
                 #    start using those here so we do not mix with general errors
+                wfCounters += 1
                 try:
                     pipeLine.run(docOut)
+                    wfCountersOk += 1
                 except (KeyError, TypeError) as ex:
                     msg = "%s Possibly malformed record in MongoDB. Err: %s. " % (pipeLineName, str(ex))
                     msg += "Continue to the next document."
@@ -542,10 +545,12 @@ class MSOutput(MSCore):
                     msg = "%s General error from pipeline. Err: %s. " % (pipeLineName, str(ex))
                     msg += "Will retry again in the next cycle."
                     self.logger.exception(msg)
-                    break
-                wfCounters += 1
-            self.logger.info("Processed %d workflows from pipeline: %s", wfCounters, pipeLineName)
-            wfCounterTotal += wfCounters
+                    workflowname = docOut.get("_id", "")
+                    self.alertGenericError(self.mode, workflowname, msg, str(ex), str(docOut))
+                    continue
+            self.logger.info("Successfully processed %d workflows from pipeline: %s", wfCountersOk, pipeLineName)
+            self.logger.info("Failed to proccess %d workflows from pipeline: %s", wfCounters - wfCountersOk, pipeLineName)
+            wfCounterTotal += wfCountersOk
 
         return wfCounterTotal
 
@@ -579,12 +584,12 @@ class MSOutput(MSCore):
                                         Functor(self.docCleaner)])
         # TODO:
         #    To generate the object from within the Function scope see above.
-        counter = 0
+        counterOk = 0
         for request in requestRecords:
-            counter += 1
             pipeLineName = msPipeline.getPipelineName()
             try:
                 msPipeline.run(request)
+                counterOk += 1
             except (KeyError, TypeError) as ex:
                 msg = "%s Possibly broken read from ReqMgr2 API or other. Err: %s." % (pipeLineName, str(ex))
                 msg += " Continue to the next document."
@@ -594,8 +599,10 @@ class MSOutput(MSCore):
                 msg = "%s General Error from pipeline. Err: %s. " % (pipeLineName, str(ex))
                 msg += "Giving up Now."
                 self.logger.exception(str(ex))
-                break
-        return counter
+                workflowname = request.get("_id", "")
+                self.alertGenericError(self.mode, workflowname, msg, str(ex), str(request))
+                continue
+        return counterOk
 
     def docTransformer(self, doc):
         """
@@ -1023,3 +1030,24 @@ class MSOutput(MSCore):
         self.logger.critical(alertDescription)
         if self.msConfig["sendNotification"] and not isRelVal:
             self.sendAlert(alertName, alertSeverity, alertSummary, alertDescription, self.alertServiceName)
+
+    def alertGenericError(self, caller, workflowname, msg, exMsg, document):
+        """
+        Send an alert to Prometheus in the case of a generic error with ms-output
+
+        :param caller: str, indicates if the error comes from Producer or Consumer
+        :param workflowname: str, representing the workflow name
+        :param msg: str, context about the error
+        :param exMsg: str, excetpion message
+        :param document: str, serialized mongodb document
+        :return: none
+        """
+        alertName = "ms-output: Generic MSOutput error inside {} while processing workflow '{}'".format(caller, workflowname)
+        alertSeverity = "high"
+        alertSummary = "[MSOutput] Generic MSOutput error inside {} while processing workflow '{}'".format(caller, workflowname)
+        alertDescription = "wf: {}\n\nmsg: {}\n\nex: {}\n\n{}".format(workflowname, msg, exMsg, document)
+        self.logger.error("%s\n%s\n%s", alertName, alertSummary, alertDescription)
+        if self.msConfig["sendNotification"]:
+            self.sendAlert(alertName, alertSeverity, alertSummary, alertDescription, self.alertServiceName)
+
+

{reqmgr2ms-output-2.3.8rc8 → reqmgr2ms-output-2.3.8rc10}/src/python/WMCore/MicroService/Service/RestApiHub.py

@@ -39,4 +39,8 @@ class RestApiHub(RESTApi):
         # the mount point should match service configuration, e.g. in MSPileup we define
         # mount point as /ms-pileup/data, e.g. http://.../ms-pileup/data
         if 'ms-pileup' in mount:
+            # add new API "pileup" and it will be accessible as /ms-pileup/data/pileup
             self._add({"pileup": Data(app, self, config, mount)})
+        elif 'ms-transferor' in mount:
+            # add new API "transferor" and it will be accessible as /ms-transferor/data/transferor
+            self._add({"transferor": Data(app, self, config, mount)})

{reqmgr2ms-output-2.3.8rc8 → reqmgr2ms-output-2.3.8rc10}/src/python/WMCore/Services/WorkQueue/WorkQueue.py

@@ -237,34 +237,6 @@ class WorkQueue(object):
         elements = [x['id'] for x in data.get('rows', []) if x['key'][1] not in nonCancelableElements]
         return self.updateElements(*elements, Status='CancelRequested')
 
-    def updateSiteLists(self, wf, siteWhiteList=None, siteBlackList=None):
-        """
-        Update site list parameters in elements matching a given workflow and a list of element statuse
-
-        :param wf: workflow name
-        :param siteWhiteList: optional list of strings, new site white list
-        :param siteBlackList: optional list of strings, new site black list
-        :return: None
-        """
-        # Update elements in Available status
-        data = self.db.loadView('WorkQueue', 'jobStatusByRequest',
-                                {'reduce': False})
-        states = ['Available']
-        elementsToUpdate = [x['id'] for x in data.get('rows', []) if x['key'][-1] in states and wf in x['key']]
-        if elementsToUpdate:
-            self.logger.info("Updating %d elements in status %s for workflow %s", len(elementsToUpdate), states, wf)
-            self.updateElements(*elementsToUpdate, SiteWhiteList=siteWhiteList, SiteBlackList=siteBlackList)
-        # Update the spec, if it exists
-        if self.db.documentExists(wf):
-            wmspec = WMWorkloadHelper()
-            # update local workqueue couchDB
-            wmspec.load(self.hostWithAuth + "/%s/%s/spec" % (self.db.name, wf))
-            wmspec.setSiteWhiteList(siteWhiteList)
-            wmspec.setSiteBlackList(siteBlackList)
-            dummy_values = {'name': wmspec.name()}
-            wmspec.saveCouch(self.hostWithAuth, self.db.name, dummy_values)
-        return
-
     def updatePriority(self, wf, priority):
         """Update priority of a workflow, this implies
            updating the spec and the priority of the Available elements"""
@@ -284,6 +256,40 @@ class WorkQueue(object):
             wmspec.saveCouch(self.hostWithAuth, self.db.name, dummy_values)
         return
 
+    def updateElementsByWorkflow(self, workload, updateParams, status=None):
+        """
+        Update all available WorkQueue elements of a given workflow  with a set
+        of arguments provided through the `updateParams` dictionary
+        :param workload:     A workflow spec
+        :param updateParams: A dictionary with parameters  to be updated
+        :param status:       A list of allowed WorkQueue elements statuses to be considered for updating
+                             Default: None - do not filter by status
+        :return:             No value, raises exceptions from internal methods in case of errors.
+        """
+        # Fetch the whole view with Workqueue elements per given workflow
+        wfName = workload.name()
+        data = self.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus',
+                                {'startkey': [wfName], 'endkey': [wfName, {}],
+                                 'reduce': False})
+
+        # Fetch only a list of WorkQueue element Ids && Filter them by allowed status
+        if status:
+            elementsToUpdate = [x['id'] for x in data.get('rows', []) if x['value']['Status'] in status]
+        else:
+            elementsToUpdate = [x['id'] for x in data.get('rows', [])]
+
+        # Update all WorkQueue elements with the parameters provided in a single push
+        if elementsToUpdate:
+            self.updateElements(*elementsToUpdate, **updateParams)
+
+        # Update the spec, if it exists
+        if self.db.documentExists(wfName):
+            # Update all workload parameters based on the full reqArgs dictionary
+            workload.updateWorkloadArgs(updateParams)
+            # Commit the changes of the current workload object to the database:
+            workload.saveCouchUrl(workload.specUrl())
+        return
+
     def getWorkflowNames(self, inboxFlag=False):
         """Get workflow names from workqueue db"""
         if inboxFlag:

{reqmgr2ms-output-2.3.8rc8 → reqmgr2ms-output-2.3.8rc10}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.8rc8'
+__version__ = '2.3.8rc10'
 __all__ = []

{reqmgr2ms-output-2.3.8rc8 → reqmgr2ms-output-2.3.8rc10/src/python/reqmgr2ms_output.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.8rc8
+Version: 2.3.8rc10
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group