reqmgr2ms-monitor 2.3.8rc6__tar.gz → 2.3.8rc9__tar.gz

{reqmgr2ms-monitor-2.3.8rc6/src/python/reqmgr2ms_monitor.egg-info → reqmgr2ms-monitor-2.3.8rc9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.8rc6
+Version: 2.3.8rc9
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc9}/src/python/Utils/Patterns.py

@@ -1,7 +1,7 @@
 """
 Patterns module provides set of CS patterns
 """
-
+import re
 
 class Singleton(type):
     """Implementation of Singleton class"""
@@ -11,3 +11,14 @@ class Singleton(type):
             cls._instances[cls] = \
                     super(Singleton, cls).__call__(*args, **kwargs)
         return cls._instances[cls]
+
+
+def getDomainName(urlStr):
+    """
+    Given a URL string, return the domain name.
+    :param urlStr: URL string
+    :return: a string with the domain name (e.g. "cmsweb-prod")
+    """
+    domainPattern = re.compile(r'https?://([^/]+)\.cern\.ch')
+    match = domainPattern.search(urlStr)
+    return match.group(1) if match else ""

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc9}/src/python/WMCore/MicroService/MSCore/MSManager.py

@@ -34,6 +34,7 @@ from WMCore.MicroService.Tools.Common import getMSLogger
 from WMCore.MicroService.MSCore.TaskManager import start_new_thread
 from Utils.Utilities import strToBool
 
+
 def daemon(func, reqStatus, interval, logger):
     "Daemon to perform given function action for all request in our store"
     while True:
@@ -398,16 +399,18 @@ class MSManager(object):
         :param doc: input JSON doc for HTTP POST request
         :return: list of results
         """
+        res = []
         if 'pileup' in self.services:
-            res = []
             if 'pileupName' in doc:
                 # this is create POST request
                 res = self.msPileup.createPileup(doc)
             if 'query' in doc:
                 # this is POST request to get data for a given JSON query
                 res = self.msPileup.queryDatabase(doc)
-            return res
-        return []
+        elif 'transferor' in self.services:
+            if 'workflow' in doc:
+                res = self.msTransferor.updateSites(doc)
+        return res
 
     def update(self, doc):
         """

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc9}/src/python/WMCore/Services/RequestDB/RequestDBWriter.py

@@ -7,13 +7,6 @@ class RequestDBWriter(RequestDBReader):
         # set the connection for local couchDB call
         # inherited from WMStatsReader
         self._commonInit(couchURL, couchapp)
-        self._propertyNeedToBeEncoded = ["RequestTransition",
-                                         "PriorityTransition",
-                                         "SiteWhitelist",
-                                         "SiteBlacklist",
-                                         "BlockWhitelist",
-                                         "InputDatasetTypes",
-                                         "OutputDatasets"]
 
     def insertGenericRequest(self, doc):
 
@@ -31,7 +24,7 @@ class RequestDBWriter(RequestDBReader):
 
     def updateRequestStats(self, request, stats):
         """
-        This functionis only available ReqMgr couch app currutly (not T0)
+        This function is only available ReqMgr couch app currently (not T0)
         WQ specific function
         param: stats: dict of {'total_jobs': 0, 'input_lumis': 0,
                                'input_events': 0, 'input_num_files': 0}

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc9}/src/python/WMCore/Services/Rucio/Rucio.py

@@ -16,7 +16,7 @@ from copy import deepcopy
 from rucio.client import Client
 from rucio.common.exception import (AccountNotFound, DataIdentifierNotFound, AccessDenied, DuplicateRule,
                                     DataIdentifierAlreadyExists, DuplicateContent, InvalidRSEExpression,
-                                    UnsupportedOperation, FileAlreadyExists, RuleNotFound, RSENotFound)
+                                    UnsupportedOperation, FileAlreadyExists, RuleNotFound, RSENotFound, RuleReplaceFailed)
 from Utils.MemoryCache import MemoryCache
 from Utils.IteratorTools import grouper
 from WMCore.Services.Rucio.RucioUtils import (validateMetaData, weightedChoice,
@@ -454,6 +454,30 @@ class Rucio(object):
             self.logger.error("Exception closing container/block: %s. Error: %s", name, str(ex))
         return response
 
+    def moveReplicationRule(self, ruleId, rseExpression, account):
+        """
+        Perform move operation for provided rule id and rse expression
+        :param ruleId: rule id
+        :param rseExpression: rse expression
+        :param account: rucio quota account
+        :return: it returns either an empty list or a list with a string id for the rule created
+        Please note, we made return type from this wrapper compatible with createReplicateRule
+        """
+        ruleIds = []
+        try:
+            rid = self.cli.move_replication_rule(ruleId, rseExpression, account)
+            ruleIds.append(rid)
+        except RuleNotFound as ex:
+            msg = "RuleNotFound move DID replication rule. Error: %s" % str(ex)
+            raise WMRucioException(msg) from ex
+        except RuleReplaceFailed as ex:
+            msg = "RuleReplaceFailed move DID replication rule. Error: %s" % str(ex)
+            raise WMRucioException(msg) from ex
+        except Exception as ex:
+            msg = "Unsupported exception from Rucio API. Error: %s" % str(ex)
+            raise WMRucioException(msg) from ex
+        return ruleIds
+
     def createReplicationRule(self, names, rseExpression, scope='cms', copies=1, **kwargs):
         """
         _createReplicationRule_

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc9}/src/python/WMCore/Services/WorkQueue/WorkQueue.py

@@ -203,7 +203,7 @@ class WorkQueue(object):
         eleParams[self.eleKey] = updatedParams
         conflictIDs = self.db.updateBulkDocumentsWithConflictHandle(elementIds, eleParams, maxConflictLimit=20)
         if conflictIDs:
-            raise CouchConflictError("WQ update failed with conflict", data=updatedParams, result=conflictIDs)
+            raise CouchConflictError("WQ update failed with conflict", data=updatedParams, result=conflictIDs, status=409)
         return
 
     def getAvailableWorkflows(self):
@@ -237,6 +237,34 @@ class WorkQueue(object):
         elements = [x['id'] for x in data.get('rows', []) if x['key'][1] not in nonCancelableElements]
         return self.updateElements(*elements, Status='CancelRequested')
 
+    def updateSiteLists(self, wf, siteWhiteList=None, siteBlackList=None):
+        """
+        Update site list parameters in elements matching a given workflow and a list of element statuse
+
+        :param wf: workflow name
+        :param siteWhiteList: optional list of strings, new site white list
+        :param siteBlackList: optional list of strings, new site black list
+        :return: None
+        """
+        # Update elements in Available status
+        data = self.db.loadView('WorkQueue', 'jobStatusByRequest',
+                                {'reduce': False})
+        states = ['Available']
+        elementsToUpdate = [x['id'] for x in data.get('rows', []) if x['key'][-1] in states and wf in x['key']]
+        if elementsToUpdate:
+            self.logger.info("Updating %d elements in status %s for workflow %s", len(elementsToUpdate), states, wf)
+            self.updateElements(*elementsToUpdate, SiteWhiteList=siteWhiteList, SiteBlackList=siteBlackList)
+        # Update the spec, if it exists
+        if self.db.documentExists(wf):
+            wmspec = WMWorkloadHelper()
+            # update local workqueue couchDB
+            wmspec.load(self.hostWithAuth + "/%s/%s/spec" % (self.db.name, wf))
+            wmspec.setSiteWhiteList(siteWhiteList)
+            wmspec.setSiteBlackList(siteBlackList)
+            dummy_values = {'name': wmspec.name()}
+            wmspec.saveCouch(self.hostWithAuth, self.db.name, dummy_values)
+        return
+
     def updatePriority(self, wf, priority):
         """Update priority of a workflow, this implies
            updating the spec and the priority of the Available elements"""
@@ -307,7 +335,6 @@ class WorkQueue(object):
                                                 'Jobs': x['value']['sum']}
         return result
 
-
     def _retrieveWorkflowStatus(self, data):
         workflowsStatus = {}
 
@@ -318,7 +345,6 @@ class WorkQueue(object):
                 workflowsStatus[workflow] = status
         return workflowsStatus
 
-
     def getWorkflowStatusFromWQE(self, stale=True):
         """
         only checks workqueue db not inbox db.

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc9}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.8rc6'
+__version__ = '2.3.8rc9'
 __all__ = []

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc9/src/python/reqmgr2ms_monitor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.8rc6
+Version: 2.3.8rc9
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group