reqmgr2ms-monitor 2.3.8rc6__tar.gz → 2.3.8rc10__tar.gz

{reqmgr2ms-monitor-2.3.8rc6/src/python/reqmgr2ms_monitor.egg-info → reqmgr2ms-monitor-2.3.8rc10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.8rc6
+Version: 2.3.8rc10
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc10}/src/python/Utils/Patterns.py

@@ -1,7 +1,7 @@
 """
 Patterns module provides set of CS patterns
 """
-
+import re
 
 class Singleton(type):
     """Implementation of Singleton class"""
@@ -11,3 +11,14 @@ class Singleton(type):
             cls._instances[cls] = \
                     super(Singleton, cls).__call__(*args, **kwargs)
         return cls._instances[cls]
+
+
+def getDomainName(urlStr):
+    """
+    Given a URL string, return the domain name.
+    :param urlStr: URL string
+    :return: a string with the domain name (e.g. "cmsweb-prod")
+    """
+    domainPattern = re.compile(r'https?://([^/]+)\.cern\.ch')
+    match = domainPattern.search(urlStr)
+    return match.group(1) if match else ""

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc10}/src/python/WMCore/MicroService/MSCore/MSManager.py

@@ -34,6 +34,7 @@ from WMCore.MicroService.Tools.Common import getMSLogger
 from WMCore.MicroService.MSCore.TaskManager import start_new_thread
 from Utils.Utilities import strToBool
 
+
 def daemon(func, reqStatus, interval, logger):
     "Daemon to perform given function action for all request in our store"
     while True:
@@ -398,16 +399,18 @@ class MSManager(object):
         :param doc: input JSON doc for HTTP POST request
         :return: list of results
         """
+        res = []
         if 'pileup' in self.services:
-            res = []
             if 'pileupName' in doc:
                 # this is create POST request
                 res = self.msPileup.createPileup(doc)
             if 'query' in doc:
                 # this is POST request to get data for a given JSON query
                 res = self.msPileup.queryDatabase(doc)
-            return res
-        return []
+        elif 'transferor' in self.services:
+            if 'workflow' in doc:
+                res = self.msTransferor.updateSites(doc)
+        return res
 
     def update(self, doc):
         """

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc10}/src/python/WMCore/MicroService/Service/RestApiHub.py

@@ -39,4 +39,8 @@ class RestApiHub(RESTApi):
         # the mount point should match service configuration, e.g. in MSPileup we define
         # mount point as /ms-pileup/data, e.g. http://.../ms-pileup/data
         if 'ms-pileup' in mount:
+            # add new API "pileup" and it will be accessible as /ms-pileup/data/pileup
             self._add({"pileup": Data(app, self, config, mount)})
+        elif 'ms-transferor' in mount:
+            # add new API "transferor" and it will be accessible as /ms-transferor/data/transferor
+            self._add({"transferor": Data(app, self, config, mount)})

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc10}/src/python/WMCore/Services/RequestDB/RequestDBWriter.py

@@ -7,13 +7,6 @@ class RequestDBWriter(RequestDBReader):
         # set the connection for local couchDB call
         # inherited from WMStatsReader
         self._commonInit(couchURL, couchapp)
-        self._propertyNeedToBeEncoded = ["RequestTransition",
-                                         "PriorityTransition",
-                                         "SiteWhitelist",
-                                         "SiteBlacklist",
-                                         "BlockWhitelist",
-                                         "InputDatasetTypes",
-                                         "OutputDatasets"]
 
     def insertGenericRequest(self, doc):
 
@@ -31,7 +24,7 @@ class RequestDBWriter(RequestDBReader):
 
     def updateRequestStats(self, request, stats):
         """
-        This functionis only available ReqMgr couch app currutly (not T0)
+        This function is only available ReqMgr couch app currently (not T0)
         WQ specific function
         param: stats: dict of {'total_jobs': 0, 'input_lumis': 0,
                                'input_events': 0, 'input_num_files': 0}

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc10}/src/python/WMCore/Services/Rucio/Rucio.py

@@ -16,7 +16,7 @@ from copy import deepcopy
 from rucio.client import Client
 from rucio.common.exception import (AccountNotFound, DataIdentifierNotFound, AccessDenied, DuplicateRule,
                                     DataIdentifierAlreadyExists, DuplicateContent, InvalidRSEExpression,
-                                    UnsupportedOperation, FileAlreadyExists, RuleNotFound, RSENotFound)
+                                    UnsupportedOperation, FileAlreadyExists, RuleNotFound, RSENotFound, RuleReplaceFailed)
 from Utils.MemoryCache import MemoryCache
 from Utils.IteratorTools import grouper
 from WMCore.Services.Rucio.RucioUtils import (validateMetaData, weightedChoice,
@@ -454,6 +454,30 @@ class Rucio(object):
             self.logger.error("Exception closing container/block: %s. Error: %s", name, str(ex))
         return response
 
+    def moveReplicationRule(self, ruleId, rseExpression, account):
+        """
+        Perform move operation for provided rule id and rse expression
+        :param ruleId: rule id
+        :param rseExpression: rse expression
+        :param account: rucio quota account
+        :return: it returns either an empty list or a list with a string id for the rule created
+        Please note, we made return type from this wrapper compatible with createReplicateRule
+        """
+        ruleIds = []
+        try:
+            rid = self.cli.move_replication_rule(ruleId, rseExpression, account)
+            ruleIds.append(rid)
+        except RuleNotFound as ex:
+            msg = "RuleNotFound move DID replication rule. Error: %s" % str(ex)
+            raise WMRucioException(msg) from ex
+        except RuleReplaceFailed as ex:
+            msg = "RuleReplaceFailed move DID replication rule. Error: %s" % str(ex)
+            raise WMRucioException(msg) from ex
+        except Exception as ex:
+            msg = "Unsupported exception from Rucio API. Error: %s" % str(ex)
+            raise WMRucioException(msg) from ex
+        return ruleIds
+
     def createReplicationRule(self, names, rseExpression, scope='cms', copies=1, **kwargs):
         """
         _createReplicationRule_

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc10}/src/python/WMCore/Services/WorkQueue/WorkQueue.py

@@ -203,7 +203,7 @@ class WorkQueue(object):
         eleParams[self.eleKey] = updatedParams
         conflictIDs = self.db.updateBulkDocumentsWithConflictHandle(elementIds, eleParams, maxConflictLimit=20)
         if conflictIDs:
-            raise CouchConflictError("WQ update failed with conflict", data=updatedParams, result=conflictIDs)
+            raise CouchConflictError("WQ update failed with conflict", data=updatedParams, result=conflictIDs, status=409)
         return
 
     def getAvailableWorkflows(self):
@@ -256,6 +256,40 @@ class WorkQueue(object):
             wmspec.saveCouch(self.hostWithAuth, self.db.name, dummy_values)
         return
 
+    def updateElementsByWorkflow(self, workload, updateParams, status=None):
+        """
+        Update all available WorkQueue elements of a given workflow  with a set
+        of arguments provided through the `updateParams` dictionary
+        :param workload:     A workflow spec
+        :param updateParams: A dictionary with parameters  to be updated
+        :param status:       A list of allowed WorkQueue elements statuses to be considered for updating
+                             Default: None - do not filter by status
+        :return:             No value, raises exceptions from internal methods in case of errors.
+        """
+        # Fetch the whole view with Workqueue elements per given workflow
+        wfName = workload.name()
+        data = self.db.loadView('WorkQueue', 'elementsDetailByWorkflowAndStatus',
+                                {'startkey': [wfName], 'endkey': [wfName, {}],
+                                 'reduce': False})
+
+        # Fetch only a list of WorkQueue element Ids && Filter them by allowed status
+        if status:
+            elementsToUpdate = [x['id'] for x in data.get('rows', []) if x['value']['Status'] in status]
+        else:
+            elementsToUpdate = [x['id'] for x in data.get('rows', [])]
+
+        # Update all WorkQueue elements with the parameters provided in a single push
+        if elementsToUpdate:
+            self.updateElements(*elementsToUpdate, **updateParams)
+
+        # Update the spec, if it exists
+        if self.db.documentExists(wfName):
+            # Update all workload parameters based on the full reqArgs dictionary
+            workload.updateWorkloadArgs(updateParams)
+            # Commit the changes of the current workload object to the database:
+            workload.saveCouchUrl(workload.specUrl())
+        return
+
     def getWorkflowNames(self, inboxFlag=False):
         """Get workflow names from workqueue db"""
         if inboxFlag:
@@ -307,7 +341,6 @@ class WorkQueue(object):
                                                 'Jobs': x['value']['sum']}
         return result
 
-
     def _retrieveWorkflowStatus(self, data):
         workflowsStatus = {}
 
@@ -318,7 +351,6 @@ class WorkQueue(object):
                 workflowsStatus[workflow] = status
         return workflowsStatus
 
-
     def getWorkflowStatusFromWQE(self, stale=True):
         """
         only checks workqueue db not inbox db.

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc10}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.8rc6'
+__version__ = '2.3.8rc10'
 __all__ = []

{reqmgr2ms-monitor-2.3.8rc6 → reqmgr2ms-monitor-2.3.8rc10/src/python/reqmgr2ms_monitor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.8rc6
+Version: 2.3.8rc10
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group