reqmgr2ms-monitor 2.3.2rc2__tar.gz → 2.3.2rc4__tar.gz

{reqmgr2ms-monitor-2.3.2rc2/src/python/reqmgr2ms_monitor.egg-info → reqmgr2ms-monitor-2.3.2rc4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.2rc2
+Version: 2.3.2rc4
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-monitor-2.3.2rc2 → reqmgr2ms-monitor-2.3.2rc4}/setup_dependencies.py

@@ -223,7 +223,8 @@ dependencies = {
         'packages': ['WMCore.Credential', 'WMCore.Algorithms+', 'WMCore.WMSpec+',
                      'WMCore.JobSplitting', 'WMCore.Services+', 'Utils+'],
         'systems': ['wmc-database', 'wmc-runtime'],
-        'modules': ['WMCore.WMBS.File', 'WMCore.WMBS.WMBSBase', 'WMCore.WMBS.__init__'],
+        'modules': ['WMCore.WMBS.File', 'WMCore.WMBS.WMBSBase', 'WMCore.WMBS.__init__',
+                    'WMCore.BossAir.Plugins.BasePlugin', 'WMCore.BossAir.Plugins.__init__'],
     },
     'wmclient': {
         'systems': ['wmc-runtime', 'wmc-database']

{reqmgr2ms-monitor-2.3.2rc2 → reqmgr2ms-monitor-2.3.2rc4}/src/python/WMCore/Lexicon.py

@@ -26,7 +26,7 @@ from WMCore.WMException import WMException, WMEXCEPTION_START_STR, WMEXCEPTION_E
 # i.e production should end with v[number]
 PRIMARY_DS = {'re': '^[a-zA-Z][a-zA-Z0-9\-_]*$', 'maxLength': 99}
 PROCESSED_DS = {'re': '[a-zA-Z0-9\.\-_]+', 'maxLength': 199}
-TIER = {'re': '[A-Z\-_]+', 'maxLength': 99}
+TIER = {'re': '[A-Z0-9\-_]+', 'maxLength': 99}
 BLOCK_STR = {'re': '#[a-zA-Z0-9\.\-_]+', 'maxLength': 100}
 
 lfnParts = {
@@ -103,7 +103,7 @@ def searchblock(candidate):
     return check(regexp, candidate)
 
 
-SEARCHDATASET_RE = r'^/(\*|[a-zA-Z\*][a-zA-Z0-9_\*\-]{0,100})(/(\*|[a-zA-Z0-9_\.\-\*]{1,199})){0,1}(/(\*|[A-Z\-\*]{1,50})){0,1}$'
+SEARCHDATASET_RE = r'^/(\*|[a-zA-Z\*][a-zA-Z0-9_\*\-]{0,100})(/(\*|[a-zA-Z0-9_\.\-\*]{1,199})){0,1}(/(\*|[A-Z0-9\-\*]{1,99})){0,1}$'
 
 
 def searchdataset(candidate):
@@ -202,7 +202,7 @@ def globalTag(candidate):
     return check(r'[a-zA-Z0-9\s\.\-_:]{1,100}$', candidate)
 
 
-DATASET_RE = r'^/[a-zA-Z0-9\-_]{1,99}/[a-zA-Z0-9\.\-_]{1,199}/[A-Z\-]{1,50}$'
+DATASET_RE = r'^/[a-zA-Z0-9\-_]{1,99}/[a-zA-Z0-9\.\-_]{1,199}/[A-Z0-9\-]{1,99}$'
 
 
 def dataset(candidate):

reqmgr2ms-monitor-2.3.2rc4/src/python/WMCore/Services/WMStats/WMStatsPycurl.py

@@ -0,0 +1,145 @@
+"""
+This module provides some functions to create concurrent HTTP requests
+for CouchDB, as an alternative to the sequential modules available under:
+* WMCore/Services/WMStats/WMStatsReader.getTaskJobSummaryByRequest
+* WMCore/Services/WMStats/WMStatsReader.jobDetailByTasks
+
+Documentation for the parameters supported in views query can be found at:
+https://docs.couchdb.org/en/stable/api/ddoc/views.html#get--db-_design-ddoc-_view-view
+"""
+import json
+
+from urllib.parse import urljoin, urlencode
+from Utils.CertTools import ckey, cert
+from WMCore.Services.pycurl_manager import getdata as multi_getdata
+
+
+def getTaskJobSummaryByRequestPycurl(rowsSummary, sampleSize, serviceOpts):
+    """
+    Pycurl-based implementation of WMStatsReader.getTaskJobSummaryByRequest
+    :param rowsSummary: a dictionary with rows from CouchDB
+    :param sampleSize: integer with number of documents to retrieve
+    :param serviceOpts: dictionary with CouchDB options (url, db name and couchapp name)
+    :return: a dictionary with job detail
+    """
+    paramQueries = []
+    for row in rowsSummary['rows']:
+        thisQuery = {"startkey": [], "endkey": [], "numOfError": 0}
+        # row["key"] = ['workflow', 'task', 'jobstatus', 'exitCode', 'site']
+        thisQuery["startkey"] = row["key"][:4]
+        if row["key"][4]:
+            thisQuery["startkey"].append(row["key"][4])  # site
+
+        thisQuery["endkey"] = []
+        thisQuery["endkey"].extend(thisQuery["startkey"])
+        thisQuery["endkey"].append({})
+        # append amount of errors matching the 5 keys above
+        thisQuery["numOfError"] = row["value"]
+        # add query to the pool
+        paramQueries.append(thisQuery)
+
+    return jobDetailByTasksPycurl(paramQueries, sampleSize, serviceOpts)
+
+
+def jobDetailByTasksPycurl(queries, limit, serviceOpts):
+    """
+    Pycurl-based implementation of WMStatsReader.jobDetailByTasks.
+    In short, for each tuple of errors, it returns "limit" number of documents.
+    :param queries: list of CouchDB query parameters
+    :param limit: number of documents to retrieve from CouchDB
+    :param serviceOpts: dictionary with CouchDB options (url, db name and couchapp name)
+    :return: dictionary with job detail information, in a format like:
+        {"WORKFLOW_NAME":
+            {"/WORKFLOW_NAME/TASK_NAME":
+                {"JOB_STATE":
+                    {"EXIT_CODE":
+                        {"SITE_NAME":
+                            "samples": [
+                                {"_id": "123abc",
+                                 "_rev": "12-abc",
+                                 "wmbsid": 2006208,
+                                 "type": "jobsummary",
+                                 "retrycount": 3,
+                                 "errors": {"JobSubmit": [{
+                                    "type": "SubmitFailed",
+                                    "details": "The job can blah blah",
+                                    "exitCode": 71103}]
+                                    },
+                                "timestamp": 1698749157,
+                                ... etc etc}],
+                            "errorCount": 1}
+                        }
+                    }
+                },
+            {"/WORKFLOW_NAME/TASK_NAME-2":
+                  {"JOB_STATE": {... etc etc
+
+    A decoded example of this query would be:
+        scurl "https://cmsweb-test9.cern.ch/couchdb/wmstats/_design/WMStatsErl3/_view/jobsByStatusWorkflow?reduce=false&include_docs=true&startkey=["WORKFLOW_NAME","/WORKFLOW_NAME/TASK_NAME","JOB_STATE",EXIT_CODE,"SITE_NAME"]&endkey=["WORKFLOW_NAME","/WORKFLOW_NAME/TASK_NAME","JOB_STATE",EXIT_CODE,"SITE_NAME",{}]&limit=1&stale=update_after"
+    while encoding the url would result in:
+        scurl "https://****.cern.ch/couchdb/wmstats/_design/WMStatsErl3/_view/jobsByStatusWorkflow?reduce=false&include_docs=true&startkey=%5B%22WORKFLOW_NAME%22%2C+%22%2FWORKFLOW_NAME%2FTASK_NAME%22%2C+%22JOB_STATE%22%2C+EXIT_CODE%2C+%22SITE_NAME%22%5D&endkey=%5B%22WORKFLOW_NAME%22%2C+%22%2FWORKFLOW_NAME%2FTASK_NAME%22%2C+%22JOB_STATE%22%2C+EXIT_CODE%2C+%22SITE_NAME%22%2C+%7B%7D%5D&limit=1&stale=update_after"
+
+    Example of output is:
+        {"total_rows":4764135,"offset":4056263,"rows":[
+        {"id":"12703ce5-xxx","key":["WORKFLOW_NAME","/WORKFLOW_NAME/TASK_NAME","jobfailed",8006,"SITE_NAME","https://xxx.cern.ch/couchdb/acdcserver","vocms0255.cern.ch",["Fatal Exception"]],
+         "value":{"id":"12703ce5-xxx","rev":"4-xxx"},
+         "doc":{"_id":"12703ce5-xx","_rev":"4-xxx","wmbsid":2709786,"type":"jobsummary", ...
+
+    where (for the examples above):
+        WORKFLOW_NAME is, e.g.: pdmvserv_Run2017G_DoubleMuon_UL2017_MiniAODv2_BParking_230917_124108_9876
+        TASK_NAME is, e.g.: DataProcessing
+        SITE_NAME is, e.g.: T2_US_Wisconsin
+        JOB_STATE is, e.g.: jobfailed
+        EXIT_CODE is, e.g.: 8006
+    """
+    uri = f"couchdb/{serviceOpts['dbName']}/_design/{serviceOpts['couchapp']}/_view/jobsByStatusWorkflow"
+    baseUrl = urljoin(serviceOpts['couchURL'], uri)
+
+    encoder = json.JSONEncoder()
+    urlsPool = []
+    for query in queries:
+        options = {'include_docs': encoder.encode(True),
+                   'reduce': encoder.encode(False),
+                   'startkey': encoder.encode(query["startkey"]),
+                   'endkey': encoder.encode(query["endkey"]),
+                   'limit': encoder.encode(limit)}
+        # we cannot encode the 'stale' parameter
+        options.setdefault("stale", "update_after")
+        # encode url data for the GET request
+        thisUrl = f"{baseUrl}?{urlencode(options, doseq=True)}"
+        urlsPool.append(thisUrl)
+
+    jobInfoDoc = {}
+    # now run all of these calls in parallel
+    for response in multi_getdata(urlsPool, ckey(), cert()):
+        if 'error' in response:
+            raise RuntimeError(f"Unexpected error in HTTP call. Details: {response}")
+        data = json.loads(response.get('data', ''))
+        if 'error' in data:
+            raise RuntimeError(f"CouchDB query failed. Details: {data}")
+
+        for row in data.get('rows', []):
+            keys = row['key']
+            workflow = keys[0]
+            task = keys[1]
+            jobStatus = keys[2]
+            exitCode = keys[3]
+            site = keys[4]
+
+            jobInfoDoc.setdefault(workflow, {})
+            jobInfoDoc[workflow].setdefault(task, {})
+            jobInfoDoc[workflow][task].setdefault(jobStatus, {})
+            jobInfoDoc[workflow][task][jobStatus].setdefault(exitCode, {})
+            jobInfoDoc[workflow][task][jobStatus][exitCode].setdefault(site, {})
+            finalStruct = jobInfoDoc[workflow][task][jobStatus][exitCode][site]
+            finalStruct.setdefault("samples", [])
+            finalStruct["samples"].append(row["doc"])
+            # now painfully find out the number of errors based on the original query
+            finalStruct.setdefault("errorCount", 0)
+            keysJson = json.dumps(keys[:5])
+            for query in queries:
+                if json.dumps(query["startkey"]) in keysJson:
+                    finalStruct["errorCount"] = query["numOfError"]
+                    break
+
+    return jobInfoDoc

{reqmgr2ms-monitor-2.3.2rc2 → reqmgr2ms-monitor-2.3.2rc4}/src/python/WMCore/Services/WMStats/WMStatsReader.py

@@ -1,15 +1,20 @@
 from __future__ import division, print_function
 
+import time
+
 from builtins import object
 from future.utils import viewitems
 
 import logging
+
 from Utils.IteratorTools import nestedDictUpdate, grouper
 from WMCore.Database.CMSCouch import CouchServer
 from WMCore.Lexicon import splitCouchServiceURL, sanitizeURL
 from WMCore.Services.RequestDB.RequestDBReader import RequestDBReader
 from WMCore.Services.WMStats.DataStruct.RequestInfoCollection import RequestInfo
-from WMCore.ReqMgr.DataStructs.RequestStatus import T0_ACTIVE_STATUS, WMSTATS_JOB_INFO, WMSTATS_NO_JOB_INFO
+from WMCore.ReqMgr.DataStructs.RequestStatus import T0_ACTIVE_STATUS
+from WMCore.Services.WMStats.WMStatsPycurl import getTaskJobSummaryByRequestPycurl
+
 
 REQUEST_PROPERTY_MAP = {
     "_id": "_id",
@@ -70,6 +75,9 @@ class WMStatsReader(object):
         else:
             self.reqDB = None
         self.logger = logger if logger else logging.getLogger()
+        self.serviceOpts = {"couchURL": self.couchURL,
+                            "dbName": self.dbName,
+                            "couchapp": self.couchapp}
 
     def _sanitizeURL(self, couchURL):
         return sanitizeURL(couchURL)['url']
@@ -370,26 +378,35 @@ class WMStatsReader(object):
         reqInfoInstance = RequestInfo(requestInfo[requestName])
         return reqInfoInstance.isWorkflowFinished()
 
-    def getTaskJobSummaryByRequest(self, requestName, sampleSize=1):
-
+    def getTaskJobSummaryByRequest(self, requestName, sampleSize=1, usePycurl=True):
+        reqStart = time.time()
         options = {'reduce': True, 'group_level': 5, 'startkey': [requestName],
                    'endkey': [requestName, {}]}
         results = self._getCouchView("jobsByStatusWorkflow", options)
+
         jobDetails = {}
-        for row in results['rows']:
-            # row["key"] = ['workflow', 'task', 'jobstatus', 'exitCode', 'site']
-            startKey = row["key"][:4]
-            endKey = []
-            site = row["key"][4]
-            if site:
-                startKey.append(site)
-
-            endKey.extend(startKey)
-            endKey.append({})
-            numOfError = row["value"]
-
-            jobInfo = self.jobDetailByTasks(startKey, endKey, numOfError, sampleSize)
-            jobDetails = nestedDictUpdate(jobDetails, jobInfo)
+        if usePycurl is True:
+            jobDetails = getTaskJobSummaryByRequestPycurl(results, sampleSize, self.serviceOpts)
+        else:
+            # then it is sequential
+            for row in results['rows']:
+                # row["key"] = ['workflow', 'task', 'jobstatus', 'exitCode', 'site']
+                startKey = row["key"][:4]
+                endKey = []
+                site = row["key"][4]
+                if site:
+                    startKey.append(site)
+
+                endKey.extend(startKey)
+                endKey.append({})
+                numOfError = row["value"]
+
+                jobInfo = self.jobDetailByTasks(startKey, endKey, numOfError, sampleSize)
+                jobDetails = nestedDictUpdate(jobDetails, jobInfo)
+        callRuntime = round((time.time() - reqStart), 3)
+        msg = f"Retrieved job details (pycurl mode: {usePycurl}) for {requestName} in "
+        msg += f"{callRuntime} seconds, with a total of {len(results['rows']) + 1} CouchDB calls"
+        print(msg)
         return jobDetails
 
     def jobDetailByTasks(self, startKey, endKey, numOfError, limit=1):

{reqmgr2ms-monitor-2.3.2rc2 → reqmgr2ms-monitor-2.3.2rc4}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.2rc2'
+__version__ = '2.3.2rc4'
 __all__ = []

{reqmgr2ms-monitor-2.3.2rc2 → reqmgr2ms-monitor-2.3.2rc4/src/python/reqmgr2ms_monitor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.2rc2
+Version: 2.3.2rc4
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-monitor-2.3.2rc2 → reqmgr2ms-monitor-2.3.2rc4}/src/python/reqmgr2ms_monitor.egg-info/SOURCES.txt

@@ -251,6 +251,7 @@ src/python/WMCore/Services/WMArchive/WMArchive.py
 src/python/WMCore/Services/WMArchive/__init__.py
 src/python/WMCore/Services/WMBS/WMBS.py
 src/python/WMCore/Services/WMBS/__init__.py
+src/python/WMCore/Services/WMStats/WMStatsPycurl.py
 src/python/WMCore/Services/WMStats/WMStatsReader.py
 src/python/WMCore/Services/WMStats/WMStatsWriter.py
 src/python/WMCore/Services/WMStats/__init__.py