reqmgr2ms-monitor 2.3.2rc1__tar.gz → 2.3.2rc2__tar.gz

{reqmgr2ms-monitor-2.3.2rc1/src/python/reqmgr2ms_monitor.egg-info → reqmgr2ms-monitor-2.3.2rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.2rc1
+Version: 2.3.2rc2
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

reqmgr2ms-monitor-2.3.2rc2/src/python/WMCore/Services/DBS/DBSConcurrency.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+File       : DBSConcurrency.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: dedicated module to holds DBS related functions executed
+concurrent calls to DBS APIs.
+"""
+
+import json
+import urllib
+from WMCore.Services.pycurl_manager import getdata as multi_getdata
+
+
+def getBlockInfo4PU(blockNames, ckey, cert):
+    """
+    Fetch block information details, file list and number of events, from DBS
+    server. Here we use concrete set of parameters for DBS to use in this case, i.e.
+    we must look-up only valid files and get full details from the DBS API (in order
+    to get number of events).
+    :param blockNames: list of block names
+    :return: dictionary of {block: {"FileList": list of strings, "NumberOfEvents": integer}, ...}
+    """
+    urls = []
+    for blk in blockNames:
+        # need to encode block name properly
+        block = urllib.parse.quote_plus(blk)
+        url = f"https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/files?detail=true&validFileOnly=1&block_name={block}"
+        urls.append(url)
+    # place concurrent calls to DBS, please note that multi_getdata is generator, therefore
+    # it does not put DBS results into the memory until this generator is iterated
+    results = multi_getdata(urls, ckey, cert)
+    # parse output of getdata in some form
+    blockInfo = {}
+    for row in results:
+        blk = row['url'].split('block_name=')[-1]
+        block = urllib.parse.unquote_plus(blk)
+        data = json.loads(row['data'])
+        files = [r['logical_file_name'] for r in data]
+        nevents = sum([r['event_count'] for r in data])
+        blockInfo[block] = {'FileList': files, 'NumberOfEvents': nevents}
+    return blockInfo

{reqmgr2ms-monitor-2.3.2rc1 → reqmgr2ms-monitor-2.3.2rc2}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.2rc1'
+__version__ = '2.3.2rc2'
 __all__ = []

{reqmgr2ms-monitor-2.3.2rc1 → reqmgr2ms-monitor-2.3.2rc2/src/python/reqmgr2ms_monitor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-monitor
-Version: 2.3.2rc1
+Version: 2.3.2rc2
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-monitor-2.3.2rc1 → reqmgr2ms-monitor-2.3.2rc2}/src/python/reqmgr2ms_monitor.egg-info/SOURCES.txt

@@ -200,6 +200,7 @@ src/python/WMCore/Services/AlertManager/__init__.py
 src/python/WMCore/Services/CRIC/CRIC.py
 src/python/WMCore/Services/CRIC/__init__.py
 src/python/WMCore/Services/DBS/DBS3Reader.py
+src/python/WMCore/Services/DBS/DBSConcurrency.py
 src/python/WMCore/Services/DBS/DBSErrors.py
 src/python/WMCore/Services/DBS/DBSReader.py
 src/python/WMCore/Services/DBS/DBSUtils.py