reqmgr2ms-output 2.3.1rc4__tar.gz → 2.3.2rc2__tar.gz

{reqmgr2ms-output-2.3.1rc4/src/python/reqmgr2ms_output.egg-info → reqmgr2ms-output-2.3.2rc2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.1rc4
+Version: 2.3.2rc2
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-output-2.3.1rc4 → reqmgr2ms-output-2.3.2rc2}/src/python/Utils/FileTools.py

@@ -6,6 +6,7 @@ Utilities related to file handling
 
 import io
 import os
+import glob
 import stat
 import subprocess
 import time
@@ -13,6 +14,37 @@ import zlib
 
 from Utils.Utilities import decodeBytesToUnicode
 
+
+def findFiles(path, pat):
+    """
+    Find files within given path and matching given pattern.
+    :param path: starting directory path (string)
+    :param pat: match pattern (string), e.g. *.py or name of the file
+    :return: matched file names
+    """
+    files = []
+    for idir, _, _ in os.walk(path):
+        files.extend(glob.glob(os.path.join(idir, pat)))
+    return files
+
+
+def tarMode(tfile, opMode):
+    """
+    Extract proper mode of operation for given tar file. For instance,
+    if op='r' and tfile name is file.tar.gz we should get 'r:gz',
+    while if tfile name is file.tar.bz2 we should get 'r':bz2', while
+    if tfile name is file.tar we should get 'r', etc.
+    :param opMode: mode of operation (string), e.g. 'r', or 'w'
+    :param tfile: sandbox tar file name (string)
+    :return: mode of operation
+    """
+    ext = tfile.split(".")[-1]
+    if ext == 'tar':
+        return opMode
+    mode = opMode + ':' + ext
+    return mode
+
+
 def calculateChecksums(filename):
     """
     _calculateChecksums_
@@ -108,6 +140,7 @@ def findMagicStr(filename, matchString):
             if matchString in line:
                 yield line
 
+
 def getFullPath(name, envPath="PATH"):
     """
     :param name: file name

{reqmgr2ms-output-2.3.1rc4 → reqmgr2ms-output-2.3.2rc2}/src/python/WMCore/MicroService/Tools/Common.py

@@ -106,26 +106,6 @@ def dbsInfo(datasets, dbsUrl):
     return datasetBlocks, datasetSizes, datasetTransfers
 
 
-def getPileupDocs(mspileupUrl, queryDict):
-    """
-    Fetch documents from MSPileup according to the query passed in.
-    :param mspileupUrl: string with the MSPileup url
-    :param queryDict: dictionary with the MongoDB query to run
-    :return: returns a list with all the pileup objects, or raises
-        an exception in case of failure
-    """
-    mgr = RequestHandler()
-    headers = {'Content-Type': 'application/json'}
-    data = mgr.getdata(mspileupUrl, queryDict, headers, verb='POST',
-                       ckey=ckey(), cert=cert(), encode=True, decode=True)
-    if data and data.get("result", []):
-        if "error" in data["result"][0]:
-            msg = f"Failed to retrieve MSPileup documents with query: {queryDict}"
-            msg += f" and error message: {data}"
-            raise RuntimeError(msg)
-    return data["result"]
-
-
 def getPileupDatasetSizes(datasets, phedexUrl):
     """
     Given a list of datasets, find all their blocks with replicas

reqmgr2ms-output-2.3.2rc2/src/python/WMCore/Services/DBS/DBSConcurrency.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+File       : DBSConcurrency.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: dedicated module to holds DBS related functions executed
+concurrent calls to DBS APIs.
+"""
+
+import json
+import urllib
+from WMCore.Services.pycurl_manager import getdata as multi_getdata
+
+
+def getBlockInfo4PU(blockNames, ckey, cert):
+    """
+    Fetch block information details, file list and number of events, from DBS
+    server. Here we use concrete set of parameters for DBS to use in this case, i.e.
+    we must look-up only valid files and get full details from the DBS API (in order
+    to get number of events).
+    :param blockNames: list of block names
+    :return: dictionary of {block: {"FileList": list of strings, "NumberOfEvents": integer}, ...}
+    """
+    urls = []
+    for blk in blockNames:
+        # need to encode block name properly
+        block = urllib.parse.quote_plus(blk)
+        url = f"https://cmsweb-prod.cern.ch/dbs/prod/global/DBSReader/files?detail=true&validFileOnly=1&block_name={block}"
+        urls.append(url)
+    # place concurrent calls to DBS, please note that multi_getdata is generator, therefore
+    # it does not put DBS results into the memory until this generator is iterated
+    results = multi_getdata(urls, ckey, cert)
+    # parse output of getdata in some form
+    blockInfo = {}
+    for row in results:
+        blk = row['url'].split('block_name=')[-1]
+        block = urllib.parse.unquote_plus(blk)
+        data = json.loads(row['data'])
+        files = [r['logical_file_name'] for r in data]
+        nevents = sum([r['event_count'] for r in data])
+        blockInfo[block] = {'FileList': files, 'NumberOfEvents': nevents}
+    return blockInfo

{reqmgr2ms-output-2.3.1rc4 → reqmgr2ms-output-2.3.2rc2}/src/python/WMCore/Services/MSPileup/MSPileupUtils.py

@@ -21,7 +21,6 @@ def getPileupDocs(mspileupUrl, queryDict=None, method='GET'):
     queryDict = queryDict or {}
     mgr = RequestHandler()
     headers = {'Content-Type': 'application/json'}
-    # TODO: Use GET instead?
     data = mgr.getdata(mspileupUrl, queryDict, headers, verb=method,
                        ckey=ckey(), cert=cert(), encode=True, decode=True)
     if data and data.get("result", []):

{reqmgr2ms-output-2.3.1rc4 → reqmgr2ms-output-2.3.2rc2}/src/python/WMCore/__init__.py

@@ -6,5 +6,5 @@ Core libraries for Workload Management Packages
 
 """
 
-__version__ = '2.3.1rc4'
+__version__ = '2.3.2rc2'
 __all__ = []

{reqmgr2ms-output-2.3.1rc4 → reqmgr2ms-output-2.3.2rc2/src/python/reqmgr2ms_output.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: reqmgr2ms-output
-Version: 2.3.1rc4
+Version: 2.3.2rc2
 Summary: UNKNOWN
 Home-page: https://github.com/dmwm/WMCore
 Maintainer: CMS DMWM Group

{reqmgr2ms-output-2.3.1rc4 → reqmgr2ms-output-2.3.2rc2}/src/python/reqmgr2ms_output.egg-info/SOURCES.txt

@@ -203,6 +203,7 @@ src/python/WMCore/Services/AlertManager/__init__.py
 src/python/WMCore/Services/CRIC/CRIC.py
 src/python/WMCore/Services/CRIC/__init__.py
 src/python/WMCore/Services/DBS/DBS3Reader.py
+src/python/WMCore/Services/DBS/DBSConcurrency.py
 src/python/WMCore/Services/DBS/DBSErrors.py
 src/python/WMCore/Services/DBS/DBSReader.py
 src/python/WMCore/Services/DBS/DBSUtils.py