wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/Services/DBS/DBSConcurrency.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+"""
+File       : DBSConcurrency.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: dedicated module to holds DBS related functions executed
+concurrent calls to DBS APIs.
+"""
+
+import json
+import urllib
+from WMCore.Services.pycurl_manager import getdata as multi_getdata
+
+
+def getBlockInfo4PU(blockNames, dbsUrl, ckey, cert):
+    """
+    Fetch block information details, file list and number of events, from DBS
+    server. Here we use concrete set of parameters for DBS to use in this case, i.e.
+    we must look-up only valid files and get full details from the DBS API (in order
+    to get number of events).
+    :param blockNames: list of block names
+    :param dbsUrl: dbs URL
+    :param ckey: user keyfile
+    :param cert: user certificate
+    :return: dictionary of {block: {"FileList": list of strings, "NumberOfEvents": integer}, ...}
+    """
+    urls = []
+    for blk in blockNames:
+        # need to encode block name properly
+        block = urllib.parse.quote_plus(blk)
+        url = f"{dbsUrl}/files?detail=true&validFileOnly=1&block_name={block}"
+        urls.append(url)
+    # place concurrent calls to DBS, please note that multi_getdata is generator, therefore
+    # it does not put DBS results into the memory until this generator is iterated
+    results = multi_getdata(urls, ckey, cert)
+    # parse output of getdata in some form
+    blockInfo = {}
+    for row in results:
+        blk = row['url'].split('block_name=')[-1]
+        block = urllib.parse.unquote_plus(blk)
+        data = json.loads(row['data'])
+        files = [r['logical_file_name'] for r in data]
+        nevents = sum([r['event_count'] for r in data])
+        blockInfo[block] = {'FileList': files, 'NumberOfEvents': nevents}
+    return blockInfo

WMCore/Services/DBS/DBSErrors.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""
+DBSErrors represents generic class to handle DBS Go server errors
+"""
+
+import json
+
+from WMCore.Services.DBS.ProdException import ProdException
+
+
+def formatEx(excepInst):
+    """
+    _formatEx_
+
+    given a DbdException instance, generate a simple message from it
+    """
+    msg = "%s:%s %s" % (excepInst.__class__.__name__,
+                        excepInst.getErrorMessage(),
+                        excepInst.getErrorCode(),
+                        )
+    return msg
+
+
+def formatEx3(excepInst):
+    """
+    _formatEx_
+
+    given a DbdException instance, generate a simple message from it
+    """
+    msg = "%s:%s" % (excepInst.__class__.__name__, str(excepInst))
+    return msg
+
+
+class DataMgmtError(ProdException):
+    """
+    _DataMgmtError_
+
+    General Exception from DataMgmt Interface
+
+    """
+    def __init__(self, message, errorNo=1000, **data):
+        ProdException.__init__(self, message, errorNo, **data)
+
+
+class DBSWriterError(DataMgmtError):
+    """
+    _DBSWriterError_
+
+    Generic Exception for DBS Write Error
+
+    """
+    def __init__(self, msg, **data):
+        DataMgmtError.__init__(self, msg, 1001, **data)
+
+
+class DBSReaderError(DataMgmtError):
+    """
+    _DBSReaderError_
+
+    Generic Exception for DBS Read Error
+
+    """
+    def __init__(self, msg, **data):
+        DataMgmtError.__init__(self, msg, 1002, **data)
+
+
+class DBSError():
+    """
+    DBSError provides generic interface for DBS (Go-based) errors
+    """
+    def __init__(self, ex):
+        try:
+            if hasattr(ex, "body"):
+                # case of DBSClient HTTPError
+                self.data = json.loads(ex.body)[0]
+            else:
+                self.data = json.loads(ex)[0]
+        except Exception as exp:
+            self.data = str(ex)
+
+    def getHttpCode(self):
+        """
+        :return: HTTP error code
+        """
+        if isinstance(self.data, dict):
+            return self.data['http']['code']
+        return 0
+
+    def getServerCode(self):
+        """
+        :return: DBS server error code which is defined here
+-        https://github.com/dmwm/dbs2go/blob/master/dbs/errors.go
+        """
+        if isinstance(self.data, dict):
+            return self.data['error']['code']
+        return 0
+
+    def getMessage(self):
+        """
+        :return: DBS server error message (consice output, last error in DBS error chain)
+        """
+        if isinstance(self.data, dict):
+            return self.data['error']['message']
+        return ""
+
+    def getReason(self):
+        """
+        :return: DBS server error reason (expanded message)
+        """
+        if isinstance(self.data, dict):
+            return self.data['error']['reason']
+        return ""

WMCore/Services/DBS/DBSReader.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+"""
+_DBSReader_
+
+Readonly DBS Interface
+
+"""
+
+from WMCore.Services.DBS.DBSErrors import DBSReaderError
+from WMCore.Services.DBS.DBS3Reader import DBS3Reader
+
+
+def DBSReader(endpoint, **kwargs):
+    """Function to find and instantiate desired DBSReader object"""
+
+    try:
+        dbs = DBS3Reader(endpoint, **kwargs)
+        # if this doesn't throw endpoint is dbs3
+        dbs.dbs.serverinfo()
+        return dbs
+    except Exception as ex:
+        msg = 'Instantiating DBS3Reader failed with %s\n' % str(ex)
+        raise DBSReaderError("Can't contact DBS at %s, got errors %s" % (endpoint, msg))

WMCore/Services/DBS/DBSUtils.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+"""
+_DBSUtils_
+
+set of common utilities for DBS3Reader
+
+"""
+import json
+import urllib
+import logging
+from urllib.parse import urlparse, parse_qs, quote_plus
+from collections import defaultdict
+
+from Utils.CertTools import cert, ckey
+from dbs.apis.dbsClient import aggFileLumis, aggFileParents
+from WMCore.Services.pycurl_manager import getdata as multi_getdata
+from WMCore.Services.pycurl_manager import RequestHandler
+from Utils.PortForward import PortForward
+
+
+def dbsListFileParents(dbsUrl, blocks):
+    """
+    Concurrent counter part of DBS listFileParents API
+
+    :param dbsUrl: DBS URL
+    :param blocks: list of blocks
+    :return: list of file parents
+    """
+    urls = ['%s/fileparents?block_name=%s' % (dbsUrl, quote_plus(b)) for b in blocks]
+    func = aggFileParents
+    uKey = 'block_name'
+    return getUrls(urls, func, uKey)
+
+
+def dbsListFileLumis(dbsUrl, blocks):
+    """
+    Concurrent counter part of DBS listFileLumis API
+
+    :param dbsUrl: DBS URL
+    :param blocks: list of blocks
+    :return: list of file lumis
+    """
+    urls = ['%s/filelumis?block_name=%s' % (dbsUrl, quote_plus(b)) for b in blocks]
+    func = aggFileLumis
+    uKey = 'block_name'
+    return getUrls(urls, func, uKey)
+
+
+def dbsBlockOrigin(dbsUrl, blocks):
+    """
+    Concurrent counter part of DBS files API
+
+    :param dbsUrl: DBS URL
+    :param blocks: list of blocks
+    :return: list of block origins for a given parent lfns
+    """
+    urls = ['%s/blockorigin?block_name=%s' % (dbsUrl, quote_plus(b)) for b in blocks]
+    func = None
+    uKey = 'block_name'
+    return getUrls(urls, func, uKey)
+
+
+def dbsParentFilesGivenParentDataset(dbsUrl, parentDataset, fInfo):
+    """
+    Obtain parent files for given fileInfo object
+
+    :param dbsUrl: DBS URL
+    :param parentDataset: parent dataset name
+    :param fInfo: file info object
+    :return: list of parent files for given file info object
+    """
+    portForwarder = PortForward(8443)
+    urls = []
+    for fileInfo in fInfo:
+        run = fileInfo['run_num']
+        lumis = urllib.parse.quote_plus(str(fileInfo['lumi_section_num']))
+        url = f'{dbsUrl}/files?dataset={parentDataset}&run_num={run}&lumi_list={lumis}'
+        urls.append(portForwarder(url))
+    func = None
+    uKey = None
+    rdict = getUrls(urls, func, uKey)
+    parentFiles = defaultdict(set)
+    for fileInfo in fInfo:
+        run = fileInfo['run_num']
+        lumis = urllib.parse.quote_plus(str(fileInfo['lumi_section_num']))
+        url = f'{dbsUrl}/files?dataset={parentDataset}&run_num={run}&lumi_list={lumis}'
+        url = portForwarder(url)
+        if url in rdict:
+            pFileList = rdict[url]
+            pFiles = {x['logical_file_name'] for x in pFileList}
+            parentFiles[fileInfo['logical_file_name']] = \
+                parentFiles[fileInfo['logical_file_name']].union(pFiles)
+    return parentFiles
+
+
+def getUrls(urls, aggFunc, uKey=None):
+    """
+    Perform parallel DBS calls for given set of urls and apply given aggregation
+    function to the results.
+
+    :param urls: list of DBS urls to call
+    :param aggFunc: aggregation function
+    :param uKey: url parameter to use for final dictionary
+    :return: dictionary of resuls where keys are urls and values are obtained results
+    """
+    data = multi_getdata(urls, ckey(), cert())
+
+    rdict = {}
+    for row in data:
+        url = row['url']
+        code = int(row.get('code', 200))
+        error = row.get('error')
+        if code != 200:
+            msg = f"Fail to query {url}. Error: {code} {error}"
+            raise RuntimeError(msg)
+        if uKey:
+            key = urlParams(url).get(uKey)
+        else:
+            key = url
+        data = row.get('data', [])
+        res = json.loads(data)
+        if aggFunc:
+            rdict[key] = aggFunc(res)
+        else:
+            rdict[key] = res
+    return rdict
+
+
+def urlParams(url):
+    """
+    Return dictionary of URL parameters
+
+    :param url: URL link
+    :return: dictionary of URL parameters
+    """
+    parsedUrl = urlparse(url)
+    rdict = parse_qs(parsedUrl.query)
+    for key, vals in rdict.items():
+        if len(vals) == 1:
+            rdict[key] = vals[0]
+    return rdict
+
+def DBSErrors(dbsUrl):
+    """
+    Fetch and return all DBS server errors
+    :param dbsUrl: DBS url to use
+    :return: dictionary of DBS server errors and their meaning
+    """
+    dbsErrors = {}
+    method = "GET"
+    payload = {}
+    headers = {'Content-Type': 'application/json'}
+    mgr = RequestHandler()
+    # dbs server url for error codes
+    rurl = f"{dbsUrl}/errors"
+    data = ""   # initialize data variable to be used in logging.error if necessary
+    try:
+        data = mgr.getdata(rurl, payload, headers, verb=method, ckey=ckey(), cert=cert())
+        # check if we received proper data from DBS server
+        if 'code' in str(data) and 'meaning' in str(data):
+            errorCodes = json.loads(data)
+            for row in errorCodes:
+                dbsErrors[row['code']] = row['meaning']
+    except Exception as exp:
+        logging.error("Unable to query DBS url %s, error %s, data=%s", dbsUrl, str(exp), data)
+    return dbsErrors