wmglobalqueue 2.3.10__py3-none-any.whl

WMCore/Services/DBS/DBSConcurrency.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+"""
+File       : DBSConcurrency.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: dedicated module to holds DBS related functions executed
+concurrent calls to DBS APIs.
+"""
+
+import json
+import urllib
+from WMCore.Services.pycurl_manager import getdata as multi_getdata
+
+
+def getBlockInfo4PU(blockNames, dbsUrl, ckey, cert):
+    """
+    Fetch block information details, file list and number of events, from DBS
+    server. Here we use concrete set of parameters for DBS to use in this case, i.e.
+    we must look-up only valid files and get full details from the DBS API (in order
+    to get number of events).
+    :param blockNames: list of block names
+    :param dbsUrl: dbs URL
+    :param ckey: user keyfile
+    :param cert: user certificate
+    :return: dictionary of {block: {"FileList": list of strings, "NumberOfEvents": integer}, ...}
+    """
+    urls = []
+    for blk in blockNames:
+        # need to encode block name properly
+        block = urllib.parse.quote_plus(blk)
+        url = f"{dbsUrl}/files?detail=true&validFileOnly=1&block_name={block}"
+        urls.append(url)
+    # place concurrent calls to DBS, please note that multi_getdata is generator, therefore
+    # it does not put DBS results into the memory until this generator is iterated
+    results = multi_getdata(urls, ckey, cert)
+    # parse output of getdata in some form
+    blockInfo = {}
+    for row in results:
+        blk = row['url'].split('block_name=')[-1]
+        block = urllib.parse.unquote_plus(blk)
+        data = json.loads(row['data'])
+        files = [r['logical_file_name'] for r in data]
+        nevents = sum([r['event_count'] for r in data])
+        blockInfo[block] = {'FileList': files, 'NumberOfEvents': nevents}
+    return blockInfo

WMCore/Services/DBS/DBSErrors.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""
+DBSErrors represents generic class to handle DBS Go server errors
+"""
+
+
+import json
+
+from WMCore.Services.DBS.ProdException import ProdException
+
+
+def formatEx(excepInst):
+    """
+    _formatEx_
+
+    given a DbdException instance, generate a simple message from it
+    """
+    msg = "%s:%s %s" % (excepInst.__class__.__name__,
+                        excepInst.getErrorMessage(),
+                        excepInst.getErrorCode(),
+                        )
+    return msg
+
+
+def formatEx3(excepInst):
+    """
+    _formatEx_
+
+    given a DbdException instance, generate a simple message from it
+    """
+    msg = "%s:%s" % (excepInst.__class__.__name__, str(excepInst))
+    return msg
+
+
+class DataMgmtError(ProdException):
+    """
+    _DataMgmtError_
+
+    General Exception from DataMgmt Interface
+
+    """
+    def __init__(self, message, errorNo=1000, **data):
+        ProdException.__init__(self, message, errorNo, **data)
+
+
+class DBSWriterError(DataMgmtError):
+    """
+    _DBSWriterError_
+
+    Generic Exception for DBS Write Error
+
+    """
+    def __init__(self, msg, **data):
+        DataMgmtError.__init__(self, msg, 1001, **data)
+
+
+class DBSReaderError(DataMgmtError):
+    """
+    _DBSReaderError_
+
+    Generic Exception for DBS Read Error
+
+    """
+    def __init__(self, msg, **data):
+        DataMgmtError.__init__(self, msg, 1002, **data)
+
+
+class DBSError():
+    """
+    DBSError provides generic interface for DBS (Go-based) errors
+    """
+    def __init__(self, ex):
+        try:
+            if hasattr(ex, "body"):
+                # case of DBSClient HTTPError
+                self.data = json.loads(ex.body)[0]
+            else:
+                self.data = json.loads(ex)[0]
+        except Exception as exp:
+            self.data = str(ex)
+
+    def getHttpCode(self):
+        """
+        :return: HTTP error code
+        """
+        if isinstance(self.data, dict):
+            return self.data['http']['code']
+        return 0
+
+    def getServerCode(self):
+        """
+        :return: DBS server error code which is defined here
+-        https://github.com/dmwm/dbs2go/blob/master/dbs/errors.go
+        """
+        if isinstance(self.data, dict):
+            return self.data['error']['code']
+        return 0
+
+    def getMessage(self):
+        """
+        :return: DBS server error message (consice output, last error in DBS error chain)
+        """
+        if isinstance(self.data, dict):
+            return self.data['error']['message']
+        return ""
+
+    def getReason(self):
+        """
+        :return: DBS server error reason (expanded message)
+        """
+        if isinstance(self.data, dict):
+            return self.data['error']['reason']
+        return ""

WMCore/Services/DBS/DBSReader.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+"""
+_DBSReader_
+
+Readonly DBS Interface
+
+"""
+
+from WMCore.Services.DBS.DBSErrors import DBSReaderError
+from WMCore.Services.DBS.DBS3Reader import DBS3Reader
+
+
+def DBSReader(endpoint, **kwargs):
+    """Function to find and instantiate desired DBSReader object"""
+
+    try:
+        dbs = DBS3Reader(endpoint, **kwargs)
+        # if this doesn't throw endpoint is dbs3
+        dbs.dbs.serverinfo()
+        return dbs
+    except Exception as ex:
+        msg = 'Instantiating DBS3Reader failed with %s\n' % str(ex)
+        raise DBSReaderError("Can't contact DBS at %s, got errors %s" % (endpoint, msg))

WMCore/Services/DBS/DBSUtils.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+"""
+_DBSUtils_
+
+set of common utilities for DBS3Reader
+
+"""
+import json
+import urllib
+from urllib.parse import urlparse, parse_qs, quote_plus
+from collections import defaultdict
+
+from Utils.CertTools import cert, ckey
+from dbs.apis.dbsClient import aggFileLumis, aggFileParents
+from WMCore.Services.pycurl_manager import getdata as multi_getdata
+from Utils.PortForward import PortForward
+
+
+def dbsListFileParents(dbsUrl, blocks):
+    """
+    Concurrent counter part of DBS listFileParents API
+
+    :param dbsUrl: DBS URL
+    :param blocks: list of blocks
+    :return: list of file parents
+    """
+    urls = ['%s/fileparents?block_name=%s' % (dbsUrl, quote_plus(b)) for b in blocks]
+    func = aggFileParents
+    uKey = 'block_name'
+    return getUrls(urls, func, uKey)
+
+
+def dbsListFileLumis(dbsUrl, blocks):
+    """
+    Concurrent counter part of DBS listFileLumis API
+
+    :param dbsUrl: DBS URL
+    :param blocks: list of blocks
+    :return: list of file lumis
+    """
+    urls = ['%s/filelumis?block_name=%s' % (dbsUrl, quote_plus(b)) for b in blocks]
+    func = aggFileLumis
+    uKey = 'block_name'
+    return getUrls(urls, func, uKey)
+
+
+def dbsBlockOrigin(dbsUrl, blocks):
+    """
+    Concurrent counter part of DBS files API
+
+    :param dbsUrl: DBS URL
+    :param blocks: list of blocks
+    :return: list of block origins for a given parent lfns
+    """
+    urls = ['%s/blockorigin?block_name=%s' % (dbsUrl, quote_plus(b)) for b in blocks]
+    func = None
+    uKey = 'block_name'
+    return getUrls(urls, func, uKey)
+
+
+def dbsParentFilesGivenParentDataset(dbsUrl, parentDataset, fInfo):
+    """
+    Obtain parent files for given fileInfo object
+
+    :param dbsUrl: DBS URL
+    :param parentDataset: parent dataset name
+    :param fInfo: file info object
+    :return: list of parent files for given file info object
+    """
+    portForwarder = PortForward(8443)
+    urls = []
+    for fileInfo in fInfo:
+        run = fileInfo['run_num']
+        lumis = urllib.parse.quote_plus(str(fileInfo['lumi_section_num']))
+        url = f'{dbsUrl}/files?dataset={parentDataset}&run_num={run}&lumi_list={lumis}'
+        urls.append(portForwarder(url))
+    func = None
+    uKey = None
+    rdict = getUrls(urls, func, uKey)
+    parentFiles = defaultdict(set)
+    for fileInfo in fInfo:
+        run = fileInfo['run_num']
+        lumis = urllib.parse.quote_plus(str(fileInfo['lumi_section_num']))
+        url = f'{dbsUrl}/files?dataset={parentDataset}&run_num={run}&lumi_list={lumis}'
+        url = portForwarder(url)
+        if url in rdict:
+            pFileList = rdict[url]
+            pFiles = {x['logical_file_name'] for x in pFileList}
+            parentFiles[fileInfo['logical_file_name']] = \
+                parentFiles[fileInfo['logical_file_name']].union(pFiles)
+    return parentFiles
+
+
+def getUrls(urls, aggFunc, uKey=None):
+    """
+    Perform parallel DBS calls for given set of urls and apply given aggregation
+    function to the results.
+
+    :param urls: list of DBS urls to call
+    :param aggFunc: aggregation function
+    :param uKey: url parameter to use for final dictionary
+    :return: dictionary of resuls where keys are urls and values are obtained results
+    """
+    data = multi_getdata(urls, ckey(), cert())
+
+    rdict = {}
+    for row in data:
+        url = row['url']
+        code = int(row.get('code', 200))
+        error = row.get('error')
+        if code != 200:
+            msg = f"Fail to query {url}. Error: {code} {error}"
+            raise RuntimeError(msg)
+        if uKey:
+            key = urlParams(url).get(uKey)
+        else:
+            key = url
+        data = row.get('data', [])
+        res = json.loads(data)
+        if aggFunc:
+            rdict[key] = aggFunc(res)
+        else:
+            rdict[key] = res
+    return rdict
+
+
+def urlParams(url):
+    """
+    Return dictionary of URL parameters
+
+    :param url: URL link
+    :return: dictionary of URL parameters
+    """
+    parsedUrl = urlparse(url)
+    rdict = parse_qs(parsedUrl.query)
+    for key, vals in rdict.items():
+        if len(vals) == 1:
+            rdict[key] = vals[0]
+    return rdict

WMCore/Services/DBS/DBSWriterObjects.py

@@ -0,0 +1,381 @@
+#!/usr/bin/env python
+"""
+_DBSWriterObjects_
+
+Functions to instantiate and return DBS Objects and insert them
+into DBS if required
+
+"""
+from __future__ import print_function
+
+from builtins import int
+
+import logging
+
+from DBSAPI.dbsException import *
+from DBSAPI.dbsApiException import *
+from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
+from DBSAPI.dbsAlgorithm import DbsAlgorithm
+from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet
+from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
+from DBSAPI.dbsFile import DbsFile
+from DBSAPI.dbsFileBlock import DbsFileBlock
+from DBSAPI.dbsStorageElement import DbsStorageElement
+from DBSAPI.dbsRun import DbsRun
+from DBSAPI.dbsLumiSection import DbsLumiSection
+
+def makeTierList(dataTier):
+    """
+    _makeTierList_
+
+    Standard tool to split data tiers if they contain - chars
+    *** Do not use outside of this module ***
+
+    """
+    tierList = dataTier.split("-")
+    return tierList
+
+def createPrimaryDataset(datasetInfo, apiRef = None):
+    """
+    _createPrimaryDataset_
+
+    Create and return a Primary Dataset object.
+    If apiRef is not None, it is used to insert the dataset into the
+    DBS
+
+    """
+    if 'PrimaryDatasetType' in datasetInfo:
+        PrimaryDatasetType = datasetInfo['PrimaryDatasetType']
+    else:
+        PrimaryDatasetType = 'mc'
+
+    logging.debug("Inserting PrimaryDataset %s with Type %s", datasetInfo["PrimaryDataset"], PrimaryDatasetType)
+    primary = DbsPrimaryDataset(Name = datasetInfo["PrimaryDataset"], Type=PrimaryDatasetType)
+
+    if apiRef != None:
+        apiRef.insertPrimaryDataset(primary)
+    return primary
+
+
+def createAlgorithm(datasetInfo, configMetadata = None, apiRef = None):
+    """
+    _createAlgorithm_
+
+    Create an algorithm assuming that datasetInfo is a
+    ProdCommon.MCPayloads.DatasetInfo like dictionary
+
+    """
+
+    exeName = datasetInfo['ApplicationName']
+    appVersion = datasetInfo['ApplicationVersion']
+    appFamily = datasetInfo["ApplicationFamily"]
+
+    #
+    # HACK:  Problem with large PSets (is this still relevant ?)
+    #
+    # Repacker jobs have no PSetContent/PSetHash
+    #
+    psetContent = datasetInfo.get('PSetContent',None)
+    if psetContent == None:
+        psetContent = "PSET_CONTENT_NOT_AVAILABLE"
+    psetHash = datasetInfo.get('PSetHash',None)
+    if psetHash == None:
+        psetHash = "NO_PSET_HASH"
+    else:
+        if psetHash.find(";"):
+            # no need for fake hash in new schema
+            psetHash = psetHash.split(";")[0]
+            psetHash = psetHash.replace("hash=", "")
+
+    ## No more hacks
+    #msg = ">>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"
+    #msg += "TEST HACK USED FOR PSetContent\n"
+    #msg += ">>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+    #logging.warning(msg)
+    #print msg
+    #psetContent = "This is not a PSet"
+
+    #
+    # HACK: 100 char limit on cfg file name
+    if configMetadata != None:
+        cfgName = configMetadata['name']
+        if len(cfgName) > 100:
+            msg = ">>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"
+            msg += "TEST HACK USED FOR Config File Name"
+            msg += ">>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+            logging.warning(msg)
+            print(msg)
+            configMetadata['name'] = cfgName[-99]
+
+        psetInstance = DbsQueryableParameterSet(
+            Hash = psetHash,
+            Name = configMetadata['name'],
+            Version = configMetadata['version'],
+            Type = configMetadata['Type'],
+            Annotation = configMetadata['annotation'],
+            Content = psetContent,
+            )
+
+
+        algorithmInstance = DbsAlgorithm(
+            ExecutableName = exeName,
+            ApplicationVersion = appVersion,
+            ApplicationFamily = appFamily,
+            ParameterSetID = psetInstance
+            )
+    else:
+        psetInstance = DbsQueryableParameterSet(
+                    Hash = psetHash)
+        algorithmInstance = DbsAlgorithm(
+            ExecutableName = exeName,
+            ApplicationVersion = appVersion,
+            ApplicationFamily = appFamily,
+            ParameterSetID = psetInstance
+            )
+
+    if apiRef != None:
+        apiRef.insertAlgorithm(algorithmInstance)
+    return algorithmInstance
+
+def createAlgorithmForInsert(datasetInfo):
+    """
+    _createPartialAlgorithm_
+
+    Create an Algorithm instance that uses the minimal info needed
+    to insert a file
+
+    """
+    exeName = datasetInfo['ApplicationName']
+    appVersion = datasetInfo['ApplicationVersion']
+    appFamily = datasetInfo["ApplicationFamily"]
+
+    #
+    # Repacker jobs have no PsetContent/PSetHash
+    #
+    psetContent = datasetInfo.get('PSetContent',None)
+    if psetContent == None:
+        psetContent = "PSET_CONTENT_NOT_AVAILABLE"
+    psetHash = datasetInfo.get('PSetHash',None)
+    if psetHash == None:
+        psetHash = "NO_PSET_HASH"
+    else:
+        if psetHash.find(";"):
+            # no need for fake hash in new schema
+            psetHash = psetHash.split(";")[0]
+            psetHash = psetHash.replace("hash=", "")
+
+    psetInstance = DbsQueryableParameterSet(
+        Hash = psetHash)
+    algorithmInstance = DbsAlgorithm(
+        ExecutableName = exeName,
+        ApplicationVersion = appVersion,
+        ApplicationFamily = appFamily,
+        ParameterSetID = psetInstance
+        )
+    return algorithmInstance
+
+def createMergeAlgorithm(datasetInfo, apiRef = None):
+    """
+    _createMergeAlgorithm_
+
+    Create a DbsAlgorithm for a merge dataset
+
+    """
+    exeName = datasetInfo['ApplicationName']
+    version = datasetInfo['ApplicationVersion']
+    family = datasetInfo.get('ApplicationFamily', None)
+    if (family == None) or not (family) :
+        family = datasetInfo['OutputModuleName']
+
+
+    mergeAlgo = DbsAlgorithm (
+        ExecutableName = exeName,
+        ApplicationVersion = version,
+        ApplicationFamily = family,
+        )
+
+    if apiRef != None:
+        apiRef.insertAlgorithm(mergeAlgo)
+    return mergeAlgo
+
+
+
+
+def createProcessedDataset(primaryDataset, algorithm, datasetInfo,
+                           apiRef = None):
+    """
+    _createProcessedDataset_
+
+
+    """
+
+    physicsGroup = datasetInfo.get("PhysicsGroup", "NoGroup")
+    status = datasetInfo.get("Status", "VALID")
+    dataTier = datasetInfo['DataTier']
+    globalTag = datasetInfo.get('Conditions', None)
+    if globalTag is None: globalTag = ''
+
+    parents = []
+    inputDataset = datasetInfo.get('ParentDataset', None)
+    if inputDataset != None:
+        parents.append(inputDataset)
+
+    tierList = makeTierList(datasetInfo['DataTier'])
+
+    name = datasetInfo['ProcessedDataset']
+    algolist=[]
+    if algorithm not in ('', None):
+        algolist=list(algorithm)
+
+    processedDataset = DbsProcessedDataset (
+        PrimaryDataset = primaryDataset,
+        AlgoList=algolist,
+        Name = name,
+        TierList = tierList,
+        ParentList = parents,
+        PhysicsGroup = physicsGroup,
+        Status = status,
+        GlobalTag = globalTag,
+        )
+
+    if apiRef != None:
+        apiRef.insertProcessedDataset(processedDataset)
+    #
+    logging.debug("PrimaryDataset: %s ProcessedDataset: %s DataTierList: %s  requested by PhysicsGroup: %s ", primaryDataset['Name'], name, tierList, physicsGroup)
+    return processedDataset
+
+
+
+
+def createDBSFiles(fjrFileInfo, jobType = None, apiRef = None):
+    """
+    _createDBSFiles_
+
+    Create a list of DBS File instances from the file details contained
+    in a FwkJobRep.FileInfo instance describing an output file
+    Does not insert files, returns as list of DbsFile objects
+    Does insert runs and lumisections if DBS API reference is passed
+
+    """
+    results = []
+    inputLFNs = [ x['LFN'] for x in fjrFileInfo.inputFiles]
+    checksum = fjrFileInfo.checksums['cksum']
+    adler32sum = fjrFileInfo.checksums.get('adler32', '')
+
+    nEvents = int(fjrFileInfo['TotalEvents'])
+
+    if len(fjrFileInfo.dataset)<=0:
+        logging.error("No dataset info found in FWJobReport!")
+        return results
+
+    #  //
+    # // Set FileType
+    #//
+    if 'FileType' in fjrFileInfo:
+        fileType = fjrFileInfo['FileType']
+    else:
+        fileType = 'EDM'
+
+    #
+    # FIXME: at this point I should use the mc or data event type from
+    #        the jobreport. Until this is supported by the framework,
+    #        we use the workaround that mc job reports have an empty
+    #        lumisections list (stripped in DBSInterface)
+    #
+    lumiList = []
+    if ( len(fjrFileInfo.getLumiSections()) > 0 ):
+
+        #
+        # insert runs (for data files from detector)
+        #
+        if ( apiRef != None ):
+
+            for runinfo in fjrFileInfo.runs:
+
+                run = DbsRun(
+                    RunNumber = int(runinfo),
+                    NumberOfEvents = 0,
+                    NumberOfLumiSections = 0,
+                    TotalLuminosity = 0,
+                    StoreNumber = 0,
+                    StartOfRun = 0,
+                    EndOfRun = 0,
+                    )
+
+                apiRef.insertRun(run)
+
+        #
+        # insert lumisections (for data files from detector)
+        # associate files with lumisections (for all data files)
+        #
+        for lumiinfo in fjrFileInfo.getLumiSections():
+
+            lumi = DbsLumiSection(
+                LumiSectionNumber = int(lumiinfo['LumiSectionNumber']),
+                StartEventNumber = 0,
+                EndEventNumber = 0,
+                LumiStartTime = 0,
+                LumiEndTime = 0,
+                RunNumber = int(lumiinfo['RunNumber']),
+                )
+
+            # Isnt needed, causes monster slowdown
+            #if ( apiRef != None ):
+            #    apiRef.insertLumiSection(lumi)
+
+            lumiList.append(lumi)
+
+            logging.debug("Lumi associated to file is: %s" % ([x for x in lumiList]))
+
+    #  //
+    # // Dataset info related to files and creation of DbsFile object
+    #//
+    for dataset in fjrFileInfo.dataset:
+
+        primary = createPrimaryDataset(dataset)
+        if jobType == "Merge":
+            algo = createMergeAlgorithm(dataset)
+        else:
+            algo = createAlgorithmForInsert(dataset)
+
+        processed = createProcessedDataset(primary, algo, dataset)
+
+        dbsFileInstance = DbsFile(
+            Checksum = checksum,
+            Adler32 = adler32sum,
+            NumberOfEvents = nEvents,
+            LogicalFileName = fjrFileInfo['LFN'],
+            FileSize = int(fjrFileInfo['Size']),
+            Status = "VALID",
+            ValidationStatus = 'VALID',
+            FileType = fileType,
+            Dataset = processed,
+            TierList = makeTierList(dataset['DataTier']),
+            AlgoList = [algo],
+            LumiList = lumiList,
+            ParentList = inputLFNs,
+            BranchList = fjrFileInfo.branches,
+            )
+
+        results.append(dbsFileInstance)
+    return results
+
+
+def createDBSStorageElement(pnn):
+    """
+    _createDBSStorageElement_
+
+    """
+    return DbsStorageElement(Name = pnn)
+
+
+def createDBSFileBlock(blockName):
+    """
+    _createDBSFileBlock_
+
+    return a DbsFileBlock object with the block name provided
+
+    NOTE: This method DOES NOT create a new block in DBS
+
+    """
+    return DbsFileBlock(Name=blockName)