wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/Services/DBS/DBSWriterObjects.py

@@ -0,0 +1,381 @@
+#!/usr/bin/env python
+"""
+_DBSWriterObjects_
+
+Functions to instantiate and return DBS Objects and insert them
+into DBS if required
+
+"""
+from __future__ import print_function
+
+from builtins import int
+
+import logging
+
+from DBSAPI.dbsException import *
+from DBSAPI.dbsApiException import *
+from DBSAPI.dbsPrimaryDataset import DbsPrimaryDataset
+from DBSAPI.dbsAlgorithm import DbsAlgorithm
+from DBSAPI.dbsQueryableParameterSet import DbsQueryableParameterSet
+from DBSAPI.dbsProcessedDataset import DbsProcessedDataset
+from DBSAPI.dbsFile import DbsFile
+from DBSAPI.dbsFileBlock import DbsFileBlock
+from DBSAPI.dbsStorageElement import DbsStorageElement
+from DBSAPI.dbsRun import DbsRun
+from DBSAPI.dbsLumiSection import DbsLumiSection
+
+def makeTierList(dataTier):
+    """
+    _makeTierList_
+
+    Standard tool to split data tiers if they contain - chars
+    *** Do not use outside of this module ***
+
+    """
+    tierList = dataTier.split("-")
+    return tierList
+
+def createPrimaryDataset(datasetInfo, apiRef = None):
+    """
+    _createPrimaryDataset_
+
+    Create and return a Primary Dataset object.
+    If apiRef is not None, it is used to insert the dataset into the
+    DBS
+
+    """
+    if 'PrimaryDatasetType' in datasetInfo:
+        PrimaryDatasetType = datasetInfo['PrimaryDatasetType']
+    else:
+        PrimaryDatasetType = 'mc'
+
+    logging.debug("Inserting PrimaryDataset %s with Type %s", datasetInfo["PrimaryDataset"], PrimaryDatasetType)
+    primary = DbsPrimaryDataset(Name = datasetInfo["PrimaryDataset"], Type=PrimaryDatasetType)
+
+    if apiRef != None:
+        apiRef.insertPrimaryDataset(primary)
+    return primary
+
+
+def createAlgorithm(datasetInfo, configMetadata = None, apiRef = None):
+    """
+    _createAlgorithm_
+
+    Create an algorithm assuming that datasetInfo is a
+    ProdCommon.MCPayloads.DatasetInfo like dictionary
+
+    """
+
+    exeName = datasetInfo['ApplicationName']
+    appVersion = datasetInfo['ApplicationVersion']
+    appFamily = datasetInfo["ApplicationFamily"]
+
+    #
+    # HACK:  Problem with large PSets (is this still relevant ?)
+    #
+    # Repacker jobs have no PSetContent/PSetHash
+    #
+    psetContent = datasetInfo.get('PSetContent',None)
+    if psetContent == None:
+        psetContent = "PSET_CONTENT_NOT_AVAILABLE"
+    psetHash = datasetInfo.get('PSetHash',None)
+    if psetHash == None:
+        psetHash = "NO_PSET_HASH"
+    else:
+        if psetHash.find(";"):
+            # no need for fake hash in new schema
+            psetHash = psetHash.split(";")[0]
+            psetHash = psetHash.replace("hash=", "")
+
+    ## No more hacks
+    #msg = ">>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"
+    #msg += "TEST HACK USED FOR PSetContent\n"
+    #msg += ">>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+    #logging.warning(msg)
+    #print msg
+    #psetContent = "This is not a PSet"
+
+    #
+    # HACK: 100 char limit on cfg file name
+    if configMetadata != None:
+        cfgName = configMetadata['name']
+        if len(cfgName) > 100:
+            msg = ">>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"
+            msg += "TEST HACK USED FOR Config File Name"
+            msg += ">>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+            logging.warning(msg)
+            print(msg)
+            configMetadata['name'] = cfgName[-99]
+
+        psetInstance = DbsQueryableParameterSet(
+            Hash = psetHash,
+            Name = configMetadata['name'],
+            Version = configMetadata['version'],
+            Type = configMetadata['Type'],
+            Annotation = configMetadata['annotation'],
+            Content = psetContent,
+            )
+
+
+        algorithmInstance = DbsAlgorithm(
+            ExecutableName = exeName,
+            ApplicationVersion = appVersion,
+            ApplicationFamily = appFamily,
+            ParameterSetID = psetInstance
+            )
+    else:
+        psetInstance = DbsQueryableParameterSet(
+                    Hash = psetHash)
+        algorithmInstance = DbsAlgorithm(
+            ExecutableName = exeName,
+            ApplicationVersion = appVersion,
+            ApplicationFamily = appFamily,
+            ParameterSetID = psetInstance
+            )
+
+    if apiRef != None:
+        apiRef.insertAlgorithm(algorithmInstance)
+    return algorithmInstance
+
+def createAlgorithmForInsert(datasetInfo):
+    """
+    _createPartialAlgorithm_
+
+    Create an Algorithm instance that uses the minimal info needed
+    to insert a file
+
+    """
+    exeName = datasetInfo['ApplicationName']
+    appVersion = datasetInfo['ApplicationVersion']
+    appFamily = datasetInfo["ApplicationFamily"]
+
+    #
+    # Repacker jobs have no PsetContent/PSetHash
+    #
+    psetContent = datasetInfo.get('PSetContent',None)
+    if psetContent == None:
+        psetContent = "PSET_CONTENT_NOT_AVAILABLE"
+    psetHash = datasetInfo.get('PSetHash',None)
+    if psetHash == None:
+        psetHash = "NO_PSET_HASH"
+    else:
+        if psetHash.find(";"):
+            # no need for fake hash in new schema
+            psetHash = psetHash.split(";")[0]
+            psetHash = psetHash.replace("hash=", "")
+
+    psetInstance = DbsQueryableParameterSet(
+        Hash = psetHash)
+    algorithmInstance = DbsAlgorithm(
+        ExecutableName = exeName,
+        ApplicationVersion = appVersion,
+        ApplicationFamily = appFamily,
+        ParameterSetID = psetInstance
+        )
+    return algorithmInstance
+
+def createMergeAlgorithm(datasetInfo, apiRef = None):
+    """
+    _createMergeAlgorithm_
+
+    Create a DbsAlgorithm for a merge dataset
+
+    """
+    exeName = datasetInfo['ApplicationName']
+    version = datasetInfo['ApplicationVersion']
+    family = datasetInfo.get('ApplicationFamily', None)
+    if (family == None) or not (family) :
+        family = datasetInfo['OutputModuleName']
+
+
+    mergeAlgo = DbsAlgorithm (
+        ExecutableName = exeName,
+        ApplicationVersion = version,
+        ApplicationFamily = family,
+        )
+
+    if apiRef != None:
+        apiRef.insertAlgorithm(mergeAlgo)
+    return mergeAlgo
+
+
+
+
+def createProcessedDataset(primaryDataset, algorithm, datasetInfo,
+                           apiRef = None):
+    """
+    _createProcessedDataset_
+
+
+    """
+
+    physicsGroup = datasetInfo.get("PhysicsGroup", "NoGroup")
+    status = datasetInfo.get("Status", "VALID")
+    dataTier = datasetInfo['DataTier']
+    globalTag = datasetInfo.get('Conditions', None)
+    if globalTag is None: globalTag = ''
+
+    parents = []
+    inputDataset = datasetInfo.get('ParentDataset', None)
+    if inputDataset != None:
+        parents.append(inputDataset)
+
+    tierList = makeTierList(datasetInfo['DataTier'])
+
+    name = datasetInfo['ProcessedDataset']
+    algolist=[]
+    if algorithm not in ('', None):
+        algolist=list(algorithm)
+
+    processedDataset = DbsProcessedDataset (
+        PrimaryDataset = primaryDataset,
+        AlgoList=algolist,
+        Name = name,
+        TierList = tierList,
+        ParentList = parents,
+        PhysicsGroup = physicsGroup,
+        Status = status,
+        GlobalTag = globalTag,
+        )
+
+    if apiRef != None:
+        apiRef.insertProcessedDataset(processedDataset)
+    #
+    logging.debug("PrimaryDataset: %s ProcessedDataset: %s DataTierList: %s  requested by PhysicsGroup: %s ", primaryDataset['Name'], name, tierList, physicsGroup)
+    return processedDataset
+
+
+
+
+def createDBSFiles(fjrFileInfo, jobType = None, apiRef = None):
+    """
+    _createDBSFiles_
+
+    Create a list of DBS File instances from the file details contained
+    in a FwkJobRep.FileInfo instance describing an output file
+    Does not insert files, returns as list of DbsFile objects
+    Does insert runs and lumisections if DBS API reference is passed
+
+    """
+    results = []
+    inputLFNs = [ x['LFN'] for x in fjrFileInfo.inputFiles]
+    checksum = fjrFileInfo.checksums['cksum']
+    adler32sum = fjrFileInfo.checksums.get('adler32', '')
+
+    nEvents = int(fjrFileInfo['TotalEvents'])
+
+    if len(fjrFileInfo.dataset)<=0:
+        logging.error("No dataset info found in FWJobReport!")
+        return results
+
+    #  //
+    # // Set FileType
+    #//
+    if 'FileType' in fjrFileInfo:
+        fileType = fjrFileInfo['FileType']
+    else:
+        fileType = 'EDM'
+
+    #
+    # FIXME: at this point I should use the mc or data event type from
+    #        the jobreport. Until this is supported by the framework,
+    #        we use the workaround that mc job reports have an empty
+    #        lumisections list (stripped in DBSInterface)
+    #
+    lumiList = []
+    if ( len(fjrFileInfo.getLumiSections()) > 0 ):
+
+        #
+        # insert runs (for data files from detector)
+        #
+        if ( apiRef != None ):
+
+            for runinfo in fjrFileInfo.runs:
+
+                run = DbsRun(
+                    RunNumber = int(runinfo),
+                    NumberOfEvents = 0,
+                    NumberOfLumiSections = 0,
+                    TotalLuminosity = 0,
+                    StoreNumber = 0,
+                    StartOfRun = 0,
+                    EndOfRun = 0,
+                    )
+
+                apiRef.insertRun(run)
+
+        #
+        # insert lumisections (for data files from detector)
+        # associate files with lumisections (for all data files)
+        #
+        for lumiinfo in fjrFileInfo.getLumiSections():
+
+            lumi = DbsLumiSection(
+                LumiSectionNumber = int(lumiinfo['LumiSectionNumber']),
+                StartEventNumber = 0,
+                EndEventNumber = 0,
+                LumiStartTime = 0,
+                LumiEndTime = 0,
+                RunNumber = int(lumiinfo['RunNumber']),
+                )
+
+            # Isnt needed, causes monster slowdown
+            #if ( apiRef != None ):
+            #    apiRef.insertLumiSection(lumi)
+
+            lumiList.append(lumi)
+
+            logging.debug("Lumi associated to file is: %s" % ([x for x in lumiList]))
+
+    #  //
+    # // Dataset info related to files and creation of DbsFile object
+    #//
+    for dataset in fjrFileInfo.dataset:
+
+        primary = createPrimaryDataset(dataset)
+        if jobType == "Merge":
+            algo = createMergeAlgorithm(dataset)
+        else:
+            algo = createAlgorithmForInsert(dataset)
+
+        processed = createProcessedDataset(primary, algo, dataset)
+
+        dbsFileInstance = DbsFile(
+            Checksum = checksum,
+            Adler32 = adler32sum,
+            NumberOfEvents = nEvents,
+            LogicalFileName = fjrFileInfo['LFN'],
+            FileSize = int(fjrFileInfo['Size']),
+            Status = "VALID",
+            ValidationStatus = 'VALID',
+            FileType = fileType,
+            Dataset = processed,
+            TierList = makeTierList(dataset['DataTier']),
+            AlgoList = [algo],
+            LumiList = lumiList,
+            ParentList = inputLFNs,
+            BranchList = fjrFileInfo.branches,
+            )
+
+        results.append(dbsFileInstance)
+    return results
+
+
+def createDBSStorageElement(pnn):
+    """
+    _createDBSStorageElement_
+
+    """
+    return DbsStorageElement(Name = pnn)
+
+
+def createDBSFileBlock(blockName):
+    """
+    _createDBSFileBlock_
+
+    return a DbsFileBlock object with the block name provided
+
+    NOTE: This method DOES NOT create a new block in DBS
+
+    """
+    return DbsFileBlock(Name=blockName)

WMCore/Services/DBS/ProdException.py

@@ -0,0 +1,133 @@
+#!/usr/bin/python
+"""
+_ProdException_
+
+General Exception class for Prod modules
+
+"""
+
+from future.utils import viewitems
+
+try:
+    import exceptions
+except ImportError:
+    import builtins as exceptions
+
+import inspect
+import logging
+
+
+class ProdException(exceptions.Exception):
+    """
+    _ProdException_
+
+    Exception class which works out details of where
+    it was raised.
+
+    """
+    def __init__(self, message, errorNo=1000,**data):
+        self.name = str(self.__class__.__name__)
+        exceptions.Exception.__init__(self, self.name,
+                                      message)
+
+        #  //
+        # // Init data dictionary with defaults
+        #//
+        self.data = {}
+        self.data.setdefault("ClassName", None)
+        self.data.setdefault("ModuleName", None)
+        self.data.setdefault("MethodName", None)
+        self.data.setdefault("ClassInstance", None)
+        self.data.setdefault("FileName", None)
+        self.data.setdefault("LineNumber", None)
+        if errorNo==None:
+            self.data.setdefault("ErrorNr",0)
+        else:
+            self.data.setdefault("ErrorNr",errorNo)
+
+        self.message = message
+        self.data.update(data)
+
+        #  //
+        # // Automatically determine the module name
+        #//  if not set
+        if self.data['ModuleName'] == None:
+            frame = inspect.currentframe()
+            lastframe = inspect.getouterframes(frame)[1][0]
+            excepModule = inspect.getmodule(lastframe)
+            if excepModule != None:
+                modName = excepModule.__name__
+                self.data['ModuleName'] = modName
+
+
+        #  //
+        # // Find out where the exception came from
+        #//
+        stack = inspect.stack(1)[1]
+        self.data['FileName'] = stack[1]
+        self.data['LineNumber'] = stack[2]
+        self.data['MethodName'] = stack[3]
+
+        #  //
+        # // ClassName if ClassInstance is passed
+        #//
+        if self.data['ClassInstance'] != None:
+            self.data['ClassName'] = \
+              self.data['ClassInstance'].__class__.__name__
+
+        logging.error(str(self))
+
+    def __getitem__(self, key):
+        """
+        make exception look like a dictionary
+        """
+        return self.data[key]
+
+    def __setitem__(self, key, value):
+        """
+        make exception look like a dictionary
+        """
+        self.data[key] = value
+
+    def addInfo(self, **data):
+        """
+        _addInfo_
+
+        Add key=value information pairs to an
+        exception instance
+        """
+        for key, value in viewitems(data):
+            self.data[key] = value
+        return
+
+    def xml(self):
+        """create a xml string rep of this exception"""
+        strg ="<Exception>\n"
+        strg +="<Object>\n"
+        strg += "%s\n" % self.name
+        strg +="</Object>\n"
+        strg +="<Message>\n"
+        strg += self.message
+        strg +="</Message>\n"
+        strg +="<DataItems>\n"
+        for key, value in viewitems(self.data):
+            strg +="<DataItem>\n"
+            strg += "<Key>\n"
+            strg += str(key)
+            strg += "</Key>\n"
+            strg += "<Value>\n"
+            strg += str(value)
+            strg += "</Value>\n"
+            strg +="</DataItem>\n"
+        strg +="</DataItems>\n"
+        strg +="</Exception>\n"
+        logging.error(strg)
+        return strg
+
+    def __str__(self):
+        """create a string rep of this exception"""
+        strg = "%s\n" % self.name
+        strg += "Message: %s\n" % self.message
+        for key, value in viewitems(self.data):
+            strg += "\t%s : %s\n" % (key, value, )
+        return strg

WMCore/Services/DBS/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""
+_DBSUpload.DBSInterface_
+
+DBS APIs for DBSUpload
+
+"""
+__all__ = []

WMCore/Services/FWJRDB/FWJRDBAPI.py

@@ -0,0 +1,118 @@
+from __future__ import (division, print_function)
+
+from builtins import str, bytes, object
+
+from WMCore.Database.CMSCouch import CouchServer, Database
+from WMCore.Lexicon import splitCouchServiceURL
+
+class FWJRDBAPI(object):
+
+    def __init__(self, couchURL, dbName=None):
+        """
+        setting up comon variables for inherited class.
+        inherited class should call this in their init function
+        """
+        if isinstance(couchURL, Database):
+            self.couchDB = couchURL
+            self.couchURL = self.couchDB['host']
+            self.dbName = self.couchDB.name
+            self.couchServer = CouchServer(self.couchURL)
+        else:
+            if dbName == None:
+                self.couchURL, self.dbName = splitCouchServiceURL(couchURL)
+            else:
+                self.couchURL = couchURL
+                self.dbName = dbName
+            self.couchServer = CouchServer(self.couchURL)
+            self.couchDB = self.couchServer.connectDatabase(self.dbName, False)
+        self.couchapp = "FWJRDump"
+        self.defaultStale = {"stale": "update_after"}
+
+
+    def setDefaultStaleOptions(self, options):
+        if not options:
+            options = {}
+        if 'stale' not in options:
+            options.update(self.defaultStale)
+        return options
+
+    def _setNoStale(self):
+        """
+        Use this only for the unittest
+        """
+        self.defaultStale = {}
+
+    def _getCouchView(self, view, options, keys = []):
+
+        options = self.setDefaultStaleOptions(options)
+
+        if keys and isinstance(keys, (str, bytes)):
+            keys = [keys]
+        return self.couchDB.loadView(self.couchapp, view, options, keys)
+
+
+    def _filterCouchInfo(self, couchInfo):
+        # remove the couch specific information
+        for key in ['_rev', '_attachments']:
+            if  key in couchInfo:
+                del couchInfo[key]
+        return
+
+
+    def _formatCouchData(self, data, key = "id", detail = True, filterCouch = True,
+                         returnDict = False):
+        result = {}
+        for row in data['rows']:
+            if 'error' in row:
+                continue
+            if "doc" in row:
+                if filterCouch:
+                    self._filterCouchInfo(row["doc"])
+                result[row[key]] = row["doc"]
+            else:
+                result[row[key]] = row["value"]
+        if detail or returnDict:
+            return result
+        else:
+            return list(result)
+
+    def getFWJRByArchiveStatus(self, status, limit=None, skip=None):
+        """
+        'status': list of the status or status string
+        """
+        options = {}
+        options["include_docs"] = True
+
+        if limit != None:
+            options["limit"] = limit
+        if skip != None:
+            options["skip"] = skip
+        keys = status
+        return self._getCouchView("reportsByArchiveStatus", options, keys)
+
+    def updateArchiveUploadedStatus(self, docID):
+
+        return self.couchDB.updateDocument(docID, self.couchapp, "archiveStatus")
+
+    def isAllFWJRArchived(self, workflow):
+        keys = [[workflow, "ready"]]
+        options = {"reduce": True, "group": True}
+        result = self._getCouchView("byWorkflowAndArchiveStatus", options, keys)
+        if len(result["rows"]) == 0:
+            return True
+        else:
+            return False
+
+
+    def outputByWorkflowName(self):
+
+        options = {"group": True, "stale": "ok", "reduce": True}
+        # site of data should be relatively small (~1M) for put in the memory
+        # If not, find a way to stream
+        return self._getCouchView("outputByWorkflowName", options)
+
+
+    def getFWJRWithSkippedFiles(self):
+        options = {"reduce": True, "group": True, "include_docs": False}
+        return self._getCouchView("skippedFileInfoByTaskAndSite", options)
+