wmglobalqueue 2.3.10__py3-none-any.whl

WMCore/Services/TagCollector/TagCollector.py

@@ -0,0 +1,155 @@
+from __future__ import (division, print_function)
+from future import standard_library
+
+from Utils.Utilities import decodeBytesToUnicode
+
+standard_library.install_aliases()
+
+import logging
+from urllib.parse import urlparse
+
+from collections import defaultdict
+from WMCore.Services.Service import Service
+from WMCore.Services.TagCollector.XMLUtils import xml_parser
+
+class TagCollector(Service):
+    """
+    Class which provides interface to CMS TagCollector web-service.
+    Provides non-deprecated CMSSW releases in all their ScramArchs (not only prod)
+    """
+
+    def __init__(self, url=None, logger=None, configDict=None, **kwargs):
+        """
+        responseType will be either xml or json
+        """
+        defaultURL = "https://cmssdt.cern.ch/SDT/cgi-bin/ReleasesXML"
+        url = url or defaultURL
+        parsedUrl = urlparse(url)
+        self.cFileUrlPath = parsedUrl.path.replace("/", "_")
+        # all releases types and all their archs
+        self.tcArgs = kwargs
+        self.tcArgs.setdefault("anytype", 1)
+        self.tcArgs.setdefault("anyarch", 1)
+
+        configDict = configDict or {}
+        configDict.setdefault('endpoint', url)
+        configDict.setdefault("timeout", 300)
+        configDict.setdefault('cacheduration', 1)
+        configDict['logger'] = logger if logger else logging.getLogger()
+        super(TagCollector, self).__init__(configDict)
+        self['logger'].debug("Initializing TagCollector with url: %s", self['endpoint'])
+
+    def _getResult(self, callname="", clearCache=False,
+                   args=None, verb="GET", encoder=None, decoder=None,
+                   contentType=None):
+        """
+        _getResult_
+
+        retrieve JSON/XML formatted information given the service name and the
+        argument dictionaries
+
+        TODO: Probably want to move this up into Service
+        """
+        if not args:
+            args = self.tcArgs
+
+        cFile = '%s_%s'% (self.cFileUrlPath, callname.replace("/", "_"))
+        # If no callname or url path, the base host is getting queried
+        if cFile == '_':
+            cFile = 'baseRequest'
+
+        if clearCache:
+            self.clearCache(cFile, args, verb)
+
+        # Note cFile is just the base name pattern, args 
+        # are also considered for the end filename in the method below
+        f = self.refreshCache(cFile, callname, args, encoder=encoder, decoder=decodeBytesToUnicode,
+                              verb=verb, contentType=contentType)
+        result = f.read()
+        f.close()
+
+        # overhead from REST model which returns results as strings or None
+        # therefore they can be encoded by JSON to None, etc.
+        if result == 'None':
+            return
+        if result and decoder:
+            result = decoder(result)
+        return result
+
+    def data(self):
+        "Fetch data from tag collector or local cache"
+        data = self._getResult()
+        pkey = 'architecture'
+        for row in xml_parser(data, pkey):
+            yield row[pkey]
+
+    def releases(self, arch=None):
+        "Yield CMS releases known in tag collector"
+        arr = []
+        for row in self.data():
+            if arch:
+                if arch == row['name']:
+                    for item in row['project']:
+                        arr.append(item['label'])
+            else:
+                for item in row['project']:
+                    arr.append(item['label'])
+        return list(set(arr))
+
+    def architectures(self):
+        "Yield CMS architectures known in tag collector"
+        arr = []
+        for row in self.data():
+            arr.append(row['name'])
+        return list(set(arr))
+
+    def releases_by_architecture(self):
+        "returns CMS architectures and realease in dictionary format"
+        arch_dict = defaultdict(list)
+        for row in self.data():
+            releases = set()
+            for item in row['project']:
+                releases.add(item['label'])
+            arch_dict[row['name']].extend(list(releases))
+        return dict(arch_dict)
+
+
+    def defaultMicroArchVersionNumberByRelease(self, default_microarch=0):
+        """
+        Yield default microarchitecture by CMS release known in tag collector.
+
+        :param default_microarch: int, default microar when not found in TagCollector XML file.
+        :return: dictionary ("release": default_microarch (or default_microarch if not found)).
+        """
+        rel_microarchs = {}
+        for row in self.data():
+            for item in row['project']:
+                microarch = item.get('default_micro_arch', default_microarch)
+                if isinstance(microarch, str):
+                    # format of this value is "arch-vN"
+                    microarch = int((microarch.split("-")[-1]).lstrip("v"))
+                rel_microarchs[item['label']] = microarch
+        return rel_microarchs
+
+    def getGreaterMicroarchVersionNumber(self, releases=None, default_microarch=0, rel_microarchs=None):
+        """
+        Return the greated default microarchitecture, given a list of releases.
+
+        :param releases: str, comma-separated list of releases.
+        :param default_microarch: int, default microar when not found in TagCollector XML file.
+        :return: int, greater default microarchitecture version number.
+        """
+        if releases is None:
+            return default_microarch
+
+        if rel_microarchs is None:
+            rel_microarchs = self.defaultMicroArchVersionNumberByRelease()
+
+        releases = releases.split(",")
+        microarch = default_microarch
+        for r in releases:
+            m = rel_microarchs.get(r, default_microarch)
+            if m > microarch:
+                microarch = m
+
+        return microarch

WMCore/Services/TagCollector/XMLUtils.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+# encoding: utf-8
+
+"""
+File       : XMLUtils.py
+Author     : Valentin Kuznetsov <vkuznet AT gmail dot com>
+Description: Set of utilities for RequestManager code
+"""
+
+from __future__ import (division, print_function)
+from builtins import str, bytes
+from Utils.Utilities import encodeUnicodeToBytes
+
+from io import BytesIO
+import re
+import xml.etree.cElementTree as ET
+
+int_number_pattern = re.compile(r'(^[0-9-]$|^[0-9-][0-9]*$)')
+float_number_pattern = re.compile(r'(^[-]?\d+\.\d*$|^\d*\.{1,1}\d+$)')
+
+
+def adjust_value(value):
+    """
+    Change null value to None.
+    """
+    pat_float = float_number_pattern
+    pat_integer = int_number_pattern
+    if isinstance(value, str):
+        if value == 'null' or value == '(null)':
+            return None
+        elif pat_float.match(value):
+            return float(value)
+        elif pat_integer.match(value):
+            return int(value)
+        else:
+            return value
+    else:
+        return value
+
+
+def xml_parser(data, prim_key):
+    """
+    Generic XML parser
+
+    :param data: can be of type "file object", unicode string or bytes string
+    """
+    if isinstance(data, (str, bytes)):
+        stream = BytesIO()
+        data = encodeUnicodeToBytes(data, "ignore")
+        stream.write(data)
+        stream.seek(0)
+    else:
+        stream = data
+
+    context = ET.iterparse(stream)
+    for event, elem in context:
+        row = {}
+        key = elem.tag
+        if key != prim_key:
+            continue
+        row[key] = elem.attrib
+        get_children(elem, event, row, key)
+        elem.clear()
+        yield row
+
+
+def get_children(elem, event, row, key):
+    """
+    xml_parser helper function. It gets recursively information about
+    children for given element tag. Information is stored into provided
+    row for given key. The change of notations can be applied during
+    parsing step by using provided notations dictionary.
+    """
+    for child in list(elem):
+        child_key = child.tag
+        child_data = child.attrib
+        if not child_data:
+            child_dict = adjust_value(child.text)
+        else:
+            child_dict = child_data
+
+        if list(child):  # we got grand-children
+            if child_dict:
+                row[key][child_key] = child_dict
+            else:
+                row[key][child_key] = {}
+            if isinstance(child_dict, dict):
+                newdict = {child_key: child_dict}
+            else:
+                newdict = {child_key: {}}
+            get_children(child, event, newdict, child_key)
+            row[key][child_key] = newdict[child_key]
+        else:
+            if not isinstance(row[key], dict):
+                row[key] = {}
+            row[key].setdefault(child_key, [])
+            row[key][child_key].append(child_dict)
+        child.clear()

WMCore/Services/UUIDLib.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+from builtins import str
+import uuid
+
+
+def makeUUID():
+    """
+    _makeUUID_
+
+    Makes a UUID from the uuid class, returns it
+    """
+    return str(uuid.uuid4())

WMCore/Services/UserFileCache/UserFileCache.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+"""
+_UserFileCache_
+
+API for UserFileCache service
+"""
+
+from builtins import str, bytes
+
+import os
+import json
+import shutil
+import hashlib
+import tarfile
+import tempfile
+
+from Utils.Utilities import encodeUnicodeToBytesConditional
+from Utils.PythonVersion import PY3
+
+from WMCore.Services.Service import Service
+
+
+def calculateChecksum(tarfile_, exclude=None):
+    """
+    Calculate the checksum of the tar file in input.
+
+    The tarfile_ input parameter could be a string or a file object (anything compatible
+    with the fileobj parameter of tarfile.open).
+
+    The exclude parameter could be a list of strings, or a callable that takes as input
+    the output of  the list of tarfile.getmembers() and return a list of strings.
+    The exclude param is interpreted as a list of files that will not be taken into consideration
+    when calculating the checksum.
+
+    The output is the checksum of the tar input file.
+
+    The checksum is calculated taking into consideration the names of the objects
+    in the tarfile (files, directories etc) and the content of each file.
+
+    Each file is exctracted, read, and then deleted right after the input is passed
+    to the hasher object. The file is read in chuncks of 4096 bytes to avoid memory
+    issues.
+    """
+    if exclude==None: #[] is a dangerous value for a param
+        exclude = []
+
+    hasher = hashlib.sha256()
+
+    ## "massage" out the input parameters
+    if isinstance(tarfile_, (str, bytes)):
+        tar = tarfile.open(tarfile_, mode='r')
+    else:
+        tar = tarfile.open(fileobj=tarfile_, mode='r')
+
+    if exclude and hasattr(exclude, '__call__'):
+        excludeList = exclude(tar.getmembers())
+    else:
+        excludeList = exclude
+
+
+    tmpDir = tempfile.mkdtemp()
+    try:
+        for tarmember in tar:
+            if tarmember.name in excludeList:
+                continue
+            hasher.update(encodeUnicodeToBytesConditional(tarmember.name, condition=PY3))
+            if tarmember.isfile() and tarmember.name.split('.')[-1]!='pkl':
+                tar.extractall(path=tmpDir, members=[tarmember])
+                fn = os.path.join(tmpDir, tarmember.name)
+                with open(fn, 'rb') as fd:
+                    while True:
+                        buf = fd.read(4096)
+                        if not buf:
+                            break
+                        hasher.update(buf)
+                os.remove(fn)
+    finally:
+        #never leave tmddir around
+        shutil.rmtree(tmpDir)
+    checksum = hasher.hexdigest()
+
+    return checksum
+
+
+class UserFileCache(Service):
+    """
+    API for UserFileCache service
+    """
+    # Should be filled out with other methods: download, exists
+
+    def __init__(self, mydict=None):
+        if mydict==None: #dangerous {} default value
+            mydict = {}
+        mydict['endpoint'] =  mydict.get('endpoint', 'https://cmsweb.cern.ch/crabcache/')
+        Service.__init__(self, mydict)
+        self['requests']['accept_type'] = 'application/json'
+
+        if 'proxyfilename' in mydict:
+            #in case there is some code I have not updated in ticket #3780. Should not be required... but...
+            self['logger'].warning('The UserFileCache proxyfilename parameter has been replace with the more'
+                                   ' general (ckey/cert) pair.')
+
+    def downloadLog(self, fileName, output):
+        """
+        """
+        url = self['endpoint'] + 'logfile?name=%s' % os.path.split(fileName)[1]
+
+        self['logger'].info('Fetching URL %s' % url)
+        fileName, dummyHeader = self['requests'].downloadFile(output, str(url)) #unicode broke pycurl.setopt
+        self['logger'].debug('Wrote %s' % output)
+        return fileName
+
+    def uploadLog(self, fileName, uploadName=None):
+        """
+        """
+        if not uploadName:
+            uploadName = os.path.split(fileName)[1]
+        params = [('name', uploadName)]
+
+        resString = self["requests"].uploadFile(fileName=fileName, fieldName='inputfile',
+                                                url=self['endpoint'] + 'logfile',
+                                                params=params, verb='PUT')
+
+        return json.loads(resString)['result'][0]
+
+    def removeFile(self, haskey):
+        result=self['requests'].makeRequest(uri = 'info', data = {'subresource':'fileremove', 'hashkey': haskey})
+        return json.loads(result[0])['result'][0]
+
+    def download(self, hashkey, output, username=None):
+        """
+        Download tarfile with the provided hashkey.
+        """
+        url = self['endpoint'] + 'file?hashkey=%s' % hashkey
+        if username:
+            url = url + '&username=%s' % username
+
+        self['logger'].info('Fetching URL %s' % url)
+        fileName, dummyHeader = self['requests'].downloadFile(output, str(url)) # unicode broke pycurl.setopt
+        self['logger'].debug('Wrote %s' % fileName)
+        return fileName
+
+    def upload(self, fileName, excludeList = None):
+        """
+        Upload the tarfile fileName to the user file cache. Returns the hash of the content of the file
+        which can be used to retrieve the file later on.
+        """
+        if excludeList==None: #pylint says [] is a dangerous default value
+            excludeList = []
+
+        #The parameter newchecksum tells the crabcache to use the new algorithm. It's there
+        #to guarantee backward compatibility.
+        params = [('hashkey', calculateChecksum(fileName, excludeList)), ('newchecksum', '2')]
+
+        resString = self["requests"].uploadFile(fileName=fileName, fieldName='inputfile',
+                                                url=self['endpoint'] + 'file',
+                                                params=params, verb='PUT')
+
+        return json.loads(resString)['result'][0]
+

WMCore/Services/UserFileCache/__init__.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env python
+"""
+_UserFileCache_
+
+UserFileCache interface package
+
+"""
+__all__ = []

WMCore/Services/WMAgent/WMAgent.py

@@ -0,0 +1,63 @@
+import json
+
+from WMCore.Services.Service import Service
+
+
+class WMAgent(Service):
+
+    """
+    API for dealing with retrieving information from WMAgent RESTModel Service
+    """
+
+    def __init__(self, dict={}):
+        """
+        responseType will be JSON
+        """
+
+        dict.setdefault("accept_type", "application/json")
+        dict.setdefault("content_type", "application/json")
+        self.encoder = json.dumps
+        self.decoder = json.loads
+
+        Service.__init__(self, dict)
+
+    def _getResult(self, callname, clearCache = True,
+                   args = None, verb="GET", contentType = None):
+        """
+        _getResult_
+
+        retrieve JSON/XML formatted information given the service name and the
+        argument dictionaries
+
+        TODO: Probably want to move this up into Service
+        """
+        result = ''
+        # make base file name from call name.
+        file = callname.replace("/", "_")
+        if clearCache:
+            self.clearCache(file, args, verb)
+
+        # can't pass the decoder here since refreshCache wright to file
+        f = self.refreshCache(file, callname, args, encoder = self.encoder,
+                              verb = verb, contentType = contentType)
+        result = f.read()
+        f.close()
+        result = self.decoder(result)
+
+        return result
+
+    def getAgentStatus(self, detail = False):
+        """
+        """
+        callname = 'agentstatus'
+        # TODO support detail flag if it is needed
+        # need to convert to boolean on serverside
+        #args = {'detail': detail}
+        args = {}
+        return self._getResult(callname, args = args)
+
+    def getACDCInfo(self):
+        """
+        """
+        callname = 'acdclink'
+        return self._getResult(callname)