wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/Services/Rucio/RucioUtils.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+This module contains a few basic WMCore-related utilitarian
+Rucio data structures and functions
+"""
+
+import random
+
+RUCIO_VALID_PROJECT = ("Production", "RelVal", "Tier0", "Test", "User")
+# grouping values are extracted from:
+# https://github.com/rucio/rucio/blob/master/lib/rucio/common/schema/cms.py#L117
+GROUPING_DSET = "DATASET"
+GROUPING_ALL = "ALL"
+RUCIO_RULES_PRIORITY = {"low": 2, "normal": 3, "high": 4, "reserved": 5}
+# number of copies to be defined when creating replication rules
+NUM_COPIES_DEFAULT = 1
+NUM_COPIES_NANO = 2
+
+def validateMetaData(did, metaDict, logger):
+    """
+    This function can be extended in the future, for now it will only
+    validate the DID creation metadata, more specifically only the
+    "project" parameter
+    :param did: the DID that will be inserted
+    :param metaDict: a dictionary with all the DID metadata data to be inserted
+    :param logger: a logger object
+    :return: False if validation fails, otherwise True
+    """
+    if metaDict.get("project", "Production") in RUCIO_VALID_PROJECT:
+        return True
+    msg = f"DID: {did} has an invalid 'project' meta-data value: {metaDict['project']}"
+    msg += f"The supported 'project' values are: {str(RUCIO_VALID_PROJECT)}"
+    logger.error(msg)
+    return False
+
+
+def weightedChoice(rses, rseWeights):
+    """
+    Given a list of items and their respective weights (quota in this case),
+    perform a weighted selection.
+    :param rses: a list of tuples with the RSE name and whether it requires approval
+    :param rseWeights: a list with RSE weights (quota)
+    :return: a tuple from the choices list
+    """
+    listChoice = random.choices(population=rses, weights=rseWeights, k=1)
+    # return only the tuple, not the list with a tuple item
+    return listChoice[0]
+
+
+def isTapeRSE(rseName):
+    """
+    Given an RSE name, return True if it's a Tape RSE (rse_type=TAPE), otherwise False
+    :param rseName: string with the RSE name
+    :return: True or False
+    """
+    # NOTE: a more reliable - but more expensive - way to know that would be
+    # to query `get_rse` and evaluate the rse_type parameter
+    return rseName.endswith("_Tape")
+
+
+def dropTapeRSEs(listRSEs):
+    """
+    Method to parse a list of RSE names and return only those that
+    are not a rse_type=TAPE, so in general only Disk endpoints
+    :param listRSEs: list with the RSE names
+    :return: a new list with only DISK RSE names
+    """
+    diskRSEs = []
+    for rse in listRSEs:
+        if isTapeRSE(rse):
+            continue
+        diskRSEs.append(rse)
+    return diskRSEs

WMCore/Services/RucioConMon/RucioConMon.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+# coding=utf-8
+"""
+Rucio Consistency Monitor Service class developed on top of the native WMCore
+Service APIs providing custom output and handling as necessary for the
+CMS Workload Management system
+"""
+
+from __future__ import division, print_function, absolute_import
+from future import standard_library
+
+from urllib.parse import urlencode
+
+import json
+import logging
+
+from WMCore.Services.Service import Service
+from Utils.Utilities import decodeBytesToUnicode
+
+
+standard_library.install_aliases()
+
+
+class RucioConMon(Service):
+    """
+    API for dealing with retrieving information from Rucio Consistency Monitor
+    """
+
+    def __init__(self, url, logger=None, configDict=None):
+        """
+        Init method for the RucioConMon Class
+        """
+        configDict = configDict or {}
+        configDict.setdefault('endpoint', url)
+        configDict.setdefault('cacheduration', 1)  # in hours
+        configDict.setdefault('accept_type', 'application/json')
+        configDict.setdefault('content_type', 'application/json')
+        configDict['logger'] = logger if logger else logging.getLogger()
+        super(RucioConMon, self).__init__(configDict)
+        self['logger'].debug("Initializing RucioConMon with url: %s", self['endpoint'])
+
+    def _getResult(self, uri, callname="", clearCache=False, args=None):
+        """
+        Either fetch data from the cache file or query the data-service
+        :param uri: The endpoint uri
+        :param callname: alias for caller function
+        :param clearCache: parameter to control the cache behavior
+        :param args: additional parameters to HTTP request call
+        :return:    A dictionary
+        """
+
+        # NOTE: Unlike the common case we are not using the callname for building
+        #       apiUrl. we  are using it only for giving proper name of the cache
+        #       file.
+        # NOTE: The 'callname' should not contain '/', otherwise the cache is
+        #       tried to be created with a a non existing subdirectory structure
+
+        cachedApi = "%s.json" % callname
+        # apiUrl = '%s?json&preset=%s' % (uri, callname)
+        apiUrl = uri
+
+        self['logger'].debug('Fetching data from %s, with args %s', apiUrl, args)
+        if args:
+            apiUrl = "%s&%s" % (apiUrl, urlencode(args, doseq=True))
+
+        if clearCache:
+            self.clearCache(cachedApi, args)
+        with self.refreshCache(cachedApi, apiUrl, decoder=True, binary=False) as istream:
+            return json.load(istream)
+
+    def _getResultZipped(self, uri, callname="", clearCache=True):
+        """
+        This method retrieves gzipped content, instead of the standard json format.
+        :param uri: The endpoint uri
+        :param callname: alias for caller function
+        :param clearCache: parameter to control the cache behavior
+        :return: yields a single record from the data retrieved
+        """
+        cachedApi = callname
+        if clearCache:
+            self.clearCache(cachedApi)
+
+        with self.refreshCache(cachedApi, uri, decoder=False, binary=True) as istream:
+            for line in istream:
+                line = decodeBytesToUnicode(line).replace("\n", "")
+                yield line
+
+    def getRSEStats(self):
+        """
+        Gets the latest statistics from the RucioConMon, together with the last
+        update timestamps for all RSEs known to CMS Rucio
+        :return: A dictionary
+        """
+        uri = "stats"
+        rseStats = self._getResult(uri, callname='stats')
+        return rseStats
+
+    def getRSEUnmerged(self, rseName, zipped=False):
+        """
+        Gets the list of all unmerged files in an RSE
+        :param rseName: The RSE whose list of unmerged files to be retrieved
+        :param zipped:  If True the interface providing the zipped lists will be called
+        :return: a generator of unmerged files for the RSE in question
+        """
+        # NOTE: The default API provided by Rucio Consistency Monitor is in a form of a
+        #       zipped file/stream. Currently we are using the newly provided json API
+        #       But in in case we figure out the data is too big we may need to
+        #       implement the method with the zipped API and use disc cache for
+        #       reading/streaming from file. This will prevent any set arithmetic
+        #       in the future.
+        if zipped:
+            uri = "files?rse=%s&format=raw" % rseName
+            callname = '{}.zipped'.format(rseName)
+            rseUnmerged = self._getResultZipped(uri, callname=callname, clearCache=True)
+        else:
+            uri = "files?rse=%s&format=json" % rseName
+            callname = '{}.json'.format(rseName)
+            rseUnmerged = self._getResult(uri, callname=callname)
+        # now lazily return items
+        for item in rseUnmerged:
+            yield item

WMCore/Services/Service.py

@@ -0,0 +1,400 @@
+#!/usr/bin/env python
+"""
+_Service_
+
+A Service talks to some http(s) accessible service that provides information and
+caches the result of these queries. The cache will be refreshed if the file is
+older than a timeout set in the instance of Service.
+
+It has a cache path, cache duration, an endpoint (the url the
+service exists on) a logger and an accept type (json, xml etc) and method
+(GET/POST).
+
+The Service satisfies two caching cases:
+
+1. set a defined query, cache results, poll for new ones
+2. use a changing query, cache results to a file depending on the query, poll
+   for new ones
+
+Data maybe passed to the remote service either via adding the query string to
+the URL (for GET's) or by passing a dictionary to either the service constructor
+(case 1.) or by passing the data as a dictionary to the refreshCache,
+forceCache, clearCache calls. By default the cache lasts 30 minutes.
+
+Calling refreshCache/forceRefresh will return an open file object, the cache
+file. Once done with it you should close the object.
+
+The service has a default timeout to receive a response from the remote service
+of 300 seconds. Over ride this by passing in a timeout via the configuration
+dict, set to None if you want to turn off the timeout.
+
+If you just want to retrieve the data without caching use the Requests class
+directly.
+
+The Service class provides two layers of caching:
+    1. Caching from httplib2 is provided via Request, this respects etag and
+    expires, but the cache will be lost if the service raises an exception or
+    similar.
+    2. Internal caching which respects an internal cache duration. If the remote
+    service fails to respond the second layer cache will be used until the cache
+    dies.
+
+In tabular form:
+
+httplib2 cache  |   yes    |   yes    |    no    |     no     |
+----------------+----------+----------+----------+------------+
+service cache   |    no    |   yes    |   yes    |     no     |
+----------------+----------+----------+----------+------------+
+result          |  cached  |  cached  |  cached  | not cached |
+"""
+
+from builtins import str
+from future import standard_library
+standard_library.install_aliases()
+
+import datetime
+import json
+import logging
+import os
+import time
+from io import BytesIO, StringIO
+from http.client import HTTPException
+
+from Utils.PythonVersion import PY3
+from WMCore.Services.Requests import Requests, JSONRequests
+from WMCore.WMException import WMException
+
+try:
+    from httplib2 import HttpLib2Error
+except ImportError:
+    # Mock HttpLib2Error since we don't want that WMCore depend on httplib2 using pycurl
+    class HttpLib2Error(Exception):
+        pass
+
+
+def isfile(obj):
+    """
+    Check whether obj is a file-like object (file, StringIO)"
+    """
+    return hasattr(obj, 'flush')
+
+
+def cache_expired(cache, delta=0):
+    """
+    Is the cache expired? At delta hours (default 0) in the future.
+    """
+    if isfile(cache):
+        # already opened file-like object, assume we need to refresh
+        return True
+    else:
+        # then it's a file name
+        if not os.path.exists(cache):
+            return True
+
+        delta = datetime.timedelta(hours=delta)
+        t = datetime.datetime.now() - delta
+        # cache file mtime has been set to cache expiry time
+        if os.path.getmtime(cache) < time.mktime(t.timetuple()):
+            return True
+
+    return False
+
+
+def _makeHash(inputdata):
+    """
+    Turn the input data into json and hash the string. This is simple and
+    means that the input data must be json-serialisable, which is good.
+    """
+    json_hash = json.dumps(inputdata)
+    return json_hash.__hash__()
+
+
+class Service(dict):
+    def __init__(self, cfg_dict=None):
+        super(Service, self).__init__()
+        cfg_dict = cfg_dict or {}
+        # The following should read the configuration class
+        for a in ['endpoint']:
+            assert a in list(cfg_dict), "Can't have a service without a %s" % a
+
+        # if end point ends without '/', add that
+        if not cfg_dict['endpoint'].endswith('/'):
+            cfg_dict['endpoint'] = cfg_dict['endpoint'].strip() + '/'
+
+        # set up defaults
+        self.setdefault("inputdata", {})
+        self.setdefault("cacheduration", 0.5)
+        self.supportVerbList = ('GET', 'POST', 'PUT', 'DELETE')
+        # this value should be only set when whole service class uses
+        # the same verb ('GET', 'POST', 'PUT', 'DELETE')
+        self.setdefault("method", None)
+
+        # Set a timeout for the socket
+        self.setdefault("timeout", 300)
+
+        # then update with the incoming dict
+        self.update(cfg_dict)
+
+        self['service_name'] = self.__class__.__name__  # used for cache naming
+
+        # Get the request class, to instantiate later
+        # either passed as param to __init__, determine via scheme or default
+        if isinstance(self.get('requests'), type):
+            requests = self['requests']
+        elif self.get('accept_type') == "application/json" and self.get('content_type') == "application/json":
+            requests = JSONRequests
+        else:
+            requests = Requests
+        # Instantiate a Request
+        try:
+            self["requests"] = requests(cfg_dict['endpoint'], cfg_dict)
+        except WMException as ex:
+            msg = str(ex)
+            self["logger"].exception(msg)
+            raise
+
+        # cachepath will be modified - i.e. hostname added
+        self['cachepath'] = self["requests"]["cachepath"]
+
+        if 'logger' not in self:
+            if self['cachepath']:
+                logfile = os.path.join(self['cachepath'], '%s.log' % self.__class__.__name__.lower())
+            else:
+                logfile = None
+            logging.basicConfig(level=logging.DEBUG,
+                                format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
+                                datefmt='%m-%d %H:%M',
+                                filename=logfile,
+                                filemode='w')
+            self['logger'] = logging.getLogger(self.__class__.__name__)
+            self['requests']['logger'] = self['logger']
+
+        msg = "Service '%s' initialized with the following settings: %s"
+        self['logger'].debug(msg, self.__class__.__name__, self)
+
+    def cacheFileName(self, cachefile, verb='GET', inputdata=None):
+        """
+        Calculate the cache filename for a given query.
+        """
+        # if not caching to disk return StringIO object
+        if not self['cachepath'] or not cachefile:
+            return StringIO() if PY3 else BytesIO()
+
+        inputdata = inputdata or {}
+        if inputdata:
+            hash_ = _makeHash(inputdata)
+        else:
+            hash_ = _makeHash(self['inputdata'])
+        cachefile = "%s/%s_%s_%s" % (self["cachepath"], hash_, verb, cachefile)
+
+        return cachefile
+
+    def refreshCache(self, cachefile, url='', inputdata=None, openfile=True,
+                     encoder=True, decoder=True, verb='GET', contentType=None, incoming_headers=None, binary=False):
+        """
+        See if the cache has expired. If it has make a new request to the
+        service for the input data. Return the cachefile as an open file object.
+        If cachefile is None returns StringIO.
+
+        :param cachefile: cache file name
+        :param url: url value
+        :param inputdata: HTTP (JSON) payload string
+        :param openfile: flag to use open file, boolean
+        :param encoder: flag to use encoder, boolean
+        :param decoder: flag to use decode, boolean
+        :param verb: HTTP method, string
+        :param contentType: HTTP content type value, string
+        :param incoming_headers: set of HTTP headers, dict
+        :param binary: use binary mode to decode HTTP data, boolean
+        :return: either file object or StringIO
+        """
+        inputdata = inputdata or {}
+        incoming_headers = incoming_headers or {}
+        verb = self._verbCheck(verb)
+
+        cachefile = self.cacheFileName(cachefile, verb, inputdata)
+
+        if cache_expired(cachefile, self["cacheduration"]):
+            self.getData(cachefile, url, inputdata, incoming_headers, encoder, decoder, verb, contentType, binary=binary)
+        else:
+            self['logger'].debug('Data is from the Service cache')
+
+        # cachefile may be filename or file object
+        if openfile and not isfile(cachefile):
+            if binary:
+                return open(cachefile, 'rb')
+            return open(cachefile, 'r', encoding='utf-8')
+        else:
+            return cachefile
+
+    def forceRefresh(self, cachefile, url='', inputdata=None, openfile=True,
+                     encoder=True, decoder=True, verb='GET',
+                     contentType=None, incoming_headers=None, binary=False):
+        """
+        Make a new request to the service for the input data, regardless of the
+        cache state. Return the cachefile as an open file object.
+        If cachefile is None returns StringIO.
+
+        :param cachefile: cache file name
+        :param url: url value
+        :param inputdata: HTTP (JSON) payload string
+        :param openfile: flag to use open file, boolean
+        :param encoder: flag to use encoder, boolean
+        :param decoder: flag to use decode, boolean
+        :param verb: HTTP method, string
+        :param contentType: HTTP content type value, string
+        :param incoming_headers: set of HTTP headers, dict
+        :param binary: use binary mode to decode HTTP data, boolean
+        :return: either file object or StringIO
+        """
+        inputdata = inputdata or {}
+        incoming_headers = incoming_headers or {}
+        verb = self._verbCheck(verb)
+
+        cachefile = self.cacheFileName(cachefile, verb, inputdata)
+
+        self['logger'].debug("Forcing cache refresh of %s" % cachefile)
+        incoming_headers.update({'cache-control': 'no-cache'})
+        self.getData(cachefile, url, inputdata, incoming_headers,
+                     encoder, decoder, verb, contentType, force_refresh=True, binary=binary)
+        if openfile and not isfile(cachefile):
+            if binary:
+                return open(cachefile, 'rb')
+            return open(cachefile, 'r', encoding='utf-8')
+        else:
+            return cachefile
+
+    def clearCache(self, cachefile, inputdata=None, verb='GET'):
+        """
+        Delete the cache file and the httplib2 cache.
+        """
+        if not self['cachepath'] or not cachefile:
+            # nothing to clear
+            return
+
+        inputdata = inputdata or {}
+        verb = self._verbCheck(verb)
+        os.system("/bin/rm -f %s/*" % self['requests']['req_cache_path'])
+        cachefile = self.cacheFileName(cachefile, verb, inputdata)
+        try:
+            if not isfile(cachefile):
+                os.remove(cachefile)
+        except OSError:  # File doesn't exist
+            return
+
+    def getData(self, cachefile, url, inputdata=None, incoming_headers=None,
+                encoder=True, decoder=True,
+                verb='GET', contentType=None, force_refresh=False, binary=False):
+        """
+        Takes the already generated *full* path to cachefile and the url of the
+        resource. Don't need to call self.cacheFileName(cachefile, verb, inputdata)
+        here.
+
+        If cachefile is StringIO append to that
+
+        :param cachefile: cache file name
+        :param url: url value
+        :param inputdata: HTTP (JSON) payload string
+        :param encoder: flag to use encoder, boolean
+        :param decoder: flag to use decode, boolean
+        :param verb: HTTP method, string
+        :param contentType: HTTP content type value, string
+        :param force_refresh: refresh internal cache, boolean
+        :param binary: use binary mode to decode HTTP data, boolean
+        :return: None
+        """
+        inputdata = inputdata or {}
+        incoming_headers = incoming_headers or {}
+        verb = self._verbCheck(verb)
+
+        try:
+            # Get the data
+            if not inputdata:
+                inputdata = self["inputdata"]
+            self['logger'].debug('getData: \n\turl: %s\n\tverb: %s\n\tincoming_headers: %s\n\tdata: %s',
+                                 url, verb, incoming_headers, inputdata)
+            # self['logger'].debug('getData: \n\turl: %s\n\tdata: %s' % \
+            #                     (url, inputdata))
+            data, dummyStatus, dummyReason, from_cache = self["requests"].makeRequest(uri=url,
+                                                                                      verb=verb,
+                                                                                      data=inputdata,
+                                                                                      incoming_headers=incoming_headers,
+                                                                                      encoder=encoder,
+                                                                                      decoder=decoder,
+                                                                                      contentType=contentType)
+            if from_cache:
+                # If it's coming from the cache we don't need to write it to the
+                # second cache, or do we?
+                self['logger'].debug('Data is from the Requests cache')
+            else:
+                # getData have done that for us
+                if isfile(cachefile):
+                    cachefile.write(data)
+                    cachefile.seek(0, 0)  # return to beginning of file
+                elif binary:
+                    with open(cachefile, 'wb') as f:
+                        f.write(data)
+                elif isinstance(data, (dict, list)):
+                    with open(cachefile, 'w', encoding='utf-8') as f:
+                        f.write(json.dumps(data))
+                else:
+                    with open(cachefile, 'w', encoding='utf-8') as f:
+                        f.write(data)
+
+
+        except (IOError, HttpLib2Error, HTTPException) as he:
+            #
+            # Overly complicated exception handling. This is due to a request
+            # from *Ops that it is very clear that data is is being returned
+            # from a cachefile, and that cachefiles can be good/stale/dead.
+            #
+            if force_refresh or isfile(cachefile) or not os.path.exists(cachefile):
+                msg = 'The cachefile %s does not exist and the service at %s'
+                msg = msg % (cachefile, self["requests"]['host'] + url)
+                if hasattr(he, 'status') and hasattr(he, 'reason'):
+                    msg += ' is unavailable - it returned %s because %s' % (he.status,
+                                                                            he.reason)
+                    if hasattr(he, 'result'):
+                        msg += ' with result: %s\n' % he.result
+                else:
+                    msg += ' raised a %s when accessed' % he.__repr__()
+                self['logger'].warning(msg)
+                raise he
+            else:
+                cache_dead = cache_expired(cachefile, delta=self["cacheduration"])
+                if cache_dead:
+                    msg = 'The cachefile %s is dead (older than %s hours of cache duration), '
+                    msg += 'and the service at %s '
+                    msg = msg % (cachefile, self["cacheduration"], url)
+                    if hasattr(he, 'status') and hasattr(he, 'reason'):
+                        msg += 'is unavailable - it returned %s because %s' % (he.status, he.reason)
+                    else:
+                        msg += 'raised a %s when accessed' % he.__repr__()
+                    self['logger'].warning(msg)
+                    raise he
+                if self.get('usestalecache', False):
+                    # then we can return data from the cache file, without raising an exception
+                    # but with a suitable message in the log
+                    msg = 'Returning stale cache data from %s, the service at ' % cachefile
+                    if hasattr(he, 'status') and hasattr(he, 'reason'):
+                        msg += '%s returned %s because %s' % (he.url, he.status, he.reason)
+                    else:
+                        msg += '%s raised a %s when accessed' % (url, he.__repr__())
+                    self['logger'].warning(msg)
+                else:
+                    # Cache is not dead, but Service is configured to not return stale data.
+                    msg = 'The cachefile %s is stale and the service at %s ' % (cachefile, url)
+                    if hasattr(he, 'status') and hasattr(he, 'reason'):
+                        msg += 'is unavailable - it returned %s because %s' % (he.status, he.reason)
+                    else:
+                        msg += 'raised a %s when accessed' % he.__repr__()
+                    self['logger'].warning(msg)
+                    raise he
+
+    def _verbCheck(self, verb='GET'):
+        if verb.upper() in self.supportVerbList:
+            return verb.upper()
+        elif self['method'].upper() in self.supportVerbList:
+            return self['method'].upper()
+        else:
+            raise TypeError('verb parameter needs to be set')