wmglobalqueue 2.4.5.1__py3-none-any.whl

WMCore/Services/Requests.py

@@ -0,0 +1,624 @@
+#!/usr/bin/env python
+# pylint: disable=C0103,R0913,W0102
+"""
+_Requests_
+
+A set of classes to handle making http and https requests to a remote server and
+deserialising the response.
+
+The response from the remote server is cached if expires/etags are set.
+Note that the cache can have two different behaviors:
+1. when WMCORE_CACHE_DIR is defined: it defines one specific path to be used for the
+   cache files. Cache files are never automatically cleaned up and it's up to the user/
+   maintainer to do so. Note that cache directories are named after the base class.
+2. otherwise, the system will use the Operating System temp dir to store the cache files,
+   which uses `/tmp` as default for Linux systems. When using these temporary areas, the
+   cache files are automatically cleaned up when the object using it is destroyed and
+   garbage collected. Cache directories carry a random name.
+
+By default, all of the WMCore central services define the CACHE_DIR (to use /data/srv/state).
+"""
+from __future__ import division, print_function
+
+from future import standard_library
+standard_library.install_aliases()
+
+from builtins import str, bytes, object
+from future.utils import viewvalues
+
+import base64
+import logging
+import os
+import shutil
+import socket
+import stat
+import tempfile
+import traceback
+import types
+
+from urllib.parse import urlparse, urlencode
+from io import BytesIO
+from http.client import HTTPException
+from json import JSONEncoder, JSONDecoder
+
+from Utils.CertTools import getKeyCertFromEnv, getCAPathFromEnv
+from Utils.Utilities import encodeUnicodeToBytes, decodeBytesToUnicode
+from Utils.PythonVersion import PY3
+from WMCore.Algorithms import Permissions
+from WMCore.Lexicon import sanitizeURL
+from WMCore.WMException import WMException
+from WMCore.Wrappers.JsonWrapper.JSONThunker import JSONThunker
+from Utils.PortForward import portForward
+
+try:
+    from WMCore.Services.pycurl_manager import RequestHandler, ResponseHeader
+except ImportError:
+    pass
+
+try:
+    from httplib2 import ServerNotFoundError
+except ImportError:
+    # Mock ServerNotFoundError since we don't want that WMCore depend on httplib2 using pycurl
+    class ServerNotFoundError(Exception):
+        pass
+
+
+def check_server_url(srvurl):
+    """Check given url for correctness"""
+    good_name = srvurl.startswith('http://') or srvurl.startswith('https://')
+    if not good_name:
+        msg = "You must include "
+        msg += "http(s):// in your server's address, %s doesn't" % srvurl
+        raise ValueError(msg)
+
+
+class Requests(dict):
+    """
+    Generic class for sending different types of HTTP Request to a given URL
+    """
+
+    @portForward(8443)
+    def __init__(self, url='http://localhost', idict=None):
+        """
+        url should really be host - TODO fix that when have sufficient code
+        coverage and change _getURLOpener if needed
+        """
+        if not idict:
+            idict = {}
+        dict.__init__(self, idict)
+        self.pycurl = idict.get('pycurl', True)
+        self.capath = idict.get('capath', None)
+        if self.pycurl:
+            self.reqmgr = RequestHandler()
+
+        # set up defaults
+        self.setdefault("accept_type", 'text/html')
+        self.setdefault("content_type", 'application/x-www-form-urlencoded')
+        self.additionalHeaders = {}
+
+        # check for basic auth early, as if found this changes the url
+        urlComponent = sanitizeURL(url)
+        if urlComponent['username'] is not None:
+            self.addBasicAuth(urlComponent['username'], urlComponent['password'])
+            # CouchDB 3.x requires user/passwd in the source/target of replication docs
+            # More info in: https://github.com/dmwm/WMCore/pull/11001
+            url = urlComponent['url']  # remove user, password from url
+
+        self.setdefault("host", url)
+
+        # then update with the incoming dict
+        self.update(idict)
+
+        self['endpoint_components'] = urlparse(self['host'])
+
+        # If cachepath = None disable caching
+        if 'cachepath' in idict and idict['cachepath'] is None:
+            self["req_cache_path"] = None
+        else:
+            cache_dir = (self.cachePath(idict.get('cachepath'), idict.get('service_name')))
+            self["cachepath"] = cache_dir
+            self["req_cache_path"] = os.path.join(cache_dir, '.cache')
+
+        # Try to resolve the key/cert pair path in the constructor:
+        # NOTE: Calling self.getKeyCert() assures that if the pair is already provided by
+        #       the caller (e.g. the higher level Service class) through the `idict' argument
+        #       during construction time, no additional os.environ calls would be made.
+        #       If the pair is not provided anywhere: not by the higher level class
+        #       nor from the environment, Exception will be raised early during construction
+        #       time rather than later during an internel method call.
+        self.setdefault("cert", None)
+        self.setdefault("key", None)
+        self['key'], self['cert'] = self.getKeyCert()
+
+        self.setdefault('capath', None)
+        self['capath'] = self.getCAPath()
+
+        self.setdefault("timeout", 300)
+        self.setdefault("logger", logging)
+
+        check_server_url(self['host'])
+
+    def get(self, uri=None, data={}, incoming_headers={},
+            encode=True, decode=True, contentType=None):
+        """
+        GET some data
+        """
+        return self.makeRequest(uri, data, 'GET', incoming_headers,
+                                encode, decode, contentType)
+
+    def post(self, uri=None, data={}, incoming_headers={},
+             encode=True, decode=True, contentType=None):
+        """
+        POST some data
+        """
+        return self.makeRequest(uri, data, 'POST', incoming_headers,
+                                encode, decode, contentType)
+
+    def put(self, uri=None, data={}, incoming_headers={},
+            encode=True, decode=True, contentType=None):
+        """
+        PUT some data
+        """
+        return self.makeRequest(uri, data, 'PUT', incoming_headers,
+                                encode, decode, contentType)
+
+    def delete(self, uri=None, data={}, incoming_headers={},
+               encode=True, decode=True, contentType=None):
+        """
+        DELETE some data
+        """
+        return self.makeRequest(uri, data, 'DELETE', incoming_headers,
+                                encode, decode, contentType)
+
+    def makeRequest(self, uri=None, data=None, verb='GET', incoming_headers=None,
+                    encoder=True, decoder=True, contentType=None):
+        """
+        Wrapper around request helper functions.
+        """
+        data = data or {}
+        incoming_headers = incoming_headers or {}
+        data, headers = self.encodeParams(data, verb, incoming_headers, encoder, contentType)
+
+        # both httpib2/pycurl require absolute url
+        uri = self['host'] + uri
+        if self.pycurl:
+            result, response = self.makeRequest_pycurl(uri, data, verb, headers)
+        else:
+            result, response = self.makeRequest_httplib(uri, data, verb, headers)
+
+        result = self.decodeResult(result, decoder)
+        return result, response.status, response.reason, response.fromcache
+
+    def makeRequest_pycurl(self, uri, data, verb, headers):
+        """
+        Make HTTP(s) request via pycurl library. Stay complaint with
+        makeRequest_httplib method.
+        """
+        ckey, cert = self.getKeyCert()
+        capath = self.getCAPath()
+
+        headers["Accept-Encoding"] = "gzip,deflate,identity"
+
+        response, result = self.reqmgr.request(uri, data, headers, verb=verb,
+                                               ckey=ckey, cert=cert, capath=capath)
+        return result, response
+
+    def makeRequest_httplib(self, uri, data, verb, headers):
+        """
+        Make a request to the remote database. for a give URI. The type of
+        request will determine the action take by the server (be careful with
+        DELETE!). Data should be a dictionary of {dataname: datavalue}.
+
+        Returns a tuple of the data from the server, decoded using the
+        appropriate method the response status and the response reason, to be
+        used in error handling.
+
+        You can override the method to encode/decode your data by passing in an
+        encoding/decoding function to this method. Your encoded data must end up
+        as a string.
+
+        """
+        if verb == 'GET' and data:
+            uri = "%s?%s" % (uri, data)
+
+        assert isinstance(data, (str, bytes)), \
+            "Data in makeRequest is %s and not encoded to a string" % type(data)
+
+        # And now overwrite any headers that have been passed into the call:
+        # WARNING: doesn't work with deplate so only accept gzip
+        headers["Accept-Encoding"] = "gzip,identity"
+
+        # httplib2 will allow sockets to close on remote end without retrying
+        # try to send request - if this fails try again - should then succeed
+        try:
+            conn = self._getURLOpener()
+            response, result = conn.request(uri, method=verb, body=data, headers=headers)
+            if response.status == 408:  # timeout can indicate a socket error
+                raise socket.error
+        except ServerNotFoundError as ex:
+            # DNS cannot resolve this domain name, let's call it 'Service Unavailable'
+            e = HTTPException()
+            setattr(e, 'url', uri)
+            setattr(e, 'status', 503)
+            setattr(e, 'reason', 'Service Unavailable')
+            setattr(e, 'result', str(ex))
+            raise e from None
+        except (socket.error, AttributeError):
+            self['logger'].warning("Http request failed, retrying once again..")
+            # AttributeError implies initial connection error - need to close
+            # & retry. httplib2 doesn't clear httplib state before next request
+            # if this is threaded this may spoil things
+            # only have one endpoint so don't need to determine which to shut
+            for con in viewvalues(conn.connections):
+                con.close()
+            conn = self._getURLOpener()
+            # ... try again... if this fails propagate error to client
+            try:
+                response, result = conn.request(uri, method=verb, body=data, headers=headers)
+            except AttributeError:
+                msg = 'Error contacting: {}: {}'.format(self.getDomainName(), traceback.format_exc())
+                # socket/httplib really screwed up - nuclear option
+                conn.connections = {}
+                raise socket.error(msg) from None
+        if response.status >= 400:
+            e = HTTPException()
+            setattr(e, 'req_data', data)
+            setattr(e, 'req_headers', headers)
+            setattr(e, 'url', uri)
+            setattr(e, 'result', result)
+            setattr(e, 'status', response.status)
+            setattr(e, 'reason', response.reason)
+            setattr(e, 'headers', response)
+            raise e
+
+        return result, response
+
+    def encodeParams(self, data, verb, incomingHeaders, encoder, contentType):
+        """
+        Encode request parameters for usage with the 4 verbs.
+        Assume params is already encoded if it is a string and
+        uses a different encoding depending on the HTTP verb
+        (either json.dumps or urllib.urlencode)
+        """
+        # TODO: User agent should be:
+        # $client/$client_version (CMS)
+        # $http_lib/$http_lib_version $os/$os_version ($arch)
+        headers = {"Content-type": contentType if contentType else self['content_type'],
+                   "User-Agent": "WMCore.Services.Requests/v002",
+                   "Accept": self['accept_type']}
+
+        for key in self.additionalHeaders:
+            headers[key] = self.additionalHeaders[key]
+        # And now overwrite any headers that have been passed into the call:
+        # WARNING: doesn't work with deplate so only accept gzip
+        incomingHeaders["Accept-Encoding"] = "gzip,identity"
+        headers.update(incomingHeaders)
+
+        # If you're posting an attachment, the data might not be a dict
+        #   please test against ConfigCache_t if you're unsure.
+        # assert type(data) == type({}), \
+        #        "makeRequest input data must be a dict (key/value pairs)"
+        encoded_data = ''
+        if verb != 'GET' and data:
+            if isinstance(encoder, (types.MethodType, types.FunctionType)):
+                encoded_data = encoder(data)
+            elif encoder is False:
+                # Don't encode the data more than we have to
+                #  we don't want to URL encode the data blindly,
+                #  that breaks POSTing attachments... ConfigCache_t
+                # encoded_data = urllib.urlencode(data)
+                #  -- Andrew Melo 25/7/09
+                encoded_data = data
+            else:
+                # Either the encoder is set to True or it's junk, so use
+                # self.encode
+                encoded_data = self.encode(data)
+            headers["Content-Length"] = len(encoded_data)
+        elif verb != 'GET':
+            # delete requests might not have any body
+            headers["Content-Length"] = 0
+        elif verb == 'GET' and data:
+            # encode the data as a get string
+            encoded_data = urlencode(data, doseq=True)
+
+        return encoded_data, headers
+
+    def decodeResult(self, result, decoder):
+        """
+        Decode the http/pycurl request result
+        NOTE: if decoder is provided with a False value, then it means no
+        decoding is applied on the results at all
+        """
+        if isinstance(decoder, (types.MethodType, types.FunctionType)):
+            result = decoder(result)
+        elif decoder is not False:
+            result = self.decode(result)
+        return result
+
+    def encode(self, data):
+        """
+        encode data into some appropriate format, for now make it a string...
+        """
+        return urlencode(data, doseq=True)
+
+    def decode(self, data):
+        """
+        decode data to some appropriate format, for now make it a string...
+        """
+        if PY3:
+            return decodeBytesToUnicode(data)
+        return data.__str__()
+
+    def cachePath(self, given_path, service_name):
+        """Return cache location"""
+        if not service_name:
+            service_name = 'REQUESTS'
+        top = self.cacheTopPath(given_path, service_name)
+
+        # deal with multiple Services that have the same service running and
+        # with multiple users for a given Service
+        if self.getUserName() is None:
+            cachepath = os.path.join(top, self['endpoint_components'].netloc)
+        else:
+            cachepath = os.path.join(top, '%s-%s' % (self.getUserName(), self.getDomainName()))
+
+        try:
+            # only we should be able to write to this dir
+            os.makedirs(cachepath, stat.S_IRWXU)
+        except OSError:
+            if not os.path.isdir(cachepath):
+                raise
+            Permissions.owner_readwriteexec(cachepath)
+
+        return cachepath
+
+    def cacheTopPath(self, given_path, service_name):
+        """Where to cache results?
+
+        Logic:
+          o If passed in take that
+          o Is the environment variable "SERVICE_NAME"_CACHE_DIR defined?
+          o Is WMCORE_CACHE_DIR set
+          o Generate a temporary directory
+          """
+        if given_path:
+            return given_path
+        user = str(os.getuid())
+        # append user id so users don't clobber each other
+        lastbit = os.path.join('.wmcore_cache_%s' % user, service_name.lower())
+        for var in ('%s_CACHE_DIR' % service_name.upper(),
+                    'WMCORE_CACHE_DIR'):
+            if os.environ.get(var):
+                firstbit = os.environ[var]
+                break
+        else:
+            idir = tempfile.mkdtemp(prefix='.wmcore_cache_')
+            # Alan Malta in 29 Mar 2022: this seems to prematurely remove the cache
+            # directory. For details, see: https://github.com/dmwm/WMCore/pull/10915
+            self['deleteCacheOnExit'] = TempDirectory(idir)
+            return idir
+
+        return os.path.join(firstbit, lastbit)
+
+    def getDomainName(self):
+        """Parse netloc info to get hostname"""
+        return self['endpoint_components'].hostname
+
+    def getUserName(self):
+        """Parse netloc to get user"""
+        return self['endpoint_components'].username
+
+    def _getURLOpener(self):
+        """
+        method getting a secure (HTTPS) connection
+        """
+        import httplib2
+        key, cert = None, None
+        if self['endpoint_components'].scheme == 'https':
+            # only add certs to https requests
+            # if we have a key/cert add to request,
+            # if not proceed as not all https connections require them
+            try:
+                key, cert = self.getKeyCert()
+            except Exception as ex:
+                msg = 'No certificate or key found, authentication may fail'
+                self['logger'].info(msg)
+                self['logger'].debug(str(ex))
+
+        try:
+            # disable validation as we don't have a single PEM with all ca's
+            http = httplib2.Http(self['req_cache_path'], self['timeout'],
+                                 disable_ssl_certificate_validation=True)
+        except TypeError:
+            # old httplib2 versions disable validation by default
+            http = httplib2.Http(self['req_cache_path'], self['timeout'])
+
+        # Domain must be just a hostname and port. self[host] is a URL currently
+        if key or cert:
+            http.add_certificate(key=key, cert=cert, domain='')
+        return http
+
+    def addBasicAuth(self, username, password):
+        """Add basic auth headers to request"""
+        username = encodeUnicodeToBytes(username)
+        password = encodeUnicodeToBytes(password)
+        encodedauth = base64.encodebytes(b'%s:%s' % (username, password)).strip()
+        if PY3:
+            encodedauth = decodeBytesToUnicode(encodedauth)
+        auth_string = "Basic %s" % encodedauth
+        self.additionalHeaders["Authorization"] = auth_string
+
+    def getKeyCert(self):
+        """
+       _getKeyCert_
+
+       Get the user credentials if they exist, otherwise throw an exception.
+       This code was modified from DBSAPI/dbsHttpService.py
+        """
+
+        # Zeroth case is if the class has over ridden the key/cert and has it
+        # stored in self
+        if self['cert'] and self['key']:
+            key = self['key']
+            cert = self['cert']
+        else:
+            key, cert = getKeyCertFromEnv()
+
+        # Set but not found
+        if key is None or cert is None:
+            raise WMException('Request requires a host certificate and key',
+                              "WMCORE-11")
+
+        # All looks OK, still doesn't guarantee proxy's validity etc.
+        return key, cert
+
+    def getCAPath(self):
+        """
+        _getCAPath_
+
+        Return the path of the CA certificates. The check is loose in the pycurl_manager:
+        is capath == None then the server identity is not verified. To enable this check
+        you need to set either the X509_CERT_DIR variable or the cacert key of the request.
+        """
+        capath = self['capath']
+        if not capath:
+            capath = getCAPathFromEnv()
+        return capath
+
+    def uploadFile(self, fileName, url, fieldName='file1', params=[], verb='POST'):
+        """
+        Upload a file with curl streaming it directly from disk
+
+        :rtype: bytes (both py2 and py3)
+        """
+        ckey, cert = self.getKeyCert()
+        capath = self.getCAPath()
+        import pycurl
+        c = pycurl.Curl()
+        if verb == 'POST':
+            c.setopt(c.POST, 1)
+        elif verb == 'PUT':
+            c.setopt(pycurl.CUSTOMREQUEST, 'PUT')
+        else:
+            raise HTTPException("Verb %s not supported for upload." % verb)
+        c.setopt(c.URL, url)
+        fullParams = [(fieldName, (c.FORM_FILE, fileName))]
+        fullParams.extend(params)
+        c.setopt(c.HTTPPOST, fullParams)
+        bbuf = BytesIO()
+        hbuf = BytesIO()
+        c.setopt(pycurl.WRITEFUNCTION, bbuf.write)
+        c.setopt(pycurl.HEADERFUNCTION, hbuf.write)
+        if capath:
+            c.setopt(pycurl.CAPATH, capath)
+            c.setopt(pycurl.SSL_VERIFYPEER, True)
+        else:
+            c.setopt(pycurl.SSL_VERIFYPEER, False)
+        if ckey:
+            c.setopt(pycurl.SSLKEY, ckey)
+        if cert:
+            c.setopt(pycurl.SSLCERT, cert)
+        c.perform()
+        hres = hbuf.getvalue()
+        bres = bbuf.getvalue()
+        rh = ResponseHeader(hres)
+        c.close()
+        if rh.status < 200 or rh.status >= 300:
+            exc = HTTPException(bres)
+            setattr(exc, 'req_data', fullParams)
+            setattr(exc, 'url', url)
+            setattr(exc, 'result', bres)
+            setattr(exc, 'status', rh.status)
+            setattr(exc, 'reason', rh.reason)
+            setattr(exc, 'headers', rh.header)
+            raise exc
+
+        return bres
+
+    def downloadFile(self, fileName, url):
+        """
+        Download a file with curl streaming it directly to disk
+        """
+        ckey, cert = self.getKeyCert()
+        capath = self.getCAPath()
+        import pycurl
+
+        hbuf = BytesIO()
+
+        with open(fileName, "wb") as fp:
+            curl = pycurl.Curl()
+            curl.setopt(pycurl.URL, url)
+            curl.setopt(pycurl.WRITEDATA, fp)
+            curl.setopt(pycurl.HEADERFUNCTION, hbuf.write)
+            if capath:
+                curl.setopt(pycurl.CAPATH, capath)
+                curl.setopt(pycurl.SSL_VERIFYPEER, True)
+            else:
+                curl.setopt(pycurl.SSL_VERIFYPEER, False)
+            if ckey:
+                curl.setopt(pycurl.SSLKEY, ckey)
+            if cert:
+                curl.setopt(pycurl.SSLCERT, cert)
+            curl.setopt(pycurl.FOLLOWLOCATION, 1)
+            curl.perform()
+            curl.close()
+
+            header = ResponseHeader(hbuf.getvalue())
+            if header.status < 200 or header.status >= 300:
+                raise RuntimeError('Reading %s failed with code %s' % (url, header.status))
+        return fileName, header
+
+
+class JSONRequests(Requests):
+    """
+    Example implementation of Requests that encodes data to/from JSON.
+    """
+
+    def __init__(self, url='http://localhost:8080', idict={}):
+        Requests.__init__(self, url, idict)
+        self['accept_type'] = "application/json"
+        self['content_type'] = "application/json"
+
+    def encode(self, data):
+        """
+        encode data as json
+        """
+        encoder = JSONEncoder()
+        thunker = JSONThunker()
+        thunked = thunker.thunk(data)
+        return encoder.encode(thunked)
+
+    def decode(self, data):
+        """
+        decode the data to python from json
+        """
+        if data:
+            decoder = JSONDecoder()
+            thunker = JSONThunker()
+            if PY3:
+                data = decodeBytesToUnicode(data)
+            data = decoder.decode(data)
+            unthunked = thunker.unthunk(data)
+            return unthunked
+        return {}
+
+
+class TempDirectory(object):
+    """
+    Directory that cleans up after itself
+
+    Except this doesn't work, python __del__ is NOT a destructor
+
+    Leaving it anyways, since it might work sometimes
+
+    """
+
+    def __init__(self, idir):
+        self.dir = idir
+
+    def __del__(self):
+        try:
+            # it'll likely fail, but give it a try
+            shutil.rmtree(self.dir, ignore_errors=True)
+        except Exception:
+            pass