crate 1.0.0__py3-none-any.whl → 1.0.0.dev1__py3-none-any.whl

crate/client/http.py

@@ -30,11 +30,11 @@ import re
 import socket
 import ssl
 import threading
+from urllib.parse import urlparse
 from base64 import b64encode
-from datetime import date, datetime, timezone
-from decimal import Decimal
 from time import time
-from urllib.parse import urlparse
+from datetime import datetime, date, timezone
+from decimal import Decimal
 from uuid import UUID
 
 import urllib3
@@ -52,41 +52,42 @@ from urllib3.util.retry import Retry
 from verlib2 import Version
 
 from crate.client.exceptions import (
-    BlobLocationNotFoundException,
     ConnectionError,
+    BlobLocationNotFoundException,
     DigestNotFoundException,
-    IntegrityError,
     ProgrammingError,
+    IntegrityError,
 )
 
+
 logger = logging.getLogger(__name__)
 
 
-_HTTP_PAT = pat = re.compile("https?://.+", re.I)
-SRV_UNAVAILABLE_STATUSES = {502, 503, 504, 509}
-PRESERVE_ACTIVE_SERVER_EXCEPTIONS = {ConnectionResetError, BrokenPipeError}
-SSL_ONLY_ARGS = {"ca_certs", "cert_reqs", "cert_file", "key_file"}
+_HTTP_PAT = pat = re.compile('https?://.+', re.I)
+SRV_UNAVAILABLE_STATUSES = set((502, 503, 504, 509))
+PRESERVE_ACTIVE_SERVER_EXCEPTIONS = set((ConnectionResetError, BrokenPipeError))
+SSL_ONLY_ARGS = set(('ca_certs', 'cert_reqs', 'cert_file', 'key_file'))
 
 
 def super_len(o):
-    if hasattr(o, "__len__"):
+    if hasattr(o, '__len__'):
         return len(o)
-    if hasattr(o, "len"):
+    if hasattr(o, 'len'):
         return o.len
-    if hasattr(o, "fileno"):
+    if hasattr(o, 'fileno'):
         try:
             fileno = o.fileno()
         except io.UnsupportedOperation:
             pass
         else:
             return os.fstat(fileno).st_size
-    if hasattr(o, "getvalue"):
+    if hasattr(o, 'getvalue'):
         # e.g. BytesIO, cStringIO.StringI
         return len(o.getvalue())
-    return None
 
 
 class CrateJsonEncoder(json.JSONEncoder):
+
     epoch_aware = datetime(1970, 1, 1, tzinfo=timezone.utc)
     epoch_naive = datetime(1970, 1, 1)
 
@@ -98,22 +99,21 @@ class CrateJsonEncoder(json.JSONEncoder):
                 delta = o - self.epoch_aware
             else:
                 delta = o - self.epoch_naive
-            return int(
-                delta.microseconds / 1000.0
-                + (delta.seconds + delta.days * 24 * 3600) * 1000.0
-            )
+            return int(delta.microseconds / 1000.0 +
+                       (delta.seconds + delta.days * 24 * 3600) * 1000.0)
         if isinstance(o, date):
             return calendar.timegm(o.timetuple()) * 1000
         return json.JSONEncoder.default(self, o)
 
 
-class Server:
+class Server(object):
+
     def __init__(self, server, **pool_kw):
         socket_options = _get_socket_opts(
-            pool_kw.pop("socket_keepalive", False),
-            pool_kw.pop("socket_tcp_keepidle", None),
-            pool_kw.pop("socket_tcp_keepintvl", None),
-            pool_kw.pop("socket_tcp_keepcnt", None),
+            pool_kw.pop('socket_keepalive', False),
+            pool_kw.pop('socket_tcp_keepidle', None),
+            pool_kw.pop('socket_tcp_keepintvl', None),
+            pool_kw.pop('socket_tcp_keepcnt', None),
         )
         self.pool = connection_from_url(
             server,
@@ -121,57 +121,53 @@ class Server:
             **pool_kw,
         )
 
-    def request(
-        self,
-        method,
-        path,
-        data=None,
-        stream=False,
-        headers=None,
-        username=None,
-        password=None,
-        schema=None,
-        backoff_factor=0,
-        **kwargs,
-    ):
+    def request(self,
+                method,
+                path,
+                data=None,
+                stream=False,
+                headers=None,
+                username=None,
+                password=None,
+                schema=None,
+                backoff_factor=0,
+                **kwargs):
         """Send a request
 
         Always set the Content-Length and the Content-Type header.
         """
         if headers is None:
             headers = {}
-        if "Content-Length" not in headers:
+        if 'Content-Length' not in headers:
             length = super_len(data)
             if length is not None:
-                headers["Content-Length"] = length
+                headers['Content-Length'] = length
 
         # Authentication credentials
         if username is not None:
-            if "Authorization" not in headers and username is not None:
-                credentials = username + ":"
+            if 'Authorization' not in headers and username is not None:
+                credentials = username + ':'
                 if password is not None:
                     credentials += password
-                headers["Authorization"] = "Basic %s" % b64encode(
-                    credentials.encode("utf-8")
-                ).decode("utf-8")
+                headers['Authorization'] = 'Basic %s' % b64encode(credentials.encode('utf-8')).decode('utf-8')
             # For backwards compatibility with Crate <= 2.2
-            if "X-User" not in headers:
-                headers["X-User"] = username
+            if 'X-User' not in headers:
+                headers['X-User'] = username
 
         if schema is not None:
-            headers["Default-Schema"] = schema
-        headers["Accept"] = "application/json"
-        headers["Content-Type"] = "application/json"
-        kwargs["assert_same_host"] = False
-        kwargs["redirect"] = False
-        kwargs["retries"] = Retry(read=0, backoff_factor=backoff_factor)
+            headers['Default-Schema'] = schema
+        headers['Accept'] = 'application/json'
+        headers['Content-Type'] = 'application/json'
+        kwargs['assert_same_host'] = False
+        kwargs['redirect'] = False
+        kwargs['retries'] = Retry(read=0, backoff_factor=backoff_factor)
         return self.pool.urlopen(
             method,
             path,
             body=data,
             preload_content=not stream,
             headers=headers,
-            **kwargs,
+            **kwargs
         )
 
     def close(self):
@@ -180,27 +176,24 @@ class Server:
 
 def _json_from_response(response):
     try:
-        return json.loads(response.data.decode("utf-8"))
-    except ValueError as ex:
+        return json.loads(response.data.decode('utf-8'))
+    except ValueError:
         raise ProgrammingError(
-            "Invalid server response of content-type '{}':\n{}".format(
-                response.headers.get("content-type", "unknown"),
-                response.data.decode("utf-8"),
-            )
-        ) from ex
+            "Invalid server response of content-type '{}':\n{}"
+            .format(response.headers.get("content-type", "unknown"), response.data.decode('utf-8')))
 
 
 def _blob_path(table, digest):
-    return "/_blobs/{table}/{digest}".format(table=table, digest=digest)
+    return '/_blobs/{table}/{digest}'.format(table=table, digest=digest)
 
 
 def _ex_to_message(ex):
-    return getattr(ex, "message", None) or str(ex) or repr(ex)
+    return getattr(ex, 'message', None) or str(ex) or repr(ex)
 
 
 def _raise_for_status(response):
     """
-    Raise `IntegrityError` exceptions for `DuplicateKeyException` errors.
+    Properly raise `IntegrityError` exceptions for CrateDB's `DuplicateKeyException` errors.
     """
     try:
         return _raise_for_status_real(response)
@@ -211,33 +204,29 @@ def _raise_for_status(response):
 
 
 def _raise_for_status_real(response):
-    """make sure that only crate.exceptions are raised that are defined in
-    the DB-API specification"""
-    message = ""
+    """ make sure that only crate.exceptions are raised that are defined in
+    the DB-API specification """
+    message = ''
     if 400 <= response.status < 500:
-        message = "%s Client Error: %s" % (response.status, response.reason)
+        message = '%s Client Error: %s' % (response.status, response.reason)
     elif 500 <= response.status < 600:
-        message = "%s Server Error: %s" % (response.status, response.reason)
+        message = '%s Server Error: %s' % (response.status, response.reason)
     else:
         return
     if response.status == 503:
         raise ConnectionError(message)
     if response.headers.get("content-type", "").startswith("application/json"):
-        data = json.loads(response.data.decode("utf-8"))
-        error = data.get("error", {})
-        error_trace = data.get("error_trace", None)
+        data = json.loads(response.data.decode('utf-8'))
+        error = data.get('error', {})
+        error_trace = data.get('error_trace', None)
         if "results" in data:
-            errors = [
-                res["error_message"]
-                for res in data["results"]
-                if res.get("error_message")
-            ]
+            errors = [res["error_message"] for res in data["results"]
+                      if res.get("error_message")]
             if errors:
                 raise ProgrammingError("\n".join(errors))
         if isinstance(error, dict):
-            raise ProgrammingError(
-                error.get("message", ""), error_trace=error_trace
-            )
+            raise ProgrammingError(error.get('message', ''),
+                                   error_trace=error_trace)
         raise ProgrammingError(error, error_trace=error_trace)
     raise ProgrammingError(message)
 
@@ -258,9 +247,9 @@ def _server_url(server):
     http://demo.crate.io
     """
     if not _HTTP_PAT.match(server):
-        server = "http://%s" % server
+        server = 'http://%s' % server
     parsed = urlparse(server)
-    url = "%s://%s" % (parsed.scheme, parsed.netloc)
+    url = '%s://%s' % (parsed.scheme, parsed.netloc)
     return url
 
 
@@ -270,36 +259,30 @@ def _to_server_list(servers):
     return [_server_url(s) for s in servers]
 
 
-def _pool_kw_args(
-    verify_ssl_cert,
-    ca_cert,
-    client_cert,
-    client_key,
-    timeout=None,
-    pool_size=None,
-):
-    ca_cert = ca_cert or os.environ.get("REQUESTS_CA_BUNDLE", None)
+def _pool_kw_args(verify_ssl_cert, ca_cert, client_cert, client_key,
+                  timeout=None, pool_size=None):
+    ca_cert = ca_cert or os.environ.get('REQUESTS_CA_BUNDLE', None)
     if ca_cert and not os.path.exists(ca_cert):
         # Sanity check
         raise IOError('CA bundle file "{}" does not exist.'.format(ca_cert))
 
     kw = {
-        "ca_certs": ca_cert,
-        "cert_reqs": ssl.CERT_REQUIRED if verify_ssl_cert else ssl.CERT_NONE,
-        "cert_file": client_cert,
-        "key_file": client_key,
+        'ca_certs': ca_cert,
+        'cert_reqs': ssl.CERT_REQUIRED if verify_ssl_cert else ssl.CERT_NONE,
+        'cert_file': client_cert,
+        'key_file': client_key,
     }
     if timeout is not None:
         if isinstance(timeout, str):
             timeout = float(timeout)
-        kw["timeout"] = timeout
+        kw['timeout'] = timeout
     if pool_size is not None:
-        kw["maxsize"] = int(pool_size)
+        kw['maxsize'] = int(pool_size)
     return kw
 
 
 def _remove_certs_for_non_https(server, kwargs):
-    if server.lower().startswith("https"):
+    if server.lower().startswith('https'):
         return kwargs
     used_ssl_args = SSL_ONLY_ARGS & set(kwargs.keys())
     if used_ssl_args:
@@ -317,7 +300,6 @@ def _update_pool_kwargs_for_ssl_minimum_version(server, kwargs):
     """
     if Version(urllib3.__version__) >= Version("2"):
         from urllib3.util import parse_url
-
         scheme, _, host, port, *_ = parse_url(server)
         if scheme == "https":
             kwargs["ssl_minimum_version"] = ssl.TLSVersion.MINIMUM_SUPPORTED
@@ -325,21 +307,24 @@ def _update_pool_kwargs_for_ssl_minimum_version(server, kwargs):
 
 def _create_sql_payload(stmt, args, bulk_args):
     if not isinstance(stmt, str):
-        raise ValueError("stmt is not a string")
+        raise ValueError('stmt is not a string')
     if args and bulk_args:
-        raise ValueError("Cannot provide both: args and bulk_args")
+        raise ValueError('Cannot provide both: args and bulk_args')
 
-    data = {"stmt": stmt}
+    data = {
+        'stmt': stmt
+    }
     if args:
-        data["args"] = args
+        data['args'] = args
     if bulk_args:
-        data["bulk_args"] = bulk_args
+        data['bulk_args'] = bulk_args
     return json.dumps(data, cls=CrateJsonEncoder)
 
 
-def _get_socket_opts(
-    keepalive=True, tcp_keepidle=None, tcp_keepintvl=None, tcp_keepcnt=None
-):
+def _get_socket_opts(keepalive=True,
+                     tcp_keepidle=None,
+                     tcp_keepintvl=None,
+                     tcp_keepcnt=None):
     """
     Return an optional list of socket options for urllib3's HTTPConnection
     constructor.
@@ -352,23 +337,23 @@ def _get_socket_opts(
 
     # hasattr check because some options depend on system capabilities
     # see https://docs.python.org/3/library/socket.html#socket.SOMAXCONN
-    if hasattr(socket, "TCP_KEEPIDLE") and tcp_keepidle is not None:
+    if hasattr(socket, 'TCP_KEEPIDLE') and tcp_keepidle is not None:
         opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, tcp_keepidle))
-    if hasattr(socket, "TCP_KEEPINTVL") and tcp_keepintvl is not None:
+    if hasattr(socket, 'TCP_KEEPINTVL') and tcp_keepintvl is not None:
         opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, tcp_keepintvl))
-    if hasattr(socket, "TCP_KEEPCNT") and tcp_keepcnt is not None:
+    if hasattr(socket, 'TCP_KEEPCNT') and tcp_keepcnt is not None:
         opts.append((socket.IPPROTO_TCP, socket.TCP_KEEPCNT, tcp_keepcnt))
 
     # additionally use urllib3's default socket options
-    return list(HTTPConnection.default_socket_options) + opts
+    return HTTPConnection.default_socket_options + opts
 
 
-class Client:
+class Client(object):
     """
     Crate connection client using CrateDB's HTTP API.
     """
 
-    SQL_PATH = "/_sql?types=true"
+    SQL_PATH = '/_sql?types=true'
     """Crate URI path for issuing SQL statements."""
 
     retry_interval = 30
@@ -377,26 +362,25 @@ class Client:
     default_server = "http://127.0.0.1:4200"
     """Default server to use if no servers are given on instantiation."""
 
-    def __init__(
-        self,
-        servers=None,
-        timeout=None,
-        backoff_factor=0,
-        verify_ssl_cert=True,
-        ca_cert=None,
-        error_trace=False,
-        cert_file=None,
-        key_file=None,
-        ssl_relax_minimum_version=False,
-        username=None,
-        password=None,
-        schema=None,
-        pool_size=None,
-        socket_keepalive=True,
-        socket_tcp_keepidle=None,
-        socket_tcp_keepintvl=None,
-        socket_tcp_keepcnt=None,
-    ):
+    def __init__(self,
+                 servers=None,
+                 timeout=None,
+                 backoff_factor=0,
+                 verify_ssl_cert=True,
+                 ca_cert=None,
+                 error_trace=False,
+                 cert_file=None,
+                 key_file=None,
+                 ssl_relax_minimum_version=False,
+                 username=None,
+                 password=None,
+                 schema=None,
+                 pool_size=None,
+                 socket_keepalive=True,
+                 socket_tcp_keepidle=None,
+                 socket_tcp_keepintvl=None,
+                 socket_tcp_keepcnt=None,
+                 ):
         if not servers:
             servers = [self.default_server]
         else:
@@ -412,30 +396,22 @@ class Client:
                 if url.password is not None:
                     password = url.password
             except Exception as ex:
-                logger.warning(
-                    "Unable to decode credentials from database "
-                    "URI, so connecting to CrateDB without "
-                    "authentication: {ex}".format(ex=ex)
-                )
+                logger.warning("Unable to decode credentials from database "
+                               "URI, so connecting to CrateDB without "
+                               "authentication: {ex}"
+                               .format(ex=ex))
 
         self._active_servers = servers
         self._inactive_servers = []
         pool_kw = _pool_kw_args(
-            verify_ssl_cert,
-            ca_cert,
-            cert_file,
-            key_file,
-            timeout,
-            pool_size,
-        )
-        pool_kw.update(
-            {
-                "socket_keepalive": socket_keepalive,
-                "socket_tcp_keepidle": socket_tcp_keepidle,
-                "socket_tcp_keepintvl": socket_tcp_keepintvl,
-                "socket_tcp_keepcnt": socket_tcp_keepcnt,
-            }
+            verify_ssl_cert, ca_cert, cert_file, key_file, timeout, pool_size,
         )
+        pool_kw.update({
+            'socket_keepalive': socket_keepalive,
+            'socket_tcp_keepidle': socket_tcp_keepidle,
+            'socket_tcp_keepintvl': socket_tcp_keepintvl,
+            'socket_tcp_keepcnt': socket_tcp_keepcnt,
+        })
         self.ssl_relax_minimum_version = ssl_relax_minimum_version
         self.backoff_factor = backoff_factor
         self.server_pool = {}
@@ -449,7 +425,7 @@ class Client:
 
         self.path = self.SQL_PATH
         if error_trace:
-            self.path += "&error_trace=true"
+            self.path += '&error_trace=true'
 
     def close(self):
         for server in self.server_pool.values():
@@ -457,9 +433,8 @@ class Client:
 
     def _create_server(self, server, **pool_kw):
         kwargs = _remove_certs_for_non_https(server, pool_kw)
-        # After updating to urllib3 v2, optionally retain support
-        # for TLS 1.0 and TLS 1.1, in order to support connectivity
-        # to older versions of CrateDB.
+        # After updating to urllib3 v2, optionally retain support for TLS 1.0 and TLS 1.1,
+        # in order to support connectivity to older versions of CrateDB.
         if self.ssl_relax_minimum_version:
             _update_pool_kwargs_for_ssl_minimum_version(server, kwargs)
         self.server_pool[server] = Server(server, **kwargs)
@@ -476,26 +451,28 @@ class Client:
             return None
 
         data = _create_sql_payload(stmt, parameters, bulk_parameters)
-        logger.debug("Sending request to %s with payload: %s", self.path, data)
-        content = self._json_request("POST", self.path, data=data)
+        logger.debug(
+            'Sending request to %s with payload: %s', self.path, data)
+        content = self._json_request('POST', self.path, data=data)
         logger.debug("JSON response for stmt(%s): %s", stmt, content)
 
         return content
 
     def server_infos(self, server):
-        response = self._request("GET", "/", server=server)
+        response = self._request('GET', '/', server=server)
         _raise_for_status(response)
         content = _json_from_response(response)
         node_name = content.get("name")
-        node_version = content.get("version", {}).get("number", "0.0.0")
+        node_version = content.get('version', {}).get('number', '0.0.0')
         return server, node_name, node_version
 
-    def blob_put(self, table, digest, data) -> bool:
+    def blob_put(self, table, digest, data):
         """
         Stores the contents of the file like @data object in a blob under the
         given table and digest.
         """
-        response = self._request("PUT", _blob_path(table, digest), data=data)
+        response = self._request('PUT', _blob_path(table, digest),
+                                 data=data)
         if response.status == 201:
             # blob created
             return True
@@ -505,43 +482,40 @@ class Client:
         if response.status in (400, 404):
             raise BlobLocationNotFoundException(table, digest)
         _raise_for_status(response)
-        return False
 
-    def blob_del(self, table, digest) -> bool:
+    def blob_del(self, table, digest):
         """
         Deletes the blob with given digest under the given table.
         """
-        response = self._request("DELETE", _blob_path(table, digest))
+        response = self._request('DELETE', _blob_path(table, digest))
         if response.status == 204:
             return True
         if response.status == 404:
             return False
         _raise_for_status(response)
-        return False
 
     def blob_get(self, table, digest, chunk_size=1024 * 128):
         """
         Returns a file like object representing the contents of the blob
         with the given digest.
         """
-        response = self._request("GET", _blob_path(table, digest), stream=True)
+        response = self._request('GET', _blob_path(table, digest), stream=True)
         if response.status == 404:
             raise DigestNotFoundException(table, digest)
         _raise_for_status(response)
         return response.stream(amt=chunk_size)
 
-    def blob_exists(self, table, digest) -> bool:
+    def blob_exists(self, table, digest):
         """
         Returns true if the blob with the given digest exists
         under the given table.
         """
-        response = self._request("HEAD", _blob_path(table, digest))
+        response = self._request('HEAD', _blob_path(table, digest))
         if response.status == 200:
             return True
         elif response.status == 404:
             return False
         _raise_for_status(response)
-        return False
 
     def _add_server(self, server):
         with self._lock:
@@ -563,45 +537,42 @@ class Client:
                     password=self.password,
                     backoff_factor=self.backoff_factor,
                     schema=self.schema,
-                    **kwargs,
+                    **kwargs
                 )
                 redirect_location = response.get_redirect_location()
                 if redirect_location and 300 <= response.status <= 308:
                     redirect_server = _server_url(redirect_location)
                     self._add_server(redirect_server)
                     return self._request(
-                        method, path, server=redirect_server, **kwargs
-                    )
+                        method, path, server=redirect_server, **kwargs)
                 if not server and response.status in SRV_UNAVAILABLE_STATUSES:
                     with self._lock:
                         # drop server from active ones
                         self._drop_server(next_server, response.reason)
                 else:
                     return response
-            except (
-                MaxRetryError,
-                ReadTimeoutError,
-                SSLError,
-                HTTPError,
-                ProxyError,
-            ) as ex:
+            except (MaxRetryError,
+                    ReadTimeoutError,
+                    SSLError,
+                    HTTPError,
+                    ProxyError,) as ex:
                 ex_message = _ex_to_message(ex)
                 if server:
                     raise ConnectionError(
                         "Server not available, exception: %s" % ex_message
-                    ) from ex
+                    )
                 preserve_server = False
                 if isinstance(ex, ProtocolError):
                     preserve_server = any(
                         t in [type(arg) for arg in ex.args]
                         for t in PRESERVE_ACTIVE_SERVER_EXCEPTIONS
                     )
-                if not preserve_server:
+                if (not preserve_server):
                     with self._lock:
                         # drop server from active ones
                         self._drop_server(next_server, ex_message)
             except Exception as e:
-                raise ProgrammingError(_ex_to_message(e)) from e
+                raise ProgrammingError(_ex_to_message(e))
 
     def _json_request(self, method, path, data):
         """
@@ -621,7 +592,7 @@ class Client:
         """
         with self._lock:
             inactive_server_count = len(self._inactive_servers)
-            for _ in range(inactive_server_count):
+            for i in range(inactive_server_count):
                 try:
                     ts, server, message = heapq.heappop(self._inactive_servers)
                 except IndexError:
@@ -629,14 +600,12 @@ class Client:
                 else:
                     if (ts + self.retry_interval) > time():
                         # Not yet, put it back
-                        heapq.heappush(
-                            self._inactive_servers, (ts, server, message)
-                        )
+                        heapq.heappush(self._inactive_servers,
+                                       (ts, server, message))
                     else:
                         self._active_servers.append(server)
-                        logger.warning(
-                            "Restored server %s into active pool", server
-                        )
+                        logger.warning("Restored server %s into active pool",
+                                       server)
 
             # if none is old enough, use oldest
             if not self._active_servers:
@@ -670,9 +639,8 @@ class Client:
         # if this is the last server raise exception, otherwise try next
         if not self._active_servers:
             raise ConnectionError(
-                ("No more Servers available, " "exception from last server: %s")
-                % message
-            )
+                ("No more Servers available, "
+                 "exception from last server: %s") % message)
 
     def _roundrobin(self):
         """
@@ -681,4 +649,4 @@ class Client:
         self._active_servers.append(self._active_servers.pop(0))
 
     def __repr__(self):
-        return "<Client {0}>".format(str(self._active_servers))
+        return '<Client {0}>'.format(str(self._active_servers))