gate-io-api 0.0.65__py3-none-any.whl → 0.0.100__py3-none-any.whl

gate/ccxt/async_support/base/ws/client.py

@@ -1,21 +1,31 @@
 # -*- coding: utf-8 -*-
 
-from asyncio import sleep, ensure_future, wait_for, TimeoutError
-from .functions import milliseconds, iso8601, deep_extend
-from ccxt import NetworkError, RequestTimeout, NotSupported
+orjson = None
+try:
+    import orjson as orjson
+except ImportError:
+    pass
+
+import json
+
+from asyncio import sleep, ensure_future, wait_for, TimeoutError, BaseEventLoop, Future as asyncioFuture
+from .functions import milliseconds, iso8601, deep_extend, is_json_encoded_object
+from ccxt import NetworkError, RequestTimeout
 from ccxt.async_support.base.ws.future import Future
-from collections import deque
+from ccxt.async_support.base.ws.functions import gunzip, inflate
+from typing import Dict
+
+from aiohttp import WSMsgType
+
 
 class Client(object):
 
     url = None
     ws = None
-    futures = {}
+    futures: Dict[str, Future] = {}
     options = {}  # ws-specific options
     subscriptions = {}
     rejections = {}
-    message_queue = {}
-    useMessageQueue = False
     on_message_callback = None
     on_error_callback = None
     on_close_callback = None
@@ -32,14 +42,15 @@ class Client(object):
     maxPingPongMisses = 2.0  # how many missed pongs to raise a timeout
     lastPong = None
     ping = None  # ping-function if defined
+    proxy = None
     verbose = False  # verbose output
     gunzip = False
     inflate = False
     throttle = None
     connecting = False
-    asyncio_loop = None
+    asyncio_loop: BaseEventLoop = None
     ping_looper = None
-    receive_looper = None
+    decompressBinary = True  # decompress binary messages by default
 
     def __init__(self, url, on_message_callback, on_error_callback, on_close_callback, on_connected_callback, config={}):
         defaults = {
@@ -70,37 +81,25 @@ class Client(object):
         if message_hash in self.rejections:
             future.reject(self.rejections[message_hash])
             del self.rejections[message_hash]
-            del self.message_queue[message_hash]
-            return future
-        if self.useMessageQueue and message_hash in self.message_queue:
-            queue = self.message_queue[message_hash]
-            if len(queue):
-                future.resolve(queue.popleft())
-                del self.futures[message_hash]
         return future
 
+    def reusable_future(self, message_hash):
+        return self.future(message_hash)  # only used in go
+
+    def reusableFuture(self, message_hash):
+        return self.future(message_hash)  # only used in go
+
     def resolve(self, result, message_hash):
         if self.verbose and message_hash is None:
             self.log(iso8601(milliseconds()), 'resolve received None messageHash')
-
-        if self.useMessageQueue:
-            if message_hash not in self.message_queue:
-                self.message_queue[message_hash] = deque(maxlen=10)
-            queue = self.message_queue[message_hash]
-            queue.append(result)
-            if message_hash in self.futures:
-                future = self.futures[message_hash]
-                future.resolve(queue.popleft())
-                del self.futures[message_hash]
-        else:
-            if message_hash in self.futures:
-                future = self.futures[message_hash]
-                future.resolve(result)
-                del self.futures[message_hash]
+        if message_hash in self.futures:
+            future = self.futures[message_hash]
+            future.resolve(result)
+            del self.futures[message_hash]
         return result
 
     def reject(self, result, message_hash=None):
-        if message_hash:
+        if message_hash is not None:
             if message_hash in self.futures:
                 future = self.futures[message_hash]
                 future.reject(result)
@@ -113,19 +112,41 @@ class Client(object):
                 self.reject(result, message_hash)
         return result
 
-    async def receive_loop(self):
+    def receive_loop(self):
         if self.verbose:
             self.log(iso8601(milliseconds()), 'receive loop')
-        while not self.closed():
-            try:
-                message = await self.receive()
-                # self.log(iso8601(milliseconds()), 'received', message)
-                self.handle_message(message)
-            except Exception as e:
-                error = NetworkError(str(e))
-                if self.verbose:
-                    self.log(iso8601(milliseconds()), 'receive_loop', 'Exception', error)
-                self.reset(error)
+        if not self.closed():
+            # let's drain the aiohttp buffer to avoid latency
+            if self.buffer and len(self.buffer) > 1:
+                size_delta = 0
+                while len(self.buffer) > 1:
+                    message, size = self.buffer.popleft()
+                    size_delta += size
+                    self.handle_message(message)
+                # we must update the size of the last message inside WebSocketDataQueue
+                # self.receive() calls WebSocketDataQueue.read() that calls WebSocketDataQueue._read_from_buffer()
+                # which updates the size of the buffer, the _size will overflow and pause the transport
+                # make sure to set the enviroment variable AIOHTTP_NO_EXTENSIONS=Y to check
+                # print(self.connection._conn.protocol._payload._size)
+                self.buffer[0] = (self.buffer[0][0], self.buffer[0][1] + size_delta)
+
+            task = self.asyncio_loop.create_task(self.receive())
+
+            def after_interrupt(resolved: asyncioFuture):
+                exception = resolved.exception()
+                if exception is None:
+                    self.handle_message(resolved.result())
+                    self.asyncio_loop.call_soon(self.receive_loop)
+                else:
+                    error = NetworkError(str(exception))
+                    if self.verbose:
+                        self.log(iso8601(milliseconds()), 'receive_loop', 'Exception', error)
+                    self.reject(error)
+
+            task.add_done_callback(after_interrupt)
+        else:
+            # connection got terminated after the connection was made and before the receive loop ran
+            self.on_close(1006)
 
     async def open(self, session, backoff_delay=0):
         # exponential backoff for consequent connections if necessary
@@ -146,7 +167,7 @@ class Client(object):
             self.on_connected_callback(self)
             # run both loops forever
             self.ping_looper = ensure_future(self.ping_loop(), loop=self.asyncio_loop)
-            self.receive_looper = ensure_future(self.receive_loop(), loop=self.asyncio_loop)
+            self.asyncio_loop.call_soon(self.receive_loop)
         except TimeoutError:
             # connection timeout
             error = RequestTimeout('Connection timeout')
@@ -160,6 +181,23 @@ class Client(object):
                 self.log(iso8601(milliseconds()), 'NetworkError', error)
             self.on_error(error)
 
+    @property
+    def buffer(self):
+        # looks like they exposed it in C
+        # this means we can bypass it
+        # https://github.com/aio-libs/aiohttp/blob/master/aiohttp/_websocket/reader_c.pxd#L53C24-L53C31
+        # these checks are necessary to protect these errors: AttributeError: 'NoneType' object has no attribute '_buffer'
+        # upon getting an error message
+        if self.connection is None:
+            return None
+        if self.connection._conn is None:
+            return None
+        if self.connection._conn.protocol is None:
+            return None
+        if self.connection._conn.protocol._payload is None:
+            return None
+        return self.connection._conn.protocol._payload._buffer
+
     def connect(self, session, backoff_delay=0):
         if not self.connection and not self.connecting:
             self.connecting = True
@@ -170,7 +208,7 @@ class Client(object):
         if self.verbose:
             self.log(iso8601(milliseconds()), 'on_error', error)
         self.error = error
-        self.reset(error)
+        self.reject(error)
         self.on_error_callback(self, error)
         if not self.closed():
             ensure_future(self.close(1006), loop=self.asyncio_loop)
@@ -179,36 +217,128 @@ class Client(object):
         if self.verbose:
             self.log(iso8601(milliseconds()), 'on_close', code)
         if not self.error:
-            self.reset(NetworkError('Connection closed by remote server, closing code ' + str(code)))
+            self.reject(NetworkError('Connection closed by remote server, closing code ' + str(code)))
         self.on_close_callback(self, code)
-        if not self.closed():
-            ensure_future(self.close(code), loop=self.asyncio_loop)
+        ensure_future(self.aiohttp_close(), loop=self.asyncio_loop)
 
-    def reset(self, error):
-        self.message_queue = {}
-        self.reject(error)
+    def log(self, *args):
+        print(*args)
 
-    async def ping_loop(self):
-        if self.verbose:
-            self.log(iso8601(milliseconds()), 'ping loop')
+    def closed(self):
+        return (self.connection is None) or self.connection.closed
 
     def receive(self):
-        raise NotSupported('receive() not implemented')
+        return self.connection.receive()
+
+    # helper method for binary and text messages
+    def handle_text_or_binary_message(self, data):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'message', data)
+        if isinstance(data, bytes):
+            if self.decompressBinary:
+                data = data.decode()
+        # decoded = json.loads(data) if is_json_encoded_object(data) else data
+        decode = None
+        if is_json_encoded_object(data):
+            if orjson is None:
+                decode = json.loads(data)
+            else:
+                decode = orjson.loads(data)
+        else:
+            decode = data
+        self.on_message_callback(self, decode)
 
     def handle_message(self, message):
-        raise NotSupported('handle_message() not implemented')
+        # self.log(iso8601(milliseconds()), message)
+        if message.type == WSMsgType.TEXT:
+            self.handle_text_or_binary_message(message.data)
+        elif message.type == WSMsgType.BINARY:
+            data = message.data
+            if self.gunzip:
+                data = gunzip(data)
+            elif self.inflate:
+                data = inflate(data)
+            self.handle_text_or_binary_message(data)
+        # autoping is responsible for automatically replying with pong
+        # to a ping incoming from a server, we have to disable autoping
+        # with aiohttp's websockets and respond with pong manually
+        # otherwise aiohttp's websockets client won't trigger WSMsgType.PONG
+        elif message.type == WSMsgType.PING:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'ping', message)
+            ensure_future(self.connection.pong(message.data), loop=self.asyncio_loop)
+        elif message.type == WSMsgType.PONG:
+            self.lastPong = milliseconds()
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'pong', message)
+            pass
+        elif message.type == WSMsgType.CLOSE:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'close', self.closed(), message)
+            self.on_close(message.data)
+        elif message.type == WSMsgType.ERROR:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'error', message)
+            error = NetworkError(str(message))
+            self.on_error(error)
 
-    def closed(self):
-        raise NotSupported('closed() not implemented')
+    def create_connection(self, session):
+        # autoping is responsible for automatically replying with pong
+        # to a ping incoming from a server, we have to disable autoping
+        # with aiohttp's websockets and respond with pong manually
+        # otherwise aiohttp's websockets client won't trigger WSMsgType.PONG
+        # call aenter here to simulate async with otherwise we get the error "await not called with future"
+        # if connecting to a non-existent endpoint
+        if (self.proxy):
+            return session.ws_connect(self.url, autoping=False, autoclose=False, headers=self.options.get('headers'), proxy=self.proxy, max_msg_size=10485760).__aenter__()
+        return session.ws_connect(self.url, autoping=False, autoclose=False, headers=self.options.get('headers'), max_msg_size=10485760).__aenter__()
 
     async def send(self, message):
-        raise NotSupported('send() not implemented')
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'sending', message)
+        send_msg = None
+        if isinstance(message, str):
+            send_msg = message
+        else:
+            if orjson is None:
+                send_msg = json.dumps(message, separators=(',', ':'))
+            else:
+                send_msg = orjson.dumps(message).decode('utf-8')
+        return await self.connection.send_str(send_msg)
 
     async def close(self, code=1000):
-        raise NotSupported('close() not implemented')
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'closing', code)
+        for future in self.futures.values():
+            future.cancel()
+        await self.aiohttp_close()
 
-    def create_connection(self, session):
-        raise NotSupported('create_connection() not implemented')
+    async def aiohttp_close(self):
+        if not self.closed():
+            await self.connection.close()
+        # these will end automatically once self.closed() = True
+        # so we don't need to cancel them
+        if self.ping_looper:
+            self.ping_looper.cancel()
 
-    def log(self, *args):
-        print(*args)
+    async def ping_loop(self):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'ping loop')
+        while self.keepAlive and not self.closed():
+            now = milliseconds()
+            self.lastPong = now if self.lastPong is None else self.lastPong
+            if (self.lastPong + self.keepAlive * self.maxPingPongMisses) < now:
+                self.on_error(RequestTimeout('Connection to ' + self.url + ' timed out due to a ping-pong keepalive missing on time'))
+            # the following ping-clause is not necessary with aiohttp's built-in ws
+            # since it has a heartbeat option (see create_connection above)
+            # however some exchanges require a text-type ping message
+            # therefore we need this clause anyway
+            else:
+                if self.ping:
+                    try:
+                        await self.send(self.ping(self))
+                    except Exception as e:
+                        self.on_error(e)
+                else:
+                    await self.connection.ping()
+            await sleep(self.keepAlive / 1000)

gate/ccxt/async_support/base/ws/future.py

@@ -1,69 +1,46 @@
 import asyncio
-from ccxt import ExchangeClosedByUser
 
+# Test by running:
+# - python python/ccxt/pro/test/base/test_close.py
+# - python python/ccxt/pro/test/base/test_future.py
 class Future(asyncio.Future):
 
-    is_race_future = False
-
     def resolve(self, result=None):
         if not self.done():
-            try:
-                self.set_result(result)
-            except BaseException as e:
-                print("Error in Future.resolve")
-                raise e
+            self.set_result(result)
 
     def reject(self, error=None):
         if not self.done():
-            # If not an exception, wrap it in a generic Exception
-            if not isinstance(error, BaseException):
-                error = Exception(error)
-            try:
-                self.set_exception(error)
-            except BaseException as e:
-                print("Error in Future.reject")
-                raise e
+            self.set_exception(error)
 
     @classmethod
     def race(cls, futures):
         future = Future()
-        for f in futures:
-            f.is_race_future = True
         coro = asyncio.wait(futures, return_when=asyncio.FIRST_COMPLETED)
         task = asyncio.create_task(coro)
 
         def callback(done):
-            try:
-                complete, pending = done.result()
-                # check for exceptions
-                for i, f in enumerate(complete):
-                    try:
-                        f.result()
-                    except ExchangeClosedByUser as e:
-                        if len(pending) == 0 and i == len(complete) - 1:
-                            future.reject(e)
-                        # wait for all the sub promises to be reject before rejecting future
-                        continue
-                    except asyncio.CancelledError as e:
-                        continue
-                    except Exception as e:
-                        future.reject(e)
-                        return
-                # no exceptions return first result
-                futures_list = list(complete)
-
-                are_all_canceled = all([f.cancelled() for f in futures_list])
-                if are_all_canceled:
-                    future.reject(ExchangeClosedByUser('Connection closed by the user'))
-                    return
-
-                first = futures_list[0]
-
-                first_result = first.result()
-                future.resolve(first_result)
-            except asyncio.CancelledError as e:
-                future.reject(e)
-            except Exception as e:
-                future.reject(e)
+            complete, _ = done.result()
+            # check for exceptions
+            exceptions = []
+            cancelled = False
+            for f in complete:
+                if f.cancelled():
+                    cancelled = True
+                else:
+                    err = f.exception()
+                    if err:
+                        exceptions.append(err)
+            # if any exceptions return with first exception
+            if future.cancelled():
+                return
+            if len(exceptions) > 0:
+                future.set_exception(exceptions[0])
+            # else return first result
+            elif cancelled:
+                future.cancel()
+            else:
+                first_result = list(complete)[0].result()
+                future.set_result(first_result)
         task.add_done_callback(callback)
         return future