ccxt-ir 4.3.46.0.3__py2.py3-none-any.whl → 4.5.0__py2.py3-none-any.whl

ccxt/async_support/base/ws/cache.py

@@ -1,5 +1,7 @@
 import collections
+import logging
 
+logger = logging.getLogger(__name__)
 
 class Delegate:
     def __init__(self, name, delegated):
@@ -23,6 +25,7 @@ class BaseCache(list):
     __contains__ = Delegate('__contains__', '_deque')
     __reversed__ = Delegate('__reversed__', '_deque')
     clear = Delegate('clear', '_deque')
+    pop = Delegate('pop', '_deque')
 
     def __init__(self, max_size=None):
         super(BaseCache, self).__init__()
@@ -150,7 +153,10 @@ class ArrayCacheBySymbolById(ArrayCache):
         if len(self._deque) == self._deque.maxlen:
             delete_item = self._deque.popleft()
             self._index.popleft()
-            del self.hashmap[delete_item['symbol']][delete_item['id']]
+            try:
+                del self.hashmap[delete_item['symbol']][delete_item['id']]
+            except Exception as e:
+                logger.error(f"Error deleting item from hashmap: {delete_item}. Error:{e}")
         self._deque.append(item)
         self._index.append(item['id'])
         if self._clear_all_updates:
@@ -184,7 +190,7 @@ class ArrayCacheBySymbolBySide(ArrayCache):
             if reference != item:
                 reference.update(item)
             item = reference
-            index = self._index.index(item['side'])
+            index = self._index.index(item['symbol'] + item['side'])
             del self._deque[index]
             del self._index[index]
         else:
@@ -194,7 +200,7 @@ class ArrayCacheBySymbolBySide(ArrayCache):
             self._index.popleft()
             del self.hashmap[delete_item['symbol']][delete_item['side']]
         self._deque.append(item)
-        self._index.append(item['side'])
+        self._index.append(item['symbol'] + item['side'])
         if self._clear_all_updates:
             self._clear_all_updates = False
             self._clear_updates_by_symbol.clear()

ccxt/async_support/base/ws/client.py

@@ -1,16 +1,28 @@
 # -*- coding: utf-8 -*-
 
-from asyncio import sleep, ensure_future, wait_for, TimeoutError
-from .functions import milliseconds, iso8601, deep_extend
-from ccxt import NetworkError, RequestTimeout, NotSupported
+orjson = None
+try:
+    import orjson as orjson
+except ImportError:
+    pass
+
+import json
+
+from asyncio import sleep, ensure_future, wait_for, TimeoutError, BaseEventLoop, Future as asyncioFuture
+from .functions import milliseconds, iso8601, deep_extend, is_json_encoded_object
+from ccxt import NetworkError, RequestTimeout
 from ccxt.async_support.base.ws.future import Future
+from ccxt.async_support.base.ws.functions import gunzip, inflate
+from typing import Dict
+
+from aiohttp import WSMsgType
 
 
 class Client(object):
 
     url = None
     ws = None
-    futures = {}
+    futures: Dict[str, Future] = {}
     options = {}  # ws-specific options
     subscriptions = {}
     rejections = {}
@@ -30,14 +42,15 @@ class Client(object):
     maxPingPongMisses = 2.0  # how many missed pongs to raise a timeout
     lastPong = None
     ping = None  # ping-function if defined
+    proxy = None
     verbose = False  # verbose output
     gunzip = False
     inflate = False
     throttle = None
     connecting = False
-    asyncio_loop = None
+    asyncio_loop: BaseEventLoop = None
     ping_looper = None
-    receive_looper = None
+    decompressBinary = True  # decompress binary messages by default
 
     def __init__(self, url, on_message_callback, on_error_callback, on_close_callback, on_connected_callback, config={}):
         defaults = {
@@ -80,7 +93,7 @@ class Client(object):
         return result
 
     def reject(self, result, message_hash=None):
-        if message_hash:
+        if message_hash is not None:
             if message_hash in self.futures:
                 future = self.futures[message_hash]
                 future.reject(result)
@@ -93,19 +106,41 @@ class Client(object):
                 self.reject(result, message_hash)
         return result
 
-    async def receive_loop(self):
+    def receive_loop(self):
         if self.verbose:
             self.log(iso8601(milliseconds()), 'receive loop')
-        while not self.closed():
-            try:
-                message = await self.receive()
-                # self.log(iso8601(milliseconds()), 'received', message)
-                self.handle_message(message)
-            except Exception as e:
-                error = NetworkError(str(e))
-                if self.verbose:
-                    self.log(iso8601(milliseconds()), 'receive_loop', 'Exception', error)
-                self.reset(error)
+        if not self.closed():
+            # let's drain the aiohttp buffer to avoid latency
+            if len(self.buffer) > 1:
+                size_delta = 0
+                while len(self.buffer) > 1:
+                    message, size = self.buffer.popleft()
+                    size_delta += size
+                    self.handle_message(message)
+                # we must update the size of the last message inside WebSocketDataQueue
+                # self.receive() calls WebSocketDataQueue.read() that calls WebSocketDataQueue._read_from_buffer()
+                # which updates the size of the buffer, the _size will overflow and pause the transport
+                # make sure to set the enviroment variable AIOHTTP_NO_EXTENSIONS=Y to check
+                # print(self.connection._conn.protocol._payload._size)
+                self.buffer[0] = (self.buffer[0][0], self.buffer[0][1] + size_delta)
+
+            task = self.asyncio_loop.create_task(self.receive())
+
+            def after_interrupt(resolved: asyncioFuture):
+                exception = resolved.exception()
+                if exception is None:
+                    self.handle_message(resolved.result())
+                    self.asyncio_loop.call_soon(self.receive_loop)
+                else:
+                    error = NetworkError(str(exception))
+                    if self.verbose:
+                        self.log(iso8601(milliseconds()), 'receive_loop', 'Exception', error)
+                    self.reject(error)
+
+            task.add_done_callback(after_interrupt)
+        else:
+            # connection got terminated after the connection was made and before the receive loop ran
+            self.on_close(1006)
 
     async def open(self, session, backoff_delay=0):
         # exponential backoff for consequent connections if necessary
@@ -126,7 +161,7 @@ class Client(object):
             self.on_connected_callback(self)
             # run both loops forever
             self.ping_looper = ensure_future(self.ping_loop(), loop=self.asyncio_loop)
-            self.receive_looper = ensure_future(self.receive_loop(), loop=self.asyncio_loop)
+            self.asyncio_loop.call_soon(self.receive_loop)
         except TimeoutError:
             # connection timeout
             error = RequestTimeout('Connection timeout')
@@ -140,6 +175,13 @@ class Client(object):
                 self.log(iso8601(milliseconds()), 'NetworkError', error)
             self.on_error(error)
 
+    @property
+    def buffer(self):
+        # looks like they exposed it in C
+        # this means we can bypass it
+        # https://github.com/aio-libs/aiohttp/blob/master/aiohttp/_websocket/reader_c.pxd#L53C24-L53C31
+        return self.connection._conn.protocol._payload._buffer
+
     def connect(self, session, backoff_delay=0):
         if not self.connection and not self.connecting:
             self.connecting = True
@@ -150,7 +192,7 @@ class Client(object):
         if self.verbose:
             self.log(iso8601(milliseconds()), 'on_error', error)
         self.error = error
-        self.reset(error)
+        self.reject(error)
         self.on_error_callback(self, error)
         if not self.closed():
             ensure_future(self.close(1006), loop=self.asyncio_loop)
@@ -159,35 +201,128 @@ class Client(object):
         if self.verbose:
             self.log(iso8601(milliseconds()), 'on_close', code)
         if not self.error:
-            self.reset(NetworkError('Connection closed by remote server, closing code ' + str(code)))
+            self.reject(NetworkError('Connection closed by remote server, closing code ' + str(code)))
         self.on_close_callback(self, code)
-        if not self.closed():
-            ensure_future(self.close(code), loop=self.asyncio_loop)
+        ensure_future(self.aiohttp_close(), loop=self.asyncio_loop)
 
-    def reset(self, error):
-        self.reject(error)
+    def log(self, *args):
+        print(*args)
 
-    async def ping_loop(self):
-        if self.verbose:
-            self.log(iso8601(milliseconds()), 'ping loop')
+    def closed(self):
+        return (self.connection is None) or self.connection.closed
 
     def receive(self):
-        raise NotSupported('receive() not implemented')
+        return self.connection.receive()
+
+    # helper method for binary and text messages
+    def handle_text_or_binary_message(self, data):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'message', data)
+        if isinstance(data, bytes):
+            if self.decompressBinary:
+                data = data.decode()
+        # decoded = json.loads(data) if is_json_encoded_object(data) else data
+        decode = None
+        if is_json_encoded_object(data):
+            if orjson is None:
+                decode = json.loads(data)
+            else:
+                decode = orjson.loads(data)
+        else:
+            decode = data
+        self.on_message_callback(self, decode)
 
     def handle_message(self, message):
-        raise NotSupported('handle_message() not implemented')
+        # self.log(iso8601(milliseconds()), message)
+        if message.type == WSMsgType.TEXT:
+            self.handle_text_or_binary_message(message.data)
+        elif message.type == WSMsgType.BINARY:
+            data = message.data
+            if self.gunzip:
+                data = gunzip(data)
+            elif self.inflate:
+                data = inflate(data)
+            self.handle_text_or_binary_message(data)
+        # autoping is responsible for automatically replying with pong
+        # to a ping incoming from a server, we have to disable autoping
+        # with aiohttp's websockets and respond with pong manually
+        # otherwise aiohttp's websockets client won't trigger WSMsgType.PONG
+        elif message.type == WSMsgType.PING:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'ping', message)
+            ensure_future(self.connection.pong(message.data), loop=self.asyncio_loop)
+        elif message.type == WSMsgType.PONG:
+            self.lastPong = milliseconds()
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'pong', message)
+            pass
+        elif message.type == WSMsgType.CLOSE:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'close', self.closed(), message)
+            self.on_close(message.data)
+        elif message.type == WSMsgType.ERROR:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'error', message)
+            error = NetworkError(str(message))
+            self.on_error(error)
 
-    def closed(self):
-        raise NotSupported('closed() not implemented')
+    def create_connection(self, session):
+        # autoping is responsible for automatically replying with pong
+        # to a ping incoming from a server, we have to disable autoping
+        # with aiohttp's websockets and respond with pong manually
+        # otherwise aiohttp's websockets client won't trigger WSMsgType.PONG
+        # call aenter here to simulate async with otherwise we get the error "await not called with future"
+        # if connecting to a non-existent endpoint
+        if (self.proxy):
+            return session.ws_connect(self.url, autoping=False, autoclose=False, headers=self.options.get('headers'), proxy=self.proxy, max_msg_size=10485760).__aenter__()
+        return session.ws_connect(self.url, autoping=False, autoclose=False, headers=self.options.get('headers'), max_msg_size=10485760).__aenter__()
 
     async def send(self, message):
-        raise NotSupported('send() not implemented')
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'sending', message)
+        send_msg = None
+        if isinstance(message, str):
+            send_msg = message
+        else:
+            if orjson is None:
+                send_msg = json.dumps(message, separators=(',', ':'))
+            else:
+                send_msg = orjson.dumps(message).decode('utf-8')
+        return await self.connection.send_str(send_msg)
 
     async def close(self, code=1000):
-        raise NotSupported('close() not implemented')
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'closing', code)
+        for future in self.futures.values():
+            future.cancel()
+        await self.aiohttp_close()
 
-    def create_connection(self, session):
-        raise NotSupported('create_connection() not implemented')
+    async def aiohttp_close(self):
+        if not self.closed():
+            await self.connection.close()
+        # these will end automatically once self.closed() = True
+        # so we don't need to cancel them
+        if self.ping_looper:
+            self.ping_looper.cancel()
 
-    def log(self, *args):
-        print(*args)
+    async def ping_loop(self):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'ping loop')
+        while self.keepAlive and not self.closed():
+            now = milliseconds()
+            self.lastPong = now if self.lastPong is None else self.lastPong
+            if (self.lastPong + self.keepAlive * self.maxPingPongMisses) < now:
+                self.on_error(RequestTimeout('Connection to ' + self.url + ' timed out due to a ping-pong keepalive missing on time'))
+            # the following ping-clause is not necessary with aiohttp's built-in ws
+            # since it has a heartbeat option (see create_connection above)
+            # however some exchanges require a text-type ping message
+            # therefore we need this clause anyway
+            else:
+                if self.ping:
+                    try:
+                        await self.send(self.ping(self))
+                    except Exception as e:
+                        self.on_error(e)
+                else:
+                    await self.connection.ping()
+            await sleep(self.keepAlive / 1000)

ccxt/async_support/base/ws/future.py

@@ -1,10 +1,10 @@
 import asyncio
-from ccxt import ExchangeClosedByUser
 
+# Test by running:
+# - python python/ccxt/pro/test/base/test_close.py
+# - python python/ccxt/pro/test/base/test_future.py
 class Future(asyncio.Future):
 
-    is_race_future = False
-
     def resolve(self, result=None):
         if not self.done():
             self.set_result(result)
@@ -16,43 +16,31 @@ class Future(asyncio.Future):
     @classmethod
     def race(cls, futures):
         future = Future()
-        for f in futures:
-            f.is_race_future = True
         coro = asyncio.wait(futures, return_when=asyncio.FIRST_COMPLETED)
         task = asyncio.create_task(coro)
 
         def callback(done):
-            try:
-                complete, pending = done.result()
-                # check for exceptions
-                for i, f in enumerate(complete):
-                    try:
-                        f.result()
-                    except ExchangeClosedByUser as e:
-                        if len(pending) == 0 and i == len(complete) - 1:
-                            future.reject(e)
-                        # wait for all the sub promises to be reject before rejecting future
-                        continue
-                    except asyncio.CancelledError as e:
-                        continue
-                    except Exception as e:
-                        future.reject(e)
-                        return
-                # no exceptions return first result
-                futures_list = list(complete)
-
-                are_all_canceled = all([f.cancelled() for f in futures_list])
-                if are_all_canceled:
-                    future.reject(ExchangeClosedByUser('Connection closed by the user'))
-                    return
-
-                first = futures_list[0]
-
-                first_result = first.result()
-                future.resolve(first_result)
-            except asyncio.CancelledError as e:
-                future.reject(e)
-            except Exception as e:
-                future.reject(e)
+            complete, _ = done.result()
+            # check for exceptions
+            exceptions = []
+            cancelled = False
+            for f in complete:
+                if f.cancelled():
+                    cancelled = True
+                else:
+                    err = f.exception()
+                    if err:
+                        exceptions.append(err)
+            # if any exceptions return with first exception
+            if future.cancelled():
+                return
+            if len(exceptions) > 0:
+                future.set_exception(exceptions[0])
+            # else return first result
+            elif cancelled:
+                future.cancel()
+            else:
+                first_result = list(complete)[0].result()
+                future.set_result(first_result)
         task.add_done_callback(callback)
         return future

ccxt/async_support/bequant.py

@@ -5,19 +5,21 @@
 
 from ccxt.async_support.hitbtc import hitbtc
 from ccxt.abstract.bequant import ImplicitAPI
+from ccxt.base.types import Any
 
 
 class bequant(hitbtc, ImplicitAPI):
 
-    def describe(self):
+    def describe(self) -> Any:
         return self.deep_extend(super(bequant, self).describe(), {
             'id': 'bequant',
             'name': 'Bequant',
-            'countries': ['MT'],  # Malta
             'pro': True,
+            'countries': ['MT'],  # Malta
             'urls': {
-                'logo': 'https://user-images.githubusercontent.com/1294454/55248342-a75dfe00-525a-11e9-8aa2-05e9dca943c6.jpg',
+                'logo': 'https://github.com/user-attachments/assets/0583ef1f-29fe-4b7c-8189-63565a0e2867',
                 'api': {
+                    # v3
                     'public': 'https://api.bequant.io/api/3',
                     'private': 'https://api.bequant.io/api/3',
                 },