ccxt-ir 4.3.46.0.1__py2.py3-none-any.whl

ccxt/async_support/base/throttler.py

@@ -0,0 +1,50 @@
+import asyncio
+import collections
+from time import time
+
+
+class Throttler:
+    def __init__(self, config, loop=None):
+        self.loop = loop
+        self.config = {
+            'refillRate': 1.0,
+            'delay': 0.001,
+            'cost': 1.0,
+            'tokens': 0,
+            'maxCapacity': 2000,
+            'capacity': 1.0,
+        }
+        self.config.update(config)
+        self.queue = collections.deque()
+        self.running = False
+
+    async def looper(self):
+        last_timestamp = time() * 1000
+        while self.running:
+            future, cost = self.queue[0]
+            cost = self.config['cost'] if cost is None else cost
+            if self.config['tokens'] >= 0:
+                self.config['tokens'] -= cost
+                if not future.done():
+                    future.set_result(None)
+                self.queue.popleft()
+                # context switch
+                await asyncio.sleep(0)
+                if len(self.queue) == 0:
+                    self.running = False
+            else:
+                await asyncio.sleep(self.config['delay'])
+                now = time() * 1000
+                elapsed = now - last_timestamp
+                last_timestamp = now
+                self.config['tokens'] = min(self.config['tokens'] + elapsed * self.config['refillRate'], self.config['capacity'])
+
+    def __call__(self, cost=None):
+        future = asyncio.Future()
+        if len(self.queue) > self.config['maxCapacity']:
+            raise RuntimeError('throttle queue is over maxCapacity (' + str(int(self.config['maxCapacity'])) + '), see https://github.com/ccxt/ccxt/issues/11645#issuecomment-1195695526')
+        self.queue.append((future, cost))
+        if not self.running:
+            self.running = True
+            asyncio.ensure_future(self.looper(), loop=self.loop)
+        return future

ccxt/async_support/base/ws/__init__.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+
+from ccxt.base import errors
+
+# -----------------------------------------------------------------------------
+
+from ccxt.base import decimal_to_precision
+
+from ccxt import BaseError                  # noqa: F401
+from ccxt import ExchangeError              # noqa: F401
+from ccxt import NotSupported               # noqa: F401
+from ccxt import AuthenticationError        # noqa: F401
+from ccxt import PermissionDenied           # noqa: F401
+from ccxt import AccountSuspended           # noqa: F401
+from ccxt import InvalidNonce               # noqa: F401
+from ccxt import InsufficientFunds          # noqa: F401
+from ccxt import InvalidOrder               # noqa: F401
+from ccxt import OrderNotFound              # noqa: F401
+from ccxt import OrderNotCached             # noqa: F401
+from ccxt import DuplicateOrderId           # noqa: F401
+from ccxt import CancelPending              # noqa: F401
+from ccxt import NetworkError               # noqa: F401
+from ccxt import DDoSProtection             # noqa: F401
+from ccxt import RateLimitExceeded          # noqa: F401
+from ccxt import RequestTimeout             # noqa: F401
+from ccxt import ExchangeNotAvailable       # noqa: F401
+from ccxt import OnMaintenance              # noqa: F401
+from ccxt import InvalidAddress             # noqa: F401
+from ccxt import AddressPending             # noqa: F401
+from ccxt import ArgumentsRequired          # noqa: F401
+from ccxt import BadRequest                 # noqa: F401
+from ccxt import BadResponse                # noqa: F401
+from ccxt import NullResponse               # noqa: F401
+from ccxt import OrderImmediatelyFillable   # noqa: F401
+from ccxt import OrderNotFillable           # noqa: F401
+
+
+__all__ = decimal_to_precision.__all__ + errors.__all__  # noqa: F405

ccxt/async_support/base/ws/aiohttp_client.py

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+
+import json
+from asyncio import sleep, ensure_future
+from aiohttp import WSMsgType
+from .functions import milliseconds, iso8601, is_json_encoded_object
+from ccxt.async_support.base.ws.client import Client
+from ccxt.async_support.base.ws.functions import gunzip, inflate
+from ccxt import NetworkError, RequestTimeout, ExchangeClosedByUser
+
+
+class AiohttpClient(Client):
+
+    proxy = None
+
+    def closed(self):
+        return (self.connection is None) or self.connection.closed
+
+    def receive(self):
+        return self.connection.receive()
+
+    # helper method for binary and text messages
+    def handle_text_or_binary_message(self, data):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'message', data)
+        if isinstance(data, bytes):
+            data = data.decode()
+        decoded = json.loads(data) if is_json_encoded_object(data) else data
+        self.on_message_callback(self, decoded)
+
+    def handle_message(self, message):
+        # self.log(iso8601(milliseconds()), message)
+        if message.type == WSMsgType.TEXT:
+            self.handle_text_or_binary_message(message.data)
+        elif message.type == WSMsgType.BINARY:
+            data = message.data
+            if self.gunzip:
+                data = gunzip(data)
+            elif self.inflate:
+                data = inflate(data)
+            self.handle_text_or_binary_message(data)
+        # autoping is responsible for automatically replying with pong
+        # to a ping incoming from a server, we have to disable autoping
+        # with aiohttp's websockets and respond with pong manually
+        # otherwise aiohttp's websockets client won't trigger WSMsgType.PONG
+        elif message.type == WSMsgType.PING:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'ping', message)
+            ensure_future(self.connection.pong(message.data), loop=self.asyncio_loop)
+        elif message.type == WSMsgType.PONG:
+            self.lastPong = milliseconds()
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'pong', message)
+            pass
+        elif message.type == WSMsgType.CLOSE:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'close', self.closed(), message)
+            self.on_close(message.data)
+        elif message.type == WSMsgType.CLOSED:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'closed', self.closed(), message)
+            self.on_close(1000)
+        elif message.type == WSMsgType.ERROR:
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'error', message)
+            error = NetworkError(str(message))
+            self.on_error(error)
+
+    def create_connection(self, session):
+        # autoping is responsible for automatically replying with pong
+        # to a ping incoming from a server, we have to disable autoping
+        # with aiohttp's websockets and respond with pong manually
+        # otherwise aiohttp's websockets client won't trigger WSMsgType.PONG
+        # call aenter here to simulate async with otherwise we get the error "await not called with future"
+        # if connecting to a non-existent endpoint
+        if (self.proxy):
+            return session.ws_connect(self.url, autoping=False, autoclose=False, headers=self.options.get('headers'), proxy=self.proxy).__aenter__()
+        return session.ws_connect(self.url, autoping=False, autoclose=False, headers=self.options.get('headers')).__aenter__()
+
+    async def send(self, message):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'sending', message)
+        return await self.connection.send_str(message if isinstance(message, str) else json.dumps(message, separators=(',', ':')))
+
+    async def close(self, code=1000):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'closing', code)
+        if not self.closed():
+            await self.connection.close()
+        # these will end automatically once self.closed() = True
+        # so we don't need to cancel them
+        if self.ping_looper:
+            self.ping_looper.cancel()
+        if self.receive_looper:
+            self.receive_looper.cancel()  # cancel all pending futures stored in self.futures
+        for key in self.futures:
+            future = self.futures[key]
+            if not future.done():
+                if future.is_race_future:
+                    future.cancel()  # this is an "internal" future so we want to cancel it silently
+                else:
+                    future.reject(ExchangeClosedByUser('Connection closed by the user'))
+
+
+    async def ping_loop(self):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'ping loop')
+        while self.keepAlive and not self.closed():
+            now = milliseconds()
+            self.lastPong = now if self.lastPong is None else self.lastPong
+            if (self.lastPong + self.keepAlive * self.maxPingPongMisses) < now:
+                self.on_error(RequestTimeout('Connection to ' + self.url + ' timed out due to a ping-pong keepalive missing on time'))
+            # the following ping-clause is not necessary with aiohttp's built-in ws
+            # since it has a heartbeat option (see create_connection above)
+            # however some exchanges require a text-type ping message
+            # therefore we need this clause anyway
+            else:
+                if self.ping:
+                    try:
+                        await self.send(self.ping(self))
+                    except Exception as e:
+                        self.on_error(e)
+                else:
+                    await self.connection.ping()
+            await sleep(self.keepAlive / 1000)

ccxt/async_support/base/ws/cache.py

@@ -0,0 +1,212 @@
+import collections
+
+
+class Delegate:
+    def __init__(self, name, delegated):
+        self.name = name
+        self.delegated = delegated
+
+    def __get__(self, instance, owner):
+        deque = getattr(instance, self.delegated)
+        return getattr(deque, self.name)
+
+
+class BaseCache(list):
+    # implicitly called magic methods don't invoke __getattribute__
+    # https://docs.python.org/3/reference/datamodel.html#special-method-lookup
+    # all method lookups obey the descriptor protocol
+    # this is how the implicit api is defined in ccxt
+    __iter__ = Delegate('__iter__', '_deque')
+    __setitem__ = Delegate('__setitem__', '_deque')
+    __delitem__ = Delegate('__delitem__', '_deque')
+    __len__ = Delegate('__len__', '_deque')
+    __contains__ = Delegate('__contains__', '_deque')
+    __reversed__ = Delegate('__reversed__', '_deque')
+    clear = Delegate('clear', '_deque')
+
+    def __init__(self, max_size=None):
+        super(BaseCache, self).__init__()
+        self.max_size = max_size
+        self._deque = collections.deque([], max_size)
+
+    def __eq__(self, other):
+        return list(self) == other
+
+    def __repr__(self):
+        return str(list(self))
+
+    def __add__(self, other):
+        return list(self) + other
+
+    def __getitem__(self, item):
+        # deque doesn't support slicing
+        deque = super(list, self).__getattribute__('_deque')
+        if isinstance(item, slice):
+            start, stop, step = item.indices(len(deque))
+            return [deque[i] for i in range(start, stop, step)]
+        else:
+            return deque[item]
+
+    # to be overriden
+    def getLimit(self, symbol, limit):
+        pass
+
+    # support transpiled snake_case calls
+    def get_limit(self, symbol, limit):
+        return self.getLimit(symbol, limit)
+
+
+class ArrayCache(BaseCache):
+    def __init__(self, max_size=None):
+        super(ArrayCache, self).__init__(max_size)
+        self._nested_new_updates_by_symbol = False
+        self._new_updates_by_symbol = {}
+        self._clear_updates_by_symbol = {}
+        self._all_new_updates = 0
+        self._clear_all_updates = False
+
+    def getLimit(self, symbol, limit):
+        if symbol is None:
+            new_updates_value = self._all_new_updates
+            self._clear_all_updates = True
+        else:
+            new_updates_value = self._new_updates_by_symbol.get(symbol)
+            if new_updates_value is not None and self._nested_new_updates_by_symbol:
+                new_updates_value = len(new_updates_value)
+            self._clear_updates_by_symbol[symbol] = True
+
+        if new_updates_value is None:
+            return limit
+        elif limit is not None:
+            return min(new_updates_value, limit)
+        else:
+            return new_updates_value
+
+    def append(self, item):
+        self._deque.append(item)
+        if self._clear_all_updates:
+            self._clear_all_updates = False
+            self._clear_updates_by_symbol.clear()
+            self._all_new_updates = 0
+            self._new_updates_by_symbol.clear()
+        if self._clear_updates_by_symbol.get(item['symbol']):
+            self._clear_updates_by_symbol[item['symbol']] = False
+            self._new_updates_by_symbol[item['symbol']] = 0
+        self._new_updates_by_symbol[item['symbol']] = self._new_updates_by_symbol.get(item['symbol'], 0) + 1
+        self._all_new_updates = (self._all_new_updates or 0) + 1
+
+
+class ArrayCacheByTimestamp(BaseCache):
+    def __init__(self, max_size=None):
+        super(ArrayCacheByTimestamp, self).__init__(max_size)
+        self.hashmap = {}
+        self._size_tracker = set()
+        self._new_updates = 0
+        self._clear_updates = False
+
+    def getLimit(self, symbol, limit):
+        self._clear_updates = True
+        if limit is None:
+            return self._new_updates
+        return min(self._new_updates, limit)
+
+    def append(self, item):
+        if item[0] in self.hashmap:
+            reference = self.hashmap[item[0]]
+            if reference != item:
+                reference[0:len(item)] = item
+        else:
+            self.hashmap[item[0]] = item
+            if len(self._deque) == self._deque.maxlen:
+                delete_reference = self._deque.popleft()
+                del self.hashmap[delete_reference[0]]
+            self._deque.append(item)
+        if self._clear_updates:
+            self._clear_updates = False
+            self._size_tracker.clear()
+        self._size_tracker.add(item[0])
+        self._new_updates = len(self._size_tracker)
+
+
+class ArrayCacheBySymbolById(ArrayCache):
+    def __init__(self, max_size=None):
+        super(ArrayCacheBySymbolById, self).__init__(max_size)
+        self._nested_new_updates_by_symbol = True
+        self.hashmap = {}
+        self._index = collections.deque([], max_size)
+
+    def append(self, item):
+        by_id = self.hashmap.setdefault(item['symbol'], {})
+        if item['id'] in by_id:
+            reference = by_id[item['id']]
+            if reference != item:
+                reference.update(item)
+            item = reference
+            index = self._index.index(item['id'])
+            del self._deque[index]
+            del self._index[index]
+        else:
+            by_id[item['id']] = item
+        if len(self._deque) == self._deque.maxlen:
+            delete_item = self._deque.popleft()
+            self._index.popleft()
+            del self.hashmap[delete_item['symbol']][delete_item['id']]
+        self._deque.append(item)
+        self._index.append(item['id'])
+        if self._clear_all_updates:
+            self._clear_all_updates = False
+            self._clear_updates_by_symbol.clear()
+            self._all_new_updates = 0
+            self._new_updates_by_symbol.clear()
+        if item['symbol'] not in self._new_updates_by_symbol:
+            self._new_updates_by_symbol[item['symbol']] = set()
+        if self._clear_updates_by_symbol.get(item['symbol']):
+            self._clear_updates_by_symbol[item['symbol']] = False
+            self._new_updates_by_symbol[item['symbol']].clear()
+        id_set = self._new_updates_by_symbol[item['symbol']]
+        before_length = len(id_set)
+        id_set.add(item['id'])
+        after_length = len(id_set)
+        self._all_new_updates = (self._all_new_updates or 0) + (after_length - before_length)
+
+
+class ArrayCacheBySymbolBySide(ArrayCache):
+    def __init__(self, max_size=None):
+        super(ArrayCacheBySymbolBySide, self).__init__(max_size)
+        self._nested_new_updates_by_symbol = True
+        self.hashmap = {}
+        self._index = collections.deque([], max_size)
+
+    def append(self, item):
+        by_side = self.hashmap.setdefault(item['symbol'], {})
+        if item['side'] in by_side:
+            reference = by_side[item['side']]
+            if reference != item:
+                reference.update(item)
+            item = reference
+            index = self._index.index(item['side'])
+            del self._deque[index]
+            del self._index[index]
+        else:
+            by_side[item['side']] = item
+        if len(self._deque) == self._deque.maxlen:
+            delete_item = self._deque.popleft()
+            self._index.popleft()
+            del self.hashmap[delete_item['symbol']][delete_item['side']]
+        self._deque.append(item)
+        self._index.append(item['side'])
+        if self._clear_all_updates:
+            self._clear_all_updates = False
+            self._clear_updates_by_symbol.clear()
+            self._all_new_updates = 0
+            self._new_updates_by_symbol.clear()
+        if item['symbol'] not in self._new_updates_by_symbol:
+            self._new_updates_by_symbol[item['symbol']] = set()
+        if self._clear_updates_by_symbol.get(item['symbol']):
+            self._clear_updates_by_symbol[item['symbol']] = False
+            self._new_updates_by_symbol[item['symbol']].clear()
+        side_set = self._new_updates_by_symbol[item['symbol']]
+        before_length = len(side_set)
+        side_set.add(item['side'])
+        after_length = len(side_set)
+        self._all_new_updates = (self._all_new_updates or 0) + (after_length - before_length)

ccxt/async_support/base/ws/client.py

@@ -0,0 +1,193 @@
+# -*- coding: utf-8 -*-
+
+from asyncio import sleep, ensure_future, wait_for, TimeoutError
+from .functions import milliseconds, iso8601, deep_extend
+from ccxt import NetworkError, RequestTimeout, NotSupported
+from ccxt.async_support.base.ws.future import Future
+
+
+class Client(object):
+
+    url = None
+    ws = None
+    futures = {}
+    options = {}  # ws-specific options
+    subscriptions = {}
+    rejections = {}
+    on_message_callback = None
+    on_error_callback = None
+    on_close_callback = None
+    on_connected_callback = None
+    connectionStarted = None
+    connectionEstablished = None
+    isConnected = False
+    connectionTimeout = 10000  # ms, false to disable
+    connection = None
+    error = None  # low-level networking exception, if any
+    connected = None  # connection-related Future
+    keepAlive = 5000
+    heartbeat = True
+    maxPingPongMisses = 2.0  # how many missed pongs to raise a timeout
+    lastPong = None
+    ping = None  # ping-function if defined
+    verbose = False  # verbose output
+    gunzip = False
+    inflate = False
+    throttle = None
+    connecting = False
+    asyncio_loop = None
+    ping_looper = None
+    receive_looper = None
+
+    def __init__(self, url, on_message_callback, on_error_callback, on_close_callback, on_connected_callback, config={}):
+        defaults = {
+            'url': url,
+            'futures': {},
+            'subscriptions': {},
+            'rejections': {},
+            'on_message_callback': on_message_callback,
+            'on_error_callback': on_error_callback,
+            'on_close_callback': on_close_callback,
+            'on_connected_callback': on_connected_callback,
+        }
+        settings = {}
+        settings.update(defaults)
+        settings.update(config)
+        for key in settings:
+            if hasattr(self, key) and isinstance(getattr(self, key), dict):
+                setattr(self, key, deep_extend(getattr(self, key), settings[key]))
+            else:
+                setattr(self, key, settings[key])
+        # connection-related Future
+        self.connected = Future()
+
+    def future(self, message_hash):
+        if message_hash not in self.futures or self.futures[message_hash].cancelled():
+            self.futures[message_hash] = Future()
+        future = self.futures[message_hash]
+        if message_hash in self.rejections:
+            future.reject(self.rejections[message_hash])
+            del self.rejections[message_hash]
+        return future
+
+    def resolve(self, result, message_hash):
+        if self.verbose and message_hash is None:
+            self.log(iso8601(milliseconds()), 'resolve received None messageHash')
+        if message_hash in self.futures:
+            future = self.futures[message_hash]
+            future.resolve(result)
+            del self.futures[message_hash]
+        return result
+
+    def reject(self, result, message_hash=None):
+        if message_hash:
+            if message_hash in self.futures:
+                future = self.futures[message_hash]
+                future.reject(result)
+                del self.futures[message_hash]
+            else:
+                self.rejections[message_hash] = result
+        else:
+            message_hashes = list(self.futures.keys())
+            for message_hash in message_hashes:
+                self.reject(result, message_hash)
+        return result
+
+    async def receive_loop(self):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'receive loop')
+        while not self.closed():
+            try:
+                message = await self.receive()
+                # self.log(iso8601(milliseconds()), 'received', message)
+                self.handle_message(message)
+            except Exception as e:
+                error = NetworkError(str(e))
+                if self.verbose:
+                    self.log(iso8601(milliseconds()), 'receive_loop', 'Exception', error)
+                self.reset(error)
+
+    async def open(self, session, backoff_delay=0):
+        # exponential backoff for consequent connections if necessary
+        if backoff_delay:
+            await sleep(backoff_delay)
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'connecting to', self.url, 'with timeout', self.connectionTimeout, 'ms')
+        self.connectionStarted = milliseconds()
+        try:
+            coroutine = self.create_connection(session)
+            self.connection = await wait_for(coroutine, timeout=int(self.connectionTimeout / 1000))
+            self.connecting = False
+            self.connectionEstablished = milliseconds()
+            self.isConnected = True
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'connected')
+            self.connected.resolve(self.url)
+            self.on_connected_callback(self)
+            # run both loops forever
+            self.ping_looper = ensure_future(self.ping_loop(), loop=self.asyncio_loop)
+            self.receive_looper = ensure_future(self.receive_loop(), loop=self.asyncio_loop)
+        except TimeoutError:
+            # connection timeout
+            error = RequestTimeout('Connection timeout')
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'RequestTimeout', error)
+            self.on_error(error)
+        except Exception as e:
+            # connection failed or rejected (ConnectionRefusedError, ClientConnectorError)
+            error = NetworkError(e)
+            if self.verbose:
+                self.log(iso8601(milliseconds()), 'NetworkError', error)
+            self.on_error(error)
+
+    def connect(self, session, backoff_delay=0):
+        if not self.connection and not self.connecting:
+            self.connecting = True
+            ensure_future(self.open(session, backoff_delay), loop=self.asyncio_loop)
+        return self.connected
+
+    def on_error(self, error):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'on_error', error)
+        self.error = error
+        self.reset(error)
+        self.on_error_callback(self, error)
+        if not self.closed():
+            ensure_future(self.close(1006), loop=self.asyncio_loop)
+
+    def on_close(self, code):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'on_close', code)
+        if not self.error:
+            self.reset(NetworkError('Connection closed by remote server, closing code ' + str(code)))
+        self.on_close_callback(self, code)
+        if not self.closed():
+            ensure_future(self.close(code), loop=self.asyncio_loop)
+
+    def reset(self, error):
+        self.reject(error)
+
+    async def ping_loop(self):
+        if self.verbose:
+            self.log(iso8601(milliseconds()), 'ping loop')
+
+    def receive(self):
+        raise NotSupported('receive() not implemented')
+
+    def handle_message(self, message):
+        raise NotSupported('handle_message() not implemented')
+
+    def closed(self):
+        raise NotSupported('closed() not implemented')
+
+    async def send(self, message):
+        raise NotSupported('send() not implemented')
+
+    async def close(self, code=1000):
+        raise NotSupported('close() not implemented')
+
+    def create_connection(self, session):
+        raise NotSupported('create_connection() not implemented')
+
+    def log(self, *args):
+        print(*args)