opencode-pyneruntime 6.6.4__py3-none-any.whl

pynecore/providers/ccxt.py

@@ -0,0 +1,664 @@
+from typing import Any, Callable
+from dataclasses import dataclass
+import re
+from datetime import datetime, UTC, timedelta, time
+from pathlib import Path
+import tomllib
+
+from pynecore.core.plugin import override, ProviderError, Broker
+from pynecore.core.plugin.live_provider import LiveProviderConfig, LiveProviderPlugin
+from pynecore.core.syminfo import SymInfo, SymInfoInterval, SymInfoSession
+from ..types.ohlcv import OHLCV
+
+__all__ = ['CCXTProvider', 'CCXTError']
+
+
+class CCXTError(ProviderError):
+    """Raised when a CCXT library call fails in a user-actionable way.
+
+    Wraps the underlying :class:`ccxt.BaseError` (and the unknown-exchange
+    ``AttributeError``) raised during client creation, market loading and
+    candle downloads so the ``pyne data`` CLI reports an exchange / network /
+    auth failure as a clean one-line error instead of a traceback. The live
+    streaming path is intentionally left unwrapped so its errors still reach
+    the :class:`LiveProviderPlugin` reconnect logic.
+    """
+
+_KNOWN_LIMITS = {
+    'binance': 1000,
+    'bitget': {
+        '1w': 12,
+        '1d': 300,
+        '4h': 1000,
+        'default': 200
+    },
+    'bitmex': 500,
+    'bybit': 200,
+    'coinbase': 300,
+    'kraken': 720,
+    'kucoin': 1500,
+    'okex': 200,
+    'huobi': 2000,
+}
+
+_PYNECORE_ONLY_CONFIG_KEYS: frozenset[str] = frozenset({
+    'sandbox',
+    'default_type',
+    'symbol_map',
+})
+
+# Fallback tick size when an exchange exposes no usable price precision at all.
+# Eight decimals is the common crypto default; an over-fine tick only adds
+# display precision, while a too-coarse one would silently round away real
+# price moves, so erring fine is the safe direction.
+_DEFAULT_MINTICK: float = 1e-8
+
+
+def add_space_before_uppercase(s):
+    return re.sub(r'(?<!^)([A-Z])', r' \1', s)
+
+
+@dataclass
+class CCXTConfig(LiveProviderConfig):
+    """CCXT provider configuration.
+
+    Fields map to CCXT constructor keyword arguments, so adding a new
+    exchange-specific setting is usually as simple as declaring it here
+    with the same name CCXT uses — ``vars(config)`` is filtered for
+    truthy values and spread into the CCXT client constructor. Inherits
+    ``symbol_map`` from :class:`LiveProviderConfig` so ``ccxt.toml`` can
+    declare TradingView→native symbol translations for
+    :meth:`ProviderPlugin.resolve_symbol`.
+    """
+
+    apiKey: str = ""
+    """Default API key for all exchanges"""
+
+    secret: str = ""
+    """Default API secret for all exchanges"""
+
+    password: str = ""
+    """Default API password (required by some exchanges like KuCoin)"""
+
+    sandbox: bool = False
+    """Enable the exchange's testnet / demo endpoint via ``set_sandbox_mode``."""
+
+    default_type: str = "swap"
+    """Default market type — ``"spot"`` / ``"swap"`` / ``"future"`` / ``"margin"``."""
+
+
+class CCXTProvider(LiveProviderPlugin[CCXTConfig]):
+    """CCXT-based market-data provider for ~100 crypto exchanges.
+
+    Wraps the CCXT library for symbol discovery and historical candle
+    downloads, and CCXT Pro for real-time WebSocket OHLCV streaming.
+    One plugin instance serves every CCXT-registered exchange — the
+    exchange is selected from the symbol prefix (`BYBIT:BTCUSDT` etc.).
+
+    **Supported**
+
+    - Symbol discovery and market metadata across all CCXT exchanges
+    - Historical candle downloads with per-exchange bar-limit hints
+    - Real-time OHLCV WebSocket streaming (CCXT Pro)
+    - Sandbox / testnet endpoints via the `sandbox` config flag
+    - Forwards any extra field on `CCXTConfig` as a CCXT constructor
+      kwarg — add exchange-specific settings without touching the plugin
+
+    **Limitations**
+
+    - **No order execution** — this plugin is market-data only. For live
+      trading use a dedicated exchange `BrokerPlugin` (Capital.com,
+      Interactive Brokers, Bybit, Binance).
+    - API credentials are per-exchange; the defaults on `CCXTConfig`
+      apply to every exchange unless overridden via CCXT's own routing.
+    """
+
+    plugin_name = "CCXT"
+    Config = CCXTConfig
+    multi_broker = True
+
+    # ``watch_ohlcv`` is trade-driven on many exchanges (ccxt.pro builds
+    # candles from trades where no native kline stream pushes idle
+    # updates), so an illiquid 24/7 pair can legitimately stay silent for
+    # several bars. The framework default (3 bars) would reconnect-churn
+    # on every quiet stretch; 30 bars keeps the dead-feed safety net while
+    # making false positives rare.
+    feed_timeout_bars = 30
+
+    @classmethod
+    @override
+    def get_list_of_brokers(cls) -> list[Broker]:
+        """Return the exchanges CCXT can serve, with their display names.
+
+        The id is the CCXT exchange id (e.g. ``"binance"``); the name is the
+        exchange's human-readable title (e.g. ``"Binance"``), read from a
+        throwaway instance — construction is offline and cheap (~100 exchanges
+        in a fraction of a second).
+        """
+        try:
+            import ccxt
+        except ImportError:
+            raise ImportError("CCXT is not installed. Please install it using `pip install ccxt`.")
+        brokers: list[Broker] = []
+        for exchange_id in ccxt.exchanges:
+            try:
+                name = getattr(ccxt, exchange_id)().name or ""
+            except Exception:  # noqa: BLE001 - a single bad exchange must not drop the list
+                name = ""
+            brokers.append(Broker(id=exchange_id, name=name))
+        return brokers
+
+    @classmethod
+    @override
+    def to_tradingview_timeframe(cls, timeframe: str) -> str:
+        """
+        Convert CCXT timeframe fmt to TradingView fmt.
+
+        :param timeframe: Timeframe in CCXT fmt (e.g. "1m", "5m", "1h", "1d", "1w", "1M")
+        :return: Timeframe in TradingView fmt (e.g. "1", "5", "60", "1D", "1W", "1M")
+        :raises ValueError: If timeframe fmt is invalid
+        """
+        if len(timeframe) < 2:
+            raise ValueError(f"Invalid timeframe fmt: {timeframe}")
+
+        unit = timeframe[-1]
+        value = timeframe[:-1]
+
+        if not value.isdigit() or int(value) <= 0:
+            raise ValueError(f"Invalid timeframe value: {value}")
+
+        if unit == 'm':
+            return value
+        elif unit == 'h':
+            return str(int(value) * 60)
+        elif unit == 'd':
+            return f"{value}D"
+        elif unit == 'w':
+            return f"{value}W"
+        elif unit == 'M':
+            return f"{value}M"
+        else:
+            raise ValueError(f"Invalid timeframe fmt: {timeframe}")
+
+    @classmethod
+    @override
+    def to_exchange_timeframe(cls, timeframe: str) -> str:
+        """
+        Convert TradingView timeframe fmt to CCXT fmt.
+
+        :param timeframe: Timeframe in TradingView fmt (e.g. "1", "5", "60", "1D", "1W", "1M")
+        :return: Timeframe in CCXT fmt (e.g. "1m", "5m", "1h", "1d", "1w", "1M")
+        :raises ValueError: If timeframe fmt is invalid
+        """
+        if timeframe.isdigit():
+            mins = int(timeframe)
+            if mins <= 0:
+                raise ValueError(f"Invalid timeframe value: {timeframe}")
+            if mins >= 60 and mins % 60 == 0:
+                return f"{mins // 60}h"
+            return f"{mins}m"
+
+        if len(timeframe) < 2:
+            raise ValueError(f"Invalid timeframe fmt: {timeframe}")
+
+        unit = timeframe[-1].upper()
+        value = timeframe[:-1]
+
+        if not value.isdigit() or int(value) <= 0:
+            raise ValueError(f"Invalid timeframe value: {value}")
+
+        if unit == 'D':
+            return f"{value}d"
+        elif unit == 'W':
+            return f"{value}w"
+        elif unit == 'M':
+            return f"{value}M"
+        else:
+            raise ValueError(f"Invalid timeframe fmt: {timeframe}")
+
+    @override
+    def __init__(self, *, symbol: str | None = None, timeframe: str | None = None,
+                 ohlcv_dir: Path | None = None, config: object | None = None):
+        """
+        :param symbol: The symbol to get data for (e.g. "binance:BTC/USDT")
+        :param timeframe: The timeframe to get data for in TradingView fmt
+        :param ohlcv_dir: The directory to save OHLCV data
+        :param config: Pre-loaded CCXTConfig instance
+        """
+        try:
+            import ccxt
+        except ImportError:
+            raise ImportError("CCXT is not installed. Please install it using `pip install ccxt`.")
+
+        super().__init__(symbol=symbol, timeframe=timeframe, ohlcv_dir=ohlcv_dir, config=config)
+
+        # Check symbol fmt
+        try:
+            if symbol is None:
+                raise ValueError("Error: Symbol not provided!")
+            xchg, symbol = symbol.split(':', 1)
+        except (ValueError, AttributeError):
+            xchg = symbol
+            symbol = None
+
+        if not xchg:
+            raise ValueError("Error: Exchange name not provided! Use 'exchange:symbol' fmt! "
+                             "(or simple exchange, if you want to list symbols)")
+
+        self.symbol = symbol
+        self._exchange_name = xchg.lower()
+        exchange_name = self._exchange_name
+
+        # Build exchange config from the Config dataclass + optional exchange-specific TOML sections
+        exchange_config = {}
+        if self.config:
+            exchange_config = {
+                k: v for k, v in vars(self.config).items()
+                if v and k not in _PYNECORE_ONLY_CONFIG_KEYS
+            }
+
+            # Check for exchange-specific override in the raw TOML
+            if self.ohlcv_path:
+                config_dir = self.ohlcv_path.parent.parent / 'config'
+            else:
+                config_dir = None
+
+            if config_dir:
+                toml_path = config_dir / 'plugins' / 'ccxt.toml'
+                if toml_path.exists():
+                    with open(toml_path, 'rb') as f:
+                        raw_toml = tomllib.load(f)
+                    if exchange_name in raw_toml and isinstance(raw_toml[exchange_name], dict):
+                        exchange_config = raw_toml[exchange_name]
+
+        self._async_client = None
+        self._last_bar_timestamp: int | None = None
+        self._last_bar_ohlcv: OHLCV | None = None
+        self._exchange_config: dict[str, Any] = dict(exchange_config)
+
+        # Create the CCXT client
+        try:
+            self._client: ccxt.Exchange = getattr(ccxt, exchange_name)({
+                'enableRateLimit': True,
+                'adjustForTimeDifference': True,
+                **exchange_config
+            })
+        except AttributeError:
+            raise CCXTError(
+                f"Unknown exchange: {exchange_name!r}. Run "
+                f"`pyne data download ccxt --list-brokers` to see available exchanges."
+            ) from None
+        if self.config and getattr(self.config, 'sandbox', False):
+            try:
+                self._client.set_sandbox_mode(True)
+            except ccxt.NotSupported:
+                pass
+
+    @override
+    def normalize_symbol(self, symbol: str) -> str:
+        """Strip exchange prefix: ``"binance:BTC/USDT"`` → ``"BTC/USDT"``."""
+        return self.symbol or symbol
+
+    @override
+    def get_list_of_symbols(self, *args, **kwargs) -> list[str]:
+        """Get list of symbols."""
+        import ccxt
+        try:
+            self._client.load_markets()
+        except ccxt.BaseError as exc:
+            raise CCXTError(f"{self._client.id}: {exc}") from exc
+        return self._client.symbols or []
+
+    @staticmethod
+    def _create_24_7_sessions() -> tuple[
+        list[SymInfoInterval], list[SymInfoSession], list[SymInfoSession]
+    ]:
+        """
+        Create 24/7 opening hours and sessions for crypto markets.
+
+        :return: Tuple of (opening_hours, session_starts, session_ends).
+        """
+        return cls._create_24_7_sessions()
+
+    @staticmethod
+    def _create_24_7_sessions() -> tuple[
+        list[SymInfoInterval], list[SymInfoSession], list[SymInfoSession]
+    ]:
+        """
+        Create 24/7 opening hours and sessions for crypto markets.
+
+        :return: Tuple of (opening_hours, session_starts, session_ends).
+        """
+        opening_hours = []
+        session_starts = []
+        session_ends = []
+        for i in range(7):
+            opening_hours.append(
+                SymInfoInterval(day=i, start=time(hour=0, minute=0),
+                                end=time(hour=23, minute=59, second=59)))
+            session_starts.append(SymInfoSession(day=i, time=time(hour=0, minute=0)))
+            session_ends.append(SymInfoSession(day=i, time=time(hour=23, minute=59, second=59)))
+        return opening_hours, session_starts, session_ends
+
+    def _derive_mintick(self, market_details: dict) -> float:
+        """Derive the minimum price increment (tick size) for a market.
+
+        CCXT exposes price precision in three different ``precisionMode``
+        flavours, and not every exchange populates ``precision.price`` at all:
+
+        * ``TICK_SIZE`` -- ``precision.price`` already *is* the tick size.
+        * ``DECIMAL_PLACES`` -- ``precision.price`` is a decimal-place count.
+        * ``SIGNIFICANT_DIGITS`` -- ``precision.price`` cannot map to a fixed
+          tick (it depends on the price magnitude) and is often ``None`` (e.g.
+          bitvavo).
+
+        For the latter two we fall back to the raw ``info['tickSize']`` the
+        exchange ships, then to ``limits.price.min``, and finally to a
+        conservative default so symbol info never crashes on a missing or
+        ambiguous precision.
+
+        :param market_details: A CCXT market dict.
+        :return: The tick size as a positive float.
+        """
+        import ccxt
+
+        precision_price = market_details.get('precision', {}).get('price')
+        mode = self._client.precisionMode
+
+        if precision_price is not None:
+            if mode == ccxt.TICK_SIZE:
+                return float(precision_price)
+            if mode == ccxt.DECIMAL_PLACES:
+                return 10.0 ** -int(precision_price)
+            # SIGNIFICANT_DIGITS: not a fixed tick -- fall through to raw values.
+
+        for raw in (market_details.get('info', {}).get('tickSize'),
+                    market_details.get('limits', {}).get('price', {}).get('min')):
+            if raw is None:
+                continue
+            try:
+                tick = float(raw)
+            except (TypeError, ValueError):
+                continue
+            if tick > 0.0:
+                return tick
+
+        import logging
+        logging.getLogger(__name__).warning(
+            "%s: no usable price precision for %r; defaulting tick size to %g",
+            self._client.id, self.symbol, _DEFAULT_MINTICK,
+        )
+        return _DEFAULT_MINTICK
+
+    def _derive_qty_step(self, market_details: dict) -> float:
+        """Derive the minimum order quantity step (``mincontract``) for a market.
+
+        Mirrors :meth:`_derive_mintick` on the amount axis: with ``TICK_SIZE``
+        precision ``precision.amount`` already *is* the step, with
+        ``DECIMAL_PLACES`` it is a decimal-place count, and
+        ``SIGNIFICANT_DIGITS`` cannot map to a fixed step. The raw fallback is
+        ``limits.amount.min`` — TV defines ``mincontract`` as the smallest
+        tradable amount, which on most exchanges equals the amount step (e.g.
+        Binance ``LOT_SIZE`` ``stepSize`` / ``minQty``).
+
+        :param market_details: A CCXT market dict.
+        :return: The quantity step, or ``0.0`` when the exchange does not
+            expose one (the symbol info chain then falls back to volume
+            analysis / heuristics).
+        """
+        import ccxt
+
+        precision_amount = market_details.get('precision', {}).get('amount')
+        mode = self._client.precisionMode
+
+        if precision_amount is not None:
+            if mode == ccxt.TICK_SIZE:
+                return float(precision_amount)
+            if mode == ccxt.DECIMAL_PLACES:
+                return 10.0 ** -int(precision_amount)
+            # SIGNIFICANT_DIGITS: not a fixed step -- fall through to raw values.
+
+        raw = market_details.get('limits', {}).get('amount', {}).get('min')
+        try:
+            step = float(raw)
+        except (TypeError, ValueError):
+            return 0.0
+        return step if step > 0.0 else 0.0
+
+    @override
+    def update_symbol_info(self) -> SymInfo:
+        """Update symbol info from the exchange."""
+        import ccxt
+        try:
+            self._client.load_markets()
+        except ccxt.BaseError as exc:
+            raise CCXTError(f"{self._client.id}: {exc}") from exc
+        assert self._client.markets
+        try:
+            market_details = self._client.markets[self.symbol]
+        except KeyError:
+            raise CCXTError(
+                f"Unknown symbol {self.symbol!r} on {self._client.id}."
+            ) from None
+
+        opening_hours, session_starts, session_ends = self._create_24_7_sessions()
+
+        # Derive pricescale from the tick size. ``round_to_mintick`` divides by
+        # pricescale directly and ignores minmove, so the tick must be an
+        # integer reciprocal and minmove must stay 1.
+        mintick = self._derive_mintick(market_details)
+        pricescale = max(1, int(round(1.0 / mintick)))
+        if abs(mintick * pricescale - 1.0) > 1e-9:
+            raise CCXTError(
+                f"{self._client.id}: price tick {mintick} for {self.symbol!r} "
+                f"is not an integer reciprocal; cannot derive pricescale."
+            )
+        minmove = 1
+
+        try:
+            ticker = market_details['info']['symbol']
+        except KeyError:
+            try:
+                ticker = market_details['symbol']
+            except KeyError:
+                ticker = market_details['id']
+
+        assert self._client.id
+        return SymInfo(
+            prefix=self._client.id.upper(),
+            description=f"{market_details['base']} / {market_details['quote']} "
+                        f"{add_space_before_uppercase(market_details['info'].get('contractType', 'Spot'))}",
+            ticker=ticker,
+            currency=market_details['quote'],
+            basecurrency=market_details['base'],
+            period=self.timeframe or "1D",
+            type="crypto",
+            mintick=mintick,
+            pricescale=pricescale,
+            minmove=minmove,
+            pointvalue=market_details.get('contractSize') or 1.0,
+            mincontract=self._derive_qty_step(market_details),
+            timezone=self.timezone,
+            opening_hours=opening_hours,
+            session_starts=session_starts,
+            session_ends=session_ends,
+            taker_fee=market_details.get('taker'),
+            maker_fee=market_details.get('maker'),
+        )
+
+    @override
+    def download_ohlcv(self, time_from: datetime, time_to: datetime,
+                       on_progress: Callable[[datetime], None] | None = None,
+                       limit: int | None = None, with_extra: bool = False):
+        """
+        Download OHLCV data.
+
+        :param time_from: The start time.
+        :param time_to: The end time.
+        :param on_progress: Optional callback to call on progress.
+        :param limit: Override the automatic chunk size.
+        :param with_extra: Ignored; CCXT OHLCV has no extra per-bar fields.
+        """
+        import ccxt
+
+        assert self.symbol is not None
+        assert self.xchg_timeframe is not None
+
+        tf: datetime = time_from.replace(tzinfo=None)
+        tt: datetime = time_to.replace(tzinfo=None)
+
+        if limit is None:
+            assert self._client.id
+            limit_config = _KNOWN_LIMITS.get(self._client.id, 100)
+
+            if isinstance(limit_config, dict):
+                limit = limit_config.get(self.xchg_timeframe, limit_config.get('default', 100))
+            else:
+                limit = limit_config
+
+        try:
+            while tf < tt:
+                if on_progress:
+                    on_progress(tf)
+
+                try:
+                    res: list = self._client.fetch_ohlcv(
+                        symbol=self.symbol,
+                        limit=limit,
+                        timeframe=self.xchg_timeframe,
+                        since=self._client.parse8601(tf.isoformat())
+                    )
+                except ccxt.BaseError as exc:
+                    raise CCXTError(f"{self._client.id}: {exc}") from exc
+
+                if not res:
+                    tf += timedelta(days=1)
+
+                for r in res:
+                    t = int(r[0] / 1000)
+                    dt = datetime.fromtimestamp(t, UTC).replace(tzinfo=None)
+                    if dt > tt:
+                        raise StopIteration
+
+                    ohlcv = OHLCV(
+                        timestamp=t,
+                        open=float(r[1]),
+                        high=float(r[2]),
+                        low=float(r[3]),
+                        close=float(r[4]),
+                        volume=float(r[5]),
+                    )
+
+                    self.save_ohlcv_data(ohlcv)
+                    tf = dt + timedelta(minutes=1)
+
+        except StopIteration:
+            pass
+
+        if on_progress:
+            on_progress(tt)
+
+    # --- LiveProviderPlugin methods ---
+
+    @override
+    async def connect(self) -> None:
+        """Establish async CCXT Pro connection for live OHLCV streaming."""
+        try:
+            import ccxt.pro as ccxtpro
+        except ImportError:
+            raise ImportError(
+                "CCXT Pro is required for live data. Install it with: pip install ccxt"
+            )
+
+        # ``connect()`` runs on every reconnect too, so the candle-close
+        # tracker must be wiped: a leftover ``_last_bar_*`` from the dead
+        # websocket would make the first ``watch_ohlcv`` after a forced
+        # reconnect (e.g. the stale-feed watchdog) emit that pre-outage
+        # partial bar as a confirmed close once a new-timestamp candle
+        # arrives.
+        self._last_bar_timestamp = None
+        self._last_bar_ohlcv = None
+
+        exchange_name = self._client.id
+
+        exchange_config: dict[str, Any] = {'enableRateLimit': True}
+        exchange_config.update(self._exchange_config)
+
+        if self.config:
+            exchange_config.update({
+                k: v for k, v in vars(self.config).items()
+                if v and k not in _PYNECORE_ONLY_CONFIG_KEYS
+            })
+
+        self._async_client = getattr(ccxtpro, exchange_name)(exchange_config)
+
+        if self.config and self.config.sandbox:
+            try:
+                self._async_client.set_sandbox_mode(True)
+            except Exception as exc:  # noqa: BLE001
+                import logging
+                logging.getLogger(__name__).warning(
+                    "Exchange %r does not support sandbox mode: %s",
+                    exchange_name, exc,
+                )
+
+        if self.config and self.config.default_type:
+            self._async_client.options = {
+                **getattr(self._async_client, 'options', {}),
+                'defaultType': self.config.default_type,
+            }
+
+    @override
+    async def disconnect(self) -> None:
+        """Close the async CCXT connection."""
+        if hasattr(self, '_async_client') and self._async_client:
+            await self._async_client.close()
+            self._async_client = None
+
+    @property
+    @override
+    def is_connected(self) -> bool:
+        """Whether the async CCXT connection is active."""
+        return hasattr(self, '_async_client') and self._async_client is not None
+
+    @override
+    async def watch_ohlcv(self, symbol: str, timeframe: str) -> OHLCV:
+        """
+        Wait for the next OHLCV update from the exchange websocket.
+
+        Detects bar closure by tracking timestamp changes: when a new bar
+        timestamp appears, the previous bar is returned as closed. Intra-bar
+        updates (same timestamp) are returned with ``is_closed=False``.
+
+        :param symbol: Symbol in CCXT format (e.g. "BTC/USDT:USDT").
+        :param timeframe: Timeframe in TradingView format (e.g. "1D", "1", "4H").
+        :return: OHLCV with ``is_closed=True`` for a final bar, ``False`` for intra-bar updates.
+        """
+        xchg_tf = self.to_exchange_timeframe(timeframe)
+
+        while True:
+            candles = await self._async_client.watch_ohlcv(symbol, xchg_tf)
+            last = candles[-1]
+            timestamp = int(last[0] / 1000)
+
+            current_ohlcv = OHLCV(
+                timestamp=timestamp,
+                open=float(last[1]),
+                high=float(last[2]),
+                low=float(last[3]),
+                close=float(last[4]),
+                volume=float(last[5]),
+                is_closed=False,
+            )
+
+            if (self._last_bar_timestamp is not None
+                    and timestamp != self._last_bar_timestamp):
+                assert self._last_bar_ohlcv is not None
+                closed_bar = self._last_bar_ohlcv._replace(is_closed=True)
+                self._last_bar_timestamp = timestamp
+                self._last_bar_ohlcv = current_ohlcv
+                return closed_bar
+
+            self._last_bar_timestamp = timestamp
+            self._last_bar_ohlcv = current_ohlcv
+            return current_ohlcv