sahiapi 0.1.0__py3-none-any.whl

sahiapi/sahi/feed.py

@@ -0,0 +1,668 @@
+"""``SahiFeed``: the two live WebSocket streams.
+
+Sahi has two streams, and :class:`SahiFeed` owns one connection for each:
+
+* the **tick stream** (``WSS /v1/stream/marketdata``), binary protobuf, which you
+  subscribe to per instrument at a mode (LTP, quote, or depth); and
+* the **order-updates stream** (``WSS /v1/stream/orders``), JSON, which pushes
+  your account's order events with no subscription.
+
+Both share a token and the connection rules in :mod:`~sahiapi.sahi.ws_client`.
+Ticks update a local cache as they arrive, so :meth:`get_ltp`, :meth:`get_quote`
+and :meth:`get_depth` return the latest value even before you start consuming.
+Once the tick stream has closed fatally (an expired token above all) those
+reads raise that error instead of serving a frozen price, so a caller that only
+polls the cache learns of it too.
+Your callbacks run on the thread that calls :meth:`consume`, one at a time, never
+on a socket thread, so a slow callback cannot stall the connection or race
+another callback.
+
+What waits for :meth:`consume` is bounded. Every tick is a full snapshot for
+its instrument at its mode, so only the newest tick per instrument and mode is
+held for the callback: a callback that keeps up sees every tick, a slow one
+sees fewer, fresher ticks, and memory stays proportional to the subscriptions
+whether or not :meth:`consume` is ever called. Order updates are events, so
+they wait in order up to a cap; past it the oldest is dropped and the
+``on_connect`` hook is queued once, so the Stream + Snapshot reconcile below
+fills the gap exactly as it does after a reconnect.
+
+Subscriptions are connection-local server-side and are not replayed after a drop,
+so the feed keeps its own subscription table and re-sends it on every reconnect.
+The table holds only ids the server has acknowledged: a subscribe request waits
+in a pending batch until its ``ack`` arrives, and a batch the server rejects
+(an unknown id, the per-connection cap) is dropped whole and reported through
+the callback as an ``error`` event naming its mode and ids. The server refuses
+such a request without applying any of it, so recording ids on send would let
+one bad id silence every id replayed with it after a reconnect. Pending batches
+are replayed one frame each, after the acknowledged table, so a bad batch can
+fail only itself.
+The order stream replays nothing either, so it offers an ``on_connect`` hook for
+the API reference's Stream + Snapshot bootstrap: on every (re)connect, fetch the
+order book and reconcile by ``last_modified_at``. A fatal close — an expired
+token above all — stops both streams and is raised from :meth:`consume`, and
+so does an exception escaping one of your callbacks: :meth:`consume` never
+returns or raises with a stream still running behind it.
+
+Thread-safety: the subscription table and tick cache are read and written from
+both caller threads (subscribe/unsubscribe/get_*) and the socket thread (tick
+arrival, reconnect replay), so a single lock guards them. It is never held across
+a network send, so a blocked socket cannot stall a caller. The server answers
+subscribe requests in the order it receives them, with no request id, so a second
+lock serializes subscribe frames, held from queuing a batch through writing it:
+the wire order is then always the pending order, whichever threads subscribe or
+replay, and a write stuck on a dead link delays only the next subscribe.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import queue
+import threading
+from collections import deque
+from dataclasses import dataclass, field
+from typing import Any, Callable, Final
+
+from . import constants, env
+from .client import SahiAPI
+from .exceptions import (
+    SahiFeedConnectionException,
+    SahiFeedException,
+    SahiFeedNotSubscribedException,
+)
+from .proto import codec
+from .ws_client import WebSocketConnection
+
+__all__ = ["SahiFeed"]
+
+logger = logging.getLogger(__name__)
+
+DataCallback = Callable[[dict[str, Any], dict[str, Any]], None]
+ConnectCallback = Callable[[], None]
+
+_MARKETDATA_PATH = "/v1/stream/marketdata"
+_ORDERS_PATH = "/v1/stream/orders"
+
+# Order updates waiting for consume() beyond this many drop the oldest and queue
+# a resync (the on_connect hook), as the reference prescribes for a 4409.
+_ORDER_QUEUE_LIMIT: Final = 1000
+
+
+@dataclass
+class _Batch:
+    """One subscribe request awaiting the server's ack or error."""
+
+    mode: str
+    ids: list[int]
+    # True for a batch the replay built from already-acknowledged ids: on error
+    # those ids leave the table, since the server no longer holds them.
+    held: bool = field(default=False)
+
+
+@dataclass(frozen=True)
+class _TickReady:
+    """Dispatch marker: the newest tick for this (bucket, instrument) awaits delivery."""
+
+    key: tuple[str, int]
+
+
+class _OrdersReady:
+    """Dispatch marker: the order queue has at least one item to deliver."""
+
+
+_ORDERS_READY = _OrdersReady()
+_ORDER_CONNECT: Final = ("connect",)
+
+
+# Tick payload type -> the cache bucket it updates.
+_TICK_CACHE_BUCKET = {
+    "ltp_tick": "ltp",
+    "quote_tick": "quote",
+    "depth_tick": "depth",
+}
+
+
+class SahiFeed:
+    """Live market-data and order-update streams for a :class:`SahiAPI`.
+
+    Args:
+        api: A constructed :class:`SahiAPI`. The feed borrows its token and host;
+            it never calls back into the REST client.
+
+    Rotating the API's token with :meth:`SahiAPI.set_access_token` does not affect
+    a running feed. A feed connection carries the token it was started with until
+    it is stopped and started again.
+    """
+
+    def __init__(self, api: SahiAPI) -> None:
+        self._access_token = api.access_token
+        ws_base = env.websocket_base_url(env.API_BASE_URL)
+        self._marketdata_url = ws_base + _MARKETDATA_PATH
+        self._orders_url = ws_base + _ORDERS_PATH
+
+        self._marketdata: WebSocketConnection | None = None
+        self._orders: WebSocketConnection | None = None
+        self._marketdata_callback: DataCallback | None = None
+        self._orders_callback: DataCallback | None = None
+        self._orders_connect_callback: ConnectCallback | None = None
+
+        # Guards _subscriptions, _cache and _pending, which the caller threads
+        # and the socket thread both touch. Held only for the dict operation,
+        # never over a send, so it cannot couple caller latency to the network.
+        self._lock = threading.Lock()
+        # Serializes subscribe frames: held from queuing a batch through writing
+        # it, by subscribe calls and the reconnect replay alike, so the wire
+        # order matches _pending (the server answers in wire order, and that is
+        # how acks are matched). Taken before _lock, never inside it.
+        self._subscribe_lock = threading.Lock()
+        # The connection epoch whose reconnect replay has run; guarded by
+        # _subscribe_lock. A subscribe call sends directly only on that epoch;
+        # on a newer one (the socket is up but its replay is still to come) it
+        # leaves the batch for the replay, which sends the held table first.
+        self._replayed_epoch: int | None = None
+        # instrument_id -> mode for ids the server has acknowledged, replayed
+        # after a reconnect.
+        self._subscriptions: dict[int, str] = {}
+        # Subscribe requests sent (or queued) and not yet answered, in send order.
+        # The server answers each with one ack or one error, in that order.
+        self._pending: deque[_Batch] = deque()
+        # bucket ("ltp"/"quote"/"depth") -> instrument_id -> latest decoded tick.
+        self._cache: dict[str, dict[int, dict[str, Any]]] = {"ltp": {}, "quote": {}, "depth": {}}
+
+        # Hand-off to consume(): markers and one-off events (acks, errors, the
+        # fatal error, the stop sentinel). The payloads live in the two bounded
+        # structures below, guarded by _lock.
+        self._dispatch: queue.Queue[Any] = queue.Queue()
+        # (bucket, instrument_id) -> the newest (tick, meta) not yet delivered.
+        self._latest_tick: dict[tuple[str, int], tuple[dict[str, Any], dict[str, Any]]] = {}
+        # Order frames and connect markers, in arrival order, at most
+        # _ORDER_QUEUE_LIMIT long.
+        self._order_queue: deque[tuple[Any, ...]] = deque()
+        self._order_connects_queued = 0
+
+    # -- market-data subscriptions -------------------------------------------
+
+    def subscribe_ltp(
+        self, instrument_ids: list[int], on_data_received: DataCallback | None = None
+    ) -> None:
+        """Subscribe these instruments for last-traded-price ticks.
+
+        Opens the tick connection on first use. ``on_data_received`` sets the one
+        market-data callback shared by all tick modes (LTP/quote/depth); passing
+        it here replaces any previously registered callback, and omitting it keeps
+        the current one. Ticks reach the callback from :meth:`consume`, never on
+        the socket thread.
+
+        The call does not wait for the server. The ids count as subscribed at
+        once (:meth:`get_ltp` returns ``None`` until a tick lands), and enter the
+        replayed table when the server's ``ack`` arrives. The server refuses a
+        request whole if any id is unknown or the per-connection cap would be
+        passed; that batch is then dropped, none of its ids are subscribed, and
+        the callback receives an ``error`` event whose ``meta`` carries the
+        ``mode`` and ``instrument_ids`` of the refused batch.
+        """
+        self._subscribe(constants.MODE_LTP, instrument_ids, on_data_received)
+
+    def subscribe_quote(
+        self, instrument_ids: list[int], on_data_received: DataCallback | None = None
+    ) -> None:
+        """Subscribe these instruments for full-quote ticks.
+
+        See :meth:`subscribe_ltp` for the shared-callback and connection behaviour
+        of ``on_data_received``.
+        """
+        self._subscribe(constants.MODE_QUOTE, instrument_ids, on_data_received)
+
+    def subscribe_depth(
+        self, instrument_ids: list[int], on_data_received: DataCallback | None = None
+    ) -> None:
+        """Subscribe these instruments for depth ticks (quote plus market depth).
+
+        See :meth:`subscribe_ltp` for the shared-callback and connection behaviour
+        of ``on_data_received``.
+        """
+        self._subscribe(constants.MODE_DEPTH, instrument_ids, on_data_received)
+
+    def change_mode(self, instrument_ids: list[int], mode: str) -> None:
+        """Change the tick mode for these instruments, subscribing any that are new.
+
+        The new mode is recorded in the subscription table, so it survives a
+        reconnect. Ids not previously subscribed are added at ``mode`` rather than
+        rejected.
+
+        This sends a ``subscribe`` frame, not the wire's ``mode_change``: the
+        server's subscribe is idempotent and re-points ids it already holds,
+        while its mode_change rejects the whole request with ``NOT_SUBSCRIBED``
+        if any id is new, which would leave the table and the server disagreeing.
+        """
+        self._subscribe(mode, instrument_ids, None)
+
+    def unsubscribe(self, instrument_ids: list[int]) -> None:
+        """Stop ticks for these instruments and drop their cached values.
+
+        Afterwards :meth:`get_ltp` / :meth:`get_quote` / :meth:`get_depth` raise
+        :class:`SahiFeedNotSubscribedException` for them until re-subscribed.
+        """
+        dropped = set(instrument_ids)
+        # Under the subscribe lock, so the frame is ordered with every subscribe
+        # frame: it cannot slip in between a reconnect replay's frames and reach
+        # the server ahead of the one re-subscribing these very ids, nor cross
+        # a concurrent subscribe of the same id.
+        with self._subscribe_lock:
+            with self._lock:
+                for instrument_id in dropped:
+                    self._subscriptions.pop(instrument_id, None)
+                    for bucket_name, bucket in self._cache.items():
+                        bucket.pop(instrument_id, None)
+                        self._latest_tick.pop((bucket_name, instrument_id), None)
+                # An id still awaiting its ack must not come back when that ack lands.
+                for batch in self._pending:
+                    batch.ids = [i for i in batch.ids if i not in dropped]
+            if self._marketdata is not None:
+                self._send_marketdata(codec.encode_unsubscribe(instrument_ids))
+
+    def _subscribe(
+        self, mode: str, instrument_ids: list[int], on_data_received: DataCallback | None
+    ) -> None:
+        if on_data_received is not None:
+            self._marketdata_callback = on_data_received
+        ids = list(dict.fromkeys(instrument_ids))
+        if not ids:
+            return
+        codec.encode_subscribe(mode, ids)  # validates the mode before any state changes
+        batch = _Batch(mode, ids)
+        self._ensure_marketdata()
+        with self._subscribe_lock:
+            with self._lock:
+                self._pending.append(batch)
+            self._send_batch_locked(batch)
+
+    # -- cache reads ----------------------------------------------------------
+
+    def get_ltp(self, instrument_id: int) -> dict[str, Any] | None:
+        """Return the latest LTP tick for an instrument, or ``None`` if none has arrived.
+
+        Raises:
+            SahiFeedException: The error that closed the tick stream fatally, if
+                it has (for example :class:`SahiFeedAuthExpiredException` on an
+                expired token), so a cache-only caller does not read a frozen price.
+            SahiFeedNotSubscribedException: If the instrument is not subscribed.
+        """
+        return self._cached("ltp", instrument_id)
+
+    def get_quote(self, instrument_id: int) -> dict[str, Any] | None:
+        """Return the latest quote tick for an instrument, or ``None`` if none yet.
+
+        Raises:
+            SahiFeedException: The error that closed the tick stream fatally, if
+                it has (for example :class:`SahiFeedAuthExpiredException` on an
+                expired token), so a cache-only caller does not read a frozen price.
+            SahiFeedNotSubscribedException: If the instrument is not subscribed.
+        """
+        return self._cached("quote", instrument_id)
+
+    def get_depth(self, instrument_id: int) -> dict[str, Any] | None:
+        """Return the latest depth tick for an instrument, or ``None`` if none yet.
+
+        Raises:
+            SahiFeedException: The error that closed the tick stream fatally, if
+                it has (for example :class:`SahiFeedAuthExpiredException` on an
+                expired token), so a cache-only caller does not read a frozen price.
+            SahiFeedNotSubscribedException: If the instrument is not subscribed.
+        """
+        return self._cached("depth", instrument_id)
+
+    def _cached(self, bucket: str, instrument_id: int) -> dict[str, Any] | None:
+        conn = self._marketdata
+        if conn is not None:
+            fatal = conn.fatal_error
+            if fatal is not None:
+                raise fatal
+        with self._lock:
+            if not self._is_subscribed(instrument_id):
+                raise SahiFeedNotSubscribedException(
+                    f"instrument {instrument_id} is not subscribed"
+                )
+            return self._cache[bucket].get(instrument_id)
+
+    def _is_subscribed(self, instrument_id: int) -> bool:
+        """Acknowledged or awaiting its ack. Call with the lock held."""
+        if instrument_id in self._subscriptions:
+            return True
+        return any(instrument_id in batch.ids for batch in self._pending)
+
+    # -- order updates --------------------------------------------------------
+
+    def subscribe_order_updates(
+        self, on_data_received: DataCallback, *, on_connect: ConnectCallback | None = None
+    ) -> None:
+        """Start the order-updates stream.
+
+        There is no per-instrument subscription: the server pushes every order
+        event for the account as soon as the socket is open. Each frame is an
+        order object shaped like an element of ``GET /v1/orders``; an update may
+        arrive more than once, and nothing is replayed for time the socket was
+        down. The API reference's remedy is Stream + Snapshot, which
+        ``on_connect`` exists for:
+
+        1. ``on_connect`` fires after every (re)connect, on the :meth:`consume`
+           thread, before any frame from that connection.
+        2. In it, fetch ``api.get_order_list()``.
+        3. Keep frames that arrive meanwhile; reconcile per ``order_id`` by the
+           higher ``last_modified_at``. Duplicates fall out of that step.
+
+        Args:
+            on_data_received: Receives ``(order, meta)`` per frame, where ``meta``
+                is ``{"stream": "orders", "type": "order"}``.
+            on_connect: Optional; called with no arguments after each connect,
+                including reconnects after 4408 and 4409, and once more when
+                frames had to be dropped because :meth:`consume` fell more than
+                a thousand frames behind: the same snapshot repairs both gaps.
+        """
+        self._orders_callback = on_data_received
+        self._orders_connect_callback = on_connect
+        if self._orders is None:
+            self._orders = WebSocketConnection(
+                self._orders_url,
+                access_token=self._access_token,
+                binary=False,
+                on_message=self._on_orders_message,
+                on_open=self._on_orders_open,
+                on_fatal=self._on_fatal,
+                name="sahi-orders",
+            )
+        self._orders.start()
+
+    # -- run loop -------------------------------------------------------------
+
+    def consume(self) -> None:
+        """Dispatch callbacks on the calling thread until :meth:`stop`.
+
+        Blocks. Each queued message runs its callback here, one at a time, so your
+        callbacks never run concurrently or on a socket thread. Returns when
+        :meth:`stop` is called.
+
+        Ticks are delivered newest-first per instrument and mode: if several
+        arrived for one instrument while an earlier callback ran, only the
+        latest is delivered, since each is a full snapshot. Order updates are
+        delivered in arrival order.
+
+        However it ends, both streams are stopped first: on a fatal stream
+        close, on an exception from one of your callbacks, or on an interrupt.
+        A stream is never left reconnecting in the background with nobody
+        draining it, so the caller need not add a ``finally: stop()``. An
+        exception from a callback is re-raised as-is, not swallowed: a bug in
+        the callback should surface, and a callback that wants to survive a
+        transient failure (a database hiccup, say) catches it in its own body,
+        where it knows what to retry.
+
+        Raises:
+            SahiFeedException: If a stream closes fatally — most importantly
+                :class:`SahiFeedAuthExpiredException` when the token expires (4401).
+            Exception: Whatever one of your callbacks raised.
+        """
+        try:
+            while True:
+                item = self._dispatch.get()
+                if item is None:  # stop sentinel
+                    return
+                if isinstance(item, SahiFeedException):
+                    raise item
+                if isinstance(item, _TickReady):
+                    self._deliver_tick(item.key)
+                elif item is _ORDERS_READY:
+                    self._deliver_order()
+                else:
+                    callback, data, meta = item
+                    callback(data, meta)
+        finally:
+            self.stop()
+
+    def _deliver_tick(self, key: tuple[str, int]) -> None:
+        with self._lock:
+            entry = self._latest_tick.pop(key, None)
+        callback = self._marketdata_callback
+        if entry is None or callback is None:
+            return  # unsubscribed meanwhile, or the callback was cleared
+        data, meta = entry
+        callback(data, meta)
+
+    def _deliver_order(self) -> None:
+        with self._lock:
+            if not self._order_queue:
+                return
+            item = self._order_queue.popleft()
+            if item is _ORDER_CONNECT:
+                self._order_connects_queued -= 1
+            if self._order_queue:
+                # One item per marker keeps ticks and orders interleaved fairly.
+                self._dispatch.put(_ORDERS_READY)
+        if item is _ORDER_CONNECT:
+            connect = self._orders_connect_callback
+            if connect is not None:
+                connect()
+            return
+        callback = self._orders_callback
+        if callback is not None:
+            callback(item[0], item[1])
+
+    def _queue_order_item(self, item: tuple[Any, ...]) -> None:
+        """Append to the order queue, dropping the oldest and queueing a resync past the cap."""
+        with self._lock:
+            was_empty = not self._order_queue
+            frames_waiting = len(self._order_queue) - self._order_connects_queued
+            if item is not _ORDER_CONNECT and frames_waiting >= _ORDER_QUEUE_LIMIT:
+                self._drop_oldest_order_frame_locked()
+            self._order_queue.append(item)
+            if item is _ORDER_CONNECT:
+                self._order_connects_queued += 1
+            if was_empty:
+                self._dispatch.put(_ORDERS_READY)
+
+    def _drop_oldest_order_frame_locked(self) -> None:
+        """Drop the oldest frame (never a connect marker) and make sure a resync is queued."""
+        markers_in_front = 0
+        while self._order_queue and self._order_queue[0] is _ORDER_CONNECT:
+            self._order_queue.popleft()
+            markers_in_front += 1
+        if self._order_queue:
+            self._order_queue.popleft()
+        for _ in range(markers_in_front):
+            self._order_queue.appendleft(_ORDER_CONNECT)
+        if self._order_connects_queued == 0:
+            # Recover the way a 4409 does, but locally: the snapshot the hook
+            # fetches carries the final state of every dropped frame. Queued
+            # once per overflow episode, at the front, so recovery runs first.
+            self._order_queue.appendleft(_ORDER_CONNECT)
+            self._order_connects_queued += 1
+            logger.warning(
+                "order updates: consume() is more than %d frames behind; dropping the "
+                "oldest and queueing on_connect to resync",
+                _ORDER_QUEUE_LIMIT,
+            )
+
+    def stop(self) -> None:
+        """Close both streams and unblock :meth:`consume`."""
+        if self._marketdata is not None:
+            self._marketdata.stop()
+        if self._orders is not None:
+            self._orders.stop()
+        self._dispatch.put(None)
+
+    # -- internals ------------------------------------------------------------
+
+    def _ensure_marketdata(self) -> None:
+        if self._marketdata is None:
+            self._marketdata = WebSocketConnection(
+                self._marketdata_url,
+                access_token=self._access_token,
+                binary=True,
+                on_message=self._on_marketdata_message,
+                on_open=self._replay_subscriptions,
+                on_fatal=self._on_fatal,
+                name="sahi-marketdata",
+            )
+        self._marketdata.start()
+
+    def _send_marketdata(self, frame: bytes) -> None:
+        # Best-effort immediate send for frames that need no reply tracking
+        # (unsubscribe). If the socket is still connecting the frame is safely
+        # dropped: a reconnect wipes the server's table anyway.
+        if self._marketdata is None:
+            return
+        try:
+            self._marketdata.send(frame)
+        except SahiFeedConnectionException:
+            pass
+
+    def _send_batch_locked(self, batch: _Batch) -> None:
+        """Send a just-queued batch now, if this is its turn. Call with
+        ``_subscribe_lock`` held.
+
+        The batch goes out only on the connection epoch whose replay has run:
+        on a socket that is not up yet, or is up but still to be replayed, it
+        stays queued and the replay sends it after the held table, in pending
+        order. The subscribe lock is held through the write, so no other
+        subscribe or replay can put a frame on the wire between this batch
+        being queued and sent: the server answers in wire order, and a frame
+        out of pending order would pair its ack with the wrong batch.
+        """
+        conn = self._marketdata
+        if conn is None:
+            return
+        epoch = conn.epoch
+        if epoch is None or epoch != self._replayed_epoch:
+            return  # the replay for this socket carries it
+        with self._lock:
+            frame = codec.encode_subscribe(batch.mode, list(batch.ids))
+        try:
+            conn.send(frame, epoch=epoch)
+        except SahiFeedConnectionException:
+            pass  # dropped meanwhile: the next replay sends it
+
+    def _replay_subscriptions(self) -> None:
+        """Re-send the subscription state after a (re)connect.
+
+        Runs on the socket thread. Acknowledged ids go first, one frame per mode;
+        then every batch still awaiting an answer, one frame each, so a batch
+        the server rejects takes nothing else down with it. The order is also
+        the order the server will answer in, which is how acks are matched, so
+        the frames are written under the subscribe lock: a subscribe call
+        arriving meanwhile queues behind them and is sent, in pending order,
+        once the replay has run. Until then :meth:`_send_batch_locked` leaves
+        the batch for this replay to carry. The table and cache lock is held
+        only to snapshot the state, so the writes block no reader.
+        """
+        conn = self._marketdata
+        if conn is None:
+            return
+        with self._subscribe_lock:
+            with self._lock:
+                epoch = conn.epoch
+                if epoch is None:
+                    return
+                by_mode: dict[str, list[int]] = {}
+                for instrument_id, mode in self._subscriptions.items():
+                    by_mode.setdefault(mode, []).append(instrument_id)
+                held = [_Batch(mode, ids, held=True) for mode, ids in by_mode.items()]
+                # Batches the old socket answered were popped; the rest start
+                # over on this socket.
+                self._pending = deque(held + [b for b in self._pending if b.ids and not b.held])
+                frames = [codec.encode_subscribe(b.mode, list(b.ids)) for b in self._pending]
+            self._replayed_epoch = epoch
+            for frame in frames:
+                conn.send(frame, epoch=epoch)
+
+    def _apply_ack(self, data: dict[str, Any]) -> None:
+        if data.get("action") != "SUBSCRIBE":
+            return  # unsubscribe acks need no bookkeeping; nothing else is sent
+        with self._lock:
+            if not self._pending:
+                return
+            batch = self._pending.popleft()
+            acked = set(data.get("instrument_ids") or [])
+            if not set(batch.ids) <= acked:
+                logger.warning(
+                    "marketdata ack for %s did not cover the batch %s; applying anyway",
+                    sorted(acked),
+                    batch.ids,
+                )
+            for instrument_id in batch.ids:
+                self._subscriptions[instrument_id] = batch.mode
+
+    def _apply_error(self, data: dict[str, Any]) -> _Batch | None:
+        """Drop the batch the server refused. Returns it, or None if none was pending."""
+        with self._lock:
+            if not self._pending:
+                return None
+            batch = self._pending.popleft()
+            if batch.held:
+                # A replayed, previously acknowledged batch the server now refuses:
+                # it no longer holds these ids, so neither does the table.
+                for instrument_id in batch.ids:
+                    self._subscriptions.pop(instrument_id, None)
+                    for bucket in self._cache.values():
+                        bucket.pop(instrument_id, None)
+            return batch
+
+    def _on_marketdata_message(self, frame: Any) -> None:
+        raw = frame if isinstance(frame, bytes) else bytes(frame, "latin-1")
+        msg_type, data = codec.decode_server_message(raw)
+        meta: dict[str, Any] = {"stream": "marketdata", "type": msg_type}
+        bucket = _TICK_CACHE_BUCKET.get(msg_type)
+        if bucket is not None:
+            instrument_id = data.get("instrument_id")
+            if isinstance(instrument_id, int):
+                meta["instrument_id"] = instrument_id
+                # Only cache while subscribed or awaiting the ack (the server sends
+                # a warm-start tick before the ack): a tick already in flight when
+                # the caller unsubscribed must not repopulate the dropped entry.
+                with self._lock:
+                    if self._is_subscribed(instrument_id):
+                        self._cache[bucket][instrument_id] = data
+                        if self._marketdata_callback is not None:
+                            # Coalesce: only the newest tick per key waits, and
+                            # the key is queued once until it is delivered.
+                            key = (bucket, instrument_id)
+                            waiting = key in self._latest_tick
+                            self._latest_tick[key] = (data, meta)
+                            if not waiting:
+                                self._dispatch.put(_TickReady(key))
+            return  # delivered through the coalesced path above, if at all
+        if msg_type == "ack":
+            self._apply_ack(data)
+        elif msg_type == "error":
+            batch = self._apply_error(data)
+            if batch is not None:
+                meta["mode"] = batch.mode
+                meta["instrument_ids"] = list(batch.ids)
+            logger.warning(
+                "marketdata subscribe refused: %s %s (mode %s, ids %s)",
+                data.get("code"),
+                data.get("message"),
+                meta.get("mode"),
+                meta.get("instrument_ids"),
+            )
+        if self._marketdata_callback is not None:
+            self._dispatch.put((self._marketdata_callback, data, meta))
+
+    def _on_orders_open(self) -> None:
+        # Hand the connect event to consume() so the user's snapshot fetch runs on
+        # their thread, ordered before any frame from this connection.
+        if self._orders_connect_callback is not None:
+            self._queue_order_item(_ORDER_CONNECT)
+
+    def _on_orders_message(self, frame: Any) -> None:
+        text = frame if isinstance(frame, str) else frame.decode()
+        try:
+            data = json.loads(text)
+        except ValueError:
+            return  # ignore a non-JSON frame rather than crash the stream
+        if self._orders_callback is not None:
+            self._queue_order_item((data, {"stream": "orders", "type": "order"}))
+
+    def _on_fatal(self, error: SahiFeedException) -> None:
+        # Runs on the failing stream's socket thread. Only hand the error to
+        # consume(); consume() stops both streams — doing it here would join this
+        # very thread and deadlock.
+        self._dispatch.put(error)