back-trader-python 1.4.0__py3-none-any.whl

backtrader/channels/live_validator.py

@@ -0,0 +1,294 @@
+"""Live data validator for real-time event streams.
+
+Validates incoming events for data quality issues such as out-of-order
+timestamps, time jumps, invalid prices/volumes, and stale data detection.
+
+Example::
+
+    validator = LiveDataValidator()
+    if validator.validate(event):
+        process(event)
+    else:
+        log_rejected(event)
+
+    print(validator.get_anomaly_report())
+"""
+
+import math
+import time
+
+from ..utils.log_message import get_logger
+
+logger = get_logger(__name__)
+
+__all__ = ["LiveDataValidator"]
+
+
+class LiveDataValidator:
+    """Real-time data quality validator.
+
+    Checks each incoming event for:
+    - Out-of-order timestamps
+    - Large time jumps (>1 hour by default)
+    - Invalid prices (<=0) or volumes (<0) for tick data
+    - Invalid order book structure
+    - Stale data detection
+
+    Args:
+        max_time_jump: Maximum allowed timestamp gap in seconds (default: 3600).
+        max_clock_drift: Maximum allowed drift from wall clock in seconds (default: 60).
+        enable_clock_check: Whether to check against wall clock time (default: False).
+    """
+
+    def __init__(self, max_time_jump=3600, max_clock_drift=60.0, enable_clock_check=False):
+        """Initialize the live data validator.
+
+        Args:
+            max_time_jump: Maximum allowed time jump between events (seconds).
+            max_clock_drift: Maximum allowed clock drift across symbols (seconds).
+            enable_clock_check: Whether to enable clock synchronization checks.
+        """
+        max_time_jump = self._coerce_number(max_time_jump)
+        max_clock_drift = self._coerce_number(max_clock_drift)
+        if max_time_jump is None or max_time_jump < 0:
+            raise ValueError("max_time_jump must be a non-negative number")
+        if max_clock_drift is None or max_clock_drift < 0:
+            raise ValueError("max_clock_drift must be a non-negative number")
+        self._max_time_jump = max_time_jump
+        self._max_clock_drift = max_clock_drift
+        self._enable_clock_check = enable_clock_check
+        self._last_timestamps = {}
+        self._anomaly_count = {}
+        self._total_validated = 0
+        self._total_rejected = 0
+
+    def validate(self, event):
+        """Validate an incoming event.
+
+        Args:
+            event: Event wrapper with timestamp, channel_type, channel_name, data.
+
+        Returns:
+            True if the event passes validation, False if rejected.
+        """
+        self._total_validated += 1
+        key = self._event_key(event)
+        if event is None:
+            self._record_anomaly(key, "invalid_event")
+            self._total_rejected += 1
+            return False
+        ts = self._coerce_number(getattr(event, "timestamp", None))
+        if ts is None or ts < 0:
+            self._record_anomaly(key, "invalid_timestamp")
+            self._total_rejected += 1
+            return False
+
+        # 1. Timestamp order check
+        last_ts = self._last_timestamps.get(key, 0)
+        if last_ts > 0 and ts < last_ts:
+            self._record_anomaly(key, "out_of_order")
+            self._total_rejected += 1
+            return False
+
+        # 2. Time jump check (warn but don't reject unless extreme)
+        if last_ts > 0 and ts - last_ts > self._max_time_jump:
+            self._record_anomaly(key, "time_jump")
+            logger.warning(
+                "Time jump detected for %s: %.1fs gap",
+                key,
+                ts - last_ts,
+            )
+
+        # 3. Wall clock drift check
+        if self._enable_clock_check:
+            now = time.time()
+            drift = abs(ts - now)
+            if drift > self._max_clock_drift:
+                self._record_anomaly(key, "clock_drift")
+                logger.warning(
+                    "Clock drift for %s: %.1fs from wall clock",
+                    key,
+                    drift,
+                )
+
+        # 4. Data-specific validation
+        data = getattr(event, "data", None)
+        channel_type = getattr(event, "channel_type", "")
+
+        if data is not None:
+            if channel_type == "tick":
+                if not self._validate_tick(key, data):
+                    self._total_rejected += 1
+                    return False
+            elif channel_type == "orderbook":
+                if not self._validate_orderbook(key, data):
+                    self._total_rejected += 1
+                    return False
+            elif channel_type == "funding":
+                if not self._validate_funding(key, data):
+                    self._total_rejected += 1
+                    return False
+
+        self._last_timestamps[key] = ts
+        return True
+
+    def _validate_tick(self, key, tick):
+        """Validate tick data for price and volume constraints.
+
+        Checks that price is positive and volume is non-negative. Rejects
+        ticks that violate these constraints and records the anomaly.
+
+        Args:
+            key: Tuple of (channel_type, channel_name) for tracking.
+            tick: TickEvent instance to validate.
+
+        Returns:
+            True if the tick is valid, False otherwise.
+        """
+        raw_price = getattr(tick, "price", None)
+        raw_volume = getattr(tick, "volume", None)
+        price = self._coerce_number(raw_price) if raw_price is not None else None
+        volume = self._coerce_number(raw_volume) if raw_volume is not None else None
+
+        if raw_price is not None and (price is None or price <= 0):
+            self._record_anomaly(key, "invalid_price")
+            return False
+        if raw_volume is not None and (volume is None or volume < 0):
+            self._record_anomaly(key, "invalid_volume")
+            return False
+        return True
+
+    def _validate_orderbook(self, key, ob):
+        """Validate order book data structure and integrity.
+
+        Checks that the order book is not empty and that there are no
+        crossed books (best bid >= best ask), which would indicate
+        data corruption or invalid market state.
+
+        Args:
+            key: Tuple of (channel_type, channel_name) for tracking.
+            ob: OrderBookSnapshot instance to validate.
+
+        Returns:
+            True if the order book is valid, False otherwise.
+        """
+        bids = getattr(ob, "bids", None)
+        asks = getattr(ob, "asks", None)
+
+        if not bids and not asks:
+            self._record_anomaly(key, "empty_orderbook")
+            return False
+
+        # Check crossed book (best bid >= best ask)
+        if bids and asks:
+            best_bid = self._extract_orderbook_price(bids[0])
+            best_ask = self._extract_orderbook_price(asks[0])
+            if best_bid is None or best_ask is None:
+                self._record_anomaly(key, "invalid_orderbook_price")
+                return False
+            if best_bid >= best_ask:
+                self._record_anomaly(key, "crossed_book")
+                logger.warning(
+                    "Crossed order book for %s: bid=%.2f >= ask=%.2f", key, best_bid, best_ask
+                )
+        return True
+
+    def _validate_funding(self, key, funding):
+        """Validate funding rate data for extreme values.
+
+        Checks if the funding rate is within reasonable bounds
+        (absolute value <= 0.1). Extreme funding rates are logged
+        as warnings but not rejected.
+
+        Args:
+            key: Tuple of (channel_type, channel_name) for tracking.
+            funding: FundingEvent instance to validate.
+
+        Returns:
+            Always returns True (warnings are logged, not rejected).
+        """
+        raw_rate = getattr(funding, "rate", None)
+        if raw_rate is None:
+            return True
+        rate = self._coerce_number(raw_rate)
+        if rate is None:
+            self._record_anomaly(key, "invalid_funding_rate")
+            return False
+        if abs(rate) > 0.1:
+            self._record_anomaly(key, "extreme_funding_rate")
+            logger.warning("Extreme funding rate for %s: %.6f", key, rate)
+        return True
+
+    @staticmethod
+    def _coerce_number(value):
+        if isinstance(value, bool):
+            return None
+        try:
+            number = float(value)
+        except (TypeError, ValueError):
+            return None
+        if not math.isfinite(number):
+            return None
+        return number
+
+    @staticmethod
+    def _event_key(event):
+        return (
+            str(getattr(event, "channel_type", "") or ""),
+            str(getattr(event, "channel_name", "") or ""),
+        )
+
+    def _extract_orderbook_price(self, level):
+        if isinstance(level, (list, tuple)):
+            if not level:
+                return None
+            return self._coerce_number(level[0])
+        return self._coerce_number(getattr(level, "price", None))
+
+    def _record_anomaly(self, key, anomaly_type):
+        """Record an anomaly occurrence for later reporting.
+
+        Args:
+            key: Tuple of (channel_type, channel_name) for tracking.
+            anomaly_type: String identifying the type of anomaly.
+        """
+        if key not in self._anomaly_count:
+            self._anomaly_count[key] = {}
+        self._anomaly_count[key][anomaly_type] = self._anomaly_count[key].get(anomaly_type, 0) + 1
+
+    def get_anomaly_report(self):
+        """Get the anomaly report.
+
+        Returns:
+            Dict mapping (channel_type, channel_name) to {anomaly_type: count}.
+        """
+        return dict(self._anomaly_count)
+
+    @property
+    def stats(self):
+        """Validation statistics."""
+        return {
+            "total_validated": self._total_validated,
+            "total_rejected": self._total_rejected,
+            "rejection_rate": (
+                self._total_rejected / self._total_validated if self._total_validated > 0 else 0.0
+            ),
+            "anomaly_types": {str(k): v for k, v in self._anomaly_count.items()},
+        }
+
+    def reset(self):
+        """Reset all state."""
+        self._last_timestamps.clear()
+        self._anomaly_count.clear()
+        self._total_validated = 0
+        self._total_rejected = 0
+
+    def __repr__(self):
+        """Return a string representation of the validator.
+
+        Returns:
+            str: Representation showing validation statistics.
+        """
+        return (
+            f"LiveDataValidator(validated={self._total_validated}, rejected={self._total_rejected})"
+        )

backtrader/channels/orderbook.py

@@ -0,0 +1,257 @@
+"""OrderBook data channel for order book depth snapshots.
+
+Provides OrderBookChannel for loading, validating, and buffering order book
+data from CSV/JSONL files or other sources.
+
+Example:
+    Loading order book data from CSV::
+
+        channel = OrderBookChannel(
+            symbol='BTC/USDT',
+            dataname='data/btc_ob_20210101.csv',
+            depth=20
+        )
+        for event in channel.load():
+            print(event.best_bid, event.best_ask, event.spread)
+"""
+
+import csv
+import gzip
+import json
+import math
+from typing import IO, Iterator, List, Tuple
+
+from ..channel import DataChannel, DataValidationResult
+from ..events import OrderBookSnapshot
+from ..utils.log_message import get_logger
+
+logger = get_logger(__name__)
+
+
+class OrderBookChannel(DataChannel):
+    """Order book depth snapshot channel.
+
+    Loads order book data from CSV or JSONL files. Supports configurable
+    depth levels and validates bid/ask ordering and spread.
+
+    CSV format expects columns: timestamp, bids, asks
+    where bids/asks are JSON-encoded lists of [price, qty] pairs.
+
+    JSONL format expects one JSON object per line with fields:
+    timestamp, symbol, bids, asks.
+
+    Args:
+        symbol: Trading pair symbol (e.g., 'BTC/USDT').
+        dataname: Path to the data file (CSV or JSONL).
+        depth: Maximum order book depth to retain.
+        maxlen: Maximum buffer size.
+        validate: Whether to validate incoming events.
+        auto_fix: Whether to auto-fix invalid data.
+        **kwargs: Additional parameters passed to DataChannel.
+    """
+
+    channel_type = "orderbook"
+
+    def __init__(
+        self, symbol, dataname=None, depth=20, maxlen=10000, validate=True, auto_fix=True, **kwargs
+    ):
+        """Initialize the order book channel.
+
+        Args:
+            symbol: Trading pair symbol (e.g., 'BTC/USDT:USDT').
+            dataname: Optional data name for the feed.
+            depth: Order book depth (number of price levels).
+            maxlen: Maximum number of events to buffer.
+            validate: Whether to validate incoming events.
+            auto_fix: Whether to attempt auto-fixing invalid data.
+            **kwargs: Additional arguments passed to parent.
+        """
+        super().__init__(
+            symbol=symbol,
+            maxlen=maxlen,
+            validate=validate,
+            auto_fix=auto_fix,
+            dataname=dataname,
+            **kwargs,
+        )
+        self._dataname = dataname
+        self._depth = depth
+        self._last_best_bid = None
+        self._last_best_ask = None
+
+    @property
+    def depth(self):
+        """Configured maximum order book depth."""
+        return self._depth
+
+    def _validate_event(self, event) -> DataValidationResult:
+        """Validate order book specific fields beyond base validation.
+
+        Additional checks:
+            - Spread consistency with previous snapshots
+            - Depth within configured limits
+        """
+        result = super()._validate_event(event)
+        if not result.valid:
+            return result
+
+        # Truncate to configured depth
+        if len(event.bids) > self._depth:
+            event.bids = event.bids[: self._depth]
+        if len(event.asks) > self._depth:
+            event.asks = event.asks[: self._depth]
+
+        # Track best bid/ask for anomaly detection
+        if event.bids and event.asks:
+            self._last_best_bid = event.bids[0][0]
+            self._last_best_ask = event.asks[0][0]
+
+        return result
+
+    def load(self) -> Iterator[OrderBookSnapshot]:
+        """Load order book events from file.
+
+        Supports CSV and JSONL formats. File format is auto-detected
+        by extension (.jsonl or .csv/.csv.gz).
+
+        Yields:
+            OrderBookSnapshot instances.
+        """
+        dataname = self._dataname
+        if dataname is None:
+            raise ValueError("dataname (file path) is required for loading")
+
+        if dataname.endswith((".jsonl", ".jsonl.gz")):
+            yield from self._load_jsonl()
+        else:
+            yield from self._load_csv()
+
+    def _load_csv(self) -> Iterator[OrderBookSnapshot]:
+        """Load order book events from CSV format.
+
+        Supports both plain CSV and gzip-compressed CSV (.csv.gz).
+        Expects columns: timestamp, bids, asks (where bids/asks are
+        JSON-encoded lists of [price, qty] pairs).
+
+        Yields:
+            OrderBookSnapshot instances.
+        """
+        dataname = self._dataname
+        if dataname is None:
+            raise ValueError("dataname (file path) is required for loading")
+
+        csv_file: IO[str]
+        if dataname.endswith(".gz"):
+            csv_file = gzip.open(dataname, mode="rt", encoding="utf-8")
+        else:
+            csv_file = open(dataname, mode="r", encoding="utf-8", newline="")
+
+        with csv_file as f:
+            reader = csv.DictReader(f)
+
+            required = {"timestamp", "bids", "asks"}
+            if reader.fieldnames:
+                missing = required - set(reader.fieldnames)
+                if missing:
+                    raise ValueError(
+                        f"Missing required columns: {missing}. Found: {reader.fieldnames}"
+                    )
+
+            for row in reader:
+                try:
+                    bids = _parse_levels(row["bids"])
+                    asks = _parse_levels(row["asks"])
+
+                    ob = OrderBookSnapshot(
+                        timestamp=_parse_required_float(row["timestamp"]),
+                        symbol=row.get("symbol", self.symbol),
+                        exchange=row.get("exchange", ""),
+                        asset_type=row.get("asset_type", "spot"),
+                        bids=bids[: self._depth],
+                        asks=asks[: self._depth],
+                    )
+                    yield ob
+                except (ValueError, KeyError, json.JSONDecodeError) as e:
+                    logger.warning("Skipping invalid OB row: %s (error: %s)", row, e)
+                    continue
+
+    def _load_jsonl(self) -> Iterator[OrderBookSnapshot]:
+        """Load order book events from JSONL format.
+
+        Supports both plain JSONL and gzip-compressed JSONL (.jsonl.gz).
+        Each line should contain a JSON object with timestamp, symbol,
+        bids, and asks fields.
+
+        Yields:
+            OrderBookSnapshot instances.
+        """
+        dataname = self._dataname
+        if dataname is None:
+            raise ValueError("dataname (file path) is required for loading")
+
+        jsonl_file: IO[str] = (
+            gzip.open(dataname, mode="rt", encoding="utf-8")
+            if dataname.endswith(".gz")
+            else open(dataname, mode="r", encoding="utf-8")
+        )
+
+        with jsonl_file as f:
+            for line_num, line in enumerate(f, 1):
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    data = json.loads(line)
+                    bids = _parse_level_pairs(data.get("bids", []))
+                    asks = _parse_level_pairs(data.get("asks", []))
+
+                    ob = OrderBookSnapshot(
+                        timestamp=_parse_required_float(data["timestamp"]),
+                        symbol=data.get("symbol", self.symbol),
+                        exchange=data.get("exchange", ""),
+                        asset_type=data.get("asset_type", "spot"),
+                        bids=bids[: self._depth],
+                        asks=asks[: self._depth],
+                    )
+                    yield ob
+                except (ValueError, KeyError, json.JSONDecodeError) as e:
+                    logger.warning("Skipping invalid OB JSONL line %d: %s", line_num, e)
+                    continue
+
+    def __repr__(self):
+        """Return a string representation of the channel.
+
+        Returns:
+            str: Representation showing symbol, depth, and buffer stats.
+        """
+        return (
+            f"OrderBookChannel(symbol={self.symbol!r}, "
+            f"depth={self._depth}, "
+            f"dataname={self._dataname!r}, "
+            f"events={self._event_count}, "
+            f"buffered={len(self._buffer)})"
+        )
+
+
+def _parse_levels(text: str) -> List[Tuple[float, float]]:
+    """Parse order book levels from JSON string.
+
+    Args:
+        text: JSON string like '[[50000, 1.0], [49999, 2.0]]'
+
+    Returns:
+        List of (price, quantity) tuples.
+    """
+    levels = json.loads(text)
+    return _parse_level_pairs(levels)
+
+
+def _parse_required_float(value) -> float:
+    number = float(value)
+    if not math.isfinite(number):
+        raise ValueError(f"Non-finite float value: {value}")
+    return number
+
+
+def _parse_level_pairs(levels) -> List[Tuple[float, float]]:
+    return [(_parse_required_float(p), _parse_required_float(q)) for p, q in levels]