rangebar 11.6.1__cp313-cp313-macosx_11_0_arm64.whl

rangebar/validation/gap_classification.py

@@ -0,0 +1,294 @@
+# Issue #19: Gap classification extracted from continuity.py for modularization
+"""Gap classification types and presets for range bar validation.
+
+This module provides the tiered gap classification system based on empirical
+analysis of 30-month BTC data (Issue #19). It identifies 49 legitimate market
+microstructure events and classifies gaps into severity tiers.
+
+Gap Tiers:
+- PRECISION: < 0.001% - Floating-point artifacts (always ignored)
+- NOISE: 0.001% - 0.01% - Tick-level noise (logged, not flagged)
+- MARKET_MOVE: 0.01% - 0.1% - Normal market movement (configurable)
+- MICROSTRUCTURE: > 0.1% - Flash crashes, liquidations (warning/error)
+- SESSION_BOUNDARY: > threshold*2 - Definite session break (always error)
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum, IntEnum
+from typing import Literal
+
+from rangebar.constants import _CRYPTO_BASES, _FOREX_CURRENCIES
+
+__all__ = [
+    "ASSET_CLASS_MULTIPLIERS",
+    "VALIDATION_PRESETS",
+    "AssetClass",
+    "GapTier",
+    "TierThresholds",
+    "ValidationPreset",
+    "detect_asset_class",
+]
+
+
+# ============================================================================
+# Gap Tier Enum
+# ============================================================================
+
+
+class GapTier(IntEnum):
+    """Gap severity classification for range bar continuity validation.
+
+    Tiers are based on empirical analysis of 30-month BTC data (Issue #19)
+    which identified 49 legitimate market microstructure events.
+
+    Examples
+    --------
+    >>> gap_pct = 0.05  # 0.05% gap
+    >>> if gap_pct < 0.00001:
+    ...     tier = GapTier.PRECISION
+    >>> elif gap_pct < 0.0001:
+    ...     tier = GapTier.NOISE
+    >>> elif gap_pct < 0.001:
+    ...     tier = GapTier.MARKET_MOVE
+    >>> elif gap_pct < threshold * 2:
+    ...     tier = GapTier.MICROSTRUCTURE
+    >>> else:
+    ...     tier = GapTier.SESSION_BOUNDARY
+    """
+
+    PRECISION = 1  # < 0.001% - Floating-point artifacts (always ignored)
+    NOISE = 2  # 0.001% - 0.01% - Tick-level noise (logged, not flagged)
+    MARKET_MOVE = 3  # 0.01% - 0.1% - Normal market movement (configurable)
+    MICROSTRUCTURE = 4  # > 0.1% - Flash crashes, liquidations (warning/error)
+    SESSION_BOUNDARY = 5  # > threshold*2 - Definite session break (always error)
+
+
+# ============================================================================
+# Asset Class Enum
+# ============================================================================
+
+
+class AssetClass(Enum):
+    """Asset class for tolerance calibration.
+
+    Different asset classes have different typical gap magnitudes:
+    - Crypto: 24/7 markets, flash crashes possible, baseline tolerance
+    - Forex: Session-based, weekend gaps, tighter tolerance
+    - Equities: Overnight gaps, circuit breakers, looser tolerance
+
+    Examples
+    --------
+    >>> from rangebar import detect_asset_class, AssetClass
+    >>> detect_asset_class("BTCUSDT")
+    <AssetClass.CRYPTO: 'crypto'>
+    >>> detect_asset_class("EURUSD")
+    <AssetClass.FOREX: 'forex'>
+    """
+
+    CRYPTO = "crypto"  # 24/7 markets, flash crashes possible
+    FOREX = "forex"  # Session-based, weekend gaps
+    EQUITIES = "equities"  # Overnight gaps, circuit breakers
+    UNKNOWN = "unknown"  # Fallback to crypto defaults
+
+
+# Tolerance multipliers by asset class (relative to baseline)
+ASSET_CLASS_MULTIPLIERS: dict[AssetClass, float] = {
+    AssetClass.CRYPTO: 1.0,  # Baseline
+    AssetClass.FOREX: 0.5,  # Tighter (more stable)
+    AssetClass.EQUITIES: 1.5,  # Looser (overnight gaps)
+    AssetClass.UNKNOWN: 1.0,  # Default to crypto
+}
+
+
+def detect_asset_class(symbol: str) -> AssetClass:
+    """Auto-detect asset class from symbol pattern.
+
+    Detection Rules:
+    - Crypto: Contains common crypto bases (BTC, ETH, BNB, SOL, etc.)
+             or ends with USDT/BUSD/USDC
+    - Forex: Standard 6-char pairs (EURUSD, GBPJPY, etc.)
+             or commodity symbols (XAU, XAG, BRENT, WTI)
+    - Unknown: Fallback for unrecognized patterns
+
+    Parameters
+    ----------
+    symbol : str
+        Trading symbol (case-insensitive)
+
+    Returns
+    -------
+    AssetClass
+        Detected asset class
+
+    Examples
+    --------
+    >>> detect_asset_class("BTCUSDT")
+    <AssetClass.CRYPTO: 'crypto'>
+    >>> detect_asset_class("EURUSD")
+    <AssetClass.FOREX: 'forex'>
+    >>> detect_asset_class("AAPL")
+    <AssetClass.UNKNOWN: 'unknown'>
+    """
+    symbol_upper = symbol.upper()
+
+    # Crypto patterns: contains known base or ends with stablecoin
+    if any(base in symbol_upper for base in _CRYPTO_BASES):
+        return AssetClass.CRYPTO
+    if symbol_upper.endswith(("USDT", "BUSD", "USDC", "TUSD", "FDUSD")):
+        return AssetClass.CRYPTO
+
+    # Forex patterns: 6-char standard pairs (e.g., EURUSD)
+    forex_pair_length = 6
+    if len(symbol_upper) == forex_pair_length:
+        base, quote = symbol_upper[:3], symbol_upper[3:]
+        if base in _FOREX_CURRENCIES and quote in _FOREX_CURRENCIES:
+            return AssetClass.FOREX
+
+    # Commodities via forex brokers
+    if any(x in symbol_upper for x in ("XAU", "XAG", "BRENT", "WTI")):
+        return AssetClass.FOREX
+
+    return AssetClass.UNKNOWN
+
+
+# ============================================================================
+# Configuration Dataclasses
+# ============================================================================
+
+
+@dataclass(frozen=True)
+class TierThresholds:
+    """Configurable boundaries between gap tiers (in percentage).
+
+    These thresholds define the boundaries for classifying gaps into tiers.
+    Values are percentages (e.g., 0.00001 = 0.001%).
+
+    Attributes
+    ----------
+    precision : float
+        Tier 1/2 boundary - gaps below this are floating-point artifacts
+    noise : float
+        Tier 2/3 boundary - gaps below this are tick-level noise
+    market_move : float
+        Tier 3/4 boundary - gaps below this are normal market movement
+    session_factor : float
+        Tier 5 multiplier - gaps > (threshold * factor) are session breaks
+
+    Examples
+    --------
+    >>> thresholds = TierThresholds()
+    >>> thresholds.precision
+    1e-05
+    >>> thresholds.noise
+    0.0001
+    """
+
+    precision: float = 0.00001  # 0.001% - Tier 1/2 boundary
+    noise: float = 0.0001  # 0.01% - Tier 2/3 boundary
+    market_move: float = 0.001  # 0.1% - Tier 3/4 boundary
+    session_factor: float = 2.0  # Tier 5 = threshold * factor
+
+
+@dataclass(frozen=True)
+class ValidationPreset:
+    """Immutable validation configuration preset.
+
+    Presets bundle tolerance, behavior mode, and tier thresholds into
+    named configurations for common use cases.
+
+    Attributes
+    ----------
+    tolerance_pct : float
+        Maximum gap percentage before flagging (e.g., 0.01 = 1%)
+    mode : Literal["error", "warn", "skip"]
+        Behavior on validation failure
+    tier_thresholds : TierThresholds
+        Boundaries for gap tier classification
+    asset_class : AssetClass | None
+        Override auto-detection if set
+    description : str
+        Human-readable description of the preset
+
+    Examples
+    --------
+    >>> preset = VALIDATION_PRESETS["research"]
+    >>> preset.tolerance_pct
+    0.02
+    >>> preset.mode
+    'warn'
+    """
+
+    tolerance_pct: float  # Max gap before flagging
+    mode: Literal["error", "warn", "skip"]  # Behavior on failure
+    tier_thresholds: TierThresholds = field(default_factory=TierThresholds)
+    asset_class: AssetClass | None = None  # Override auto-detection
+    description: str = ""
+
+
+# Named validation presets for common scenarios
+VALIDATION_PRESETS: dict[str, ValidationPreset] = {
+    # =========================================================================
+    # GENERAL-PURPOSE PRESETS
+    # =========================================================================
+    "permissive": ValidationPreset(
+        tolerance_pct=0.05,  # 5%
+        mode="warn",
+        description="Accept most microstructure events, warn on extreme gaps",
+    ),
+    "research": ValidationPreset(
+        tolerance_pct=0.02,  # 2%
+        mode="warn",
+        description="Standard exploratory analysis with monitoring",
+    ),
+    "standard": ValidationPreset(
+        tolerance_pct=0.01,  # 1%
+        mode="warn",
+        description="Balanced tolerance for production backtesting",
+    ),
+    "strict": ValidationPreset(
+        tolerance_pct=0.005,  # 0.5%
+        mode="error",
+        description="Strict validation for ML training data",
+    ),
+    "paranoid": ValidationPreset(
+        tolerance_pct=0.001,  # 0.1%
+        mode="error",
+        description="Maximum strictness (original v6.1.0 behavior)",
+    ),
+    # =========================================================================
+    # ASSET-CLASS SPECIFIC PRESETS
+    # =========================================================================
+    "crypto": ValidationPreset(
+        tolerance_pct=0.02,  # 2%
+        mode="warn",
+        asset_class=AssetClass.CRYPTO,
+        description="Crypto: Tuned for 24/7 markets with flash crashes",
+    ),
+    "forex": ValidationPreset(
+        tolerance_pct=0.01,  # 1%
+        mode="warn",
+        asset_class=AssetClass.FOREX,
+        description="Forex: Accounts for session boundaries",
+    ),
+    "equities": ValidationPreset(
+        tolerance_pct=0.03,  # 3%
+        mode="warn",
+        asset_class=AssetClass.EQUITIES,
+        description="Equities: Accounts for overnight gaps",
+    ),
+    # =========================================================================
+    # SPECIAL PRESETS
+    # =========================================================================
+    "skip": ValidationPreset(
+        tolerance_pct=0.0,
+        mode="skip",
+        description="Disable validation entirely",
+    ),
+    "audit": ValidationPreset(
+        tolerance_pct=0.002,  # 0.2%
+        mode="error",
+        description="Data quality audit mode",
+    ),
+}

rangebar/validation/post_storage.py

@@ -0,0 +1,317 @@
+"""Tier 0: Post-storage validation for cache integrity (<1 sec).
+
+Run after every cache write to verify data was stored correctly.
+This is the fastest validation tier - critical for detecting cache corruption.
+
+Issue #39: Post-storage validation to verify cached data matches computed data.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import pandas as pd
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ValidationResult:
+    """Result of post-storage validation.
+
+    Attributes
+    ----------
+    passed : bool
+        True if all validation checks passed.
+    checks : dict[str, bool]
+        Individual check results.
+    details : dict[str, Any]
+        Additional details about validation (counts, timestamps, etc.).
+    timestamp : datetime
+        When validation was performed.
+    duration_ms : float
+        How long validation took in milliseconds.
+    """
+
+    passed: bool
+    checks: dict[str, bool] = field(default_factory=dict)
+    details: dict[str, Any] = field(default_factory=dict)
+    timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
+    duration_ms: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary for logging/serialization."""
+        return {
+            "passed": self.passed,
+            "checks": self.checks,
+            "details": self.details,
+            "timestamp": self.timestamp.isoformat(),
+            "duration_ms": self.duration_ms,
+        }
+
+
+def compute_dataframe_checksum(df: pd.DataFrame) -> str:
+    """Compute a checksum for a DataFrame's key columns.
+
+    Uses MD5 hash of (timestamp, OHLCV) tuples for fast comparison.
+    xxHash64 would be faster but MD5 is available without deps.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Range bar DataFrame with DatetimeIndex and OHLCV columns.
+
+    Returns
+    -------
+    str
+        Hex digest of the checksum.
+    """
+    import pandas as pd
+
+    # Use key columns for checksum
+    key_cols = ["Open", "High", "Low", "Close", "Volume"]
+    present_cols = [c for c in key_cols if c in df.columns]
+
+    if not present_cols or df.empty:
+        return "empty"
+
+    # Create string representation of key data
+    # Include index (timestamps) and OHLCV values
+    hasher = hashlib.md5()
+
+    # Hash index (timestamps)
+    if isinstance(df.index, pd.DatetimeIndex):
+        index_str = df.index.astype(int).astype(str).str.cat(sep=",")
+    else:
+        index_str = ",".join(str(x) for x in df.index)
+    hasher.update(index_str.encode())
+
+    # Hash each column
+    for col in present_cols:
+        col_str = df[col].astype(str).str.cat(sep=",")
+        hasher.update(col_str.encode())
+
+    return hasher.hexdigest()
+
+
+def validate_post_storage(
+    expected: pd.DataFrame,
+    retrieved: pd.DataFrame | None,
+    *,
+    symbol: str = "",
+    threshold_bps: int = 0,
+) -> ValidationResult:
+    """Validate that retrieved data matches expected data after cache operation.
+
+    This is a fast (<1 sec) validation that should run after every cache write
+    to verify data integrity. It checks:
+    1. Row count matches
+    2. First timestamp matches
+    3. Last timestamp matches
+    4. Checksum matches (OHLCV data)
+
+    Parameters
+    ----------
+    expected : pd.DataFrame
+        The DataFrame that was written to cache.
+    retrieved : pd.DataFrame | None
+        The DataFrame read back from cache (None if read failed).
+    symbol : str, optional
+        Symbol for logging context.
+    threshold_bps : int, optional
+        Threshold for logging context.
+
+    Returns
+    -------
+    ValidationResult
+        Validation result with pass/fail and details.
+
+    Examples
+    --------
+    >>> from rangebar.validation.post_storage import validate_post_storage
+    >>> result = validate_post_storage(computed_df, cached_df, symbol="BTCUSDT")
+    >>> if not result.passed:
+    ...     logger.error("Post-storage validation FAILED: %s", result.checks)
+    """
+    import time
+
+    start_time = time.perf_counter()
+    checks: dict[str, bool] = {}
+    details: dict[str, Any] = {
+        "symbol": symbol,
+        "threshold_bps": threshold_bps,
+    }
+
+    # Check 1: Retrieved data exists
+    if retrieved is None:
+        checks["data_retrieved"] = False
+        duration_ms = (time.perf_counter() - start_time) * 1000
+        return ValidationResult(
+            passed=False,
+            checks=checks,
+            details={**details, "error": "No data retrieved from cache"},
+            duration_ms=duration_ms,
+        )
+    checks["data_retrieved"] = True
+
+    # Check 2: Row count matches
+    expected_count = len(expected)
+    retrieved_count = len(retrieved)
+    checks["row_count_match"] = expected_count == retrieved_count
+    details["expected_count"] = expected_count
+    details["retrieved_count"] = retrieved_count
+
+    if not checks["row_count_match"]:
+        logger.warning(
+            "Row count mismatch for %s: expected %d, got %d",
+            symbol,
+            expected_count,
+            retrieved_count,
+        )
+
+    # Check 3: First timestamp matches
+    if not expected.empty and not retrieved.empty:
+        expected_first = expected.index[0]
+        retrieved_first = retrieved.index[0]
+        checks["first_timestamp_match"] = expected_first == retrieved_first
+        details["expected_first_ts"] = str(expected_first)
+        details["retrieved_first_ts"] = str(retrieved_first)
+
+        # Check 4: Last timestamp matches
+        expected_last = expected.index[-1]
+        retrieved_last = retrieved.index[-1]
+        checks["last_timestamp_match"] = expected_last == retrieved_last
+        details["expected_last_ts"] = str(expected_last)
+        details["retrieved_last_ts"] = str(retrieved_last)
+    else:
+        checks["first_timestamp_match"] = expected.empty == retrieved.empty
+        checks["last_timestamp_match"] = expected.empty == retrieved.empty
+
+    # Check 5: Checksum matches (OHLCV data integrity)
+    expected_checksum = compute_dataframe_checksum(expected)
+    retrieved_checksum = compute_dataframe_checksum(retrieved)
+    checks["checksum_match"] = expected_checksum == retrieved_checksum
+    details["expected_checksum"] = expected_checksum[:16]  # Truncate for logging
+    details["retrieved_checksum"] = retrieved_checksum[:16]
+
+    if not checks["checksum_match"]:
+        logger.warning(
+            "Checksum mismatch for %s: data corruption detected",
+            symbol,
+        )
+
+    # Overall pass/fail
+    passed = all(checks.values())
+    duration_ms = (time.perf_counter() - start_time) * 1000
+
+    result = ValidationResult(
+        passed=passed,
+        checks=checks,
+        details=details,
+        duration_ms=duration_ms,
+    )
+
+    if passed:
+        logger.debug(
+            "Post-storage validation PASSED for %s (%d bars, %.1fms)",
+            symbol,
+            expected_count,
+            duration_ms,
+        )
+    else:
+        logger.warning(
+            "Post-storage validation FAILED for %s: %s",
+            symbol,
+            {k: v for k, v in checks.items() if not v},
+        )
+
+    return result
+
+
+def validate_ohlc_invariants(df: pd.DataFrame) -> ValidationResult:
+    """Validate OHLC price invariants.
+
+    Checks that for all bars:
+    - High >= max(Open, Close)
+    - Low <= min(Open, Close)
+
+    These invariants should always hold for valid OHLC data.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        Range bar DataFrame with Open, High, Low, Close columns.
+
+    Returns
+    -------
+    ValidationResult
+        Validation result.
+    """
+    import time
+
+    start_time = time.perf_counter()
+    checks: dict[str, bool] = {}
+    details: dict[str, Any] = {"bar_count": len(df)}
+
+    if df.empty:
+        duration_ms = (time.perf_counter() - start_time) * 1000
+        return ValidationResult(
+            passed=True,
+            checks={"empty_dataframe": True},
+            details=details,
+            duration_ms=duration_ms,
+        )
+
+    # Check required columns
+    required = ["Open", "High", "Low", "Close"]
+    missing = [c for c in required if c not in df.columns]
+    if missing:
+        duration_ms = (time.perf_counter() - start_time) * 1000
+        return ValidationResult(
+            passed=False,
+            checks={"columns_present": False},
+            details={**details, "missing_columns": missing},
+            duration_ms=duration_ms,
+        )
+    checks["columns_present"] = True
+
+    # High >= max(Open, Close)
+    high_valid = (df["High"] >= df[["Open", "Close"]].max(axis=1)).all()
+    checks["high_ge_open_close"] = bool(high_valid)
+
+    if not high_valid:
+        invalid_rows = df[df["High"] < df[["Open", "Close"]].max(axis=1)]
+        details["high_invalid_count"] = len(invalid_rows)
+        details["high_invalid_first_ts"] = str(invalid_rows.index[0])
+
+    # Low <= min(Open, Close)
+    low_valid = (df["Low"] <= df[["Open", "Close"]].min(axis=1)).all()
+    checks["low_le_open_close"] = bool(low_valid)
+
+    if not low_valid:
+        invalid_rows = df[df["Low"] > df[["Open", "Close"]].min(axis=1)]
+        details["low_invalid_count"] = len(invalid_rows)
+        details["low_invalid_first_ts"] = str(invalid_rows.index[0])
+
+    passed = all(checks.values())
+    duration_ms = (time.perf_counter() - start_time) * 1000
+
+    return ValidationResult(
+        passed=passed,
+        checks=checks,
+        details=details,
+        duration_ms=duration_ms,
+    )
+
+
+__all__ = [
+    "ValidationResult",
+    "compute_dataframe_checksum",
+    "validate_ohlc_invariants",
+    "validate_post_storage",
+]