rangebar 12.1.0__tar.gz → 12.1.2__tar.gz

{rangebar-12.1.0 → rangebar-12.1.2}/.mise.toml

@@ -27,6 +27,19 @@ RANGEBAR_FOREX_MIN_THRESHOLD = "50"
 RANGEBAR_EQUITIES_MIN_THRESHOLD = "100"
 RANGEBAR_UNKNOWN_MIN_THRESHOLD = "1"
 
+# =============================================================================
+# SSoT: Microstructure Feature Defaults (Issue #68)
+# =============================================================================
+# When include_microstructure=True, these defaults are used unless overridden.
+# INTER_BAR_LOOKBACK_COUNT: Trades in lookback window for inter-bar features.
+#   - Recommended: 100-500 (200 is good balance of signal vs compute)
+#   - Set to 0 to disable inter-bar features while keeping intra-bar
+# INCLUDE_INTRA_BAR_FEATURES: Enable ITH + statistical features within bars.
+#   - "true" enables 22 intra_* columns (ITH, complexity, statistical)
+#   - "false" disables (use if you only need lookback_* features)
+RANGEBAR_INTER_BAR_LOOKBACK_COUNT = "200"
+RANGEBAR_INCLUDE_INTRA_BAR_FEATURES = "true"
+
 # =============================================================================
 # GitHub Account Isolation
 # =============================================================================

{rangebar-12.1.0 → rangebar-12.1.2}/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [12.1.2](https://github.com/terrylica/rangebar-py/compare/v12.1.1...v12.1.2) (2026-02-04)
+
+
+### Bug Fixes
+
+* **interbar:** preserve lookback trades across bar boundaries ([#68](https://github.com/terrylica/rangebar-py/issues/68)) ([d8e3fc9](https://github.com/terrylica/rangebar-py/commit/d8e3fc94bc160c20314801e5e3be684969972888))
+
+## [12.1.1](https://github.com/terrylica/rangebar-py/compare/v12.1.0...v12.1.1) (2026-02-03)
+
+
+### Bug Fixes
+
+* **microstructure:** auto-enable v12 features when include_microstructure=True ([9288e2f](https://github.com/terrylica/rangebar-py/commit/9288e2f5bc2f412d81d1954d308cea938d027b48)), closes [#68](https://github.com/terrylica/rangebar-py/issues/68)
+
 # [12.1.0](https://github.com/terrylica/rangebar-py/compare/v12.0.2...v12.1.0) (2026-02-03)
 
 

{rangebar-12.1.0 → rangebar-12.1.2}/Cargo.lock

@@ -3337,7 +3337,7 @@ dependencies = [
 
 [[package]]
 name = "rangebar"
-version = "12.1.0"
+version = "12.1.2"
 dependencies = [
  "chrono",
  "insta",
@@ -3359,7 +3359,7 @@ dependencies = [
 
 [[package]]
 name = "rangebar-batch"
-version = "12.1.0"
+version = "12.1.2"
 dependencies = [
  "polars",
  "rangebar-core",
@@ -3371,7 +3371,7 @@ dependencies = [
 
 [[package]]
 name = "rangebar-cli"
-version = "12.1.0"
+version = "12.1.2"
 dependencies = [
  "chrono",
  "clap",
@@ -3396,7 +3396,7 @@ dependencies = [
 
 [[package]]
 name = "rangebar-config"
-version = "12.1.0"
+version = "12.1.2"
 dependencies = [
  "chrono",
  "config",
@@ -3408,7 +3408,7 @@ dependencies = [
 
 [[package]]
 name = "rangebar-core"
-version = "12.1.0"
+version = "12.1.2"
 dependencies = [
  "ahash",
  "arrow",
@@ -3425,7 +3425,7 @@ dependencies = [
 
 [[package]]
 name = "rangebar-io"
-version = "12.1.0"
+version = "12.1.2"
 dependencies = [
  "polars",
  "rangebar-core",
@@ -3435,7 +3435,7 @@ dependencies = [
 
 [[package]]
 name = "rangebar-providers"
-version = "12.1.0"
+version = "12.1.2"
 dependencies = [
  "chrono",
  "csv",
@@ -3455,7 +3455,7 @@ dependencies = [
 
 [[package]]
 name = "rangebar-py"
-version = "12.1.0"
+version = "12.1.2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -3474,7 +3474,7 @@ dependencies = [
 
 [[package]]
 name = "rangebar-streaming"
-version = "12.1.0"
+version = "12.1.2"
 dependencies = [
  "async-trait",
  "chrono",

{rangebar-12.1.0 → rangebar-12.1.2}/Cargo.toml

@@ -10,7 +10,7 @@ resolver = "2"
 [workspace.package]
 # CRITICAL: Prevent crates.io publishing - this is a PyPI-only project
 publish = false
-version = "12.1.0"
+version = "12.1.2"
 authors = ["Terry Li <terry@eonlabs.com>"]
 categories = ["algorithms", "data-structures", "finance"]
 documentation = "https://docs.rs/rangebar"

{rangebar-12.1.0 → rangebar-12.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rangebar
-Version: 12.1.0
+Version: 12.1.2
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Financial and Insurance Industry
 Classifier: License :: OSI Approved :: MIT License

{rangebar-12.1.0 → rangebar-12.1.2}/crates/rangebar-core/src/interbar.rs

@@ -145,41 +145,93 @@ pub struct TradeHistory {
     trades: VecDeque<TradeSnapshot>,
     /// Configuration for lookback
     config: InterBarConfig,
+    /// Timestamp threshold: trades with timestamp < this are protected from pruning.
+    /// Set to the oldest timestamp we might need for lookback computation.
+    /// Updated each time a new bar opens.
+    protected_until: Option<i64>,
 }
 
 impl TradeHistory {
     /// Create new trade history with given configuration
     pub fn new(config: InterBarConfig) -> Self {
         let capacity = match &config.lookback_mode {
-            LookbackMode::FixedCount(n) => *n,
-            LookbackMode::FixedWindow(_) => 1000, // Initial capacity for time-based
+            LookbackMode::FixedCount(n) => *n * 2, // 2x capacity to hold pre-bar + in-bar trades
+            LookbackMode::FixedWindow(_) => 2000, // Initial capacity for time-based
         };
         Self {
             trades: VecDeque::with_capacity(capacity),
             config,
+            protected_until: None,
         }
     }
 
     /// Push a new trade to the history buffer
     ///
-    /// Automatically prunes old entries based on lookback mode.
+    /// Automatically prunes old entries based on lookback mode, but preserves
+    /// trades needed for lookback computation (timestamp < protected_until).
     pub fn push(&mut self, trade: &AggTrade) {
         let snapshot = TradeSnapshot::from(trade);
         self.trades.push_back(snapshot);
-        self.prune(trade.timestamp);
+        self.prune();
+    }
+
+    /// Notify that a new bar has opened at the given timestamp
+    ///
+    /// This sets the protection threshold to ensure trades from before the bar
+    /// opened are preserved for lookback computation. The protection extends
+    /// until the next bar opens and calls this method again.
+    pub fn on_bar_open(&mut self, bar_open_time: i64) {
+        // Protect all trades with timestamp < bar_open_time
+        // These are the trades that can be used for lookback computation
+        self.protected_until = Some(bar_open_time);
+    }
+
+    /// Notify that the current bar has closed
+    ///
+    /// This is intentionally a no-op. We keep protection until the next bar opens,
+    /// which ensures the lookback window is available for feature computation.
+    pub fn on_bar_close(&mut self) {
+        // No-op: Keep protection until next bar opens
+        // This ensures lookback trades survive between bars
     }
 
     /// Prune old trades based on lookback configuration
-    fn prune(&mut self, current_timestamp: i64) {
+    ///
+    /// Pruning logic:
+    /// - For FixedCount(n): Keep up to 2*n trades total, but never prune trades
+    ///   with timestamp < protected_until (needed for lookback)
+    /// - For FixedWindow: Standard time-based pruning, but respect protected_until
+    fn prune(&mut self) {
         match &self.config.lookback_mode {
             LookbackMode::FixedCount(n) => {
-                while self.trades.len() > *n {
+                // Keep at most 2*n trades (n for lookback + n for next bar's lookback)
+                let max_trades = *n * 2;
+                while self.trades.len() > max_trades {
+                    // Check if front trade is protected
+                    if let Some(front) = self.trades.front() {
+                        if let Some(protected) = self.protected_until {
+                            if front.timestamp < protected {
+                                // Don't prune protected trades
+                                break;
+                            }
+                        }
+                    }
                     self.trades.pop_front();
                 }
             }
             LookbackMode::FixedWindow(window_us) => {
-                let cutoff = current_timestamp - window_us;
+                // Find the oldest trade we need
+                let newest_timestamp = self.trades.back().map(|t| t.timestamp).unwrap_or(0);
+                let cutoff = newest_timestamp - window_us;
+
                 while let Some(front) = self.trades.front() {
+                    // Respect protection
+                    if let Some(protected) = self.protected_until {
+                        if front.timestamp < protected {
+                            break;
+                        }
+                    }
+                    // Prune if outside time window
                     if front.timestamp < cutoff {
                         self.trades.pop_front();
                     } else {

{rangebar-12.1.0 → rangebar-12.1.2}/crates/rangebar-core/src/processor.rs

@@ -260,6 +260,10 @@ impl RangeBarProcessor {
         // This matches the batch path's defer_open semantics - the breaching trade
         // closes the current bar, and the NEXT trade opens the new bar.
         if self.defer_open {
+            // Issue #68: Notify history that new bar is opening (preserves pre-bar trades)
+            if let Some(ref mut history) = self.trade_history {
+                history.on_bar_open(trade.timestamp);
+            }
             self.current_bar_state = Some(if self.include_intra_bar_features {
                 RangeBarState::new_with_trade_accumulation(&trade, self.threshold_decimal_bps)
             } else {
@@ -272,6 +276,10 @@ impl RangeBarProcessor {
         match &mut self.current_bar_state {
             None => {
                 // First trade - initialize new bar
+                // Issue #68: Notify history that new bar is opening (preserves pre-bar trades)
+                if let Some(ref mut history) = self.trade_history {
+                    history.on_bar_open(trade.timestamp);
+                }
                 self.current_bar_state = Some(if self.include_intra_bar_features {
                     RangeBarState::new_with_trade_accumulation(&trade, self.threshold_decimal_bps)
                 } else {
@@ -313,9 +321,11 @@ impl RangeBarProcessor {
 
                     // Issue #59: Compute inter-bar features from lookback window
                     // Features are computed from trades BEFORE bar.open_time (no lookahead)
-                    if let Some(ref history) = self.trade_history {
+                    if let Some(ref mut history) = self.trade_history {
                         let inter_bar_features = history.compute_features(bar_state.bar.open_time);
                         bar_state.bar.set_inter_bar_features(&inter_bar_features);
+                        // Issue #68: Notify history that bar is closing (resumes normal pruning)
+                        history.on_bar_close();
                     }
 
                     // Issue #59: Compute intra-bar features from accumulated trades
@@ -451,6 +461,10 @@ impl RangeBarProcessor {
 
             if defer_open {
                 // Previous bar closed, this agg_record opens new bar
+                // Issue #68: Notify history that new bar is opening (preserves pre-bar trades)
+                if let Some(ref mut history) = self.trade_history {
+                    history.on_bar_open(agg_record.timestamp);
+                }
                 current_bar = Some(if self.include_intra_bar_features {
                     RangeBarState::new_with_trade_accumulation(agg_record, self.threshold_decimal_bps)
                 } else {
@@ -463,6 +477,10 @@ impl RangeBarProcessor {
             match current_bar {
                 None => {
                     // First bar initialization
+                    // Issue #68: Notify history that new bar is opening (preserves pre-bar trades)
+                    if let Some(ref mut history) = self.trade_history {
+                        history.on_bar_open(agg_record.timestamp);
+                    }
                     current_bar = Some(if self.include_intra_bar_features {
                         RangeBarState::new_with_trade_accumulation(agg_record, self.threshold_decimal_bps)
                     } else {
@@ -504,10 +522,12 @@ impl RangeBarProcessor {
                         bar_state.bar.compute_microstructure_features();
 
                         // Issue #59: Compute inter-bar features from lookback window
-                        if let Some(ref history) = self.trade_history {
+                        if let Some(ref mut history) = self.trade_history {
                             let inter_bar_features =
                                 history.compute_features(bar_state.bar.open_time);
                             bar_state.bar.set_inter_bar_features(&inter_bar_features);
+                            // Issue #68: Notify history that bar is closing (resumes normal pruning)
+                            history.on_bar_close();
                         }
 
                         // Issue #59: Compute intra-bar features from accumulated trades

{rangebar-12.1.0 → rangebar-12.1.2}/python/rangebar/constants.py

@@ -11,6 +11,12 @@ SSoT (Single Source of Truth) for:
 - _CRYPTO_BASES: Known crypto base symbols for asset class detection
 - _FOREX_CURRENCIES: Known forex currencies for asset class detection
 - MEM_GUARDS: Memory guard registry (Issue #49)
+
+Environment Variable SSoT (from .mise.toml):
+- RANGEBAR_CRYPTO_MIN_THRESHOLD: Minimum threshold for crypto (1000 dbps)
+- RANGEBAR_FOREX_MIN_THRESHOLD: Minimum threshold for forex (50 dbps)
+- RANGEBAR_INTER_BAR_LOOKBACK_COUNT: Lookback for inter-bar features (200)
+- RANGEBAR_INCLUDE_INTRA_BAR_FEATURES: Auto-enable intra-bar features (true)
 """
 
 from __future__ import annotations
@@ -341,4 +347,9 @@ MEM_GUARDS: dict[str, dict[str, str]] = {
         "location": "orchestration/helpers.py:304",
         "stage": "processing",
     },
+    "MEM-012": {
+        "description": "Streaming bar batches (10K) to prevent OOM",
+        "location": "orchestration/helpers.py:236",
+        "stage": "processing",
+    },
 }

{rangebar-12.1.0 → rangebar-12.1.2}/python/rangebar/orchestration/helpers.py

@@ -94,12 +94,28 @@ def _stream_range_bars_binance(
         "cm": MarketType.FuturesCM,
     }[market]
 
+    # Issue #68: Auto-enable v12 features when include_microstructure=True
+    # SSoT: Defaults from mise.toml env vars
+    import os
+
+    effective_lookback = inter_bar_lookback_count
+    if include_microstructure and effective_lookback is None:
+        effective_lookback = int(
+            os.environ.get("RANGEBAR_INTER_BAR_LOOKBACK_COUNT", "200")
+        )
+
+    enable_intra = False
+    if include_microstructure:
+        intra_env = os.environ.get("RANGEBAR_INCLUDE_INTRA_BAR_FEATURES", "true")
+        enable_intra = intra_env.lower() in ("true", "1", "yes")
+
     # Create processor with symbol for checkpoint support
     processor = RangeBarProcessor(
         threshold_decimal_bps,
         symbol=symbol,
         prevent_same_timestamp_close=prevent_same_timestamp_close,
-        inter_bar_lookback_count=inter_bar_lookback_count,
+        inter_bar_lookback_count=effective_lookback,
+        include_intra_bar_features=enable_intra,
     )
     bar_buffer: list[dict] = []
 
@@ -285,6 +301,7 @@ def _process_binance_trades(
     symbol: str | None = None,
     prevent_same_timestamp_close: bool = True,
     inter_bar_lookback_count: int | None = None,
+    include_intra_bar_features: bool = False,
 ) -> tuple[pd.DataFrame, RangeBarProcessor]:
     """Process Binance trades to range bars (internal).
 
@@ -307,6 +324,8 @@ def _process_binance_trades(
         Timestamp gating for flash crash prevention
     inter_bar_lookback_count : int, optional
         Lookback trade count for inter-bar features (Issue #59)
+    include_intra_bar_features : bool, default=False
+        Enable intra-bar features (Issue #59)
 
     Returns
     -------
@@ -348,11 +367,27 @@ def _process_binance_trades(
 
     # Use provided processor or create new one
     if processor is None:
+        # Issue #68: Auto-enable v12 features when include_microstructure=True
+        # This ensures all code paths apply defaults, not just range_bars.py
+        import os
+
+        effective_lookback = inter_bar_lookback_count
+        if include_microstructure and effective_lookback is None:
+            effective_lookback = int(
+                os.environ.get("RANGEBAR_INTER_BAR_LOOKBACK_COUNT", "200")
+            )
+
+        enable_intra = include_intra_bar_features
+        if include_microstructure and not enable_intra:
+            intra_env = os.environ.get("RANGEBAR_INCLUDE_INTRA_BAR_FEATURES", "true")
+            enable_intra = intra_env.lower() in ("true", "1", "yes")
+
         processor = RangeBarProcessor(
             threshold_decimal_bps,
             symbol=symbol,
             prevent_same_timestamp_close=prevent_same_timestamp_close,
-            inter_bar_lookback_count=inter_bar_lookback_count,
+            inter_bar_lookback_count=effective_lookback,
+            include_intra_bar_features=enable_intra,
         )
 
     # MEM-012: Stream bars in batches instead of accumulating all in memory

{rangebar-12.1.0 → rangebar-12.1.2}/python/rangebar/orchestration/range_bars.py

@@ -544,6 +544,25 @@ def get_range_bars(
         any_data_found = True
 
         # Process segment (reuse processor for state continuity within segment)
+        # Issue #68: Auto-enable v12 features when include_microstructure=True
+        # SSoT: Defaults from mise.toml env vars (RANGEBAR_INTER_BAR_LOOKBACK_COUNT,
+        #       RANGEBAR_INCLUDE_INTRA_BAR_FEATURES)
+        import os
+
+        effective_lookback = inter_bar_lookback_count
+        if include_microstructure and effective_lookback is None:
+            # Read from env (mise SSoT) with fallback
+            effective_lookback = int(
+                os.environ.get("RANGEBAR_INTER_BAR_LOOKBACK_COUNT", "200")
+            )
+
+        # Intra-bar features: auto-enable if microstructure requested
+        # SSoT from mise.toml, defaults to True when include_microstructure=True
+        enable_intra = False
+        if include_microstructure:
+            intra_env = os.environ.get("RANGEBAR_INCLUDE_INTRA_BAR_FEATURES", "true")
+            enable_intra = intra_env.lower() in ("true", "1", "yes")
+
         segment_bars, processor = _process_binance_trades(
             segment_ticks,
             threshold_decimal_bps,
@@ -552,7 +571,8 @@ def get_range_bars(
             processor=processor,
             symbol=symbol,
             prevent_same_timestamp_close=prevent_same_timestamp_close,
-            inter_bar_lookback_count=inter_bar_lookback_count,
+            inter_bar_lookback_count=effective_lookback,
+            include_intra_bar_features=enable_intra,
         )
 
         if segment_bars is not None and not segment_bars.empty:

{rangebar-12.1.0 → rangebar-12.1.2}/python/rangebar/processors/core.py

@@ -100,6 +100,7 @@ class RangeBarProcessor:
         *,
         prevent_same_timestamp_close: bool = True,
         inter_bar_lookback_count: int | None = None,
+        include_intra_bar_features: bool = False,
     ) -> None:
         """Initialize processor with given threshold.
 
@@ -112,7 +113,11 @@ class RangeBarProcessor:
         prevent_same_timestamp_close : bool, default=True
             Timestamp gating for flash crash prevention (Issue #36)
         inter_bar_lookback_count : int, optional
-            Lookback trade count for inter-bar features (Issue #59)
+            Lookback trade count for inter-bar features (Issue #59).
+            Enables 16 lookback_* features computed from trades BEFORE bar opens.
+        include_intra_bar_features : bool, default=False
+            Enable intra-bar features (Issue #59). When True, computes 22 intra_*
+            features from trades WITHIN each bar, including ITH metrics.
 
         Raises
         ------
@@ -134,11 +139,13 @@ class RangeBarProcessor:
             symbol,
             prevent_same_timestamp_close,
             inter_bar_lookback_count,
+            include_intra_bar_features,
         )
         self.threshold_decimal_bps = threshold_decimal_bps
         self.symbol = symbol
         self.prevent_same_timestamp_close = prevent_same_timestamp_close
         self.inter_bar_lookback_count = inter_bar_lookback_count
+        self.include_intra_bar_features = include_intra_bar_features
 
     @classmethod
     def from_checkpoint(cls, checkpoint: dict) -> RangeBarProcessor: