ivolatility-backtesting 2.137__tar.gz → 2.138__tar.gz

{ivolatility_backtesting-2.137 → ivolatility_backtesting-2.138}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.137
+Version: 2.138
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com

{ivolatility_backtesting-2.137 → ivolatility_backtesting-2.138}/ivolatility_backtesting/ivolatility_backtesting.py

@@ -463,6 +463,34 @@ from itertools import product
 import sys
 import gc
 from typing import Callable, Dict, List, Optional, Tuple, Union, Any
+
+
+# glibc holds freed arenas instead of returning them to the OS, so RSS grows
+# monotonically across a bulk load even though Python has freed the DataFrames
+# (gc.collect() does NOT shrink RSS — verified: 4 futures chunks 346MB, gc→346MB,
+# malloc_trim→250MB). malloc_trim(0) forces glibc to return free pages. Resolved
+# lazily once; a no-op on musl/macOS (returns None / raises → swallowed).
+_libc_malloc_trim = None
+def _release_freed_memory():
+    """gc.collect() + glibc malloc_trim(0) — actually shrinks RSS, unlike gc alone."""
+    gc.collect()
+    global _libc_malloc_trim
+    if _libc_malloc_trim is None:
+        try:
+            import ctypes
+            _libc_malloc_trim = ctypes.CDLL("libc.so.6").malloc_trim
+        except Exception:
+            _libc_malloc_trim = False  # unavailable (macOS/musl) → skip silently
+    if _libc_malloc_trim:
+        try:
+            _libc_malloc_trim(0)
+        except Exception:
+            pass
+
+# Trim once at import: if the lib is re-imported into a kernel that already ran
+# heavy work, hand back any glibc arenas before this session allocates.
+_release_freed_memory()
+
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
 
@@ -1506,6 +1534,13 @@ def run_backtest_notebook(
     def _stage(name: str):
         return timing_stage(name, enabled=bool(timing))
 
+    # Reset the global timing tracker at the START of every run. It's a module
+    # singleton, so without this the "Total time" and per-stage seconds
+    # ACCUMULATE across every backtest in the same Jupyter kernel — a 2-min
+    # hot-cache run would print 12 min because it summed prior runs.
+    if timing:
+        _timing.reset()
+
     # 0) Header + paths
     if print_config:
         print_strategy_config(base_config, mode=mode)
@@ -4231,7 +4266,33 @@ class DuckDBIndicatorManager:
     def _create_all_indicator_tables(self):
         """Create temp tables for ALL indicators."""
         import time
-        
+
+        # Start the indicator SQL from a low RSS floor: the preceding load+dedup
+        # leaves freed-but-not-returned glibc arenas, and the window-function
+        # self-joins below add their own working set on top → OOM on small
+        # containers. Return pages to the OS first.
+        _release_freed_memory()
+
+        # Cap DuckDB's working memory for the indicator SQL: iv_lean_raw does 3
+        # self-joins over the full options table (the single biggest indicator
+        # allocation — +340MB at 1GB limit, +245MB at 200MB as DuckDB spills to
+        # SSD). A tight cap trades a little speed for a much lower peak, which
+        # is what a 3GB container needs. ~⅙ of the cgroup limit, floor 256MB.
+        try:
+            _lim_mb = 512
+            for _p in ('/sys/fs/cgroup/memory.max',
+                       '/sys/fs/cgroup/memory/memory.limit_in_bytes'):
+                try:
+                    _raw = open(_p).read().strip()
+                    if _raw != 'max' and int(_raw) < (1 << 50):
+                        _lim_mb = max(256, int(int(_raw) / 6 / 1024 / 1024))
+                        break
+                except Exception:
+                    continue
+            self.conn.execute(f"SET memory_limit='{_lim_mb}MB'")
+        except Exception:
+            pass
+
         print("\n" + "="*80)
         print("🦆 DUCKDB INDICATOR CALCULATION (Memory-Efficient)")
         print("="*80)
@@ -4264,14 +4325,61 @@ class DuckDBIndicatorManager:
         print(f"\n✅ All indicators calculated in {total_time:.2f}s")
         print("="*80)
     
+    def _has_ivx_data(self):
+        """True if the `ivx` table holds rows for this symbol.
+
+        Futures only have IVX when `/futures/eod/ivx` was loaded (account
+        access + listed root). When present, the IVX-based indicators run via
+        the normal SQL path; when absent they fall back to empty tables so the
+        combined view still builds. Cached (one query per manager).
+        """
+        if getattr(self, '_ivx_present', None) is not None:
+            return self._ivx_present
+        present = False
+        try:
+            tbls = [t[0] for t in self.conn.execute("SHOW TABLES").fetchall()]
+            if 'ivx' in tbls:
+                present = self.conn.execute(
+                    "SELECT 1 FROM ivx WHERE symbol = ? LIMIT 1",
+                    [self.symbol]).fetchone() is not None
+        except Exception:
+            present = False
+        self._ivx_present = present
+        return present
+
     def _create_iv_lean_raw_table(self):
         """Create IV Lean Z-Score from raw options data (ATM strikes)."""
         import time
         t0 = time.time()
-        
+
         dte_min = self.dte_target - self.dte_tolerance
         dte_max = self.dte_target + self.dte_tolerance
-        
+
+        # input_mode: which ATM series to Z-score — parity with the IVX path
+        # (calculate_iv_lean_from_ivx): 'lean' (call−put, default), 'call',
+        # 'put', 'mean' ((call+put)/2 — IV LEVEL, e.g. futures mean-IV signal,
+        # where IVX is unavailable). The 'iv_lean' column carries the raw
+        # input value regardless of mode (same convention as the IVX path).
+        input_mode = (self.config or {}).get('input_mode', 'lean')
+        _MODE_EXPR = {
+            'lean': '(call_iv - put_iv)',
+            'call': 'call_iv',
+            'put':  'put_iv',
+            'mean': '((call_iv + put_iv) / 2.0)',
+        }
+        if input_mode not in _MODE_EXPR:
+            print(f"   ⚠️  iv_lean_raw: input_mode='{input_mode}' not supported by the "
+                  f"raw path (supports {sorted(_MODE_EXPR)}); falling back to 'lean'")
+            input_mode = 'lean'
+        _mode_expr = _MODE_EXPR[input_mode]
+        if input_mode == 'call':
+            _mode_filter = "call_iv IS NOT NULL AND call_iv > 0"
+        elif input_mode == 'put':
+            _mode_filter = "put_iv IS NOT NULL AND put_iv > 0"
+        else:
+            _mode_filter = ("call_iv IS NOT NULL AND put_iv IS NOT NULL "
+                            "AND call_iv > 0 AND put_iv > 0")
+
         # Complex SQL: find ATM strike per day, get Call/Put IV, calculate Z-Score
         _opt_tbl = _get_options_eod_table(self.config)
         sql = f"""
@@ -4285,9 +4393,14 @@ class DuckDBIndicatorManager:
             WHERE symbol = '{self.symbol}'
             GROUP BY date
         ),
-        -- Step 2: Find ATM strike per day (closest to stock price)
+        -- Step 2: Find ATM strike per day (closest to stock price).
+        -- Project ONLY date+strike (the only columns atm_strikes reads) instead
+        -- of o.* — materializing all ~27 columns × millions of rows was the
+        -- single biggest indicator-phase allocation (+365MB on a 3M-row futures
+        -- table). date+strike alone cuts that to a fraction.
         options_with_diff AS (
-            SELECT o.*,
+            SELECT o.date,
+                   o.strike,
                    u.stock_price,
                    ABS(o.strike - u.stock_price) as strike_diff
             FROM {_opt_tbl} o
@@ -4342,19 +4455,16 @@ class DuckDBIndicatorManager:
                               AND o.symbol = '{self.symbol}'
             GROUP BY c.date, c.atm_strike, c.stock_price
         ),
-        -- Step 4: Calculate IV Lean
+        -- Step 4: Build the input series for the configured input_mode
         iv_lean_daily AS (
-            SELECT 
+            SELECT
                 date,
                 stock_price,
                 call_iv,
                 put_iv,
-                (call_iv - put_iv) as iv_lean
+                {_mode_expr} as iv_lean
             FROM atm_ivs
-            WHERE call_iv IS NOT NULL 
-              AND put_iv IS NOT NULL
-              AND call_iv > 0 
-              AND put_iv > 0
+            WHERE {_mode_filter}
         ),
         -- Step 5: Rolling statistics using SELF-JOIN (exactly matches pandas logic)
         -- pandas: lookback_start <= date <= current_date
@@ -4394,7 +4504,8 @@ class DuckDBIndicatorManager:
             total_rows = self.conn.execute("SELECT COUNT(*) FROM iv_lean_raw").fetchone()[0]
             self._tables_created.add('iv_lean_raw')
             elapsed = time.time() - t0
-            print(f"   ✅ iv_lean_raw (Z from ATM options): {stats[0]} rows ({elapsed:.2f}s)")
+            print(f"   ✅ iv_lean_raw (Z from ATM options, mode={input_mode}): "
+                  f"{stats[0]} rows ({elapsed:.2f}s)")
 
             # Warn when ≥50% of dates have NULL z_score. Root cause is almost
             # always lookback_period too small: SQL hardcodes window_count ≥ 20
@@ -4447,6 +4558,19 @@ class DuckDBIndicatorManager:
     
     def _create_iv_lean_ivx_table(self):
         """Create IV Lean Z-Score from IVX data."""
+        # Futures WITHOUT loaded IVX → create the table EMPTY with the same
+        # columns so the FULL OUTER JOIN combined view still builds (a failed
+        # CREATE would kill ALL indicators, incl. iv_lean_raw/rvol which work
+        # fine for futures). When `/futures/eod/ivx` WAS loaded, fall through
+        # to the normal SQL path — it works identically to equity.
+        if (self.config or {}).get('asset_type') == 'futures' and not self._has_ivx_data():
+            self.conn.execute(
+                "CREATE OR REPLACE TEMP TABLE iv_lean_ivx "
+                "(date DATE, iv_lean DOUBLE, mean_lean DOUBLE, "
+                "std_lean DOUBLE, z_score DOUBLE)")
+            self._tables_created.add('iv_lean_ivx')
+            print('   ⏭️  iv_lean_ivx skipped — no IVX data loaded for this futures symbol')
+            return
         import time
         t0 = time.time()
         
@@ -4504,6 +4628,14 @@ class DuckDBIndicatorManager:
     
     def _create_iv_rank_table(self):
         """Create IV Rank from IVX data."""
+        if (self.config or {}).get('asset_type') == 'futures' and not self._has_ivx_data():
+            self.conn.execute(
+                "CREATE OR REPLACE TEMP TABLE iv_rank_ivx "
+                "(date DATE, atm_iv DOUBLE, iv_high DOUBLE, "
+                "iv_low DOUBLE, iv_rank DOUBLE)")
+            self._tables_created.add('iv_rank_ivx')
+            print('   ⏭️  iv_rank_ivx skipped — no IVX data loaded for this futures symbol')
+            return
         import time
         t0 = time.time()
         
@@ -4556,6 +4688,13 @@ class DuckDBIndicatorManager:
     
     def _create_iv_percentile_table(self):
         """Create IV Percentile from IVX data."""
+        if (self.config or {}).get('asset_type') == 'futures' and not self._has_ivx_data():
+            self.conn.execute(
+                "CREATE OR REPLACE TEMP TABLE iv_percentile_ivx "
+                "(date DATE, atm_iv DOUBLE, iv_percentile DOUBLE)")
+            self._tables_created.add('iv_percentile_ivx')
+            print('   ⏭️  iv_percentile_ivx skipped — no IVX data loaded for this futures symbol')
+            return
         import time
         t0 = time.time()
         
@@ -4607,11 +4746,39 @@ class DuckDBIndicatorManager:
         """Create Realized Volatility from stock data."""
         import time
         t0 = time.time()
-        
-        sql = f"""
-        CREATE OR REPLACE TEMP TABLE realized_vol AS
+
+        if (self.config or {}).get('asset_type') == 'futures':
+            # Futures: stock_eod has no rows for the root — the underlying
+            # series lives in futures_eod. Use the front-month contract
+            # (highest volume per date, same rule as the stock shim), and
+            # compute log-returns WITHIN a contract (PARTITION BY
+            # future_symbol): a March→June roll is a price-level jump, not a
+            # market move, and would spike RVol with a phantom return.
+            daily_returns_cte = f"""
+        WITH front AS (
+            SELECT date, future_symbol, close,
+                   ROW_NUMBER() OVER (
+                       PARTITION BY date
+                       ORDER BY COALESCE(volume, 0) DESC, future_symbol
+                   ) as rn
+            FROM futures_eod
+            WHERE symbol = '{self.symbol}'
+              AND close IS NOT NULL
+              AND close > 0
+        ),
+        daily_returns AS (
+            SELECT
+                date,
+                close,
+                LN(close / LAG(close) OVER (
+                    PARTITION BY future_symbol ORDER BY date)) as log_return
+            FROM front
+            WHERE rn = 1
+        ),"""
+        else:
+            daily_returns_cte = f"""
         WITH daily_returns AS (
-            SELECT 
+            SELECT
                 date,
                 close,
                 LN(close / LAG(close) OVER (ORDER BY date)) as log_return
@@ -4619,7 +4786,11 @@ class DuckDBIndicatorManager:
             WHERE symbol = '{self.symbol}'
               AND close IS NOT NULL
               AND close > 0
-        ),
+        ),"""
+
+        sql = f"""
+        CREATE OR REPLACE TEMP TABLE realized_vol AS
+        {daily_returns_cte}
         with_vol AS (
             SELECT 
                 date,
@@ -19904,10 +20075,17 @@ def _load_ivx_to_duckdb(config, cache_config, symbol, start_date, end_date):
     rows, which happens with any range longer than ~2 years.  To stay under
     the threshold and receive fast direct-CSV responses, we split the full
     range into 365-calendar-day chunks (~252 trading days each).
+
+    Asset-class aware: `asset_type='futures'` reads `/futures/eod/ivx`
+    (same `{tenor} IV Call/Put/Mean` columns as equity → normalized into the
+    SAME `ivx` table). Equity reads `/equities/eod/ivx`.
     """
     debuginfo = config.get('debuginfo', 0)
     # Regional routing (see _load_stock_to_duckdb): non-US IVX needs region.
     region = config.get('region')
+    ivx_endpoint = ('/futures/eod/ivx'
+                    if config.get('asset_type') == 'futures'
+                    else '/equities/eod/ivx')
 
     # ── build 365-day chunks ────────────────────────────────────────────────
     IVX_CHUNK_DAYS = 365  # ~252 trading days → comfortably under 500-row limit
@@ -19930,7 +20108,7 @@ def _load_ivx_to_duckdb(config, cache_config, symbol, start_date, end_date):
     for i, (c_start, c_end) in enumerate(chunks, 1):
         chunk_info = {'chunk': i, 'total': n_chunks, 'start': c_start, 'end': c_end}
         chunk_df = _api_call_logged(
-            '/equities/eod/ivx',
+            ivx_endpoint,
             cache_config,
             skip_parquet_cache=True,
             debuginfo=debuginfo,
@@ -19955,7 +20133,7 @@ def _load_ivx_to_duckdb(config, cache_config, symbol, start_date, end_date):
     # Save to DuckDB storage
     rows_saved = _save_to_duckdb_storage(
         df,
-        '/equities/eod/ivx',
+        ivx_endpoint,
         cache_config,
         debug=False,
     )
@@ -21138,117 +21316,15 @@ def _normalize_futures_options_df(df, root_symbol, root_to_category=None):
     return df
 
 
-def _futures_http_get(endpoint, params, timeout=60, retries=2, debuginfo=0):
-    """GET helper with 1 retry for the futures REST endpoints (not in SDK).
-
-    Handles the 500-row response limit (asynchronous CSV overflow path):
-      - When JSON `data` is non-empty (≤500 rows), return it directly.
-      - When `data == []` AND `status.urlForDetails` is set, the API queued
-        an asynchronous job. We:
-          1. Poll `urlForDetails` (returns list-of-1-dict with `meta.status`)
-             until `meta.status == 'COMPLETE'` (~1-3 polls, ~2-6 seconds).
-          2. Download the gzipped CSV from `/data/download/{ruid}`
-             (NOT the `urlForDetails` URL — that returns JSON status only).
-          3. Decompress, parse, return rows.
-
-    Returns a list of dicts (one per row), or None on failure.
-    """
-    import requests as _requests
-    import time as _time
-    api_key = APIManager._api_key or os.getenv('API_KEY', '')
-    if not api_key:
-        if debuginfo >= 1:
-            print(f"           ⚠️  {endpoint}: no API_KEY available")
-        return None
-    full_params = dict(params)
-    full_params['apiKey'] = api_key
-    last_err = None
-    for attempt in range(1, retries + 1):
-        try:
-            r = _requests.get(f'https://restapi.ivolatility.com{endpoint}',
-                              params=full_params, timeout=timeout)
-            if r.status_code != 200:
-                last_err = f"HTTP {r.status_code}"
-                if debuginfo >= 2:
-                    print(f"           ⚠️  {endpoint} HTTP {r.status_code} (attempt {attempt})")
-                continue
-            payload = r.json()
-            data = payload.get('data', []) or []
-            if data:
-                return data
-            # Empty data → check for overflow async CSV job
-            url_for_details = (payload.get('status') or {}).get('urlForDetails')
-            if url_for_details:
-                # 1. Poll status → wait for COMPLETE
-                ruid = None
-                n_rec = None
-                MAX_POLL_SECONDS = 60
-                POLL_INTERVAL = 2
-                for _poll in range(MAX_POLL_SECONDS // POLL_INTERVAL):
-                    try:
-                        sr = _requests.get(url_for_details, timeout=15)
-                        sj = sr.json()
-                        item = sj[0] if isinstance(sj, list) and sj else sj
-                        meta = (item or {}).get('meta', {}) if isinstance(item, dict) else {}
-                        status_val = meta.get('status')
-                        if status_val == 'COMPLETE':
-                            ruid = meta.get('ruid')
-                            n_rec = meta.get('recordsCount')
-                            break
-                        if status_val in ('FAILED', 'ERROR'):
-                            last_err = f"job {status_val}: {meta}"
-                            break
-                    except Exception as _pe:
-                        last_err = f"poll parse: {_pe}"
-                    _time.sleep(POLL_INTERVAL)
-
-                if not ruid:
-                    last_err = last_err or "polling timed out for async CSV job"
-                    if debuginfo >= 1:
-                        print(f"           ⚠️  {endpoint} CSV polling failed: {last_err}")
-                    continue
-
-                # 2. Download from /data/download/{ruid}
-                if debuginfo >= 1:
-                    print(f"           📥 {endpoint}: {n_rec} rows via async CSV "
-                          f"(ruid={ruid[:8]}...)")
-                try:
-                    file_url = f"https://restapi.ivolatility.com/data/download/{ruid}"
-                    fr = _requests.get(file_url, params={'apiKey': api_key}, timeout=120)
-                    if fr.status_code != 200:
-                        last_err = f"download HTTP {fr.status_code}"
-                        continue
-                    import io as _io, csv as _csv, gzip as _gzip
-                    body = fr.content
-                    if body[:2] == b'\x1f\x8b':
-                        body = _gzip.decompress(body)
-                    text = body.decode('utf-8', errors='replace')
-                    reader = _csv.DictReader(_io.StringIO(text))
-                    return [dict(row) for row in reader]
-                except Exception as e:
-                    last_err = f"CSV download failed: {e}"
-                    if debuginfo >= 2:
-                        print(f"           ⚠️  {endpoint} CSV download: {e}")
-                    continue
-            # Genuinely empty
-            return []
-        except Exception as e:
-            last_err = str(e)
-            if debuginfo >= 2:
-                print(f"           ⚠️  {endpoint} request failed (attempt {attempt}): {e}")
-    if debuginfo >= 1:
-        print(f"           ❌ {endpoint} gave up: {last_err}")
-    return None
-
-
 def _load_futures_prices_to_duckdb(config, cache_config, symbol,
                                    start_date, end_date, region=None,
                                    api_underlying_symbol=None):
     """Load EOD futures prices (full chain per day) from /futures/eod/prices.
 
     Output table: `futures_eod` (created if missing).
-    Chunked: ≤10 calendar days per call (one day ≈ 40 contracts for ES,
-    so 10 days stays comfortably under the 500-row API hard limit).
+    Loaded in ONE from/to range call by default (mirrors _load_stock_to_duckdb);
+    responses >500 rows arrive via the SDK's async-CSV path automatically.
+    `config['fut_price_chunk_days']` can lower the chunk size if ever needed.
 
     Args:
         symbol: user-facing alias stored in DB (`'ZB'`).
@@ -21262,7 +21338,7 @@ def _load_futures_prices_to_duckdb(config, cache_config, symbol,
     from concurrent.futures import ThreadPoolExecutor, as_completed
     debuginfo = config.get('debuginfo', 0)
 
-    FUT_PRICE_CHUNK_DAYS = 10  # ~400 rows worst case for a liquid root
+    FUT_PRICE_CHUNK_DAYS = int(config.get('fut_price_chunk_days', 100000))
     start_dt = datetime.strptime(start_date, '%Y-%m-%d')
     end_dt = datetime.strptime(end_date, '%Y-%m-%d')
     if start_dt > end_dt:
@@ -21287,30 +21363,50 @@ def _load_futures_prices_to_duckdb(config, cache_config, symbol,
     if req_symbol != symbol and debuginfo >= 1:
         print(f"           ℹ️  API prices symbol={req_symbol!r} (user alias={symbol!r})")
 
-    def _fetch(c_start, c_end):
-        params = {'symbol': req_symbol, 'from': c_start, 'to': c_end}
-        if region:
-            params['region'] = region
-        return _futures_http_get('/futures/eod/prices', params,
-                                 timeout=60, debuginfo=debuginfo)
+    def _fetch(c_start, c_end, chunk_idx=1):
+        # Through the SDK (api_call): session-level urllib3 Retry on 429/5xx
+        # plus async-CSV overflow handling come for free, same as equity.
+        try:
+            return _api_call_logged(
+                '/futures/eod/prices',
+                cache_config,
+                skip_parquet_cache=True,
+                debuginfo=debuginfo,
+                _chunk_info={'chunk': chunk_idx, 'total': len(chunks),
+                             'start': c_start, 'end': c_end},
+                symbol=req_symbol,
+                from_=c_start,
+                to=c_end,
+                # per-request SDK override: a multi-year range job can exceed
+                # the global 90s file-download timeout
+                fileDownloadTimeout=int(config.get('fut_file_download_timeout', 600)),
+                **({'region': region} if region else {}),
+            )
+        except Exception as e:
+            if debuginfo >= 1:
+                print(f"           ❌ /futures/eod/prices {c_start}→{c_end}: {e}")
+            return None
 
-    all_rows = []
+    parts = []
     if use_parallel:
-        with ThreadPoolExecutor(max_workers=max_workers) as exe:
-            futures = {exe.submit(_fetch, cs, ce): (cs, ce) for cs, ce in chunks}
+        with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as exe:
+            futures = {exe.submit(_fetch, cs, ce, i + 1): (cs, ce)
+                       for i, (cs, ce) in enumerate(chunks)}
             for f in as_completed(futures):
-                rows = f.result() or []
-                all_rows.extend(rows)
+                part = f.result()
+                if part is not None and not part.empty:
+                    parts.append(part)
     else:
-        for cs, ce in chunks:
-            rows = _fetch(cs, ce) or []
-            all_rows.extend(rows)
+        for i, (cs, ce) in enumerate(chunks):
+            part = _fetch(cs, ce, i + 1)
+            if part is not None and not part.empty:
+                parts.append(part)
 
-    if not all_rows:
+    if not parts:
         print(f"           ⚠️ No futures prices returned for {symbol}")
         return None
 
-    df = pd.DataFrame(all_rows)
+    df = pd.concat(parts, ignore_index=True)
     df = _normalize_futures_prices_df(df, symbol)
     df = df.drop(columns=[c for c in ('record_no',) if c in df.columns])
 
@@ -21345,10 +21441,13 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
                                     delete_existing=True):
     """Load futures options via /futures/eod/fut-opts-by-param into DuckDB.
 
-    The futures-options endpoint is **per-tradeDate** (no date range param),
-    so we loop trading_days × {C, P} and parallelize. Loaded rows are
-    normalized to the equity-options schema (see _normalize_futures_options_df)
-    so the existing OptionsChunkManager works unchanged.
+    The endpoint accepts `from`/`to` date ranges (verified empirically —
+    its docstring-era reputation as "per-tradeDate only" was wrong), so we
+    chunk CONSECUTIVE runs of trading_days into density-sized ranges × {C, P}
+    and parallelize. Responses >500 rows arrive via the SDK's async-CSV path.
+    Loaded rows are normalized to the equity-options schema (see
+    _normalize_futures_options_df) so the existing OptionsChunkManager works
+    unchanged.
 
     Args:
         symbol: user-facing alias stored in the DB (`'ZB'`, `'6E'`).
@@ -21423,16 +21522,145 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
     if req_symbol != symbol and debuginfo >= 1:
         print(f"           ℹ️  API underlyingSymbol={req_symbol!r} "
               f"(user alias={symbol!r}) — using internal IVOL_symbol")
+    td_strs = sorted({td.strftime('%Y-%m-%d') if hasattr(td, 'strftime')
+                      else str(td)[:10] for td in trading_days})
+
+    # Big async-CSV jobs need more than the SDK's global 90s file timeout
+    # (a full-chain multi-month job can run longer); the SDK accepts a
+    # per-request override via params (popped before the URL is built).
+    FUT_FILE_DOWNLOAD_TIMEOUT = int(config.get('fut_file_download_timeout', 600))
+
+    # Chunk size — mirror the equity priority chain (_load_options_to_duckdb).
+    # Uses the SAME `chunk_days_options` key as equity (no futures-specific
+    # alias): the concept is identical — "days per options chunk":
+    #   1. explicit chunk_days_options (rarely needed)
+    #   2. dynamic (default): real row density — from DB if present, else a
+    #      1-day probe call — sized to an environment-aware job-rows target
+    #   3. static 90d fallback (density unknown)
+    # Equity targets ≤400 rows to stay in the fast JSON path; futures chunks
+    # deliberately ride the async-CSV path (few big jobs beat thousands of
+    # tiny calls on these endpoints), so the target is a JOB size that keeps
+    # each download safely inside RAM and the download timeout.
+    #
+    # The job-rows target is sized to the ENVIRONMENT, not a constant: up to
+    # `parallel_max_workers` job DataFrames sit in RAM at once (~450 bytes/row
+    # in pandas, ~25 mixed cols), so budget ~20% of available RAM across the
+    # workers, clamped to [50k, 500k]. Mac with 8GB free → 500k rows/job
+    # (fewest requests); 3-4GB container → ~150-250k (safe). Override with
+    # config['fut_target_job_rows'].
+    _explicit_target = config.get('fut_target_job_rows')
+    if _explicit_target and int(_explicit_target) > 0:
+        FUT_TARGET_JOB_ROWS = int(_explicit_target)
+    else:
+        try:
+            import psutil as _ps
+            _avail_ram = _ps.virtual_memory().available
+        except Exception:
+            _avail_ram = 4 * 1024 ** 3  # assume a small container when unknown
+        # Inside a container psutil reads /proc/meminfo = HOST memory (a 4Gi
+        # pod on a 93Gi node sees 60+Gi "available"). Respect the cgroup
+        # limit: available = limit - current usage (v2, then v1 paths).
+        for _lim_p, _use_p in (
+                ('/sys/fs/cgroup/memory.max', '/sys/fs/cgroup/memory.current'),
+                ('/sys/fs/cgroup/memory/memory.limit_in_bytes',
+                 '/sys/fs/cgroup/memory/memory.usage_in_bytes')):
+            try:
+                _lim_raw = open(_lim_p).read().strip()
+                if _lim_raw == 'max':
+                    break  # cgroup v2 without a limit → host numbers are real
+                _lim = int(_lim_raw)
+                if _lim >= 1 << 50:
+                    break  # v1 "unlimited" sentinel
+                _used = int(open(_use_p).read().strip())
+                _avail_ram = min(_avail_ram, max(0, _lim - _used))
+                break
+            except FileNotFoundError:
+                continue
+            except Exception:
+                break
+        _n_workers = int(config.get('parallel_max_workers', 5)) or 5
+        FUT_TARGET_JOB_ROWS = int(max(50_000, min(500_000,
+            (_avail_ram * 0.20) / _n_workers / 450)))
+    _opt_tbl_probe = _get_options_eod_table(config)
+    _explicit_chunk = config.get('chunk_days_options')
+    if _explicit_chunk and int(_explicit_chunk) > 0:
+        fut_opt_chunk = int(_explicit_chunk)
+    else:
+        rows_per_day = None
+        # 2a. density from DB (max rows per day per C/P side, like equity)
+        try:
+            _dconn = _get_duckdb_storage_conn(cache_config)
+            if _dconn is not None:
+                _tbls = [t[0] for t in _dconn.execute("SHOW TABLES").fetchall()]
+                if _opt_tbl_probe in _tbls:
+                    _r = _dconn.execute(
+                        f"SELECT MAX(cnt) FROM (SELECT date, type, COUNT(*) AS cnt "
+                        f"FROM {_opt_tbl_probe} WHERE symbol = ? GROUP BY date, type)",
+                        [symbol]).fetchone()
+                    if _r and _r[0]:
+                        rows_per_day = int(_r[0])
+        except Exception:
+            pass
+        # 2b. cold start: probe 1 day (C side) to measure live density.
+        # Probe the MIDDLE of the period: edge days (e.g. Dec 31) often carry
+        # a thin chain (few listed series in the DTE band) and under-estimate
+        # density → oversized jobs.
+        if rows_per_day is None:
+            try:
+                _probe_day = td_strs[len(td_strs) // 2]
+                _pp = {'underlyingSymbol': req_symbol,
+                       'from_': _probe_day, 'to': _probe_day,
+                       'dteFrom': dte_from, 'dteTo': dte_to, 'cp': 'C',
+                       'fileDownloadTimeout': FUT_FILE_DOWNLOAD_TIMEOUT}
+                if selection_type == 'delta':
+                    _pp['deltaFrom'] = delta_from
+                    _pp['deltaTo'] = delta_to
+                else:
+                    _pp['moneynessFrom'] = moneyness_from
+                    _pp['moneynessTo'] = moneyness_to
+                if region:
+                    _pp['region'] = region
+                _pdf = _api_call_logged('/futures/eod/fut-opts-by-param',
+                                        cache_config, skip_parquet_cache=True,
+                                        debuginfo=0, **_pp)
+                if _pdf is not None and len(_pdf) > 0:
+                    rows_per_day = int(len(_pdf))
+            except Exception:
+                pass
+        if rows_per_day and rows_per_day > 0:
+            fut_opt_chunk = max(1, min(90, FUT_TARGET_JOB_ROWS // rows_per_day))
+            if debuginfo >= 1:
+                print(f"           📐 Futures chunk sizing: ~{rows_per_day:,} rows/day "
+                      f"→ {fut_opt_chunk}d chunks (target ≤{FUT_TARGET_JOB_ROWS:,} rows/job)")
+        else:
+            fut_opt_chunk = 90  # static fallback, density unknown
+
+    # Build date chunks from CONSECUTIVE runs of trading_days, then split
+    # long runs into ≤fut_opt_chunk ranges. Per-run grouping (not one
+    # min..max span) matters for gap-fill: with delete_existing=False the
+    # caller passes only the MISSING dates, and a min..max span would re-fetch
+    # dates already present in DuckDB → appended duplicates.
+    date_chunks = []
+    for run_start, run_end in _group_consecutive_dates(td_strs):
+        cur = datetime.strptime(run_start, '%Y-%m-%d')
+        run_end_dt = datetime.strptime(run_end, '%Y-%m-%d')
+        while cur <= run_end_dt:
+            ce = min(cur + timedelta(days=fut_opt_chunk - 1), run_end_dt)
+            date_chunks.append((cur.strftime('%Y-%m-%d'), ce.strftime('%Y-%m-%d')))
+            cur = ce + timedelta(days=1)
+
     all_requests = []
-    for td in trading_days:
-        td_str = td.strftime('%Y-%m-%d') if hasattr(td, 'strftime') else str(td)[:10]
+    for chunk_start, chunk_end in date_chunks:
         for cp in ('C', 'P'):
             params = {
                 'underlyingSymbol': req_symbol,
-                'tradeDate': td_str,
+                'from_': chunk_start,
+                'to': chunk_end,
                 'dteFrom': dte_from,
                 'dteTo': dte_to,
                 'cp': cp,
+                # per-request SDK override: big async jobs exceed the global 90s
+                'fileDownloadTimeout': FUT_FILE_DOWNLOAD_TIMEOUT,
             }
             if selection_type == 'delta':
                 # ⚠ CRITICAL: puts have negative delta — invert range
@@ -21451,15 +21679,32 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
 
     total = len(all_requests)
     print(f"           📡 API request: /futures/eod/fut-opts-by-param "
-          f"{len(trading_days)} days × C/P = {total} requests "
-          f"(DTE {dte_from}-{dte_to})")
+          f"{len(date_chunks)} date-chunks × C/P = {total} requests "
+          f"(chunk≤{fut_opt_chunk}d, DTE {dte_from}-{dte_to})")
 
     use_parallel = config.get('parallel_mode', True) and total > 4
     max_workers = config.get('parallel_max_workers', 5)
 
-    def _fetch(params):
-        return _futures_http_get('/futures/eod/fut-opts-by-param', params,
-                                 timeout=60, debuginfo=debuginfo)
+    def _fetch(params, chunk_idx=1):
+        # Through the SDK (api_call): session-level urllib3 Retry on 429/5xx
+        # plus async-CSV overflow handling come for free, same as equity.
+        try:
+            return _api_call_logged(
+                '/futures/eod/fut-opts-by-param',
+                cache_config,
+                skip_parquet_cache=True,
+                debuginfo=debuginfo,
+                _chunk_info={'chunk': chunk_idx, 'total': total,
+                             'start': params.get('from_', '?'),
+                             'end': params.get('to', '?')},
+                **params,
+            )
+        except Exception as e:
+            if debuginfo >= 1:
+                print(f"           ❌ fut-opts-by-param "
+                      f"{params.get('from_')}→{params.get('to')} "
+                      f"{params.get('cp')}: {e}")
+            return None
 
     # Resolve allowed categories from policy framework (see OPT_CATEGORY_POLICIES).
     # `None` means no category filtering ('all' policy).
@@ -21481,6 +21726,39 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
     sample_rows = []
     completed = 0
 
+    # Bound DuckDB's own RAM during the bulk INSERT: without a cap it grows its
+    # buffer/WAL in memory across millions of inserted rows and OOMs the kernel
+    # mid-load on small containers (verified: full-chain ES 1yr → SIGKILL).
+    # A cap makes DuckDB spill to disk instead. Sized to ~⅓ of the cgroup/host
+    # limit (the rest is the kernel + pandas buffers + services); override with
+    # config['fut_duckdb_memory_limit'] (e.g. '1GB').
+    if duck_conn is not None:
+        try:
+            _dml = config.get('fut_duckdb_memory_limit')
+            if not _dml:
+                _avail = 4 * 1024 ** 3
+                try:
+                    import psutil as _ps
+                    _avail = _ps.virtual_memory().total
+                except Exception:
+                    pass
+                for _lp, _ in (('/sys/fs/cgroup/memory.max', None),):
+                    try:
+                        _raw = open(_lp).read().strip()
+                        if _raw != 'max' and int(_raw) < (1 << 50):
+                            _avail = min(_avail, int(_raw))
+                    except Exception:
+                        pass
+                _dml = f"{max(512, int(_avail / 3 / 1024 / 1024))}MB"
+            duck_conn.execute(f"SET memory_limit='{_dml}'")
+            duck_conn.execute("SET preserve_insertion_order=false")
+            if debuginfo >= 1:
+                print(f"           🧱 DuckDB memory_limit={_dml}, "
+                      f"preserve_insertion_order=false")
+        except Exception as _e:
+            if debuginfo >= 1:
+                print(f"           ⚠️  could not set DuckDB memory_limit: {_e}")
+
     # Pre-clean existing rows for this symbol & date window so re-runs don't
     # accumulate duplicates (previous behaviour: 12+ copies after 12 runs →
     # undefined picks in get_option_by_strike_exp).
@@ -21504,11 +21782,10 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
             if debuginfo >= 1:
                 print(f"           ⚠️  pre-INSERT DELETE failed (ignored): {e}")
 
-    def _save_chunk(rows):
+    def _save_chunk(df):
         nonlocal rows_saved_total, rows_filtered_total
-        if not rows or duck_conn is None:
+        if df is None or df.empty or duck_conn is None:
             return
-        df = pd.DataFrame(rows)
         df = _normalize_futures_options_df(df, symbol, root_to_category=root_to_category)
 
         # Policy-based filter: keep only rows whose opt_category is allowed.
@@ -21554,30 +21831,142 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
                     f"CREATE TABLE {table} AS SELECT * FROM df_for_db WHERE 1=0")
             duck_conn.execute(f"INSERT INTO {table} SELECT * FROM df_for_db")
             rows_saved_total += len(df)
+            # Per-chunk insert log — same format as the equity path
+            # (_save_to_duckdb_storage debug=True), so futures runs aren't
+            # silent between the every-20-requests progress summaries.
+            _safe_print(f"[DUCKDB_STORAGE] {table}: +{len(df):,} new rows")
+            # Keep only a TINY sample (legacy return value), not the whole
+            # chunk — holding 5 full chunks pinned them in RAM for the whole
+            # load (real RAM peak with full chains).
             if len(sample_rows) < 5:
-                sample_rows.append(df)
+                sample_rows.append(df.head(5).copy())
         except Exception as e:
             if debuginfo >= 1:
                 print(f"           ⚠️  {table} DuckDB write failed: {e}")
+        finally:
+            # Free the per-chunk pandas buffers immediately — the API→DuckDB
+            # pandas DataFrame (raw + normalized + filtered copies) is the real
+            # RAM peak, and at workers>1 several coexist. gc.collect() alone does
+            # NOT shrink RSS (glibc keeps freed arenas); _release_freed_memory
+            # adds malloc_trim(0) to actually return pages to the OS.
+            try:
+                del df_for_db
+            except Exception:
+                pass
+            try:
+                del df
+            except Exception:
+                pass
+            _release_freed_memory()
+
+    # Flush DuckDB's WAL to disk every N chunks so it doesn't grow unbounded in
+    # RAM across a multi-million-row load (the other half of the OOM fix).
+    _CKPT_EVERY = 20
+    def _maybe_checkpoint():
+        if duck_conn is not None and completed % _CKPT_EVERY == 0:
+            try:
+                with _duckdb_write_lock:
+                    duck_conn.execute("CHECKPOINT")
+            except Exception:
+                pass  # best-effort; e.g. if another tx is open
 
     if use_parallel:
         with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as exe:
-            futs = {exe.submit(_fetch, p): p for p in all_requests}
+            futs = {exe.submit(_fetch, p, i + 1): p
+                    for i, p in enumerate(all_requests)}
             for f in as_completed(futs):
                 completed += 1
-                rows = f.result() or []
-                _save_chunk(rows)
+                _save_chunk(f.result())
+                _maybe_checkpoint()
                 if completed % 20 == 0 or completed == total:
                     print(f"           ⚡ {completed}/{total} requests, "
                           f"{rows_saved_total:,} rows so far")
     else:
         for i, params in enumerate(all_requests, 1):
-            rows = _fetch(params) or []
-            _save_chunk(rows)
+            completed = i
+            _save_chunk(_fetch(params, i))
+            _maybe_checkpoint()
             if i % 20 == 0 or i == total:
                 print(f"           📦 {i}/{total} requests, "
                       f"{rows_saved_total:,} rows so far")
 
+    # ── Deterministic contract selection (stable-root dedup) ───────────────
+    # Futures list several option roots at the same (date, strike, expiration,
+    # type) — e.g. quarterly ES and weekly EW3 both expiring the 3rd Friday —
+    # with INDEPENDENT quotes (observed 2.4× price gap on dead/stale series).
+    # get_option_by_strike_exp picks the first matching row, so duplicate keys
+    # make backtests depend on row insert order. A per-day "most liquid" pick
+    # is not enough either: the winning root can flip between days (observed
+    # 16/590 keys in a 5-day ES sample), switching the quote source mid-position
+    # and injecting phantom P&L jumps. So: keep ONE root per (strike,
+    # expiration, type) — the highest TOTAL volume across all loaded dates —
+    # falling back per-day to the next-best root when the winner has no quote
+    # that day. Scoped to the whole symbol so append-fills can't disagree with
+    # the original pick; also collapses duplicates left by earlier append runs.
+    if duck_conn is not None and rows_saved_total > 0:
+        try:
+            # The winner is chosen by GLOBAL liquidity (root total volume across
+            # ALL dates), so the criterion needs the whole period — but a single
+            # `DELETE ... NOT IN (winners)` over the full table materializes the
+            # multi-million-row winner set in RAM (+478MB on 3.7M rows; can't be
+            # capped — at memory_limit=300MB DuckDB itself OOMs). So:
+            #   Phase 1: compute per-root total volume once (tiny — one row/root)
+            #   Phase 2: delete dups MONTH-BY-MONTH using that global table.
+            # Each month's DELETE touches ~250k rows → +58MB peak, 8× less, and
+            # the result is identical (winner per key is the same regardless of
+            # how the DELETE is batched — verified row-for-row).
+            duck_conn.execute(f"""
+                CREATE OR REPLACE TEMP TABLE _fut_rv AS
+                SELECT strike, expiration, type, opt_root,
+                       SUM(COALESCE(volume, 0)) AS rtv
+                FROM {table} WHERE symbol = ?
+                GROUP BY strike, expiration, type, opt_root
+            """, [symbol])
+            months = [r[0] for r in duck_conn.execute(
+                f"SELECT DISTINCT date_trunc('month', date) FROM {table} "
+                f"WHERE symbol = ? ORDER BY 1", [symbol]).fetchall()]
+            removed = 0
+            for _mon in months:
+                removed += duck_conn.execute(f"""
+                    WITH joined AS (
+                        SELECT t.rowid AS rid, t.date, t.strike, t.expiration, t.type,
+                               rv.rtv,
+                               COALESCE(t.volume, -1) AS v,
+                               -COALESCE(t.ask - t.bid, 1e12) AS ns
+                        FROM {table} t
+                        JOIN _fut_rv rv USING (strike, expiration, type, opt_root)
+                        WHERE t.symbol = ?
+                          AND date_trunc('month', t.date) = ?
+                    ),
+                    winners AS (
+                        SELECT arg_max(rid, [rtv, v, ns, CAST(-rid AS DOUBLE)]) AS keep
+                        FROM joined
+                        GROUP BY date, strike, expiration, type
+                    )
+                    DELETE FROM {table}
+                    WHERE symbol = ?
+                      AND date_trunc('month', date) = ?
+                      AND rowid NOT IN (SELECT keep FROM winners)
+                """, [symbol, _mon, symbol, _mon]).fetchone()[0]
+            duck_conn.execute("DROP TABLE IF EXISTS _fut_rv")
+            if removed:
+                print(f"           🔁 Stable-root dedup: removed {removed:,} "
+                      f"duplicate (date,strike,exp,type) rows")
+        except Exception as e:
+            print(f"           ⚠️  stable-root dedup failed (rows kept as-is): {e}")
+            try:
+                duck_conn.execute("DROP TABLE IF EXISTS _fut_rv")
+            except Exception:
+                pass
+        # Flush WAL and return pages to the OS so the NEXT phase (indicator SQL)
+        # doesn't start on top of accumulated RSS → OOM.
+        try:
+            with _duckdb_write_lock:
+                duck_conn.execute("CHECKPOINT")
+        except Exception:
+            pass
+        _release_freed_memory()
+
     print(f"           💾 Saved {rows_saved_total:,} futures-options rows to DuckDB ({table})")
     if rows_filtered_total > 0:
         policy_name = config.get('opt_root_policy', 'tradable')
@@ -21888,8 +22277,8 @@ def _try_read_futures_from_duckdb_storage(config, cache_config, symbol,
                     delete_existing=False,    # APPEND, don't wipe
                 )
             elif dte_gap:
-                # DTE-only gap → re-fetch full window (per-tradeDate endpoint
-                # can't isolate DTE band, so we wipe & reload for this range)
+                # DTE-only gap → re-fetch full window (existing rows only
+                # cover the narrower DTE band, so we wipe & reload the range)
                 extended_mask = stock_df['date'] <= req_max
                 extended_trading_days = sorted(stock_df.loc[extended_mask, 'date'].unique())
                 print(f"                📦 DTE-only gap → re-fetch {len(extended_trading_days)} days "
@@ -21974,6 +22363,40 @@ def _preload_futures_to_duckdb(config, cache_config):
     print(f"\n   🔮 Futures preload: {symbol} ({region}/{mic or 'auto-MIC'}) "
           f"{start_date} → {end_date}")
 
+    # Return any glibc arenas held from a previous run in this (reused) kernel
+    # before we start allocating — Jupyter keeps the kernel alive between runs,
+    # so a prior backtest's freed-but-not-returned pages would inflate the
+    # starting RSS and bring OOM closer.
+    _release_freed_memory()
+
+    # db_force_clear means "don't trust the cache, reload from scratch". A plain
+    # per-symbol DELETE leaves the singleton DuckDB connection's buffer pool
+    # (up to memory_limit, ~1GB) resident in RAM — pure waste when we're about
+    # to refill it anyway. So CLOSE the singleton connection (its buffer pool is
+    # freed) and reopen lazily with an empty buffer. The .duckdb FILE is NOT
+    # deleted — it is shared by every symbol (equity, other futures, ivx), so
+    # the on-disk data of other symbols stays intact; only this symbol's rows
+    # are re-fetched (the loader does a scoped DELETE WHERE symbol=…).
+    if config.get('db_force_clear', False):
+        global _DUCKDB_STORAGE_CONN
+        try:
+            if _DUCKDB_STORAGE_CONN is not None:
+                try:
+                    _DUCKDB_STORAGE_CONN.execute("CHECKPOINT")  # persist to disk first
+                except Exception:
+                    pass
+                try:
+                    _DUCKDB_STORAGE_CONN.close()
+                except Exception:
+                    pass
+                _DUCKDB_STORAGE_CONN = None
+                _release_freed_memory()
+                print(f"           🧹 db_force_clear: closed DuckDB conn to drop "
+                      f"the buffer pool (file & other symbols kept)")
+        except Exception as _e:
+            if debuginfo >= 1:
+                print(f"           ⚠️  db_force_clear conn-reset skipped: {_e}")
+
     # ── 0. Fast-path: full gap-detection cache (equity-parity) ──────────────
     # Mirrors _try_read_from_duckdb_storage(). If the DB already has data
     # for this symbol/window, the helper detects what's missing, fills only
@@ -22042,6 +22465,38 @@ def _preload_futures_to_duckdb(config, cache_config):
         api_underlying_symbol=meta.get('root'))   # use IVOL_symbol for API
     print(f"           ⏱ futures-options load: {_time.time()-t0:.2f}s")
 
+    # ── 4b. Load futures IVX (best-effort) ──────────────────────────────────
+    # /futures/eod/ivx delivers the same {tenor} IV Call/Put/Mean columns as
+    # equity → lands in the shared `ivx` table. With it the IVX-based
+    # indicators (iv_rank/iv_percentile/iv_lean_ivx, input_mode='mean' via
+    # IVX) work for futures exactly like equities. If the account has no
+    # futures-IVX access (or the root has none), the call returns 0 rows and
+    # the indicator manager falls back to empty IVX tables (raw indicators
+    # still compute). NOT fatal — wrapped so preload always completes.
+    ivx_loaded = 0
+    try:
+        _ivx_df = _load_ivx_to_duckdb(
+            config, cache_config, meta.get('root', symbol),
+            extended_start, end_date)
+        ivx_loaded = 0 if _ivx_df is None else len(_ivx_df)
+        if ivx_loaded:
+            # API returns the IVOL root ('ES'); rows are stored under that
+            # symbol, which matches `symbol` for ES/NQ/CL/GC but not aliases
+            # like ZB('US')/6E('EC'). Re-stamp to the user-facing alias so the
+            # indicator SQL (WHERE symbol = '<alias>') finds them.
+            if meta.get('root') and meta['root'] != symbol:
+                _dc = _get_duckdb_storage_conn(cache_config)
+                if _dc is not None:
+                    _dc.execute("UPDATE ivx SET symbol = ? WHERE symbol = ?",
+                                [symbol, meta['root']])
+            print(f"           ✅ futures IVX: {ivx_loaded:,} rows → `ivx` table")
+        else:
+            print(f"           ⏭️  futures IVX: 0 rows (no access/data) — "
+                  f"raw indicators only")
+    except Exception as _e:
+        if debuginfo >= 1:
+            print(f"           ⏭️  futures IVX skipped: {_e}")
+
     # ── 5. ChunkManager on the futures_options_eod_close table ──────────────
     print(f"\n[3/3] 📊 Building OptionsChunkManager (futures_options_eod_close)")
     dte_target = config.get('dte_target', 35)
@@ -24030,17 +24485,39 @@ def precalculate_indicators_from_config(config, preloaded_data, param_grid=None)
             chunk_mgr._ensure_connection()
             symbol = config.get('symbol', 'SPY')
             dte_max = int(config.get('dte_target', 30)) + int(config.get('dte_tolerance', 10))
-            
-            _opts_df = chunk_mgr.conn.execute(f"""
-                SELECT * FROM {_opt_tbl} 
-                WHERE symbol = ? AND dte <= ?
-            """, [symbol, dte_max]).fetchdf()
-            
-            if not _opts_df.empty:
-                _opts_df = normalize_dataframe(_opts_df, data_type='options')
-                preloaded_data['_preloaded_options'] = _opts_df
-                _options_loaded_for_indicators = True
-                print(f"   ℹ️  Auto-loaded {len(_opts_df):,} options rows from DuckDB for indicator calculation")
+
+            # OOM guard: a full-chain symbol (futures ES ≈ 15k rows/day) can be
+            # millions of rows — `SELECT *` into pandas would blow a 3-4GB
+            # container. Count first; project only the columns the pandas
+            # indicator calculators actually read; refuse above the cap rather
+            # than OOM-kill the kernel mid-optimization.
+            _max_rows = int(config.get('indicator_autoload_max_rows', 4_000_000))
+            _row_count = chunk_mgr.conn.execute(
+                f"SELECT COUNT(*) FROM {_opt_tbl} WHERE symbol = ? AND dte <= ?",
+                [symbol, dte_max]).fetchone()[0]
+            if _row_count > _max_rows:
+                print(f"   ⚠️  Skipping options auto-load for indicators: {_row_count:,} rows "
+                      f"> cap {_max_rows:,} (indicator_autoload_max_rows). Options-based "
+                      f"pandas indicators will be SKIPPED. Options: raise the cap, narrow "
+                      f"dte_target/dte_tolerance, or use single mode (SQL indicators).")
+            else:
+                _tbl_cols = [c[1] for c in chunk_mgr.conn.execute(
+                    f"PRAGMA table_info('{_opt_tbl}')").fetchall()]
+                _needed = ['date', 'symbol', 'expiration', 'strike', 'type', 'iv',
+                           'delta', 'gamma', 'theta', 'vega', 'dte', 'volume',
+                           'open_interest', 'underlying_price', 'bid', 'ask', 'mid']
+                _proj = [c for c in _needed if c in _tbl_cols] or ['*']
+                _opts_df = chunk_mgr.conn.execute(f"""
+                    SELECT {', '.join(_proj)} FROM {_opt_tbl}
+                    WHERE symbol = ? AND dte <= ?
+                """, [symbol, dte_max]).fetchdf()
+
+                if not _opts_df.empty:
+                    _opts_df = normalize_dataframe(_opts_df, data_type='options')
+                    preloaded_data['_preloaded_options'] = _opts_df
+                    _options_loaded_for_indicators = True
+                    print(f"   ℹ️  Auto-loaded {len(_opts_df):,} options rows from DuckDB "
+                          f"for indicator calculation ({len(_proj)} cols)")
         except Exception as e:
             print(f"   ⚠️  Failed to auto-load options from DuckDB: {e}")
     
@@ -28494,6 +28971,11 @@ def close_duckdb_storage():
         print(f"[DUCKDB] 🔐 CLOSED (close_duckdb_storage) | PID={os.getpid()}", flush=True)
         _DUCKDB_STORAGE_CONN.close()
         _DUCKDB_STORAGE_CONN = None
+        # This runs at the END of every run (framework cleanup). Closing the
+        # connection frees DuckDB's buffer pool inside the process, but glibc
+        # keeps those arenas — return them to the OS so the kernel's RSS
+        # actually drops between runs, not just DuckDB's internal accounting.
+        _release_freed_memory()
 
 
 def cleanup_duckdb_runtime(config: Dict[str, Any], silent: bool = True) -> None:

{ivolatility_backtesting-2.137 → ivolatility_backtesting-2.138}/ivolatility_backtesting.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.137
+Version: 2.138
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com

{ivolatility_backtesting-2.137 → ivolatility_backtesting-2.138}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ivolatility_backtesting"
-version = "2.137"
+version = "2.138"
 description = "A universal backtesting framework for financial strategies using the IVolatility API."
 readme = "README.md"
 authors = [