ivolatility-backtesting 2.141__tar.gz → 2.142__tar.gz

{ivolatility_backtesting-2.141 → ivolatility_backtesting-2.142}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.141
+Version: 2.142
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com

{ivolatility_backtesting-2.141 → ivolatility_backtesting-2.142}/ivolatility_backtesting/ivolatility_backtesting.py

@@ -7720,8 +7720,12 @@ def _api_call_direct(endpoint: str, max_retries: int = 3, **kwargs) -> Optional[
     import time
     from http.client import RemoteDisconnected
     from requests.exceptions import ConnectionError, Timeout, RequestException
+    from pandas.errors import EmptyDataError
 
-    RETRYABLE = (RemoteDisconnected, ConnectionError, Timeout, RequestException)
+    # EmptyDataError = HTTP 200 with empty async-CSV body; a re-request
+    # regenerates the file server-side, so it is retryable like a network error
+    RETRYABLE = (RemoteDisconnected, ConnectionError, Timeout, RequestException,
+                 EmptyDataError)
 
     for attempt in range(max_retries):
         try:
@@ -7772,8 +7776,12 @@ def api_call(endpoint, cache_config=None, debug=False, max_retries=3, skip_parqu
     # 0 = silent, 1 = basic, 2 = detailed (URLs), 3 = verbose timing
     debug_level = debug if isinstance(debug, int) else (1 if debug else 0)
     
-    # Network errors that should trigger retry
+    # Network errors that should trigger retry.
+    # EmptyDataError = 200 with empty async-CSV body: the SDK's download retries
+    # don't cover the pd.read_csv step; a re-request regenerates the file
+    from pandas.errors import EmptyDataError
     RETRYABLE_ERRORS = (
+        EmptyDataError,
         RemoteDisconnected,
         ConnectionError,
         Timeout,
@@ -8314,7 +8322,12 @@ def _api_call_internal(endpoint, cache_config, debug, debug_level, skip_parquet_
         # the server without benefit — so let it fall through to return None.
         from http.client import RemoteDisconnected
         from requests.exceptions import ConnectionError, Timeout, RequestException, ChunkedEncodingError
+        from pandas.errors import EmptyDataError
 
+        # Unlike ChunkedEncodingError, EmptyDataError NEEDS a new outer request
+        # (server regenerates the file) — re-raise into the retry loop
+        if isinstance(e, EmptyDataError):
+            raise
         if isinstance(e, (RemoteDisconnected, ConnectionError, Timeout, RequestException)) \
                 and not isinstance(e, ChunkedEncodingError):
             raise  # Let retry logic handle it
@@ -12052,9 +12065,73 @@ def _auto_detect_strategy_type(config):
     return 'STRADDLE'
 
 
+def _norm_opt_type_char(position, default='C'):
+    """Normalize opt_type to 'C'/'P'. Clients pass 'call'/'CALL'/'c'/'C' —
+    never compare position['opt_type'] to 'C'/'P' directly, use this.
+    Absent opt_type derives from strategy_type, then `default`."""
+    raw = position.get('opt_type')
+    if raw:
+        s = str(raw).strip().upper()
+        if s.startswith('C'):
+            return 'C'
+        if s.startswith('P'):
+            return 'P'
+    st = str(position.get('strategy_type', '')).upper()
+    if 'PUT' in st:
+        return 'P'
+    if 'CALL' in st:
+        return 'C'
+    return default
+
+
+def _as_date_obj(value):
+    """Best-effort normalize date-ish value (date/datetime/Timestamp/'YYYY-MM-DD') to datetime.date; None if impossible."""
+    if value is None:
+        return None
+    if hasattr(value, 'date') and not isinstance(value, str):
+        try:
+            return value.date()
+        except TypeError:
+            pass
+    if isinstance(value, str):
+        from datetime import datetime as _dt
+        try:
+            return _dt.strptime(value[:10], '%Y-%m-%d').date()
+        except ValueError:
+            return None
+    import datetime as _d
+    return value if isinstance(value, _d.date) else None
+
+
+# AM-settled cash index roots (CBOE): standard 3rd-Friday monthlies settle on
+# the OPENING print (SOQ). Their weekly series (SPXW/NDXP/RUTW/…) are never
+# listed ON the 3rd Friday, so root+3rd-Friday identifies AM exactly.
+_AM_SETTLED_INDEX_ROOTS = {'SPX', 'NDX', 'RUT', 'DJX', 'XSP'}
+
+
+def _is_am_settled_position(position):
+    """True when the position's expiration settles on the MORNING open (SOQ).
+
+    Explicit position['settlement_style'] ('AM'/'PM') always wins; otherwise
+    AM = (opt_root or symbol) in _AM_SETTLED_INDEX_ROOTS and expiration is a
+    standard 3rd Friday. Everything else (equities, ETFs, weeklys, futures)
+    is PM/close-settled.
+    """
+    style = str(position.get('settlement_style') or '').upper()
+    if style == 'AM':
+        return True
+    if style == 'PM':
+        return False
+    root = str(position.get('opt_root') or position.get('symbol') or '').upper()
+    if root not in _AM_SETTLED_INDEX_ROOTS:
+        return False
+    exp = _as_date_obj(position.get('expiration'))
+    return exp is not None and exp.weekday() == 4 and 15 <= exp.day <= 21
+
+
 class PositionManager:
     """Universal Position Manager with automatic mode detection"""
-    
+
     def __init__(self, config, debug=False):
         self.positions = {}
         self.closed_trades = []
@@ -12427,7 +12504,7 @@ class PositionManager:
             # ========================================================
             short_strike = position.get('short_strike', strike)
             long_strike = position.get('long_strike', strike)
-            opt_type = position.get('opt_type', 'C')
+            opt_type = _norm_opt_type_char(position)
             
             if opt_type == 'C':
                 short_intrinsic = max(0, underlying_price - short_strike)
@@ -12457,7 +12534,7 @@ class PositionManager:
             
             # Approximation: Only front month intrinsic (back has time value)
             front_strike = position.get('front_strike', position.get('strike', strike))
-            opt_type = position.get('opt_type', 'C')
+            opt_type = _norm_opt_type_char(position)
             
             if opt_type == 'C':
                 intrinsic_value = max(0, underlying_price - front_strike) * unit_multiplier * contracts
@@ -12491,7 +12568,7 @@ class PositionManager:
             # ========================================================
             # FALLBACK: Single option or unknown strategy
             # ========================================================
-            opt_type = position.get('opt_type', 'C')
+            opt_type = _norm_opt_type_char(position)
             if opt_type == 'C':
                 intrinsic_value = max(0, underlying_price - strike) * unit_multiplier * contracts
             else:
@@ -12857,6 +12934,19 @@ class PositionManager:
                             position.get('underlying_entry_price', 0)
                         )
 
+                    # AM-settled index monthlies settle on the MORNING open
+                    # (SOQ), not the close; expiration-day open is the proxy
+                    if (underlying_price and self.config.get('am_settlement_open', True)
+                            and _is_am_settled_position(position)):
+                        _open_px = None
+                        if position_id in price_data and isinstance(price_data[position_id], dict):
+                            _open_px = price_data[position_id].get('underlying_open')
+                        if _open_px and _open_px > 0:
+                            if self.debug and abs(_open_px - underlying_price) > 1e-9:
+                                print(f"[AM SETTLEMENT] {position_id}: using expiration-day "
+                                      f"OPEN {_open_px:.2f} (SOQ proxy) instead of close {underlying_price:.2f}")
+                            underlying_price = _open_px
+
                     strike         = position.get('strike', 0)
                     strategy_type  = position.get('strategy_type', '')
                     contracts      = position.get('contracts', 1)
@@ -12881,7 +12971,8 @@ class PositionManager:
                             current_pnl = entry_premium - intrinsic_value
                         else:
                             current_pnl = intrinsic_value - entry_premium
-                        current_price = (intrinsic_value / (100 * contracts)) if contracts > 0 else 0
+                        _um = position.get('unit_multiplier', 100)
+                        current_price = (intrinsic_value / (_um * contracts)) if contracts > 0 else 0
 
                         # P&L %
                         max_risk = position.get('entry_max_risk', entry_premium)
@@ -13620,9 +13711,21 @@ class PositionManager:
                 
                 # Add directional stop fields from stock_row (NEW simple approach)
                 if stock_row is not None:
+                    # runtime_open = split-aligned open (REFERENCE §4a) — the
+                    # AM-settlement reference must not use the adjusted series
+                    _stock_open = None
+                    for _oc in ('runtime_open', 'open'):
+                        try:
+                            _ov = stock_row.get(_oc) if hasattr(stock_row, 'get') else stock_row[_oc]
+                        except (KeyError, IndexError, TypeError):
+                            _ov = None
+                        if _ov is not None and not pd.isna(_ov) and float(_ov) > 0:
+                            _stock_open = float(_ov)
+                            break
                     price_data[pos_id].update({
                         'underlying_high': stock_high,
                         'underlying_low': stock_low,
+                        'underlying_open': _stock_open,
                         'underlying_entry_price': position.get('underlying_entry_price', stock_price),
                         'current_date': current_date,
                         'symbol': position.get('symbol')
@@ -13705,13 +13808,16 @@ class PositionManager:
         pos_data = price_data.get(position_id, {})
         pnl = pos_data.get('pnl', 0)
         pnl_pct = pos_data.get('pnl_pct', 0)
-        
-        # Generate close kwargs automatically
-        strategy_type = self.config.get('strategy_type', 'STRADDLE')
+
+        # Generate close kwargs automatically — use the POSITION's strategy_type;
+        # self.config may hold a placeholder type (mixed CALL/PUT scripts), which
+        # would generate kwargs for the wrong legs and drop exit data
+        strategy_type = ((self.positions.get(position_id) or {}).get('strategy_type')
+                         or self.config.get('strategy_type', 'STRADDLE'))
         kwargs = StrategyRegistry.generate_close_position_kwargs(strategy_type, pos_data)
-        
+
         # Close position
-        self.close_position(
+        trade = self.close_position(
             position_id=position_id,
             exit_date=exit_date,
             exit_price=0.0,
@@ -13721,7 +13827,12 @@ class PositionManager:
             stat_key='signal_exits',  # For consistency with other exits
             **kwargs
         )
-        
+
+        # Return the FINAL pnl: the intrinsic guard inside close_position may
+        # have re-settled a dead-quote expiration close, and clients do
+        # `capital += pnl` with this return value
+        if trade is not None and 'pnl' in trade:
+            return trade['pnl']
         return pnl
     
     def check_profit_target(self, current_date, stock_price, options_df, get_option_func, stock_row=None):
@@ -13780,6 +13891,14 @@ class PositionManager:
         trading_days = self.config.get('_trading_days')
         
         for position_id, position in self.positions.items():
+            # No PT on/after expiration: zero-quote chain rows fake a +100%
+            # CREDIT buyback at $0; expiration settles via intrinsic instead
+            _pt_exp = _as_date_obj(position.get('expiration'))
+            _pt_now = _as_date_obj(current_date)
+            if _pt_exp is not None and _pt_now is not None and _pt_now >= _pt_exp:
+                if self.debuginfo >= 2:
+                    print(f"[PT] SKIP {position_id}: at/after expiration {_pt_exp} — settled by expiration path")
+                continue
             # ========================================================
             # Check min_days_before_check for profit target (trading days)
             # ========================================================
@@ -13904,7 +14023,57 @@ class PositionManager:
         
         return to_close
     
-    def close_position(self, position_id, exit_date, exit_price, 
+    def _intrinsic_guard_pnl(self, position, position_id, exit_date, kwargs):
+        """Return (pnl, pnl_pct) via intrinsic settlement when closing at/after
+        expiration with ALL leg exit quotes dead (zero-stub chain rows book a
+        fake -100%/+100% off market quotes otherwise); None = guard not applicable."""
+        if kwargs.get('settlement_type') == 'intrinsic':
+            return None  # already settled by check_positions
+        exp_d = _as_date_obj(position.get('expiration'))
+        exit_d = _as_date_obj(exit_date)
+        if exp_d is None or exit_d is None or exit_d < exp_d:
+            return None
+        quote_keys = [k for k in kwargs
+                      if k.endswith('_exit_bid') or k.endswith('_exit_ask')]
+        if not quote_keys:
+            return None  # nothing to judge — trust the caller's pnl
+        try:
+            if any(float(kwargs.get(k) or 0) > 0 for k in quote_keys):
+                return None  # live quotes → market close is legitimate
+        except (TypeError, ValueError):
+            return None
+        underlying_price = (kwargs.get('underlying_exit_price')
+                            or kwargs.get('underlying_price')
+                            or self._current_underlying_price
+                            or position.get('underlying_exit_price')
+                            or 0)
+        try:
+            underlying_price = float(underlying_price)
+        except (TypeError, ValueError):
+            return None
+        if underlying_price <= 0:
+            return None
+        strategy_type = position.get('strategy_type') or self.config.get('strategy_type', '')
+        contracts = position.get('contracts', 1)
+        entry_premium = abs(position.get('total_cost', 0))
+        intrinsic_value = self._calculate_intrinsic_value(
+            position=position,
+            position_id=position_id,
+            strategy_type=strategy_type,
+            underlying_price=underlying_price,
+            strike=position.get('strike', 0),
+            contracts=contracts,
+        )
+        is_sell = 'SELL' in str(strategy_type).upper() or position.get('entry_price', 0) == 0
+        pnl = (entry_premium - intrinsic_value) if is_sell else (intrinsic_value - entry_premium)
+        max_risk = position.get('entry_max_risk') or entry_premium
+        pnl_pct = (pnl / max_risk * 100) if max_risk > 0 else 0.0
+        if self.debug:
+            print(f"[INTRINSIC GUARD] {exit_date} {position_id}: dead quotes at/after expiration "
+                  f"→ intrinsic=${intrinsic_value:.2f} pnl=${pnl:.2f} ({pnl_pct:.1f}%)")
+        return pnl, pnl_pct
+
+    def close_position(self, position_id, exit_date, exit_price,
                        pnl=None, pnl_pct=None,
                        portfolio_state_data=None, **kwargs):
         """
@@ -13928,7 +14097,14 @@ class PositionManager:
         
         position = self.positions.pop(position_id)
         self._leg_price_cache.pop(position_id, None)
-        
+
+        _guard = self._intrinsic_guard_pnl(position, position_id, exit_date, kwargs)
+        if _guard is not None:
+            pnl, pnl_pct = _guard
+            kwargs.pop('pnl', None)
+            kwargs.pop('pnl_pct', None)
+            kwargs['settlement_type'] = 'intrinsic_guard'
+
         # Check if pnl provided in kwargs (takes priority over calculated)
         if pnl is None:
             pnl = kwargs.get('pnl', None)
@@ -18131,6 +18307,22 @@ def _apply_duckdb_tuning(conn):
         pass
 
 
+def _report_failed_chunks(stage, failed_chunks, config=None):
+    """Loud summary of dropped chunks; flags config['_failed_chunks'] so
+    gap-detection caches are not stamped "verified" over a holey load."""
+    _safe_print(f"           ⚠️  {stage}: {len(failed_chunks)} chunk(s) DROPPED after retries — "
+                f"cache has HOLES in these ranges:")
+    for start, end, cp, err in failed_chunks[:20]:
+        _safe_print(f"              • {start} → {end} cp={cp}: {err}")
+    if len(failed_chunks) > 20:
+        _safe_print(f"              … and {len(failed_chunks) - 20} more")
+    _safe_print(f"           ⚠️  Results on this cache are INCOMPLETE for those ranges — "
+                f"re-run preload to refetch.")
+    if isinstance(config, dict):
+        config.setdefault('_failed_chunks', []).extend(
+            (stage, s, e, c) for s, e, c, _ in failed_chunks)
+
+
 def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, dte_from, dte_to):
     """
     Fetch data for specific date and DTE range from API.
@@ -18266,16 +18458,18 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
             )
             return df
         except Exception as e:
-            if debuginfo >= 1:
-                print(f"              ❌ Error: {e}")
+            _safe_print(f"              ❌ CHUNK DROPPED [{chunk_idx}/{total_requests}] "
+                        f"{chunk_info['start']} → {chunk_info['end']} cp={params.get('cp','?')}: {e}")
+            failed_chunks.append((chunk_info['start'], chunk_info['end'], params.get('cp', '?'), str(e)))
             return None
-    
+
     # MEMFIX(gap-path): раньше весь диапазон копился в all_data и склеивался одним
     # DataFrame — на 15-летнем gap-fill это весь датасет в RAM и OOM 3GB-пода
     # (сценарий Fprater39: кэш с 3-летних прогонов есть -> идём этой веткой).
     # Теперь каждый чанк пишется в DuckDB сразу и освобождается; возвращаем
     # число сохранённых строк (int), а не DataFrame — оба вызывающих обновлены.
     rows_streamed = 0
+    failed_chunks = []  # 2.142: (start, end, cp, error) of every dropped chunk
     _gap_save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
 
     def _stream_one(df):
@@ -18314,7 +18508,10 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
             df = fetch_chunk((params, i + 1))
             _stream_one(df)
             df = None
-    
+
+    if failed_chunks:
+        _report_failed_chunks('gap-fill', failed_chunks, config)
+
     return rows_streamed
 
 
@@ -19011,6 +19208,33 @@ def _try_read_from_duckdb_storage(config, cache_config, symbol, extended_start,
                 if total_rows_saved_fallback > 0:
                     _rich_print(f"           💾 Total saved: {total_rows_saved_fallback:,} rows (streamed)")
 
+            # Post-fill re-verify: every relevant stock trading day must exist
+            # in the options table; misses are loud + flag _failed_chunks
+            if trading_days:
+                try:
+                    db.refresh_conn()
+                    _post_df = db.execute_read(
+                        f"SELECT DISTINCT date FROM {_opt_tbl} WHERE symbol = ?", [symbol]
+                    ).fetchdf()
+                    _post_dates = set(pd.to_datetime(_post_df['date']).dt.strftime('%Y-%m-%d'))
+                    _req_min = pd.to_datetime(extended_start)
+                    _req_max = pd.to_datetime(end_date)
+                    _relevant = {d for d in trading_days
+                                 if _req_min <= pd.to_datetime(d) <= _req_max}
+                    _still_missing = sorted(_relevant - _post_dates)
+                except Exception as _ce:
+                    _still_missing = None
+                    _rich_print(f"           ⚠️ Completeness check could not run: {_ce}")
+                if _still_missing:
+                    _rich_print(f"           🕳️ COMPLETENESS: {len(_still_missing)} trading day(s) "
+                                f"STILL missing in options after gap-fill: {_still_missing[:10]}"
+                                + (' …' if len(_still_missing) > 10 else ''))
+                    config.setdefault('_failed_chunks', []).extend(
+                        ('completeness', d, d, '*') for d in _still_missing)
+                elif _still_missing is not None:
+                    _rich_print(f"           ✅ COMPLETENESS: options cover all "
+                                f"{len(_relevant)} relevant trading days")
+
             # IMPORTANT: Smart gap fill above only fetches OPTIONS (stock-opts-by-param).
             # If the DB was just force-cleared (tables emptied) or IVX was never loaded,
             # IVX-based indicators (Z(ivx), IV Rank, IV Percentile) will silently disappear
@@ -19086,13 +19310,21 @@ def _try_read_from_duckdb_storage(config, cache_config, symbol, extended_start,
         # ========================================================
         # SAVE TO CACHE: Coverage verified — skip gap detection next time
         # ========================================================
-        _gap_detection_cache[_cache_key] = {
-            'stock_df': stock_df,
-            'verified_at': time.time(),
-        }
-        _gap_elapsed = time.time() - start_time
-        _rich_print(f" 💾 [CACHE SAVE] Gap detection for {symbol} cached "
-                    f"(took {_gap_elapsed:.2f}s, will be skipped next time)")
+        # 2.142: NEVER stamp "verified" when this run dropped chunks — a stamped
+        # failure would silence gap detection for the whole kernel session and
+        # freeze the holes in place until restart
+        if config.get('_failed_chunks'):
+            _rich_print(f" ⚠️ [CACHE SKIP] Gap-detection NOT cached: "
+                        f"{len(config['_failed_chunks'])} chunk(s) were dropped this run — "
+                        f"next call will re-verify and refetch")
+        else:
+            _gap_detection_cache[_cache_key] = {
+                'stock_df': stock_df,
+                'verified_at': time.time(),
+            }
+            _gap_elapsed = time.time() - start_time
+            _rich_print(f" 💾 [CACHE SAVE] Gap detection for {symbol} cached "
+                        f"(took {_gap_elapsed:.2f}s, will be skipped next time)")
         
         # ========================================================
         # 5. LOAD DATA FROM DB (RAM-efficient via ChunkManager when use_duckdb_indicators=True)
@@ -19111,17 +19343,43 @@ def _try_read_from_duckdb_storage(config, cache_config, symbol, extended_start,
         
         # Stock already loaded above
         _rich_print(f"           ✅ Stock: {len(stock_df):,} rows")
-        
+
         # Extract backtest trading days from stock data (needed for chunk planning!)
         backtest_trading_days = None
         if stock_df is not None and not stock_df.empty and 'date' in stock_df.columns:
             backtest_dates = stock_df[
-                (stock_df['date'] >= pd.Timestamp(start_date)) & 
+                (stock_df['date'] >= pd.Timestamp(start_date)) &
                 (stock_df['date'] <= pd.Timestamp(end_date))
             ]['date'].unique()
             backtest_trading_days = sorted(backtest_dates)
             _rich_print(f"           📅 Found {len(backtest_trading_days)} trading days in backtest period")
-        
+
+        # Cold path loads options BEFORE stock — this is the first moment the
+        # trading-day calendar exists, so completeness is checked here
+        if backtest_trading_days:
+            try:
+                _cc_tbl = _get_target_table(
+                    _get_options_endpoints(_get_options_snapshot_mode(config))['filtered'])
+                _cc_conn = _get_duckdb_storage_conn(cache_config)
+                _cc_df = _cc_conn.execute(
+                    f"SELECT DISTINCT date FROM {_cc_tbl} WHERE symbol = ?", [symbol]
+                ).fetchdf()
+                _cc_have = set(pd.to_datetime(_cc_df['date']).dt.strftime('%Y-%m-%d'))
+                _cc_need = {pd.Timestamp(d).strftime('%Y-%m-%d') for d in backtest_trading_days}
+                _cc_missing = sorted(_cc_need - _cc_have)
+            except Exception as _ce:
+                _cc_missing = None
+                _rich_print(f"           ⚠️ Completeness check could not run: {_ce}")
+            if _cc_missing:
+                _rich_print(f"           🕳️ COMPLETENESS: {len(_cc_missing)} trading day(s) missing "
+                            f"in options after initial load: {_cc_missing[:10]}"
+                            + (' …' if len(_cc_missing) > 10 else ''))
+                config.setdefault('_failed_chunks', []).extend(
+                    ('cold-completeness', d, d, '*') for d in _cc_missing)
+            elif _cc_missing is not None:
+                _rich_print(f"           ✅ COMPLETENESS: options cover all "
+                            f"{len(_cc_need)} backtest trading days")
+
         if use_duckdb_indicators:
             # ════════════════════════════════════════════════════════════
             # TRUE RAM-EFFICIENT MODE: Don't load full table!
@@ -19898,13 +20156,15 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                 _mf_inflight.acquire()      # ждём, пока главный поток освободит слот
             return df
         except Exception as e:
-            if debuginfo >= 1:
-                print(f"           ❌ Error fetching {params.get('from_')} {params.get('cp')}: {e}")
+            _safe_print(f"           ❌ CHUNK DROPPED [{chunk_idx}/{total_requests}] "
+                        f"{chunk_info['start']} → {chunk_info['end']} cp={params.get('cp','?')}: {e}")
+            failed_chunks.append((chunk_info['start'], chunk_info['end'], params.get('cp', '?'), str(e)))
             return None
-    
+
     # MEMORY-EFFICIENT: Insert to DuckDB immediately after each request
     rows_saved_total = 0
-    
+    failed_chunks = []  # 2.142: (start, end, cp, error) of every dropped chunk
+
     if use_parallel:
         # ========================================================
         # PARALLEL MODE - Stream to DuckDB
@@ -20095,7 +20355,10 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                 print(f"           📦 Request {i + 1}/{total_requests}: {total_rows:,} rows so far")
     
     print(f"           💾 Saved {rows_saved_total:,} rows to DuckDB (streamed, no concat!)")
-    
+
+    if failed_chunks:
+        _report_failed_chunks('options-load', failed_chunks, config)
+
     # ========================================================
     # CRITICAL CHECK: Did data actually save to DuckDB?
     # ========================================================
@@ -21416,7 +21679,7 @@ def _normalize_futures_options_df(df, root_symbol, root_to_category=None):
     # (JSON path returns numbers natively; pd.to_numeric is a no-op there.)
     for col in ('strike', 'bid', 'ask', 'price', 'price_open', 'price_high',
                 'price_low', 'iv', 'iv_interpolated', 'delta', 'gamma', 'theta',
-                'vega', 'underlying_price', 'calc_OTM',
+                'vega', 'underlying_price', 'calc_OTM', 'settle',
                 'dte', 'volume', 'open_interest'):
         if col in df.columns:
             df[col] = pd.to_numeric(df[col], errors='coerce')
@@ -21836,12 +22099,14 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
                 _mf_fut_sem.acquire()      # MEMFIX: backpressure — ждём разгрузки очереди
             return _df
         except Exception as e:
-            if debuginfo >= 1:
-                print(f"           ❌ fut-opts-by-param "
-                      f"{params.get('from_')}→{params.get('to')} "
-                      f"{params.get('cp')}: {e}")
+            _safe_print(f"           ❌ CHUNK DROPPED [{chunk_idx}/{total}] fut-opts-by-param "
+                        f"{params.get('from_')} → {params.get('to')} cp={params.get('cp','?')}: {e}")
+            failed_chunks.append((params.get('from_', '?'), params.get('to', '?'),
+                                  params.get('cp', '?'), str(e)))
             return None
 
+    failed_chunks = []  # 2.142: (start, end, cp, error) of every dropped chunk
+
     # Resolve allowed categories from policy framework (see OPT_CATEGORY_POLICIES).
     # `None` means no category filtering ('all' policy).
     allowed_categories = resolve_opt_category_policy(config)
@@ -21955,17 +22220,35 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
                 existing_cols = [c[1] for c in duck_conn.execute(
                     f"PRAGMA table_info('{table}')").fetchall()]
                 if set(existing_cols) != set(df.columns):
-                    if debuginfo >= 1:
-                        added   = set(df.columns) - set(existing_cols)
-                        removed = set(existing_cols) - set(df.columns)
-                        print(f"           🔄 Schema drift in {table}; recreating "
-                              f"(+{sorted(added)} −{sorted(removed)})")
-                    duck_conn.execute(f"DROP TABLE {table}")
-                    tables.remove(table)
+                    added   = set(df.columns) - set(existing_cols)
+                    removed = set(existing_cols) - set(df.columns)
+                    # Additive drift → ALTER ADD: DROP would nuke rows of ALL
+                    # symbols while only the current window gets re-fetched.
+                    if not removed:
+                        if debuginfo >= 1:
+                            print(f"           🔄 Schema drift in {table}; ALTER ADD "
+                                  f"{sorted(added)} (rows preserved)")
+                        for _new_col in sorted(added):
+                            _dtype = 'DOUBLE' if pd.api.types.is_numeric_dtype(df[_new_col]) else 'VARCHAR'
+                            duck_conn.execute(
+                                f'ALTER TABLE {table} ADD COLUMN "{_new_col}" {_dtype}')
+                    else:
+                        if debuginfo >= 1:
+                            print(f"           🔄 Schema drift in {table}; recreating "
+                                  f"(+{sorted(added)} −{sorted(removed)})")
+                        duck_conn.execute(f"DROP TABLE {table}")
+                        tables.remove(table)
             if table not in tables:
                 duck_conn.execute(
                     f"CREATE TABLE {table} AS SELECT * FROM df_for_db WHERE 1=0")
-            duck_conn.execute(f"INSERT INTO {table} SELECT * FROM df_for_db")
+            # Name-aligned INSERT: after ALTER ADD the table's column ORDER
+            # differs from the df — positional `SELECT *` would misalign silently
+            _ins_cols = ', '.join(
+                f'"{c}"' for c in df.columns
+                if c in {ci[1] for ci in duck_conn.execute(
+                    f"PRAGMA table_info('{table}')").fetchall()})
+            duck_conn.execute(
+                f"INSERT INTO {table} ({_ins_cols}) SELECT {_ins_cols} FROM df_for_db")
             rows_saved_total += len(df)
             # Per-chunk insert log — same format as the equity path
             # (_save_to_duckdb_storage debug=True), so futures runs aren't
@@ -22142,6 +22425,9 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
         _release_freed_memory()
 
     print(f"           💾 Saved {rows_saved_total:,} futures-options rows to DuckDB ({table})")
+
+    if failed_chunks:
+        _report_failed_chunks('futures-options-load', failed_chunks, config)
     if rows_filtered_total > 0:
         policy_name = config.get('opt_root_policy', 'tradable')
         if filtered_by_category:
@@ -22467,15 +22753,43 @@ def _try_read_futures_from_duckdb_storage(config, cache_config, symbol,
         else:
             print(f"\n        ⏭️  [4/5] No gaps — skipping API")
 
+        # Post-fill re-verify: every relevant trading day must exist in the
+        # futures-options table; misses are loud + flag _failed_chunks
+        try:
+            _post_conn = _get_duckdb_storage_conn(cache_config)
+            _post_df = _post_conn.execute(
+                f"SELECT DISTINCT date FROM {_opt_tbl} WHERE symbol = ?", [symbol]
+            ).fetchdf()
+            _post_dates = set(pd.to_datetime(_post_df['date']).dt.strftime('%Y-%m-%d'))
+            _still_missing = sorted(relevant - _post_dates)
+        except Exception as _ce:
+            _still_missing = None
+            print(f"        ⚠️ Completeness check could not run: {_ce}")
+        if _still_missing:
+            print(f"        🕳️ COMPLETENESS: {len(_still_missing)} trading day(s) STILL missing "
+                  f"in futures options after gap-fill: {_still_missing[:10]}"
+                  + (' …' if len(_still_missing) > 10 else ''))
+            config.setdefault('_failed_chunks', []).extend(
+                ('fut-completeness', d, d, '*') for d in _still_missing)
+        elif _still_missing is not None:
+            print(f"        ✅ COMPLETENESS: futures options cover all "
+                  f"{len(relevant)} relevant trading days")
+
         # Save to gap-detection cache (avoids re-running steps 1-4 next time)
-        _gap_detection_cache[cache_key] = {
-            'stock_df': stock_df,
-            'meta': meta,
-            'verified_at': _time.time(),
-        }
-        elapsed = _time.time() - start_time
-        print(f"\n        💾 [CACHE SAVE] Gap detection done in {elapsed:.2f}s "
-              f"(next call: fast-path)")
+        # 2.142: skip the stamp when chunks were dropped this run (see equity path)
+        if config.get('_failed_chunks'):
+            print(f"\n        ⚠️ [CACHE SKIP] Gap-detection NOT cached: "
+                  f"{len(config['_failed_chunks'])} chunk(s) dropped this run — "
+                  f"next call will re-verify and refetch")
+        else:
+            _gap_detection_cache[cache_key] = {
+                'stock_df': stock_df,
+                'meta': meta,
+                'verified_at': _time.time(),
+            }
+            elapsed = _time.time() - start_time
+            print(f"\n        💾 [CACHE SAVE] Gap detection done in {elapsed:.2f}s "
+                  f"(next call: fast-path)")
 
         # ── [5/5] Build preloaded dict ──────────────────────────────────────
         print(f"\n        📊 [5/5] Building ChunkManager...")
@@ -23513,6 +23827,10 @@ def preload_data_universal(config, data_requests=None, debug=False):
                             'end': chunk_end.strftime('%Y-%m-%d')
                         }
                         response = api_call(endpoint, cache_config, debug=debuginfo, _chunk_info=chunk_info, **params)
+                        if not (response and 'data' in response):
+                            # 2.142: sequential path now warns like the parallel one
+                            _safe_print(f"           ⚠️  WARNING: chunk {request_num}/{total_requests} "
+                                        f"{chunk_info['start']} → {chunk_info['end']} returned no data — skipped")
                         if response and 'data' in response:
                             data = response['data']
                             # Copy DataFrame from cache to avoid modifying original
@@ -23547,6 +23865,9 @@ def preload_data_universal(config, data_requests=None, debug=False):
             params = base_params.copy()
             chunk_info = {'chunk': 1, 'total': 1, 'start': request_start_date, 'end': request_end_date}
             response = api_call(endpoint, cache_config, debug=debuginfo, _chunk_info=chunk_info, **params)
+            if not (response and 'data' in response):
+                _safe_print(f"           ⚠️  WARNING: single request {request_start_date} → "
+                            f"{request_end_date} returned no data")
             if response and 'data' in response:
                 data = response['data']
                 # Copy DataFrame from cache to avoid modifying original
@@ -24120,6 +24441,21 @@ def get_option_by_strike_exp(options_df, strike, expiration, opt_type):
     
     if len(filtered) > 0:
         row = filtered.iloc[0].to_dict()
+        # 2.142: settlement-price fallback (futures options). Illiquid strikes
+        # carry bid=ask=0 while the exchange settlement is present (EC: 93% of
+        # rows) — the engine then valued positions at $0 (phantom -100%/+100%).
+        # When both quotes are dead and `settle` is alive, price off settle.
+        # Equity chains have no `settle` column → strict no-op there.
+        _settle = row.get('settle')
+        if _settle is not None and pd.notna(_settle):
+            try:
+                _settle = float(_settle)
+            except (TypeError, ValueError):
+                _settle = 0.0
+            if _settle > 0 and (row.get('bid') or 0) <= 0 and (row.get('ask') or 0) <= 0:
+                row['bid'] = _settle
+                row['ask'] = _settle
+                row['_priced_from_settle'] = True
         if _option_patch_ctx is not None:
             row = _apply_option_1600_patch(row, _option_patch_ctx)
         return row
@@ -30035,20 +30371,44 @@ def get_options_for_date(config, date) -> pd.DataFrame:
     # Try chunk manager first (RAM-efficient mode)
     chunk_mgr = config.get('_options_chunk_manager')
     if chunk_mgr is not None:
-        return chunk_mgr.get_options_for_date(date)
-    
+        df = chunk_mgr.get_options_for_date(date)
+        return _apply_fut_opt_pricing_mode(df, config)
+
     # Fall back to preloaded DataFrame (legacy mode)
     options_df = config.get('_preloaded_options')
     if options_df is not None and not options_df.empty:
         # Normalize date for comparison
         if isinstance(date, str):
             date = pd.Timestamp(date)
-        return options_df[options_df['date'] == date].copy()
+        return _apply_fut_opt_pricing_mode(
+            options_df[options_df['date'] == date].copy(), config)
 
     # No data: preserve options schema so callers can do df['expiration'] safely
     return pd.DataFrame(columns=_get_table_columns('options_eod_close'))
 
 
+def _apply_fut_opt_pricing_mode(df, config):
+    """config['fut_opt_pricing']='settlement': price every row with a live
+    `settle` at bid=ask=settle (CME-style marking, spread=0). Default
+    'market' = no-op; the dead-quote fallback in get_option_by_strike_exp
+    still applies either way."""
+    if df is None or not isinstance(df, pd.DataFrame) or df.empty:
+        return df
+    if config.get('fut_opt_pricing', 'market') != 'settlement':
+        return df
+    if 'settle' not in df.columns:
+        return df
+    _s = pd.to_numeric(df['settle'], errors='coerce')
+    _mask = _s.notna() & (_s > 0)
+    if _mask.any():
+        df = df.copy()
+        df.loc[_mask, 'bid'] = _s[_mask]
+        df.loc[_mask, 'ask'] = _s[_mask]
+        if 'mid' in df.columns:
+            df.loc[_mask, 'mid'] = _s[_mask]
+    return df
+
+
 class DuckDBCacheManager:
     """
     High-performance cache manager using DuckDB for direct Parquet queries.
@@ -31396,7 +31756,7 @@ def check_portfolio_delta_hedge(position_managers, options_today, get_option_fun
 
             strike = pos.get('strike')
             expiration = pos.get('expiration')
-            opt_type = pos.get('opt_type', 'put')
+            opt_type = _norm_opt_type_char(pos, default='P')
             if strike is None or expiration is None:
                 continue
 
@@ -31550,7 +31910,7 @@ def check_portfolio_stop_loss(position_managers, options_today, get_option_func,
         contracts = pos.get('contracts', 0)
         strike = pos.get('strike')
         expiration = pos.get('expiration')
-        opt_type = pos.get('opt_type', 'put')
+        opt_type = _norm_opt_type_char(pos, default='P')
         if strike is not None and expiration is not None and contracts > 0:
             option_data = get_option_func(strike, expiration, opt_type)
             if option_data:

{ivolatility_backtesting-2.141 → ivolatility_backtesting-2.142}/ivolatility_backtesting.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.141
+Version: 2.142
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com

{ivolatility_backtesting-2.141 → ivolatility_backtesting-2.142}/ivolatility_backtesting.egg-info/SOURCES.txt

@@ -7,4 +7,5 @@ ivolatility_backtesting.egg-info/PKG-INFO
 ivolatility_backtesting.egg-info/SOURCES.txt
 ivolatility_backtesting.egg-info/dependency_links.txt
 ivolatility_backtesting.egg-info/requires.txt
-ivolatility_backtesting.egg-info/top_level.txt
+ivolatility_backtesting.egg-info/top_level.txt
+tests/test_2142_fixes.py

{ivolatility_backtesting-2.141 → ivolatility_backtesting-2.142}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ivolatility_backtesting"
-version = "2.141"
+version = "2.142"
 description = "A universal backtesting framework for financial strategies using the IVolatility API."
 readme = "README.md"
 authors = [

ivolatility_backtesting-2.142/tests/test_2142_fixes.py

@@ -0,0 +1,383 @@
+# Tests for 2.142 fix set: W1 intrinsic settlement, W2 loader resilience,
+# W3 futures settle pricing, W4 AM settlement reference.
+#
+# Runs standalone (no pytest needed):  python3 tests/test_2142_fixes.py
+# Also pytest-compatible for CI:       pytest tests/test_2142_fixes.py
+import os
+import sys
+import traceback
+from datetime import date
+
+import matplotlib
+matplotlib.use('Agg')
+import pandas as pd
+
+_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(_REPO, 'ivolatility_backtesting'))
+import ivolatility_backtesting as lib  # noqa: E402
+
+
+def _mk_pm(strategy_type='CALL', **cfg_extra):
+    cfg = {'strategy_type': strategy_type, 'symbol': 'SPX'}
+    cfg.update(cfg_extra)
+    return lib.PositionManager(cfg, debug=False)
+
+
+def _open_long_call(pm, pid='p1', opt_type='call', symbol='SPX',
+                    strike=5685.0, premium=23000.0, exp=date(2025, 7, 18)):
+    pm.open_position(position_id=pid, strategy_type='CALL', symbol=symbol,
+                     entry_date=date(2025, 5, 2), entry_price=premium,
+                     quantity=100, contracts=1, total_cost=premium,
+                     strike=strike, expiration=exp,
+                     underlying_entry_price=5686.67, opt_type=opt_type)
+
+
+# ---------------------------------------------------------------- W1a
+
+def test_w1a_lowercase_call_settles_as_call():
+    pm = _mk_pm()
+    _open_long_call(pm, opt_type='call')
+    si = pm.check_positions(date(2025, 7, 18), {'p1': {'underlying_price': 6296.79}})[0]
+    assert abs(si['pnl'] - 38179.0) < 1.0, si['pnl']
+
+
+def test_w1a_all_variants_normalize():
+    for variant in ('call', 'CALL', 'Call', 'c', 'C'):
+        pm = _mk_pm()
+        _open_long_call(pm, opt_type=variant)
+        si = pm.check_positions(date(2025, 7, 18), {'p1': {'underlying_price': 6296.79}})[0]
+        assert abs(si['pnl'] - 38179.0) < 1.0, (variant, si['pnl'])
+    for variant in ('put', 'PUT', 'P'):
+        pm = _mk_pm('PUT')
+        pm.open_position(position_id='p1', strategy_type='PUT', symbol='SPX',
+                         entry_date=date(2025, 5, 2), entry_price=10000.0,
+                         quantity=100, contracts=1, total_cost=10000.0,
+                         strike=5685.0, expiration=date(2025, 7, 18),
+                         underlying_entry_price=5686.67, opt_type=variant)
+        si = pm.check_positions(date(2025, 7, 18), {'p1': {'underlying_price': 5000.0}})[0]
+        assert abs(si['pnl'] - 58500.0) < 1.0, (variant, si['pnl'])
+
+
+def test_w1a_missing_opt_type_derives_from_strategy():
+    pm = _mk_pm('PUT')
+    pm.open_position(position_id='p1', strategy_type='PUT', symbol='SPX',
+                     entry_date=date(2025, 5, 2), entry_price=10000.0,
+                     quantity=100, contracts=1, total_cost=10000.0,
+                     strike=5685.0, expiration=date(2025, 7, 18),
+                     underlying_entry_price=5686.67)  # no opt_type at all
+    si = pm.check_positions(date(2025, 7, 18), {'p1': {'underlying_price': 5000.0}})[0]
+    assert abs(si['pnl'] - 58500.0) < 1.0, si['pnl']
+
+
+# ---------------------------------------------------------------- W1b
+
+_DEAD_STUB = {'pnl': -23000.0, 'pnl_pct': -100.0, 'underlying_price': 6296.79,
+              'underlying_exit_price': 6296.79,
+              'call_exit_bid': 0.0, 'call_exit_ask': 0.0, 'call_iv_exit': -1.0}
+
+
+def test_w1b_by_signal_guard_debit():
+    pm = _mk_pm()
+    _open_long_call(pm)
+    ret = pm.close_position_by_signal('p1', date(2025, 7, 18), {'p1': dict(_DEAD_STUB)}, 'expiration')
+    tr = pm.closed_trades[-1]
+    assert abs(ret - 38179.0) < 1.0, ret
+    assert abs(tr['pnl'] - 38179.0) < 1.0, tr['pnl']
+    assert tr.get('settlement_type') == 'intrinsic_guard'
+
+
+def test_w1b_by_signal_guard_credit_pin():
+    pm = _mk_pm('STRADDLE')
+    pm.open_position(position_id='p4', strategy_type='SELL_STRADDLE', symbol='SPX',
+                     entry_date=date(2025, 6, 1), entry_price=0.0, quantity=100,
+                     contracts=1, total_cost=-5000.0, strike=6300.0,
+                     expiration=date(2025, 7, 18), entry_max_risk=20000.0,
+                     underlying_entry_price=6290.0)
+    stub = {'p4': {'pnl': 5000.0, 'pnl_pct': 100.0, 'underlying_price': 6300.0,
+                   'underlying_exit_price': 6300.0,
+                   'call_exit_bid': 0.0, 'call_exit_ask': 0.0,
+                   'put_exit_bid': 0.0, 'put_exit_ask': 0.0}}
+    ret = pm.close_position_by_signal('p4', date(2025, 7, 18), stub, 'expiration')
+    assert abs(ret - 5000.0) < 1.0, ret  # premium kept, intrinsic 0 at the pin
+
+
+def test_w1b_guard_skips_live_quotes():
+    pm = _mk_pm()
+    _open_long_call(pm)
+    live = {'p1': {'pnl': 39000.0, 'pnl_pct': 169.6, 'underlying_price': 6296.79,
+                   'underlying_exit_price': 6296.79,
+                   'call_exit_bid': 620.0, 'call_exit_ask': 622.0}}
+    ret = pm.close_position_by_signal('p1', date(2025, 7, 18), live, 'expiration')
+    tr = pm.closed_trades[-1]
+    assert abs(ret - 39000.0) < 1.0, ret  # market close honored
+    assert tr.get('settlement_type') != 'intrinsic_guard'
+
+
+def test_w1b_guard_skips_before_expiration():
+    pm = _mk_pm()
+    _open_long_call(pm)
+    dead_early = {'p1': dict(_DEAD_STUB, pnl=-8000.0, pnl_pct=-34.8)}
+    ret = pm.close_position_by_signal('p1', date(2025, 6, 20), dead_early, 'signal_exit')
+    assert abs(ret - (-8000.0)) < 1.0, ret  # pre-expiration: caller's pnl kept
+
+
+def test_w1b_strategy_type_taken_from_position():
+    # config says CALL, position is PUT: kwargs must be generated for PUT legs
+    pm = _mk_pm('CALL')
+    pm.open_position(position_id='pp', strategy_type='PUT', symbol='SPX',
+                     entry_date=date(2025, 5, 2), entry_price=9000.0,
+                     quantity=100, contracts=1, total_cost=9000.0,
+                     strike=6400.0, expiration=date(2025, 9, 19),
+                     underlying_entry_price=6300.0, opt_type='put')
+    stub = {'pp': {'pnl': 1500.0, 'pnl_pct': 16.7,
+                   'put_exit_bid': 105.0, 'put_exit_ask': 106.0}}
+    pm.close_position_by_signal('pp', date(2025, 6, 20), stub, 'signal_exit')
+    tr = pm.closed_trades[-1]
+    assert tr.get('put_exit_bid') == 105.0, tr.get('put_exit_bid')
+
+
+def test_w1b_profit_target_skipped_at_expiration():
+    pm = _mk_pm('CALL', profit_target_config={'enabled': True, 'target_pct': 50})
+    _open_long_call(pm)
+    chain = pd.DataFrame({'strike': [5685.0], 'expiration': ['2025-07-18'],
+                          'type': ['C'], 'bid': [0.0], 'ask': [0.0]})
+    to_close = pm.check_profit_target(
+        date(2025, 7, 18), 6296.79, chain,
+        lambda s, e, t: lib.get_option_by_strike_exp(chain, s, e, t))
+    assert to_close == [], to_close  # no phantom PT on expiration day
+
+
+# ---------------------------------------------------------------- W2
+
+def test_w2_api_call_retries_empty_data_error():
+    calls = {'n': 0}
+
+    def fake_internal(endpoint, cache_config, debug, debug_level,
+                      skip_parquet_cache=False, _chunk_info=None, **kwargs):
+        calls['n'] += 1
+        if calls['n'] < 3:
+            raise pd.errors.EmptyDataError('No columns to parse from file')
+        return {'data': [{'x': 1}]}
+
+    orig_internal = lib._api_call_internal
+    orig_sleep = lib.time.sleep if hasattr(lib, 'time') else None
+    import time as _t
+    orig_t_sleep = _t.sleep
+    _t.sleep = lambda *_a, **_k: None
+    lib._api_call_internal = fake_internal
+    try:
+        resp = lib.api_call('/fake/endpoint', None, debug=0, max_retries=3)
+    finally:
+        lib._api_call_internal = orig_internal
+        _t.sleep = orig_t_sleep
+    assert calls['n'] == 3, calls['n']
+    assert resp and resp['data'] == [{'x': 1}]
+
+
+def test_w2_api_call_gives_up_after_retries():
+    calls = {'n': 0}
+
+    def fake_internal(*a, **k):
+        calls['n'] += 1
+        raise pd.errors.EmptyDataError('No columns to parse from file')
+
+    import time as _t
+    orig_internal, orig_sleep = lib._api_call_internal, _t.sleep
+    _t.sleep = lambda *_a, **_k: None
+    lib._api_call_internal = fake_internal
+    try:
+        raised = False
+        try:
+            lib.api_call('/fake/endpoint', None, debug=0, max_retries=3)
+        except pd.errors.EmptyDataError:
+            raised = True
+    finally:
+        lib._api_call_internal = orig_internal
+        _t.sleep = orig_sleep
+    assert calls['n'] == 3, calls['n']
+    assert raised  # surfaces to fetch_chunk → loud drop, not a silent None
+
+
+def test_w2_api_call_direct_retries_empty():
+    calls = {'n': 0}
+
+    class FakeAPIManager:
+        @staticmethod
+        def get_method(endpoint):
+            def m(**kwargs):
+                calls['n'] += 1
+                if calls['n'] < 2:
+                    raise pd.errors.EmptyDataError('No columns to parse from file')
+                return pd.DataFrame({'x': [1]})
+            return m
+
+    class FakeHelper:
+        @staticmethod
+        def normalize_response(response, debug=False):
+            return {'data': response}
+
+    import time as _t
+    orig_mgr, orig_helper, orig_sleep = lib.APIManager, lib.APIHelper, _t.sleep
+    _t.sleep = lambda *_a, **_k: None
+    lib.APIManager, lib.APIHelper = FakeAPIManager, FakeHelper
+    try:
+        df = lib._api_call_direct('/fake', max_retries=3)
+    finally:
+        lib.APIManager, lib.APIHelper = orig_mgr, orig_helper
+        _t.sleep = orig_sleep
+    assert calls['n'] == 2 and df is not None and len(df) == 1
+
+
+# ---------------------------------------------------------------- W3
+
+def _chain_row(bid, ask, settle=None, extra=None):
+    data = {'strike': [5000.0], 'expiration': ['2026-09-14'], 'type': ['C'],
+            'bid': [bid], 'ask': [ask]}
+    if settle is not None:
+        data['settle'] = [settle]
+    if extra:
+        data.update(extra)
+    return pd.DataFrame(data)
+
+
+def test_w3_settle_fallback_dead_quotes():
+    row = lib.get_option_by_strike_exp(_chain_row(0.0, 0.0, settle=12.5),
+                                       5000.0, '2026-09-14', 'C')
+    assert row['bid'] == 12.5 and row['ask'] == 12.5
+    assert row.get('_priced_from_settle') is True
+
+
+def test_w3_settle_fallback_ignores_live_quotes():
+    row = lib.get_option_by_strike_exp(_chain_row(1.0, 1.2, settle=12.5),
+                                       5000.0, '2026-09-14', 'C')
+    assert row['bid'] == 1.0 and row['ask'] == 1.2
+    assert '_priced_from_settle' not in row
+
+
+def test_w3_no_settle_column_noop():
+    row = lib.get_option_by_strike_exp(_chain_row(0.0, 0.0),
+                                       5000.0, '2026-09-14', 'C')
+    assert row['bid'] == 0.0 and row['ask'] == 0.0
+    assert '_priced_from_settle' not in row
+
+
+def test_w3_settle_varchar_from_old_cache():
+    df = _chain_row(0.0, 0.0)
+    df['settle'] = ['12.5']  # string, as a pre-fix VARCHAR cache would give
+    row = lib.get_option_by_strike_exp(df, 5000.0, '2026-09-14', 'C')
+    assert row['bid'] == 12.5, row['bid']
+
+
+def test_w3_settlement_mode_marks_all_rows():
+    df = pd.DataFrame({'strike': [1.0, 2.0], 'expiration': ['e', 'e'],
+                       'type': ['C', 'C'], 'bid': [0.5, 0.0], 'ask': [0.7, 0.0],
+                       'mid': [0.6, 0.0], 'settle': [0.55, 0.33]})
+    out = lib._apply_fut_opt_pricing_mode(df, {'fut_opt_pricing': 'settlement'})
+    assert list(out['bid']) == [0.55, 0.33]
+    assert list(out['ask']) == [0.55, 0.33]
+    # default mode: untouched
+    out2 = lib._apply_fut_opt_pricing_mode(df, {})
+    assert list(out2['bid']) == [0.5, 0.0]
+
+
+def test_w3_normalize_casts_settle_numeric():
+    raw = pd.DataFrame({'opt_symbol': ['EC/26U 1.14C.CME'], 'strike': ['1.14'],
+                        'bid': ['0'], 'ask': ['0'], 'settle': ['0.0047'],
+                        'expiration': ['2026-09-14'], 'date': ['2026-07-24'],
+                        'type': ['C']})
+    norm = lib._normalize_futures_options_df(raw.copy(), 'EC')
+    assert norm['settle'].dtype.kind == 'f', norm['settle'].dtype
+    assert abs(float(norm['settle'].iloc[0]) - 0.0047) < 1e-9
+
+
+def test_w3_duckdb_alter_add_preserves_other_symbols():
+    # simulate the migration semantics on a real tmp DuckDB the same way
+    # _save_chunk does: ALTER ADD for additive drift + name-aligned INSERT
+    import duckdb
+    import tempfile
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        con.execute("CREATE TABLE futures_options_eod_close AS "
+                    "SELECT 'ES' AS symbol, '2026-07-01' AS date, 1.0 AS bid, 2.0 AS ask")
+        # incoming frame has a NEW column settle → additive drift
+        df_new = pd.DataFrame({'symbol': ['EC'], 'date': ['2026-07-24'],
+                               'bid': [0.0], 'ask': [0.0], 'settle': [0.0047]})
+        existing = [c[1] for c in con.execute(
+            "PRAGMA table_info('futures_options_eod_close')").fetchall()]
+        added = set(df_new.columns) - set(existing)
+        removed = set(existing) - set(df_new.columns)
+        assert added == {'settle'} and not removed
+        for c in sorted(added):
+            con.execute(f'ALTER TABLE futures_options_eod_close ADD COLUMN "{c}" DOUBLE')
+        cols = ', '.join(f'"{c}"' for c in df_new.columns)
+        con.register('df_for_db', df_new)
+        con.execute(f"INSERT INTO futures_options_eod_close ({cols}) SELECT {cols} FROM df_for_db")
+        rows = con.execute("SELECT symbol, settle FROM futures_options_eod_close ORDER BY symbol").fetchall()
+        assert len(rows) == 2, rows                       # ES row SURVIVED
+        assert rows[0][0] == 'EC' and abs(rows[0][1] - 0.0047) < 1e-9
+        assert rows[1][0] == 'ES' and rows[1][1] is None  # old row, NULL settle
+
+
+# ---------------------------------------------------------------- W4
+
+def test_w4_am_detection():
+    am = {'symbol': 'SPX', 'expiration': date(2025, 7, 18)}   # 3rd Friday
+    assert lib._is_am_settled_position(am) is True
+    weekly = {'symbol': 'SPX', 'expiration': date(2025, 7, 25)}  # 4th Friday
+    assert lib._is_am_settled_position(weekly) is False
+    equity = {'symbol': 'AAPL', 'expiration': date(2025, 7, 18)}
+    assert lib._is_am_settled_position(equity) is False
+    explicit = {'symbol': 'AAPL', 'expiration': date(2025, 7, 18),
+                'settlement_style': 'AM'}
+    assert lib._is_am_settled_position(explicit) is True
+    forced_pm = {'symbol': 'SPX', 'expiration': date(2025, 7, 18),
+                 'settlement_style': 'PM'}
+    assert lib._is_am_settled_position(forced_pm) is False
+
+
+def test_w4_settlement_uses_open_for_am():
+    pm = _mk_pm()
+    _open_long_call(pm)  # SPX, exp 2025-07-18 = 3rd Friday → AM
+    pd_data = {'p1': {'underlying_price': 6296.79, 'underlying_open': 6250.00}}
+    si = pm.check_positions(date(2025, 7, 18), pd_data)[0]
+    expected = (6250.00 - 5685.0) * 100 - 23000.0
+    assert abs(si['pnl'] - expected) < 1.0, (si['pnl'], expected)
+
+
+def test_w4_settlement_keeps_close_when_disabled():
+    pm = _mk_pm(am_settlement_open=False)
+    _open_long_call(pm)
+    pd_data = {'p1': {'underlying_price': 6296.79, 'underlying_open': 6250.00}}
+    si = pm.check_positions(date(2025, 7, 18), pd_data)[0]
+    assert abs(si['pnl'] - 38179.0) < 1.0, si['pnl']
+
+
+def test_w4_settlement_keeps_close_for_non_am():
+    pm = lib.PositionManager({'strategy_type': 'CALL', 'symbol': 'AAPL'}, debug=False)
+    pm.open_position(position_id='p1', strategy_type='CALL', symbol='AAPL',
+                     entry_date=date(2025, 5, 2), entry_price=1000.0,
+                     quantity=100, contracts=1, total_cost=1000.0,
+                     strike=200.0, expiration=date(2025, 7, 18),
+                     underlying_entry_price=201.0, opt_type='call')
+    pd_data = {'p1': {'underlying_price': 230.0, 'underlying_open': 225.0}}
+    si = pm.check_positions(date(2025, 7, 18), pd_data)[0]
+    expected = (230.0 - 200.0) * 100 - 1000.0
+    assert abs(si['pnl'] - expected) < 1.0, si['pnl']
+
+
+# ---------------------------------------------------------------- runner
+
+if __name__ == '__main__':
+    tests = [(n, f) for n, f in sorted(globals().items())
+             if n.startswith('test_') and callable(f)]
+    failed = []
+    for name, fn in tests:
+        try:
+            fn()
+            print(f"PASS  {name}")
+        except Exception:
+            failed.append(name)
+            print(f"FAIL  {name}")
+            traceback.print_exc()
+    print(f"\n{len(tests) - len(failed)}/{len(tests)} passed")
+    sys.exit(1 if failed else 0)