ivolatility-backtesting 2.143__tar.gz → 2.145__tar.gz

{ivolatility_backtesting-2.143 → ivolatility_backtesting-2.145}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.143
+Version: 2.145
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com
@@ -15,8 +15,8 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: pandas>=1.5.0
-Requires-Dist: numpy>=1.21.0
+Requires-Dist: pandas<3.0,>=1.5.0
+Requires-Dist: numpy<3.0,>=1.21.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: seaborn>=0.11.0
 Requires-Dist: ivolatility>=1.8.2

{ivolatility_backtesting-2.143 → ivolatility_backtesting-2.145}/ivolatility_backtesting/ivolatility_backtesting.py

@@ -179,8 +179,29 @@ _API_LOG_FILE = None  # Track API log file path for summary printing
 # ============================================================
 from typing import Optional, Dict, Any, Tuple
 
+_NUMERIC_COERCE_WARNED = False
+
+
+def _as_data_row(source):
+    """Normalize a stock/data row to a single record.
+
+    Callers legitimately pass either a row (``df.iloc[0]``) or a one-row slice
+    (``df[df['date'] == d]``). Only a 2-D object needs ``.iloc[0]``: applying it
+    to a Series would collapse it to its first cell.
+    """
+    if source is None:
+        return None
+    try:
+        if getattr(source, 'ndim', 1) > 1:
+            return source.iloc[0] if len(source) > 0 else None
+    except Exception:
+        return source
+    return source
+
+
 def _first_valid_numeric_value(source, columns) -> Optional[float]:
     """Return the first usable numeric value from source[column] in priority order."""
+    source = _as_data_row(source)
     if source is None:
         return None
     for col in columns:
@@ -196,7 +217,16 @@ def _first_valid_numeric_value(source, columns) -> Optional[float]:
             if value is None or pd.isna(value):
                 continue
             return float(value)
-        except Exception:
+        except Exception as exc:
+            # Shape errors here used to vanish silently and surface later as a
+            # wrong number (fallback instead of the real price). Warn once per
+            # run so the cause is visible without flooding the notebook output.
+            global _NUMERIC_COERCE_WARNED
+            if not _NUMERIC_COERCE_WARNED:
+                _NUMERIC_COERCE_WARNED = True
+                print(f"\u26a0\ufe0f  Could not read numeric field '{col}' from the data row "
+                      f"({type(exc).__name__}: {exc}). Falling back to defaults — "
+                      f"check that stock_row is a row or a one-row slice.")
             continue
     return None
 
@@ -266,14 +296,7 @@ def _resolve_runtime_underlying_price(stock_price=None, stock_row=None, options_
     Resolve the best option-aligned underlying price for runtime P&L / SL logic.
     Priority: stock_row runtime fields -> options underlying_price -> passed scalar.
     """
-    row_obj = None
-    try:
-        if stock_row is not None and hasattr(stock_row, 'iloc') and len(stock_row) > 0:
-            row_obj = stock_row.iloc[0]
-        elif stock_row is not None:
-            row_obj = stock_row
-    except Exception:
-        row_obj = stock_row
+    row_obj = _as_data_row(stock_row)
 
     value = _first_valid_numeric_value(
         row_obj,
@@ -297,14 +320,7 @@ def _resolve_runtime_underlying_price(stock_price=None, stock_row=None, options_
 
 def _resolve_runtime_stock_range(stock_row=None, fallback_price=None) -> Tuple[Optional[float], Optional[float]]:
     """Resolve runtime high/low in the same coordinate system as runtime price."""
-    row_obj = None
-    try:
-        if stock_row is not None and hasattr(stock_row, 'iloc') and len(stock_row) > 0:
-            row_obj = stock_row.iloc[0]
-        elif stock_row is not None:
-            row_obj = stock_row
-    except Exception:
-        row_obj = stock_row
+    row_obj = _as_data_row(stock_row)
 
     high = _first_valid_numeric_value(row_obj, ('runtime_high', 'high'))
     low = _first_valid_numeric_value(row_obj, ('runtime_low', 'low'))
@@ -1515,6 +1531,7 @@ def report_results_folder(
     return {"folder": folder, "strategies": strategies}
 
 
+
 def run_backtest_notebook(
     base_config: dict,
     strategy_fn,
@@ -2019,6 +2036,8 @@ def _enable_api_logging(log_file='logs/ivolatility_api.log', level=2):
 # Uses ThreadPoolExecutor for parallel api_call (preserves caching!)
 # For I/O bound tasks (HTTP), threads work well despite GIL
 from concurrent.futures import ThreadPoolExecutor, as_completed
+# On py<3.11 futures raise their own TimeoutError, not the builtin
+from concurrent.futures import TimeoutError as _FuturesTimeoutError
 _PARALLEL_AVAILABLE = True
 
 # ============================================================
@@ -13713,15 +13732,12 @@ class PositionManager:
                 if stock_row is not None:
                     # runtime_open = split-aligned open (REFERENCE §4a) — the
                     # AM-settlement reference must not use the adjusted series
-                    _stock_open = None
-                    for _oc in ('runtime_open', 'open'):
-                        try:
-                            _ov = stock_row.get(_oc) if hasattr(stock_row, 'get') else stock_row[_oc]
-                        except (KeyError, IndexError, TypeError):
-                            _ov = None
-                        if _ov is not None and not pd.isna(_ov) and float(_ov) > 0:
-                            _stock_open = float(_ov)
-                            break
+                    # stock_row may arrive as a single-row slice (stock_df[mask]) —
+                    # take the row itself, otherwise .get() yields a Series and any
+                    # boolean/float use of it raises ValueError.
+                    _stock_open = _first_valid_numeric_value(stock_row, ('runtime_open', 'open'))
+                    if _stock_open is not None and _stock_open <= 0:
+                        _stock_open = None
                     price_data[pos_id].update({
                         'underlying_high': stock_high,
                         'underlying_low': stock_low,
@@ -15706,6 +15722,14 @@ class ResultsReporter:
         print("TRADING STATISTICS")
         print("-"*80)
         print(f"Total Trades:            {m['total_trades']:>15}")
+        if not m.get('total_trades'):
+            # 0 trades is almost never a real result — it usually means the entry
+            # signal never fired (unread config key, indicators not computed).
+            print("\u26a0\ufe0f  0 trades: the entry signal never fired — this is not a "
+                  "result, it is a run that never entered. Check that indicators were "
+                  "computed (look for 'Processing N cached indicators' — 0 means the "
+                  "signal could not be evaluated) and that the data loaded (a locked "
+                  "DuckDB cache or a failed fetch also ends in 0 trades).")
         print(f"Winning Trades:          {m['winning_trades']:>15}")
         print(f"Losing Trades:           {m['losing_trades']:>15}")
         print(f"Win Rate:                {m['win_rate']:>15.2f}%  (% profitable trades)")
@@ -18471,6 +18495,7 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
     rows_streamed = 0
     failed_chunks = []  # 2.142: (start, end, cp, error) of every dropped chunk
     _gap_save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
+    _GAP_GLOBAL_TIMEOUT = 1800  # same safety net as the main parallel loader
 
     def _stream_one(df):
         nonlocal rows_streamed, total_rows
@@ -18486,23 +18511,37 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
         completed = 0
         with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as executor:
             futures = {executor.submit(fetch_chunk, (params, i+1)): i for i, params in enumerate(all_requests)}
-            
-            for future in as_completed(futures):
-                completed += 1
-                try:
-                    df = future.result(timeout=120)
-                except Exception:
+
+            try:
+                # Without the timeout a single hung HTTP call blocks as_completed
+                # forever — result(timeout) never gets a chance to fire.
+                for future in as_completed(futures, timeout=_GAP_GLOBAL_TIMEOUT):
+                    completed += 1
+                    try:
+                        df = future.result(timeout=120)
+                    except Exception:
+                        df = None
+
+                    _stream_one(df)
+                    try:
+                        future._result = None
+                    except Exception:
+                        pass
                     df = None
-                
-                _stream_one(df)
-                try:
-                    future._result = None
-                except Exception:
-                    pass
-                df = None
-                
-                if debuginfo >= 1 and (completed % 10 == 0 or completed == total_requests):
-                    _safe_print(f"              ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows streamed")
+
+                    if debuginfo >= 1 and (completed % 10 == 0 or completed == total_requests):
+                        _safe_print(f"              ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows streamed")
+            except (TimeoutError, _FuturesTimeoutError):
+                _pending = [i + 1 for f, i in futures.items() if not f.done()]
+                _safe_print(f"              ❌ [GAP TIMEOUT] as_completed timed out after "
+                            f"{_GAP_GLOBAL_TIMEOUT}s — dropping pending chunks {_pending}")
+                for f, i in futures.items():
+                    if not f.done():
+                        f.cancel()
+                        _p = all_requests[i]
+                        failed_chunks.append((_p.get('from_', _p.get('startDate', '?')),
+                                              _p.get('to', _p.get('endDate', '?')),
+                                              _p.get('cp', '?'), 'gap global timeout'))
     else:
         for i, params in enumerate(all_requests):
             df = fetch_chunk((params, i + 1))
@@ -20241,7 +20280,7 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                     _bt_logger.info(f"[DIAG] MAIN LOOP future ready completed={completed}/{total_requests} chunk_id={_chunk_id}")
                     try:
                         df = future.result(timeout=120)
-                    except TimeoutError:
+                    except (TimeoutError, _FuturesTimeoutError):
                         _safe_print(f"           ⏱️ [TIMEOUT] future.result() timed out after 120s (chunk {completed}/{total_requests})")
                         _bt_logger.info(f"[DIAG] FUTURE TIMEOUT chunk={_chunk_id} completed={completed}/{total_requests}")
                         df = None
@@ -20332,7 +20371,7 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                     if completed % 10 == 0 or completed == total_requests:
                         _wall = _time_mod.time() - _parallel_start
                         _safe_print(f"           ⚡ Progress: {completed}/{total_requests} requests, {total_rows:,} rows, wall={_wall:.1f}s")
-            except TimeoutError:
+            except (TimeoutError, _FuturesTimeoutError):
                 _wall = _time_mod.time() - _parallel_start
                 _pending = [cid for f, cid in futures.items() if not f.done()]
                 _safe_print(f"           ❌ [GLOBAL TIMEOUT] as_completed timed out after {_PARALLEL_GLOBAL_TIMEOUT}s (wall={_wall:.1f}s)")
@@ -21810,10 +21849,18 @@ def _load_futures_prices_to_duckdb(config, cache_config, symbol,
         with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as exe:
             futures = {exe.submit(_fetch, cs, ce, i + 1): (cs, ce)
                        for i, (cs, ce) in enumerate(chunks)}
-            for f in as_completed(futures):
-                part = f.result()
-                if part is not None and not part.empty:
-                    parts.append(part)
+            try:
+                # a hung HTTP call blocks as_completed forever without this
+                for f in as_completed(futures, timeout=1800):
+                    part = f.result()
+                    if part is not None and not part.empty:
+                        parts.append(part)
+            except (TimeoutError, _FuturesTimeoutError):
+                _hung = [rng for f, rng in futures.items() if not f.done()]
+                print(f"           ❌ [TIMEOUT] futures_eod load: dropping hung chunks {_hung}")
+                for f in futures:
+                    if not f.done():
+                        f.cancel()
     else:
         for i, (cs, ce) in enumerate(chunks):
             part = _fetch(cs, ce, i + 1)
@@ -22342,24 +22389,34 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
         with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as exe:
             futs = {exe.submit(_fetch, p, i + 1): p
                     for i, p in enumerate(all_requests)}
-            for f in as_completed(futs):
-                completed += 1
-                _fdf = f.result()
-                _save_chunk(_fdf)
-                if _fdf is not None and not getattr(_fdf, 'empty', True):
+            try:
+                # a hung HTTP call blocks as_completed forever without this
+                for f in as_completed(futs, timeout=1800):
+                    completed += 1
+                    _fdf = f.result()
+                    _save_chunk(_fdf)
+                    if _fdf is not None and not getattr(_fdf, 'empty', True):
+                        try:
+                            _mf_fut_sem.release()
+                        except Exception:
+                            pass
                     try:
-                        _mf_fut_sem.release()
+                        f._result = None
                     except Exception:
                         pass
-                try:
-                    f._result = None
-                except Exception:
-                    pass
-                _fdf = None
-                _maybe_checkpoint()
-                if completed % 20 == 0 or completed == total:
-                    print(f"           ⚡ {completed}/{total} requests, "
-                          f"{rows_saved_total:,} rows so far")
+                    _fdf = None
+                    _maybe_checkpoint()
+                    if completed % 20 == 0 or completed == total:
+                        print(f"           ⚡ {completed}/{total} requests, "
+                              f"{rows_saved_total:,} rows so far")
+            except (TimeoutError, _FuturesTimeoutError):
+                print(f"           ❌ [TIMEOUT] futures-options load: as_completed "
+                      f"timed out after 1800s — dropping hung chunks")
+                for f, p in futs.items():
+                    if not f.done():
+                        f.cancel()
+                        failed_chunks.append((p.get('from_', '?'), p.get('to', '?'),
+                                              p.get('cp', '?'), 'global timeout'))
     else:
         for i, params in enumerate(all_requests, 1):
             completed = i
@@ -23758,23 +23815,34 @@ def preload_data_universal(config, data_requests=None, debug=False):
                 
                 with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as executor:
                     future_to_idx = {
-                        executor.submit(fetch_chunk, req_data): i 
+                        executor.submit(fetch_chunk, req_data): i
                         for i, req_data in enumerate(all_request_data)
                     }
-                    
-                    for future in as_completed(future_to_idx):
-                        idx = future_to_idx[future]
-                        completed += 1
-                        
-                        if completed % max(1, total_requests // 5) == 0 or completed == total_requests:
-                            _safe_print(f"           ⚡ Progress: {completed}/{total_requests}...")
-                        
-                        try:
-                            result = future.result(timeout=120)
-                            results.append((idx, result))
-                        except Exception as e:
-                            req_data = all_request_data[idx]
-                            results.append((idx, (None, req_data[1], req_data[2], req_data[3], e)))
+
+                    try:
+                        # a hung HTTP call blocks as_completed forever without this
+                        for future in as_completed(future_to_idx, timeout=1800):
+                            idx = future_to_idx[future]
+                            completed += 1
+
+                            if completed % max(1, total_requests // 5) == 0 or completed == total_requests:
+                                _safe_print(f"           ⚡ Progress: {completed}/{total_requests}...")
+
+                            try:
+                                result = future.result(timeout=120)
+                                results.append((idx, result))
+                            except Exception as e:
+                                req_data = all_request_data[idx]
+                                results.append((idx, (None, req_data[1], req_data[2], req_data[3], e)))
+                    except (TimeoutError, _FuturesTimeoutError):
+                        _safe_print(f"           ❌ [TIMEOUT] as_completed timed out after 1800s — "
+                                    f"dropping hung chunks")
+                        for future, idx in future_to_idx.items():
+                            if not future.done():
+                                future.cancel()
+                                req_data = all_request_data[idx]
+                                results.append((idx, (None, req_data[1], req_data[2], req_data[3],
+                                                      TimeoutError('global timeout'))))
                 
                 # Sort by original index to maintain order
                 results.sort(key=lambda x: x[0])
@@ -25504,7 +25572,7 @@ def print_signals_table(config, indicator_cache, stock_df, save_path=None, silen
     expected_ivx_key = ('iv_lean_zscore_ivx', (_cfg_dte, int(_cfg_lb))) if _cfg_lb is not None else None
 
     if expected_raw_key is not None and expected_raw_key not in indicator_cache and should_print:
-        present_lbs = sorted({k[1][0] for k in indicator_cache if k[0] == 'iv_lean_zscore' and isinstance(k[1], tuple)})
+        present_lbs = sorted({k[1][0] for k in indicator_cache if k[0] == 'iv_lean_zscore' and isinstance(k[1], tuple) and k[1]})
         print(f"⚠️  print_signals_table: no iv_lean_zscore cache for lookback={_cfg_lb} "
               f"(cache has: {present_lbs}). Table will be empty for active config.")
 
@@ -26757,6 +26825,7 @@ def run_optimization(base_config, param_grid, strategy_function,
         (combined_results_df, baselines_dict, results_folder)
         baselines_dict: {symbol: metrics_dict}
     """
+
     import copy as _copy
 
     if symbols is None:
@@ -28614,6 +28683,8 @@ def _init_duckdb_storage_tables(conn):
         print(f"[DUCKDB] ⚠️ Legacy migration warning: {_mig_err}", flush=True)
     
     # OPTIONS_EOD_CLOSE - EOD options from stock-opts-by-param and options-rawiv (close prices)
+    # No PK: a PK's ART index is RAM-resident in DuckDB and OOMs small containers on
+    # multi-million-row tables; dedup lives in _duckdb_insert_dedup(). Do not re-add.
     _OPTIONS_EOD_SCHEMA = """(
             option_symbol VARCHAR,
             date DATE,
@@ -28636,26 +28707,32 @@ def _init_duckdb_storage_tables(conn):
             open_interest DOUBLE,
             source_endpoint VARCHAR,
             bid_eod DOUBLE,
-            ask_eod DOUBLE,
-            PRIMARY KEY (option_symbol, date)
+            ask_eod DOUBLE
         )"""
+    # Rebuild legacy caches that still carry the PK, preserving rows
+    for _opt_tbl_mig in ('options_eod_close', 'options_eod_1545'):
+        try:
+            _existing_tbls = [t[0] for t in conn.execute("SHOW TABLES").fetchall()]
+            if f'{_opt_tbl_mig}__nopk' in _existing_tbls:
+                conn.execute(f"DROP TABLE {_opt_tbl_mig}__nopk")
+            if _opt_tbl_mig not in _existing_tbls:
+                continue
+            _has_pk = conn.execute(
+                "SELECT 1 FROM information_schema.table_constraints "
+                "WHERE table_name = ? AND constraint_type = 'PRIMARY KEY'",
+                [_opt_tbl_mig]).fetchall()
+            if _has_pk:
+                _mig_rows = conn.execute(f"SELECT COUNT(*) FROM {_opt_tbl_mig}").fetchone()[0]
+                conn.execute(f"CREATE TABLE {_opt_tbl_mig}__nopk AS SELECT * FROM {_opt_tbl_mig}")
+                conn.execute(f"DROP TABLE {_opt_tbl_mig}")
+                conn.execute(f"ALTER TABLE {_opt_tbl_mig}__nopk RENAME TO {_opt_tbl_mig}")
+                print(f"[DUCKDB] 🔄 Rebuilt {_opt_tbl_mig} without PRIMARY KEY "
+                      f"({_mig_rows:,} rows preserved)", flush=True)
+        except Exception as _pk_err:
+            print(f"[DUCKDB] ⚠️ PK removal migration for {_opt_tbl_mig}: {_pk_err}", flush=True)
     conn.execute(f"CREATE TABLE IF NOT EXISTS options_eod_close {_OPTIONS_EOD_SCHEMA}")
-    
+
     # OPTIONS_EOD_1545 - 15:45 snapshot options (same schema, isolated table)
-    # Migration: recreate if table exists but lacks PRIMARY KEY (created by older code)
-    try:
-        _1545_exists = any(t[0] == 'options_eod_1545' for t in conn.execute("SHOW TABLES").fetchall())
-        if _1545_exists:
-            _1545_constraints = conn.execute(
-                "SELECT constraint_type FROM information_schema.table_constraints "
-                "WHERE table_name = 'options_eod_1545' AND constraint_type = 'PRIMARY KEY'"
-            ).fetchall()
-            if not _1545_constraints:
-                _1545_rows = conn.execute("SELECT COUNT(*) FROM options_eod_1545").fetchone()[0]
-                conn.execute("DROP TABLE options_eod_1545")
-                print(f"[DUCKDB] 🔄 Recreating options_eod_1545 with PRIMARY KEY (had {_1545_rows} rows, no PK)", flush=True)
-    except Exception as _pk_err:
-        print(f"[DUCKDB] ⚠️ PK migration check for options_eod_1545: {_pk_err}", flush=True)
     conn.execute(f"CREATE TABLE IF NOT EXISTS options_eod_1545 {_OPTIONS_EOD_SCHEMA}")
     
     # OPTIONS_INTRADAY - Minute-level options data
@@ -28820,6 +28897,37 @@ def _get_target_table(endpoint: str) -> str:
     return None
 
 
+_ANTIJOIN_DEDUP_TABLES = ('options_eod_close', 'options_eod_1545')
+
+
+def _duckdb_insert_dedup(conn, table_name, df_normalized):
+    """Insert df_normalized skipping existing keys.
+
+    options_eod_* have no PK (see _OPTIONS_EOD_SCHEMA): dedup = anti-join on
+    (option_symbol, date) scoped to the batch's date range; in-batch duplicates
+    collapse via QUALIFY. Other tables keep PK + INSERT OR IGNORE."""
+    if table_name in _ANTIJOIN_DEDUP_TABLES:
+        _dmin = df_normalized['date'].min()
+        _dmax = df_normalized['date'].max()
+        conn.execute(f"""
+            INSERT INTO {table_name}
+            SELECT * FROM (
+                SELECT * FROM df_normalized
+                QUALIFY ROW_NUMBER() OVER (PARTITION BY option_symbol, date) = 1
+            ) d
+            WHERE NOT EXISTS (
+                SELECT 1 FROM {table_name} t
+                WHERE t.date >= ? AND t.date <= ?
+                  AND t.option_symbol = d.option_symbol AND t.date = d.date
+            )
+        """, [_dmin, _dmax])
+    else:
+        conn.execute(f"""
+            INSERT OR IGNORE INTO {table_name}
+            SELECT * FROM df_normalized
+        """)
+
+
 def _save_to_duckdb_storage(df, endpoint: str, cache_config: Dict[str, Any], debug: bool = False):
     """
     Save DataFrame to DuckDB storage with deduplication.
@@ -28922,10 +29030,15 @@ def _save_to_duckdb_storage(df, endpoint: str, cache_config: Dict[str, Any], deb
             count_before = conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0]
             
             # Insert with conflict handling (skip duplicates)
-            conn.execute(f"""
-                INSERT OR IGNORE INTO {table_name} 
-                SELECT * FROM df_normalized
-            """)
+            try:
+                _duckdb_insert_dedup(conn, table_name, df_normalized)
+            except Exception:
+                # CHECKPOINT frees dirty buffer pages; retry once before failing loud
+                try:
+                    conn.execute("CHECKPOINT")
+                except Exception:
+                    pass
+                _duckdb_insert_dedup(conn, table_name, df_normalized)
             
             # Get count after insert
             count_after = conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0]
@@ -28984,10 +29097,14 @@ def _save_to_duckdb_storage_batched(df, endpoint: str, cache_config: Dict[str, A
                 batch_normalized = _normalize_for_duckdb_storage(batch, endpoint, table_name)
                 
                 # Insert batch
-                conn.execute(f"""
-                    INSERT OR IGNORE INTO {table_name} 
-                    SELECT * FROM batch_normalized
-                """)
+                try:
+                    _duckdb_insert_dedup(conn, table_name, batch_normalized)
+                except Exception:
+                    try:
+                        conn.execute("CHECKPOINT")
+                    except Exception:
+                        pass
+                    _duckdb_insert_dedup(conn, table_name, batch_normalized)
                 
                 # Progress every 5 batches
                 if debug and (i // batch_size) % 5 == 0:
@@ -31287,21 +31404,29 @@ class MarketDataManager:
             completed = 0
             with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as pool:
                 futures = {pool.submit(_fetch, req): req for req in requests_list}
-                for fut in as_completed(futures):
-                    completed += 1
-                    try:
-                        ep, df = fut.result(timeout=120)
-                    except Exception:
-                        ep = futures[fut][0]
-                        df = None
-                    if df is not None and not df.empty:
-                        if 'symbol' not in df.columns:
-                            df['symbol'] = symbol
-                        df = self._normalizer.normalize(df, ep)
-                        collected[ep].append(df)
-                        self._cache_write(ep, df, cache_key=None)
-                    if completed % max(1, total // 5) == 0 or completed == total:
-                        _safe_print(f"[MDM]   progress: {completed}/{total}")
+                try:
+                    # a hung HTTP call blocks as_completed forever without this
+                    for fut in as_completed(futures, timeout=1800):
+                        completed += 1
+                        try:
+                            ep, df = fut.result(timeout=120)
+                        except Exception:
+                            ep = futures[fut][0]
+                            df = None
+                        if df is not None and not df.empty:
+                            if 'symbol' not in df.columns:
+                                df['symbol'] = symbol
+                            df = self._normalizer.normalize(df, ep)
+                            collected[ep].append(df)
+                            self._cache_write(ep, df, cache_key=None)
+                        if completed % max(1, total // 5) == 0 or completed == total:
+                            _safe_print(f"[MDM]   progress: {completed}/{total}")
+                except (TimeoutError, _FuturesTimeoutError):
+                    _safe_print(f"[MDM] ❌ [TIMEOUT] as_completed timed out after 1800s — "
+                                f"dropping hung requests")
+                    for fut in futures:
+                        if not fut.done():
+                            fut.cancel()
         else:
             for req in requests_list:
                 ep, df = _fetch(req)

{ivolatility_backtesting-2.143 → ivolatility_backtesting-2.145}/ivolatility_backtesting.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.143
+Version: 2.145
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com
@@ -15,8 +15,8 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: pandas>=1.5.0
-Requires-Dist: numpy>=1.21.0
+Requires-Dist: pandas<3.0,>=1.5.0
+Requires-Dist: numpy<3.0,>=1.21.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: seaborn>=0.11.0
 Requires-Dist: ivolatility>=1.8.2

{ivolatility_backtesting-2.143 → ivolatility_backtesting-2.145}/ivolatility_backtesting.egg-info/SOURCES.txt

@@ -8,4 +8,5 @@ ivolatility_backtesting.egg-info/SOURCES.txt
 ivolatility_backtesting.egg-info/dependency_links.txt
 ivolatility_backtesting.egg-info/requires.txt
 ivolatility_backtesting.egg-info/top_level.txt
-tests/test_2142_fixes.py
+tests/test_2142_fixes.py
+tests/test_2144_duckdb_dedup.py

{ivolatility_backtesting-2.143 → ivolatility_backtesting-2.145}/ivolatility_backtesting.egg-info/requires.txt

@@ -1,5 +1,5 @@
-pandas>=1.5.0
-numpy>=1.21.0
+pandas<3.0,>=1.5.0
+numpy<3.0,>=1.21.0
 matplotlib>=3.5.0
 seaborn>=0.11.0
 ivolatility>=1.8.2

{ivolatility_backtesting-2.143 → ivolatility_backtesting-2.145}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ivolatility_backtesting"
-version = "2.143"
+version = "2.145"
 description = "A universal backtesting framework for financial strategies using the IVolatility API."
 readme = "README.md"
 authors = [
@@ -13,8 +13,8 @@ authors = [
 license = { file = "LICENSE" }
 requires-python = ">=3.8"
 dependencies = [
-    "pandas>=1.5.0",
-    "numpy>=1.21.0",
+    "pandas>=1.5.0,<3.0",
+    "numpy>=1.21.0,<3.0",
     "matplotlib>=3.5.0",
     "seaborn>=0.11.0",
     "ivolatility>=1.8.2",

ivolatility_backtesting-2.145/tests/test_2144_duckdb_dedup.py

@@ -0,0 +1,191 @@
+# Tests for 2.144: options_eod_* without PK — anti-join dedup in
+# _duckdb_insert_dedup() + PK-removal migration of legacy caches.
+#
+# Runs standalone (no pytest needed):  python3 tests/test_2144_duckdb_dedup.py
+# Also pytest-compatible for CI:       pytest tests/test_2144_duckdb_dedup.py
+import os
+import sys
+import tempfile
+import traceback
+from datetime import date
+
+import duckdb
+import matplotlib
+matplotlib.use('Agg')
+import pandas as pd
+
+_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(_REPO, 'ivolatility_backtesting'))
+import ivolatility_backtesting as lib  # noqa: E402
+
+
+def _tbl_has_pk(con, tbl):
+    return bool(con.execute(
+        "SELECT 1 FROM information_schema.table_constraints "
+        "WHERE table_name = ? AND constraint_type = 'PRIMARY KEY'", [tbl]).fetchall())
+
+
+def _opts_row(option_symbol, d):
+    return {
+        'option_symbol': option_symbol, 'date': d, 'symbol': 'SPX',
+        'expiration': date(2026, 3, 20), 'strike': 5000.0, 'type': 'C',
+        'bid': 1.0, 'ask': 2.0, 'price': 1.5, 'underlying_price': 5100.0,
+        'iv': 0.2, 'delta': 0.5, 'gamma': 0.01, 'theta': -0.5, 'vega': 3.0,
+        'rho': 0.1, 'dte': 74.0, 'volume': 10.0, 'open_interest': 100.0,
+        'source_endpoint': 'test', 'bid_eod': 1.0, 'ask_eod': 2.0,
+    }
+
+
+def _opts_df(rows):
+    return pd.DataFrame([_opts_row(sym, d) for sym, d in rows])
+
+
+def _count(con, tbl='options_eod_1545'):
+    return con.execute(f'SELECT COUNT(*) FROM {tbl}').fetchone()[0]
+
+
+# ---------------------------------------------------------------- schema
+
+def test_fresh_options_tables_have_no_pk_and_stock_keeps_pk():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        assert not _tbl_has_pk(con, 'options_eod_close')
+        assert not _tbl_has_pk(con, 'options_eod_1545')
+        assert _tbl_has_pk(con, 'stock_eod')
+
+
+def test_pk_migration_preserves_rows():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        con.execute("CREATE TABLE options_eod_1545 (option_symbol VARCHAR, date DATE, "
+                    "bid DOUBLE, PRIMARY KEY (option_symbol, date))")
+        con.execute("INSERT INTO options_eod_1545 VALUES ('A', '2026-01-05', 1.0), "
+                    "('B', '2026-01-06', 2.0)")
+        lib._init_duckdb_storage_tables(con)
+        assert not _tbl_has_pk(con, 'options_eod_1545')
+        assert _count(con) == 2
+        # columns of the migrated table survive as-is (additive drift handled elsewhere)
+        cols = [r[0] for r in con.execute("DESCRIBE options_eod_1545").fetchall()]
+        assert cols == ['option_symbol', 'date', 'bid']
+
+
+def test_pk_migration_drops_stale_nopk_leftover():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        con.execute("CREATE TABLE options_eod_close__nopk (x INTEGER)")
+        con.execute("CREATE TABLE options_eod_close (option_symbol VARCHAR, date DATE, "
+                    "PRIMARY KEY (option_symbol, date))")
+        con.execute("INSERT INTO options_eod_close VALUES ('A', '2026-01-05')")
+        lib._init_duckdb_storage_tables(con)
+        assert not _tbl_has_pk(con, 'options_eod_close')
+        assert _count(con, 'options_eod_close') == 1
+        tables = [t[0] for t in con.execute('SHOW TABLES').fetchall()]
+        assert 'options_eod_close__nopk' not in tables
+
+
+def test_init_idempotent_second_run():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        df = _opts_df([('SPX_C5000_1', date(2026, 1, 5))])
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df)
+        lib._init_duckdb_storage_tables(con)
+        assert _count(con) == 1
+
+
+# ---------------------------------------------------------------- dedup
+
+def test_reinsert_same_batch_adds_zero():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        df = _opts_df([('SPX_C5000_1', date(2026, 1, 5)),
+                       ('SPX_C5000_2', date(2026, 1, 5)),
+                       ('SPX_C5000_1', date(2026, 1, 6))])
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df)
+        assert _count(con) == 3
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df)
+        assert _count(con) == 3
+
+
+def test_partial_overlap_inserts_only_new_keys():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        lib._duckdb_insert_dedup(con, 'options_eod_1545',
+                                 _opts_df([('A', date(2026, 1, 5))]))
+        df2 = _opts_df([('A', date(2026, 1, 5)),      # duplicate key
+                        ('B', date(2026, 1, 5)),      # new
+                        ('A', date(2026, 1, 6))])     # same symbol, new date
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df2)
+        assert _count(con) == 3
+        rows = con.execute("SELECT option_symbol, date FROM options_eod_1545 "
+                           "ORDER BY option_symbol, date").fetchall()
+        assert rows == [('A', date(2026, 1, 5)), ('A', date(2026, 1, 6)),
+                        ('B', date(2026, 1, 5))]
+
+
+def test_inbatch_duplicate_key_collapsed_to_one():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        df = _opts_df([('A', date(2026, 1, 5)), ('A', date(2026, 1, 5))])
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df)
+        assert _count(con) == 1
+
+
+def test_non_options_table_keeps_or_ignore_path():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        cols = [r[0] for r in con.execute('DESCRIBE stock_eod').fetchall()]
+        row = {c: None for c in cols}
+        row.update({'symbol': 'SPX', 'date': date(2026, 1, 5)})
+        df_stock = pd.DataFrame([row])
+        lib._duckdb_insert_dedup(con, 'stock_eod', df_stock)
+        lib._duckdb_insert_dedup(con, 'stock_eod', df_stock)
+        assert _count(con, 'stock_eod') == 1
+
+
+# ---------------------------------------------------------------- timeouts
+
+def test_futures_timeout_caught_by_except_tuple():
+    # every as_completed() in the lib has a global timeout caught as
+    # (TimeoutError, _FuturesTimeoutError) — verify the tuple catches the
+    # class as_completed actually raises on this interpreter
+    import time as _t
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    caught = False
+    with ThreadPoolExecutor(max_workers=1) as ex:
+        f = ex.submit(_t.sleep, 3)
+        try:
+            for _ in as_completed({f: 1}, timeout=0.3):
+                pass
+        except (TimeoutError, lib._FuturesTimeoutError):
+            caught = True
+        f.cancel()
+    assert caught
+
+
+# ---------------------------------------------------------------- runner
+
+def main():
+    tests = [(n, f) for n, f in sorted(globals().items())
+             if n.startswith('test_') and callable(f)]
+    passed = failed = 0
+    for name, fn in tests:
+        try:
+            fn()
+            print(f'  ✅ {name}')
+            passed += 1
+        except Exception:
+            print(f'  ❌ {name}')
+            traceback.print_exc()
+            failed += 1
+    print(f'\n{passed} passed, {failed} failed of {len(tests)}')
+    return 1 if failed else 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())