ivolatility-backtesting 2.142__tar.gz → 2.144__tar.gz

{ivolatility_backtesting-2.142 → ivolatility_backtesting-2.144}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.142
+Version: 2.144
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com
@@ -15,8 +15,8 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: pandas>=1.5.0
-Requires-Dist: numpy>=1.21.0
+Requires-Dist: pandas<3.0,>=1.5.0
+Requires-Dist: numpy<3.0,>=1.21.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: seaborn>=0.11.0
 Requires-Dist: ivolatility>=1.8.2

{ivolatility_backtesting-2.142 → ivolatility_backtesting-2.144}/ivolatility_backtesting/ivolatility_backtesting.py

@@ -2019,6 +2019,8 @@ def _enable_api_logging(log_file='logs/ivolatility_api.log', level=2):
 # Uses ThreadPoolExecutor for parallel api_call (preserves caching!)
 # For I/O bound tasks (HTTP), threads work well despite GIL
 from concurrent.futures import ThreadPoolExecutor, as_completed
+# On py<3.11 futures raise their own TimeoutError, not the builtin
+from concurrent.futures import TimeoutError as _FuturesTimeoutError
 _PARALLEL_AVAILABLE = True
 
 # ============================================================
@@ -18471,6 +18473,7 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
     rows_streamed = 0
     failed_chunks = []  # 2.142: (start, end, cp, error) of every dropped chunk
     _gap_save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
+    _GAP_GLOBAL_TIMEOUT = 1800  # same safety net as the main parallel loader
 
     def _stream_one(df):
         nonlocal rows_streamed, total_rows
@@ -18486,23 +18489,37 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
         completed = 0
         with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as executor:
             futures = {executor.submit(fetch_chunk, (params, i+1)): i for i, params in enumerate(all_requests)}
-            
-            for future in as_completed(futures):
-                completed += 1
-                try:
-                    df = future.result(timeout=120)
-                except Exception:
+
+            try:
+                # Without the timeout a single hung HTTP call blocks as_completed
+                # forever — result(timeout) never gets a chance to fire.
+                for future in as_completed(futures, timeout=_GAP_GLOBAL_TIMEOUT):
+                    completed += 1
+                    try:
+                        df = future.result(timeout=120)
+                    except Exception:
+                        df = None
+
+                    _stream_one(df)
+                    try:
+                        future._result = None
+                    except Exception:
+                        pass
                     df = None
-                
-                _stream_one(df)
-                try:
-                    future._result = None
-                except Exception:
-                    pass
-                df = None
-                
-                if debuginfo >= 1 and (completed % 10 == 0 or completed == total_requests):
-                    _safe_print(f"              ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows streamed")
+
+                    if debuginfo >= 1 and (completed % 10 == 0 or completed == total_requests):
+                        _safe_print(f"              ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows streamed")
+            except (TimeoutError, _FuturesTimeoutError):
+                _pending = [i + 1 for f, i in futures.items() if not f.done()]
+                _safe_print(f"              ❌ [GAP TIMEOUT] as_completed timed out after "
+                            f"{_GAP_GLOBAL_TIMEOUT}s — dropping pending chunks {_pending}")
+                for f, i in futures.items():
+                    if not f.done():
+                        f.cancel()
+                        _p = all_requests[i]
+                        failed_chunks.append((_p.get('from_', _p.get('startDate', '?')),
+                                              _p.get('to', _p.get('endDate', '?')),
+                                              _p.get('cp', '?'), 'gap global timeout'))
     else:
         for i, params in enumerate(all_requests):
             df = fetch_chunk((params, i + 1))
@@ -19832,8 +19849,30 @@ def _preload_duckdb_SSD_storage(config, cache_config):
             
             # Get connection for ChunkManager
             conn = _get_duckdb_storage_conn(cache_config)
-            
+
             _opt_tbl = _get_options_eod_table(config)
+
+            # Cold-path completeness: every backtest trading day must exist in
+            # the options table; misses are loud + flag _failed_chunks
+            try:
+                _cc_df = conn.execute(
+                    f"SELECT DISTINCT date FROM {_opt_tbl} WHERE symbol = ?", [symbol]
+                ).fetchdf()
+                _cc_have = set(pd.to_datetime(_cc_df['date']).dt.strftime('%Y-%m-%d'))
+                _cc_need = {pd.Timestamp(d).strftime('%Y-%m-%d') for d in backtest_trading_days}
+                _cc_missing = sorted(_cc_need - _cc_have)
+            except Exception as _ce:
+                _cc_missing = None
+                print(f"           ⚠️ Completeness check could not run: {_ce}")
+            if _cc_missing:
+                print(f"           🕳️ COMPLETENESS: {len(_cc_missing)} trading day(s) missing "
+                      f"in options after initial load: {_cc_missing[:10]}"
+                      + (' …' if len(_cc_missing) > 10 else ''))
+                config.setdefault('_failed_chunks', []).extend(
+                    ('cold-completeness', d, d, '*') for d in _cc_missing)
+            elif _cc_missing is not None:
+                print(f"           ✅ COMPLETENESS: options cover all {len(_cc_need)} backtest trading days")
+
             chunk_mgr = OptionsChunkManager(
                 db_path=db_path,
                 symbol=symbol,
@@ -20219,7 +20258,7 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                     _bt_logger.info(f"[DIAG] MAIN LOOP future ready completed={completed}/{total_requests} chunk_id={_chunk_id}")
                     try:
                         df = future.result(timeout=120)
-                    except TimeoutError:
+                    except (TimeoutError, _FuturesTimeoutError):
                         _safe_print(f"           ⏱️ [TIMEOUT] future.result() timed out after 120s (chunk {completed}/{total_requests})")
                         _bt_logger.info(f"[DIAG] FUTURE TIMEOUT chunk={_chunk_id} completed={completed}/{total_requests}")
                         df = None
@@ -20310,7 +20349,7 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                     if completed % 10 == 0 or completed == total_requests:
                         _wall = _time_mod.time() - _parallel_start
                         _safe_print(f"           ⚡ Progress: {completed}/{total_requests} requests, {total_rows:,} rows, wall={_wall:.1f}s")
-            except TimeoutError:
+            except (TimeoutError, _FuturesTimeoutError):
                 _wall = _time_mod.time() - _parallel_start
                 _pending = [cid for f, cid in futures.items() if not f.done()]
                 _safe_print(f"           ❌ [GLOBAL TIMEOUT] as_completed timed out after {_PARALLEL_GLOBAL_TIMEOUT}s (wall={_wall:.1f}s)")
@@ -21788,10 +21827,18 @@ def _load_futures_prices_to_duckdb(config, cache_config, symbol,
         with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as exe:
             futures = {exe.submit(_fetch, cs, ce, i + 1): (cs, ce)
                        for i, (cs, ce) in enumerate(chunks)}
-            for f in as_completed(futures):
-                part = f.result()
-                if part is not None and not part.empty:
-                    parts.append(part)
+            try:
+                # a hung HTTP call blocks as_completed forever without this
+                for f in as_completed(futures, timeout=1800):
+                    part = f.result()
+                    if part is not None and not part.empty:
+                        parts.append(part)
+            except (TimeoutError, _FuturesTimeoutError):
+                _hung = [rng for f, rng in futures.items() if not f.done()]
+                print(f"           ❌ [TIMEOUT] futures_eod load: dropping hung chunks {_hung}")
+                for f in futures:
+                    if not f.done():
+                        f.cancel()
     else:
         for i, (cs, ce) in enumerate(chunks):
             part = _fetch(cs, ce, i + 1)
@@ -22320,24 +22367,34 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
         with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as exe:
             futs = {exe.submit(_fetch, p, i + 1): p
                     for i, p in enumerate(all_requests)}
-            for f in as_completed(futs):
-                completed += 1
-                _fdf = f.result()
-                _save_chunk(_fdf)
-                if _fdf is not None and not getattr(_fdf, 'empty', True):
+            try:
+                # a hung HTTP call blocks as_completed forever without this
+                for f in as_completed(futs, timeout=1800):
+                    completed += 1
+                    _fdf = f.result()
+                    _save_chunk(_fdf)
+                    if _fdf is not None and not getattr(_fdf, 'empty', True):
+                        try:
+                            _mf_fut_sem.release()
+                        except Exception:
+                            pass
                     try:
-                        _mf_fut_sem.release()
+                        f._result = None
                     except Exception:
                         pass
-                try:
-                    f._result = None
-                except Exception:
-                    pass
-                _fdf = None
-                _maybe_checkpoint()
-                if completed % 20 == 0 or completed == total:
-                    print(f"           ⚡ {completed}/{total} requests, "
-                          f"{rows_saved_total:,} rows so far")
+                    _fdf = None
+                    _maybe_checkpoint()
+                    if completed % 20 == 0 or completed == total:
+                        print(f"           ⚡ {completed}/{total} requests, "
+                              f"{rows_saved_total:,} rows so far")
+            except (TimeoutError, _FuturesTimeoutError):
+                print(f"           ❌ [TIMEOUT] futures-options load: as_completed "
+                      f"timed out after 1800s — dropping hung chunks")
+                for f, p in futs.items():
+                    if not f.done():
+                        f.cancel()
+                        failed_chunks.append((p.get('from_', '?'), p.get('to', '?'),
+                                              p.get('cp', '?'), 'global timeout'))
     else:
         for i, params in enumerate(all_requests, 1):
             completed = i
@@ -23736,23 +23793,34 @@ def preload_data_universal(config, data_requests=None, debug=False):
                 
                 with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as executor:
                     future_to_idx = {
-                        executor.submit(fetch_chunk, req_data): i 
+                        executor.submit(fetch_chunk, req_data): i
                         for i, req_data in enumerate(all_request_data)
                     }
-                    
-                    for future in as_completed(future_to_idx):
-                        idx = future_to_idx[future]
-                        completed += 1
-                        
-                        if completed % max(1, total_requests // 5) == 0 or completed == total_requests:
-                            _safe_print(f"           ⚡ Progress: {completed}/{total_requests}...")
-                        
-                        try:
-                            result = future.result(timeout=120)
-                            results.append((idx, result))
-                        except Exception as e:
-                            req_data = all_request_data[idx]
-                            results.append((idx, (None, req_data[1], req_data[2], req_data[3], e)))
+
+                    try:
+                        # a hung HTTP call blocks as_completed forever without this
+                        for future in as_completed(future_to_idx, timeout=1800):
+                            idx = future_to_idx[future]
+                            completed += 1
+
+                            if completed % max(1, total_requests // 5) == 0 or completed == total_requests:
+                                _safe_print(f"           ⚡ Progress: {completed}/{total_requests}...")
+
+                            try:
+                                result = future.result(timeout=120)
+                                results.append((idx, result))
+                            except Exception as e:
+                                req_data = all_request_data[idx]
+                                results.append((idx, (None, req_data[1], req_data[2], req_data[3], e)))
+                    except (TimeoutError, _FuturesTimeoutError):
+                        _safe_print(f"           ❌ [TIMEOUT] as_completed timed out after 1800s — "
+                                    f"dropping hung chunks")
+                        for future, idx in future_to_idx.items():
+                            if not future.done():
+                                future.cancel()
+                                req_data = all_request_data[idx]
+                                results.append((idx, (None, req_data[1], req_data[2], req_data[3],
+                                                      TimeoutError('global timeout'))))
                 
                 # Sort by original index to maintain order
                 results.sort(key=lambda x: x[0])
@@ -28592,6 +28660,8 @@ def _init_duckdb_storage_tables(conn):
         print(f"[DUCKDB] ⚠️ Legacy migration warning: {_mig_err}", flush=True)
     
     # OPTIONS_EOD_CLOSE - EOD options from stock-opts-by-param and options-rawiv (close prices)
+    # No PK: a PK's ART index is RAM-resident in DuckDB and OOMs small containers on
+    # multi-million-row tables; dedup lives in _duckdb_insert_dedup(). Do not re-add.
     _OPTIONS_EOD_SCHEMA = """(
             option_symbol VARCHAR,
             date DATE,
@@ -28614,26 +28684,32 @@ def _init_duckdb_storage_tables(conn):
             open_interest DOUBLE,
             source_endpoint VARCHAR,
             bid_eod DOUBLE,
-            ask_eod DOUBLE,
-            PRIMARY KEY (option_symbol, date)
+            ask_eod DOUBLE
         )"""
+    # Rebuild legacy caches that still carry the PK, preserving rows
+    for _opt_tbl_mig in ('options_eod_close', 'options_eod_1545'):
+        try:
+            _existing_tbls = [t[0] for t in conn.execute("SHOW TABLES").fetchall()]
+            if f'{_opt_tbl_mig}__nopk' in _existing_tbls:
+                conn.execute(f"DROP TABLE {_opt_tbl_mig}__nopk")
+            if _opt_tbl_mig not in _existing_tbls:
+                continue
+            _has_pk = conn.execute(
+                "SELECT 1 FROM information_schema.table_constraints "
+                "WHERE table_name = ? AND constraint_type = 'PRIMARY KEY'",
+                [_opt_tbl_mig]).fetchall()
+            if _has_pk:
+                _mig_rows = conn.execute(f"SELECT COUNT(*) FROM {_opt_tbl_mig}").fetchone()[0]
+                conn.execute(f"CREATE TABLE {_opt_tbl_mig}__nopk AS SELECT * FROM {_opt_tbl_mig}")
+                conn.execute(f"DROP TABLE {_opt_tbl_mig}")
+                conn.execute(f"ALTER TABLE {_opt_tbl_mig}__nopk RENAME TO {_opt_tbl_mig}")
+                print(f"[DUCKDB] 🔄 Rebuilt {_opt_tbl_mig} without PRIMARY KEY "
+                      f"({_mig_rows:,} rows preserved)", flush=True)
+        except Exception as _pk_err:
+            print(f"[DUCKDB] ⚠️ PK removal migration for {_opt_tbl_mig}: {_pk_err}", flush=True)
     conn.execute(f"CREATE TABLE IF NOT EXISTS options_eod_close {_OPTIONS_EOD_SCHEMA}")
-    
+
     # OPTIONS_EOD_1545 - 15:45 snapshot options (same schema, isolated table)
-    # Migration: recreate if table exists but lacks PRIMARY KEY (created by older code)
-    try:
-        _1545_exists = any(t[0] == 'options_eod_1545' for t in conn.execute("SHOW TABLES").fetchall())
-        if _1545_exists:
-            _1545_constraints = conn.execute(
-                "SELECT constraint_type FROM information_schema.table_constraints "
-                "WHERE table_name = 'options_eod_1545' AND constraint_type = 'PRIMARY KEY'"
-            ).fetchall()
-            if not _1545_constraints:
-                _1545_rows = conn.execute("SELECT COUNT(*) FROM options_eod_1545").fetchone()[0]
-                conn.execute("DROP TABLE options_eod_1545")
-                print(f"[DUCKDB] 🔄 Recreating options_eod_1545 with PRIMARY KEY (had {_1545_rows} rows, no PK)", flush=True)
-    except Exception as _pk_err:
-        print(f"[DUCKDB] ⚠️ PK migration check for options_eod_1545: {_pk_err}", flush=True)
     conn.execute(f"CREATE TABLE IF NOT EXISTS options_eod_1545 {_OPTIONS_EOD_SCHEMA}")
     
     # OPTIONS_INTRADAY - Minute-level options data
@@ -28798,6 +28874,37 @@ def _get_target_table(endpoint: str) -> str:
     return None
 
 
+_ANTIJOIN_DEDUP_TABLES = ('options_eod_close', 'options_eod_1545')
+
+
+def _duckdb_insert_dedup(conn, table_name, df_normalized):
+    """Insert df_normalized skipping existing keys.
+
+    options_eod_* have no PK (see _OPTIONS_EOD_SCHEMA): dedup = anti-join on
+    (option_symbol, date) scoped to the batch's date range; in-batch duplicates
+    collapse via QUALIFY. Other tables keep PK + INSERT OR IGNORE."""
+    if table_name in _ANTIJOIN_DEDUP_TABLES:
+        _dmin = df_normalized['date'].min()
+        _dmax = df_normalized['date'].max()
+        conn.execute(f"""
+            INSERT INTO {table_name}
+            SELECT * FROM (
+                SELECT * FROM df_normalized
+                QUALIFY ROW_NUMBER() OVER (PARTITION BY option_symbol, date) = 1
+            ) d
+            WHERE NOT EXISTS (
+                SELECT 1 FROM {table_name} t
+                WHERE t.date >= ? AND t.date <= ?
+                  AND t.option_symbol = d.option_symbol AND t.date = d.date
+            )
+        """, [_dmin, _dmax])
+    else:
+        conn.execute(f"""
+            INSERT OR IGNORE INTO {table_name}
+            SELECT * FROM df_normalized
+        """)
+
+
 def _save_to_duckdb_storage(df, endpoint: str, cache_config: Dict[str, Any], debug: bool = False):
     """
     Save DataFrame to DuckDB storage with deduplication.
@@ -28900,10 +29007,15 @@ def _save_to_duckdb_storage(df, endpoint: str, cache_config: Dict[str, Any], deb
             count_before = conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0]
             
             # Insert with conflict handling (skip duplicates)
-            conn.execute(f"""
-                INSERT OR IGNORE INTO {table_name} 
-                SELECT * FROM df_normalized
-            """)
+            try:
+                _duckdb_insert_dedup(conn, table_name, df_normalized)
+            except Exception:
+                # CHECKPOINT frees dirty buffer pages; retry once before failing loud
+                try:
+                    conn.execute("CHECKPOINT")
+                except Exception:
+                    pass
+                _duckdb_insert_dedup(conn, table_name, df_normalized)
             
             # Get count after insert
             count_after = conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0]
@@ -28962,10 +29074,14 @@ def _save_to_duckdb_storage_batched(df, endpoint: str, cache_config: Dict[str, A
                 batch_normalized = _normalize_for_duckdb_storage(batch, endpoint, table_name)
                 
                 # Insert batch
-                conn.execute(f"""
-                    INSERT OR IGNORE INTO {table_name} 
-                    SELECT * FROM batch_normalized
-                """)
+                try:
+                    _duckdb_insert_dedup(conn, table_name, batch_normalized)
+                except Exception:
+                    try:
+                        conn.execute("CHECKPOINT")
+                    except Exception:
+                        pass
+                    _duckdb_insert_dedup(conn, table_name, batch_normalized)
                 
                 # Progress every 5 batches
                 if debug and (i // batch_size) % 5 == 0:
@@ -31265,21 +31381,29 @@ class MarketDataManager:
             completed = 0
             with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as pool:
                 futures = {pool.submit(_fetch, req): req for req in requests_list}
-                for fut in as_completed(futures):
-                    completed += 1
-                    try:
-                        ep, df = fut.result(timeout=120)
-                    except Exception:
-                        ep = futures[fut][0]
-                        df = None
-                    if df is not None and not df.empty:
-                        if 'symbol' not in df.columns:
-                            df['symbol'] = symbol
-                        df = self._normalizer.normalize(df, ep)
-                        collected[ep].append(df)
-                        self._cache_write(ep, df, cache_key=None)
-                    if completed % max(1, total // 5) == 0 or completed == total:
-                        _safe_print(f"[MDM]   progress: {completed}/{total}")
+                try:
+                    # a hung HTTP call blocks as_completed forever without this
+                    for fut in as_completed(futures, timeout=1800):
+                        completed += 1
+                        try:
+                            ep, df = fut.result(timeout=120)
+                        except Exception:
+                            ep = futures[fut][0]
+                            df = None
+                        if df is not None and not df.empty:
+                            if 'symbol' not in df.columns:
+                                df['symbol'] = symbol
+                            df = self._normalizer.normalize(df, ep)
+                            collected[ep].append(df)
+                            self._cache_write(ep, df, cache_key=None)
+                        if completed % max(1, total // 5) == 0 or completed == total:
+                            _safe_print(f"[MDM]   progress: {completed}/{total}")
+                except (TimeoutError, _FuturesTimeoutError):
+                    _safe_print(f"[MDM] ❌ [TIMEOUT] as_completed timed out after 1800s — "
+                                f"dropping hung requests")
+                    for fut in futures:
+                        if not fut.done():
+                            fut.cancel()
         else:
             for req in requests_list:
                 ep, df = _fetch(req)

{ivolatility_backtesting-2.142 → ivolatility_backtesting-2.144}/ivolatility_backtesting.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.142
+Version: 2.144
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com
@@ -15,8 +15,8 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: pandas>=1.5.0
-Requires-Dist: numpy>=1.21.0
+Requires-Dist: pandas<3.0,>=1.5.0
+Requires-Dist: numpy<3.0,>=1.21.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: seaborn>=0.11.0
 Requires-Dist: ivolatility>=1.8.2

{ivolatility_backtesting-2.142 → ivolatility_backtesting-2.144}/ivolatility_backtesting.egg-info/SOURCES.txt

@@ -8,4 +8,5 @@ ivolatility_backtesting.egg-info/SOURCES.txt
 ivolatility_backtesting.egg-info/dependency_links.txt
 ivolatility_backtesting.egg-info/requires.txt
 ivolatility_backtesting.egg-info/top_level.txt
-tests/test_2142_fixes.py
+tests/test_2142_fixes.py
+tests/test_2144_duckdb_dedup.py

{ivolatility_backtesting-2.142 → ivolatility_backtesting-2.144}/ivolatility_backtesting.egg-info/requires.txt

@@ -1,5 +1,5 @@
-pandas>=1.5.0
-numpy>=1.21.0
+pandas<3.0,>=1.5.0
+numpy<3.0,>=1.21.0
 matplotlib>=3.5.0
 seaborn>=0.11.0
 ivolatility>=1.8.2

{ivolatility_backtesting-2.142 → ivolatility_backtesting-2.144}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ivolatility_backtesting"
-version = "2.142"
+version = "2.144"
 description = "A universal backtesting framework for financial strategies using the IVolatility API."
 readme = "README.md"
 authors = [
@@ -13,8 +13,8 @@ authors = [
 license = { file = "LICENSE" }
 requires-python = ">=3.8"
 dependencies = [
-    "pandas>=1.5.0",
-    "numpy>=1.21.0",
+    "pandas>=1.5.0,<3.0",
+    "numpy>=1.21.0,<3.0",
     "matplotlib>=3.5.0",
     "seaborn>=0.11.0",
     "ivolatility>=1.8.2",

ivolatility_backtesting-2.144/tests/test_2144_duckdb_dedup.py

@@ -0,0 +1,191 @@
+# Tests for 2.144: options_eod_* without PK — anti-join dedup in
+# _duckdb_insert_dedup() + PK-removal migration of legacy caches.
+#
+# Runs standalone (no pytest needed):  python3 tests/test_2144_duckdb_dedup.py
+# Also pytest-compatible for CI:       pytest tests/test_2144_duckdb_dedup.py
+import os
+import sys
+import tempfile
+import traceback
+from datetime import date
+
+import duckdb
+import matplotlib
+matplotlib.use('Agg')
+import pandas as pd
+
+_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(_REPO, 'ivolatility_backtesting'))
+import ivolatility_backtesting as lib  # noqa: E402
+
+
+def _tbl_has_pk(con, tbl):
+    return bool(con.execute(
+        "SELECT 1 FROM information_schema.table_constraints "
+        "WHERE table_name = ? AND constraint_type = 'PRIMARY KEY'", [tbl]).fetchall())
+
+
+def _opts_row(option_symbol, d):
+    return {
+        'option_symbol': option_symbol, 'date': d, 'symbol': 'SPX',
+        'expiration': date(2026, 3, 20), 'strike': 5000.0, 'type': 'C',
+        'bid': 1.0, 'ask': 2.0, 'price': 1.5, 'underlying_price': 5100.0,
+        'iv': 0.2, 'delta': 0.5, 'gamma': 0.01, 'theta': -0.5, 'vega': 3.0,
+        'rho': 0.1, 'dte': 74.0, 'volume': 10.0, 'open_interest': 100.0,
+        'source_endpoint': 'test', 'bid_eod': 1.0, 'ask_eod': 2.0,
+    }
+
+
+def _opts_df(rows):
+    return pd.DataFrame([_opts_row(sym, d) for sym, d in rows])
+
+
+def _count(con, tbl='options_eod_1545'):
+    return con.execute(f'SELECT COUNT(*) FROM {tbl}').fetchone()[0]
+
+
+# ---------------------------------------------------------------- schema
+
+def test_fresh_options_tables_have_no_pk_and_stock_keeps_pk():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        assert not _tbl_has_pk(con, 'options_eod_close')
+        assert not _tbl_has_pk(con, 'options_eod_1545')
+        assert _tbl_has_pk(con, 'stock_eod')
+
+
+def test_pk_migration_preserves_rows():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        con.execute("CREATE TABLE options_eod_1545 (option_symbol VARCHAR, date DATE, "
+                    "bid DOUBLE, PRIMARY KEY (option_symbol, date))")
+        con.execute("INSERT INTO options_eod_1545 VALUES ('A', '2026-01-05', 1.0), "
+                    "('B', '2026-01-06', 2.0)")
+        lib._init_duckdb_storage_tables(con)
+        assert not _tbl_has_pk(con, 'options_eod_1545')
+        assert _count(con) == 2
+        # columns of the migrated table survive as-is (additive drift handled elsewhere)
+        cols = [r[0] for r in con.execute("DESCRIBE options_eod_1545").fetchall()]
+        assert cols == ['option_symbol', 'date', 'bid']
+
+
+def test_pk_migration_drops_stale_nopk_leftover():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        con.execute("CREATE TABLE options_eod_close__nopk (x INTEGER)")
+        con.execute("CREATE TABLE options_eod_close (option_symbol VARCHAR, date DATE, "
+                    "PRIMARY KEY (option_symbol, date))")
+        con.execute("INSERT INTO options_eod_close VALUES ('A', '2026-01-05')")
+        lib._init_duckdb_storage_tables(con)
+        assert not _tbl_has_pk(con, 'options_eod_close')
+        assert _count(con, 'options_eod_close') == 1
+        tables = [t[0] for t in con.execute('SHOW TABLES').fetchall()]
+        assert 'options_eod_close__nopk' not in tables
+
+
+def test_init_idempotent_second_run():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        df = _opts_df([('SPX_C5000_1', date(2026, 1, 5))])
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df)
+        lib._init_duckdb_storage_tables(con)
+        assert _count(con) == 1
+
+
+# ---------------------------------------------------------------- dedup
+
+def test_reinsert_same_batch_adds_zero():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        df = _opts_df([('SPX_C5000_1', date(2026, 1, 5)),
+                       ('SPX_C5000_2', date(2026, 1, 5)),
+                       ('SPX_C5000_1', date(2026, 1, 6))])
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df)
+        assert _count(con) == 3
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df)
+        assert _count(con) == 3
+
+
+def test_partial_overlap_inserts_only_new_keys():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        lib._duckdb_insert_dedup(con, 'options_eod_1545',
+                                 _opts_df([('A', date(2026, 1, 5))]))
+        df2 = _opts_df([('A', date(2026, 1, 5)),      # duplicate key
+                        ('B', date(2026, 1, 5)),      # new
+                        ('A', date(2026, 1, 6))])     # same symbol, new date
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df2)
+        assert _count(con) == 3
+        rows = con.execute("SELECT option_symbol, date FROM options_eod_1545 "
+                           "ORDER BY option_symbol, date").fetchall()
+        assert rows == [('A', date(2026, 1, 5)), ('A', date(2026, 1, 6)),
+                        ('B', date(2026, 1, 5))]
+
+
+def test_inbatch_duplicate_key_collapsed_to_one():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        df = _opts_df([('A', date(2026, 1, 5)), ('A', date(2026, 1, 5))])
+        lib._duckdb_insert_dedup(con, 'options_eod_1545', df)
+        assert _count(con) == 1
+
+
+def test_non_options_table_keeps_or_ignore_path():
+    with tempfile.TemporaryDirectory() as td:
+        con = duckdb.connect(os.path.join(td, 't.duckdb'))
+        lib._init_duckdb_storage_tables(con)
+        cols = [r[0] for r in con.execute('DESCRIBE stock_eod').fetchall()]
+        row = {c: None for c in cols}
+        row.update({'symbol': 'SPX', 'date': date(2026, 1, 5)})
+        df_stock = pd.DataFrame([row])
+        lib._duckdb_insert_dedup(con, 'stock_eod', df_stock)
+        lib._duckdb_insert_dedup(con, 'stock_eod', df_stock)
+        assert _count(con, 'stock_eod') == 1
+
+
+# ---------------------------------------------------------------- timeouts
+
+def test_futures_timeout_caught_by_except_tuple():
+    # every as_completed() in the lib has a global timeout caught as
+    # (TimeoutError, _FuturesTimeoutError) — verify the tuple catches the
+    # class as_completed actually raises on this interpreter
+    import time as _t
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+    caught = False
+    with ThreadPoolExecutor(max_workers=1) as ex:
+        f = ex.submit(_t.sleep, 3)
+        try:
+            for _ in as_completed({f: 1}, timeout=0.3):
+                pass
+        except (TimeoutError, lib._FuturesTimeoutError):
+            caught = True
+        f.cancel()
+    assert caught
+
+
+# ---------------------------------------------------------------- runner
+
+def main():
+    tests = [(n, f) for n, f in sorted(globals().items())
+             if n.startswith('test_') and callable(f)]
+    passed = failed = 0
+    for name, fn in tests:
+        try:
+            fn()
+            print(f'  ✅ {name}')
+            passed += 1
+        except Exception:
+            print(f'  ❌ {name}')
+            traceback.print_exc()
+            failed += 1
+    print(f'\n{passed} passed, {failed} failed of {len(tests)}')
+    return 1 if failed else 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())