ivolatility-backtesting 2.140__tar.gz → 2.141__tar.gz

{ivolatility_backtesting-2.140 → ivolatility_backtesting-2.141}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.140
+Version: 2.141
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com

{ivolatility_backtesting-2.140 → ivolatility_backtesting-2.141}/ivolatility_backtesting/ivolatility_backtesting.py

@@ -17965,6 +17965,7 @@ def _compute_options_chunk_days(symbol, delta_from, delta_to, dte_from, dte_to,
         if os.path.exists(db_path) and _DUCKDB_AVAILABLE:
             import duckdb as _ddb
             con = _ddb.connect(db_path, read_only=True)
+            _apply_duckdb_tuning(con)
             try:
                 row = con.execute(f"""
                     SELECT MAX(cnt) as max_per_day
@@ -18044,6 +18045,7 @@ def _probe_options_density_if_needed(
         if os.path.exists(db_path) and _DUCKDB_AVAILABLE:
             import duckdb as _ddb
             con = _ddb.connect(db_path, read_only=True)
+            _apply_duckdb_tuning(con)
             try:
                 row = con.execute(
                     f"SELECT COUNT(*) FROM {_opt_tbl} WHERE symbol = ? LIMIT 1",
@@ -18094,6 +18096,41 @@ def _probe_options_density_if_needed(
 # ============================================================
 # SMART CHUNKING: Fetch specific date/DTE range from API
 # ============================================================
+def _cgroup_duckdb_tuning(divisor: int = 10, floor_mb: int = 192, max_threads: int = 4):
+    """MEMFIX: (memory_limit_mb, threads) от cgroup контейнера, не от RAM хоста.
+    Без явного лимита DuckDB берёт 80% памяти ХОСТА — на ноде со 125GB это
+    ~100GB на буферный пул при поде в 3GB."""
+    lim_mb, threads = 512, max_threads
+    try:
+        for _p in ('/sys/fs/cgroup/memory.max', '/sys/fs/cgroup/memory/memory.limit_in_bytes'):
+            try:
+                _raw = open(_p).read().strip()
+                if _raw != 'max' and int(_raw) < (1 << 50):
+                    lim_mb = max(floor_mb, int(int(_raw) / divisor / 1024 / 1024))
+                    break
+            except Exception:
+                continue
+        try:
+            _q, _pd = open('/sys/fs/cgroup/cpu.max').read().split()[:2]
+            if _q != 'max':
+                threads = max(1, min(max_threads, int(int(_q) / int(_pd))))
+        except Exception:
+            pass
+    except Exception:
+        pass
+    return lim_mb, threads
+
+
+def _apply_duckdb_tuning(conn):
+    """MEMFIX: применить cgroup-лимиты к соединению DuckDB (best-effort)."""
+    try:
+        _lim, _thr = _cgroup_duckdb_tuning()
+        conn.execute(f"SET threads TO {_thr}")
+        conn.execute(f"SET memory_limit = '{_lim}MB'")
+    except Exception:
+        pass
+
+
 def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, dte_from, dte_to):
     """
     Fetch data for specific date and DTE range from API.
@@ -18216,8 +18253,11 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
             'end': params.get('to', params.get('endDate', '?'))
         }
         try:
+            # MEMFIX(snapshot): в 1545-режиме качаем -1545 endpoint, иначе данные
+            # close-снапшота легли бы в чужую таблицу и бэктест их не увидел
+            _snap_ep_gap = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
             df = _api_call_logged(
-                '/equities/eod/stock-opts-by-param',
+                _snap_ep_gap,
                 cache_config,
                 skip_parquet_cache=True,
                 debuginfo=api_debuginfo,
@@ -18230,6 +18270,24 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
                 print(f"              ❌ Error: {e}")
             return None
     
+    # MEMFIX(gap-path): раньше весь диапазон копился в all_data и склеивался одним
+    # DataFrame — на 15-летнем gap-fill это весь датасет в RAM и OOM 3GB-пода
+    # (сценарий Fprater39: кэш с 3-летних прогонов есть -> идём этой веткой).
+    # Теперь каждый чанк пишется в DuckDB сразу и освобождается; возвращаем
+    # число сохранённых строк (int), а не DataFrame — оба вызывающих обновлены.
+    rows_streamed = 0
+    _gap_save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
+
+    def _stream_one(df):
+        nonlocal rows_streamed, total_rows
+        if df is None or df.empty:
+            return
+        if 'symbol' not in df.columns or df['symbol'].isna().all():
+            df['symbol'] = symbol
+        rows_streamed += _save_to_duckdb_storage(
+            df, _gap_save_ep, cache_config, debug=(debuginfo >= 2))
+        total_rows += len(df)
+
     if use_parallel:
         completed = 0
         with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -18242,29 +18300,22 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
                 except Exception:
                     df = None
                 
-                if df is not None and not df.empty:
-                    all_data.append(df)
-                    total_rows += len(df)
+                _stream_one(df)
+                try:
+                    future._result = None
+                except Exception:
+                    pass
+                df = None
                 
                 if debuginfo >= 1 and (completed % 10 == 0 or completed == total_requests):
-                    _safe_print(f"              ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows")
+                    _safe_print(f"              ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows streamed")
     else:
         for i, params in enumerate(all_requests):
             df = fetch_chunk((params, i + 1))
-            if df is not None and not df.empty:
-                all_data.append(df)
-                total_rows += len(df)
-    
-    if not all_data:
-        return None
-    
-    combined_df = pd.concat(all_data, ignore_index=True)
-    
-    # Ensure symbol column
-    if 'symbol' not in combined_df.columns or combined_df['symbol'].isna().all():
-        combined_df['symbol'] = symbol
+            _stream_one(df)
+            df = None
     
-    return combined_df
+    return rows_streamed
 
 
 def _fetch_missing_dte_data(config, cache_config, symbol, start_date, end_date, dte_from, dte_to):
@@ -18336,7 +18387,7 @@ def _fetch_missing_dte_data(config, cache_config, symbol, start_date, end_date,
             chunk_idx = 1 if cp == 'C' else 2
             chunk_info = {'chunk': chunk_idx, 'total': 2, 'start': start_date, 'end': end_date}
             df = _api_call_logged(
-                '/equities/eod/stock-opts-by-param', 
+                _get_options_endpoints(_get_options_snapshot_mode(config))['filtered'],
                 cache_config, 
                 skip_parquet_cache=True,
                 debuginfo=debuginfo,
@@ -18362,7 +18413,7 @@ def _fetch_missing_dte_data(config, cache_config, symbol, start_date, end_date,
             
             rows_saved = _save_to_duckdb_storage(
                 combined_df, 
-                '/equities/eod/stock-opts-by-param', 
+                _get_options_endpoints(_get_options_snapshot_mode(config))['filtered'], 
                 cache_config, 
                 debug=False
             )
@@ -18411,6 +18462,7 @@ def _get_duckdb_coverage(config, cache_config, symbol, start_date, end_date, dte
     try:
         _opt_tbl = _get_options_eod_table(config)
         conn = duckdb.connect(db_path, read_only=True)
+        _apply_duckdb_tuning(conn)
         
         # Get options coverage
         result = conn.execute(f"""
@@ -18912,22 +18964,15 @@ def _try_read_from_duckdb_storage(config, cache_config, symbol, extended_start,
                 total_rows_saved = 0
                 for i, (range_start, range_end) in enumerate(date_ranges, 1):
                     _rich_print(f"           🔄 [{i}/{len(date_ranges)}] {range_start} → {range_end}")
-                    chunk_df = _fetch_date_range_data(
+                    rows_saved = _fetch_date_range_data(
                         config, cache_config, symbol,
                         range_start, range_end,
                         0, required_max_dte
-                    )
-                    if chunk_df is not None and not chunk_df.empty:
-                        # STREAMING: Save immediately, don't accumulate!
-                        rows_saved = _save_to_duckdb_storage(
-                            chunk_df,
-                            '/equities/eod/stock-opts-by-param',
-                            cache_config,
-                            debug=(debuginfo >= 2)
-                        )
+                    ) or 0
+                    # MEMFIX: сохранение теперь СТРИМИТСЯ внутри _fetch_date_range_data
+                    if rows_saved:
                         total_rows_saved += rows_saved
-                        _rich_print(f"              ✅ Got {len(chunk_df):,} rows → saved {rows_saved:,}")
-                        del chunk_df  # Free memory immediately
+                        _rich_print(f"              ✅ streamed {rows_saved:,} rows to DuckDB")
                     else:
                         _rich_print(f"              ⚠️ No data")
                 _rich_print(f"           💾 Total saved: {total_rows_saved:,} rows (streamed)")
@@ -18953,21 +18998,14 @@ def _try_read_from_duckdb_storage(config, cache_config, symbol, extended_start,
                 total_rows_saved_fallback = 0
                 for name, d_start, d_end, dte_from, dte_to in chunks_to_fetch:
                     _rich_print(f"           🔄 Fetching: {name} ({d_start} → {d_end}, DTE {dte_from}-{dte_to})")
-                    chunk_df = _fetch_date_range_data(
+                    rows_saved = _fetch_date_range_data(
                         config, cache_config, symbol,
                         d_start, d_end, dte_from, dte_to
-                    )
-                    if chunk_df is not None and not chunk_df.empty:
-                        # STREAMING: Save immediately, don't accumulate!
-                        rows_saved = _save_to_duckdb_storage(
-                            chunk_df,
-                            '/equities/eod/stock-opts-by-param',
-                            cache_config,
-                            debug=(debuginfo >= 2)
-                        )
+                    ) or 0
+                    # MEMFIX: сохранение теперь СТРИМИТСЯ внутри _fetch_date_range_data
+                    if rows_saved:
                         total_rows_saved_fallback += rows_saved
-                        _rich_print(f"              ✅ Got {len(chunk_df):,} rows → saved {rows_saved:,}")
-                        del chunk_df  # Free memory immediately
+                        _rich_print(f"              ✅ streamed {rows_saved:,} rows to DuckDB")
                     else:
                         _rich_print(f"              ⚠️ No data")
                 if total_rows_saved_fallback > 0:
@@ -19828,6 +19866,14 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
     # Resolve snapshot-aware endpoint ONCE for both fetch and save
     _save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
     
+    # MEMFIX(backpressure): воркеры качают быстрее, чем главный поток успевает
+    # писать чанк в DuckDB (запись ~3с), поэтому готовые, но ещё не обработанные
+    # DataFrame копились в памяти. Семафор держит очередь готовых результатов
+    # ограниченной: воркер не отдаёт результат, пока главный поток не разгрёб
+    # предыдущие. Это ограничивает пик числом воркеров, а не длиной периода.
+    import threading as _mf_thr
+    _mf_inflight = _mf_thr.Semaphore(max(2, max_workers))
+
     # Function to fetch single chunk
     def fetch_chunk(args):
         params, chunk_idx = args
@@ -19848,6 +19894,8 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                 _chunk_info=chunk_info,
                 **params
             )
+            if df is not None and not df.empty:
+                _mf_inflight.acquire()      # ждём, пока главный поток освободит слот
             return df
         except Exception as e:
             if debuginfo >= 1:
@@ -19945,6 +19993,59 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                         if len(all_data) < 5:
                             all_data.append(df)
                     
+                    if df is not None and not getattr(df, 'empty', True):
+                        try:
+                            _mf_inflight.release()
+                        except Exception:
+                            pass
+                    # MEMFIX: release the chunk once it is persisted — the Future keeps a
+                    # reference to its result until the executor block exits, so without this
+                    # the whole fetch accumulates in RAM regardless of DuckDB writes.
+                    try:
+                        future._result = None
+                    except Exception:
+                        pass
+                    df = None
+                    if completed % 25 == 0:
+                        # MEMFIX(page-cache): страницы записанного .duckdb оседают в file-кэше
+                        # cgroup и подтягивают memory.current к лимиту. Кэш вытесняемый, но
+                        # держит счётчик у потолка и сокращает буфер до OOM. Сбрасываем его:
+                        # сначала fdatasync (грязные страницы иначе не освободить), затем
+                        # POSIX_FADV_DONTNEED на файл БД и его WAL.
+                        try:
+                            import os as _os4
+                            _dbp = None
+                            try:
+                                _dbp = _get_duckdb_storage_path(cache_config)
+                            except Exception:
+                                _cd = (cache_config or {}).get('cache_dir')
+                                if _cd:
+                                    _dbp = _os4.path.join(_os4.path.expanduser(_cd), 'market_data.duckdb')
+                            for _f in ([_dbp, _dbp + '.wal'] if _dbp else []):
+                                try:
+                                    if not _os4.path.exists(_f):
+                                        continue
+                                    _fd = _os4.open(_f, _os4.O_RDONLY)
+                                    try:
+                                        try:
+                                            _os4.fdatasync(_fd)
+                                        except Exception:
+                                            pass
+                                        _os4.posix_fadvise(_fd, 0, 0, _os4.POSIX_FADV_DONTNEED)
+                                    finally:
+                                        _os4.close(_fd)
+                                except Exception:
+                                    pass
+                        except Exception:
+                            pass
+                        import gc as _gc
+                        _gc.collect()
+                        try:
+                            import ctypes as _ct
+                            _ct.CDLL("libc.so.6").malloc_trim(0)
+                        except Exception:
+                            pass
+
                     _watchdog_completed[0] = completed
                     if completed % 10 == 0 or completed == total_requests:
                         _wall = _time_mod.time() - _parallel_start
@@ -21721,7 +21822,7 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
         # Through the SDK (api_call): session-level urllib3 Retry on 429/5xx
         # plus async-CSV overflow handling come for free, same as equity.
         try:
-            return _api_call_logged(
+            _df = _api_call_logged(
                 '/futures/eod/fut-opts-by-param',
                 cache_config,
                 skip_parquet_cache=True,
@@ -21731,6 +21832,9 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
                              'end': params.get('to', '?')},
                 **params,
             )
+            if _df is not None and not _df.empty:
+                _mf_fut_sem.acquire()      # MEMFIX: backpressure — ждём разгрузки очереди
+            return _df
         except Exception as e:
             if debuginfo >= 1:
                 print(f"           ❌ fut-opts-by-param "
@@ -21901,6 +22005,33 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
                     duck_conn.execute("CHECKPOINT")
             except Exception:
                 pass  # best-effort; e.g. if another tx is open
+            # MEMFIX(page-cache): сбросить страницы .duckdb из file-кэша cgroup
+            try:
+                import os as _o5
+                _dbp5 = _get_duckdb_storage_path(cache_config)
+                for _f5 in (_dbp5, _dbp5 + '.wal'):
+                    try:
+                        if not _o5.path.exists(_f5):
+                            continue
+                        _fd5 = _o5.open(_f5, _o5.O_RDONLY)
+                        try:
+                            try:
+                                _o5.fdatasync(_fd5)
+                            except Exception:
+                                pass
+                            _o5.posix_fadvise(_fd5, 0, 0, _o5.POSIX_FADV_DONTNEED)
+                        finally:
+                            _o5.close(_fd5)
+                    except Exception:
+                        pass
+            except Exception:
+                pass
+
+    # MEMFIX: тот же класс утечки, что в equity-загрузчике — Future держит ссылку
+    # на свой DataFrame до выхода из executor-блока, а воркеры качают быстрее,
+    # чем главный поток пишет в DuckDB. Семафор + сброс f._result.
+    import threading as _mf_thr2
+    _mf_fut_sem = _mf_thr2.Semaphore(max(2, max_workers))
 
     if use_parallel:
         with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as exe:
@@ -21908,7 +22039,18 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
                     for i, p in enumerate(all_requests)}
             for f in as_completed(futs):
                 completed += 1
-                _save_chunk(f.result())
+                _fdf = f.result()
+                _save_chunk(_fdf)
+                if _fdf is not None and not getattr(_fdf, 'empty', True):
+                    try:
+                        _mf_fut_sem.release()
+                    except Exception:
+                        pass
+                try:
+                    f._result = None
+                except Exception:
+                    pass
+                _fdf = None
                 _maybe_checkpoint()
                 if completed % 20 == 0 or completed == total:
                     print(f"           ⚡ {completed}/{total} requests, "
@@ -27864,8 +28006,33 @@ def _get_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
                 print(f"[DUCKDB] 🔓 Opened: {_rel_to_project_root(db_path_abs)}", flush=True)
                 
                 # Configure for performance
-                _DUCKDB_STORAGE_CONN.execute("SET threads TO 4")
-                _DUCKDB_STORAGE_CONN.execute("SET memory_limit = '2GB'")
+                # MEMFIX: size the buffer pool from the cgroup, not a fixed 2GB.
+                # Same formula already used for the indicator connection (~1/6 of the
+                # container limit, floor 256MB): a 3GB pod must not let DuckDB alone
+                # claim 2GB. Threads follow cgroup CPU quota instead of a fixed 4.
+                _st_lim_mb = 512
+                _st_threads = 4
+                try:
+                    for _p in ('/sys/fs/cgroup/memory.max',
+                               '/sys/fs/cgroup/memory/memory.limit_in_bytes'):
+                        try:
+                            _raw = open(_p).read().strip()
+                            if _raw != 'max' and int(_raw) < (1 << 50):
+                                _st_lim_mb = max(192, int(int(_raw) / 10 / 1024 / 1024))
+                                break
+                        except Exception:
+                            continue
+                    try:
+                        _q, _pd = open('/sys/fs/cgroup/cpu.max').read().split()[:2]
+                        if _q != 'max':
+                            _st_threads = max(1, min(4, int(int(_q) / int(_pd))))
+                    except Exception:
+                        pass
+                except Exception:
+                    pass
+                _DUCKDB_STORAGE_CONN.execute(f"SET threads TO {_st_threads}")
+                _DUCKDB_STORAGE_CONN.execute(f"SET memory_limit = '{_st_lim_mb}MB'")
+                print(f"[DUCKDB] 🧱 memory_limit={_st_lim_mb}MB threads={_st_threads} (cgroup-aware)", flush=True)
                 
                 # Handle stale in-memory cache after force_clear
                 if just_cleared:
@@ -27974,7 +28141,7 @@ def _get_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
                 _DUCKDB_STORAGE_CONN = duckdb.connect(db_path, read_only=True)
                 _DUCKDB_READ_ONLY = True
                 print(f"[DUCKDB] 🔒 OPENED (read-only) | PID={os.getpid()} | {db_path}", flush=True)
-                _DUCKDB_STORAGE_CONN.execute("SET threads TO 4")
+                _apply_duckdb_tuning(_DUCKDB_STORAGE_CONN)
             except:
                 return None
     else:
@@ -29476,6 +29643,7 @@ class OptionsChunkManager:
             import duckdb
             self._conn_is_shared = False
             self.conn = duckdb.connect(self.db_path, read_only=False)
+            _apply_duckdb_tuning(self.conn)
     
     def plan_chunks(self, start_date, end_date, trading_days=None, balance_by_rows: bool = False):
         """

{ivolatility_backtesting-2.140 → ivolatility_backtesting-2.141}/ivolatility_backtesting.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.140
+Version: 2.141
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com

{ivolatility_backtesting-2.140 → ivolatility_backtesting-2.141}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ivolatility_backtesting"
-version = "2.140"
+version = "2.141"
 description = "A universal backtesting framework for financial strategies using the IVolatility API."
 readme = "README.md"
 authors = [