ivolatility-backtesting 2.139__tar.gz → 2.141__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ivolatility_backtesting
3
- Version: 2.139
3
+ Version: 2.141
4
4
  Summary: A universal backtesting framework for financial strategies using the IVolatility API.
5
5
  Author-email: IVolatility <support@ivolatility.com>
6
6
  Project-URL: Homepage, https://ivolatility.com
@@ -56,6 +56,9 @@ def _ivb_user_agent():
56
56
 
57
57
 
58
58
  _IVB_UA = _ivb_user_agent()
59
+ # ivollive/DEV: point every REST call at the internal API when IVOL_API_BASE_URL is
60
+ # set (workspace has an internal key, invalid for prod). Default stays external.
61
+ _API_BASE = __import__('os').getenv('IVOL_API_BASE_URL') or 'https://restapi.ivolatility.com'
59
62
  try:
60
63
  # _session lives in the ivolatility.ivolatility submodule; the top-level
61
64
  # package does not re-export it (same lookup as _thread_safe_session)
@@ -8142,7 +8145,7 @@ def _api_call_internal(endpoint, cache_config, debug, debug_level, skip_parquet_
8142
8145
  # TIMING: URL construction (for debug)
8143
8146
  if debug_level >= 3 or (debug and APIManager._api_key):
8144
8147
  t_url_start = time.perf_counter()
8145
- base_url = "https://restapi.ivolatility.com"
8148
+ base_url = _API_BASE
8146
8149
  url_params = {}
8147
8150
  for key, value in kwargs.items():
8148
8151
  clean_key = key.rstrip('_') if key.endswith('_') else key
@@ -9313,7 +9316,7 @@ class StopLossManager:
9313
9316
  # Log API call
9314
9317
  api_key = APIManager._api_key or "API_KEY"
9315
9318
  api_key_short = f"{api_key[:10]}...{api_key[-6:]}" if api_key and len(api_key) > 16 else api_key
9316
- full_url = f"https://restapi.ivolatility.com{endpoint}?apiKey={api_key_short}&symbol={symbol}&date={date_str}&expDate={exp_date_str}&strike={strike}&optType={api_opt_type}&minuteType={minute_interval}"
9319
+ full_url = f"{_API_BASE}{endpoint}?apiKey={api_key_short}&symbol={symbol}&date={date_str}&expDate={exp_date_str}&strike={strike}&optType={api_opt_type}&minuteType={minute_interval}"
9317
9320
  _bt_logger.info(f"[INTRADAY-OPTIONS] {full_url}")
9318
9321
 
9319
9322
  result = method(
@@ -11495,7 +11498,7 @@ class StopLossManager:
11495
11498
 
11496
11499
  # Log intraday API call with full URL
11497
11500
  api_key = APIManager._api_key or "API_KEY"
11498
- full_url = f"https://restapi.ivolatility.com/equities/intraday/stock-prices?apiKey={api_key}&symbol={symbol}&date={date_str}&minuteType={minute_interval}"
11501
+ full_url = f"{_API_BASE}/equities/intraday/stock-prices?apiKey={api_key}&symbol={symbol}&date={date_str}&minuteType={minute_interval}"
11499
11502
  if self.debuginfo >= 2:
11500
11503
  print(f" 📡 Intraday API: {full_url}")
11501
11504
  _bt_logger.info(f"[INTRADAY] {full_url}")
@@ -17962,6 +17965,7 @@ def _compute_options_chunk_days(symbol, delta_from, delta_to, dte_from, dte_to,
17962
17965
  if os.path.exists(db_path) and _DUCKDB_AVAILABLE:
17963
17966
  import duckdb as _ddb
17964
17967
  con = _ddb.connect(db_path, read_only=True)
17968
+ _apply_duckdb_tuning(con)
17965
17969
  try:
17966
17970
  row = con.execute(f"""
17967
17971
  SELECT MAX(cnt) as max_per_day
@@ -18041,6 +18045,7 @@ def _probe_options_density_if_needed(
18041
18045
  if os.path.exists(db_path) and _DUCKDB_AVAILABLE:
18042
18046
  import duckdb as _ddb
18043
18047
  con = _ddb.connect(db_path, read_only=True)
18048
+ _apply_duckdb_tuning(con)
18044
18049
  try:
18045
18050
  row = con.execute(
18046
18051
  f"SELECT COUNT(*) FROM {_opt_tbl} WHERE symbol = ? LIMIT 1",
@@ -18091,6 +18096,41 @@ def _probe_options_density_if_needed(
18091
18096
  # ============================================================
18092
18097
  # SMART CHUNKING: Fetch specific date/DTE range from API
18093
18098
  # ============================================================
18099
+ def _cgroup_duckdb_tuning(divisor: int = 10, floor_mb: int = 192, max_threads: int = 4):
18100
+ """MEMFIX: (memory_limit_mb, threads) от cgroup контейнера, не от RAM хоста.
18101
+ Без явного лимита DuckDB берёт 80% памяти ХОСТА — на ноде со 125GB это
18102
+ ~100GB на буферный пул при поде в 3GB."""
18103
+ lim_mb, threads = 512, max_threads
18104
+ try:
18105
+ for _p in ('/sys/fs/cgroup/memory.max', '/sys/fs/cgroup/memory/memory.limit_in_bytes'):
18106
+ try:
18107
+ _raw = open(_p).read().strip()
18108
+ if _raw != 'max' and int(_raw) < (1 << 50):
18109
+ lim_mb = max(floor_mb, int(int(_raw) / divisor / 1024 / 1024))
18110
+ break
18111
+ except Exception:
18112
+ continue
18113
+ try:
18114
+ _q, _pd = open('/sys/fs/cgroup/cpu.max').read().split()[:2]
18115
+ if _q != 'max':
18116
+ threads = max(1, min(max_threads, int(int(_q) / int(_pd))))
18117
+ except Exception:
18118
+ pass
18119
+ except Exception:
18120
+ pass
18121
+ return lim_mb, threads
18122
+
18123
+
18124
+ def _apply_duckdb_tuning(conn):
18125
+ """MEMFIX: применить cgroup-лимиты к соединению DuckDB (best-effort)."""
18126
+ try:
18127
+ _lim, _thr = _cgroup_duckdb_tuning()
18128
+ conn.execute(f"SET threads TO {_thr}")
18129
+ conn.execute(f"SET memory_limit = '{_lim}MB'")
18130
+ except Exception:
18131
+ pass
18132
+
18133
+
18094
18134
  def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, dte_from, dte_to):
18095
18135
  """
18096
18136
  Fetch data for specific date and DTE range from API.
@@ -18213,8 +18253,11 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
18213
18253
  'end': params.get('to', params.get('endDate', '?'))
18214
18254
  }
18215
18255
  try:
18256
+ # MEMFIX(snapshot): в 1545-режиме качаем -1545 endpoint, иначе данные
18257
+ # close-снапшота легли бы в чужую таблицу и бэктест их не увидел
18258
+ _snap_ep_gap = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
18216
18259
  df = _api_call_logged(
18217
- '/equities/eod/stock-opts-by-param',
18260
+ _snap_ep_gap,
18218
18261
  cache_config,
18219
18262
  skip_parquet_cache=True,
18220
18263
  debuginfo=api_debuginfo,
@@ -18227,6 +18270,24 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
18227
18270
  print(f" ❌ Error: {e}")
18228
18271
  return None
18229
18272
 
18273
+ # MEMFIX(gap-path): раньше весь диапазон копился в all_data и склеивался одним
18274
+ # DataFrame — на 15-летнем gap-fill это весь датасет в RAM и OOM 3GB-пода
18275
+ # (сценарий Fprater39: кэш с 3-летних прогонов есть -> идём этой веткой).
18276
+ # Теперь каждый чанк пишется в DuckDB сразу и освобождается; возвращаем
18277
+ # число сохранённых строк (int), а не DataFrame — оба вызывающих обновлены.
18278
+ rows_streamed = 0
18279
+ _gap_save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
18280
+
18281
+ def _stream_one(df):
18282
+ nonlocal rows_streamed, total_rows
18283
+ if df is None or df.empty:
18284
+ return
18285
+ if 'symbol' not in df.columns or df['symbol'].isna().all():
18286
+ df['symbol'] = symbol
18287
+ rows_streamed += _save_to_duckdb_storage(
18288
+ df, _gap_save_ep, cache_config, debug=(debuginfo >= 2))
18289
+ total_rows += len(df)
18290
+
18230
18291
  if use_parallel:
18231
18292
  completed = 0
18232
18293
  with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as executor:
@@ -18239,29 +18300,22 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
18239
18300
  except Exception:
18240
18301
  df = None
18241
18302
 
18242
- if df is not None and not df.empty:
18243
- all_data.append(df)
18244
- total_rows += len(df)
18303
+ _stream_one(df)
18304
+ try:
18305
+ future._result = None
18306
+ except Exception:
18307
+ pass
18308
+ df = None
18245
18309
 
18246
18310
  if debuginfo >= 1 and (completed % 10 == 0 or completed == total_requests):
18247
- _safe_print(f" ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows")
18311
+ _safe_print(f" ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows streamed")
18248
18312
  else:
18249
18313
  for i, params in enumerate(all_requests):
18250
18314
  df = fetch_chunk((params, i + 1))
18251
- if df is not None and not df.empty:
18252
- all_data.append(df)
18253
- total_rows += len(df)
18254
-
18255
- if not all_data:
18256
- return None
18257
-
18258
- combined_df = pd.concat(all_data, ignore_index=True)
18259
-
18260
- # Ensure symbol column
18261
- if 'symbol' not in combined_df.columns or combined_df['symbol'].isna().all():
18262
- combined_df['symbol'] = symbol
18315
+ _stream_one(df)
18316
+ df = None
18263
18317
 
18264
- return combined_df
18318
+ return rows_streamed
18265
18319
 
18266
18320
 
18267
18321
  def _fetch_missing_dte_data(config, cache_config, symbol, start_date, end_date, dte_from, dte_to):
@@ -18333,7 +18387,7 @@ def _fetch_missing_dte_data(config, cache_config, symbol, start_date, end_date,
18333
18387
  chunk_idx = 1 if cp == 'C' else 2
18334
18388
  chunk_info = {'chunk': chunk_idx, 'total': 2, 'start': start_date, 'end': end_date}
18335
18389
  df = _api_call_logged(
18336
- '/equities/eod/stock-opts-by-param',
18390
+ _get_options_endpoints(_get_options_snapshot_mode(config))['filtered'],
18337
18391
  cache_config,
18338
18392
  skip_parquet_cache=True,
18339
18393
  debuginfo=debuginfo,
@@ -18359,7 +18413,7 @@ def _fetch_missing_dte_data(config, cache_config, symbol, start_date, end_date,
18359
18413
 
18360
18414
  rows_saved = _save_to_duckdb_storage(
18361
18415
  combined_df,
18362
- '/equities/eod/stock-opts-by-param',
18416
+ _get_options_endpoints(_get_options_snapshot_mode(config))['filtered'],
18363
18417
  cache_config,
18364
18418
  debug=False
18365
18419
  )
@@ -18408,6 +18462,7 @@ def _get_duckdb_coverage(config, cache_config, symbol, start_date, end_date, dte
18408
18462
  try:
18409
18463
  _opt_tbl = _get_options_eod_table(config)
18410
18464
  conn = duckdb.connect(db_path, read_only=True)
18465
+ _apply_duckdb_tuning(conn)
18411
18466
 
18412
18467
  # Get options coverage
18413
18468
  result = conn.execute(f"""
@@ -18909,22 +18964,15 @@ def _try_read_from_duckdb_storage(config, cache_config, symbol, extended_start,
18909
18964
  total_rows_saved = 0
18910
18965
  for i, (range_start, range_end) in enumerate(date_ranges, 1):
18911
18966
  _rich_print(f" 🔄 [{i}/{len(date_ranges)}] {range_start} → {range_end}")
18912
- chunk_df = _fetch_date_range_data(
18967
+ rows_saved = _fetch_date_range_data(
18913
18968
  config, cache_config, symbol,
18914
18969
  range_start, range_end,
18915
18970
  0, required_max_dte
18916
- )
18917
- if chunk_df is not None and not chunk_df.empty:
18918
- # STREAMING: Save immediately, don't accumulate!
18919
- rows_saved = _save_to_duckdb_storage(
18920
- chunk_df,
18921
- '/equities/eod/stock-opts-by-param',
18922
- cache_config,
18923
- debug=(debuginfo >= 2)
18924
- )
18971
+ ) or 0
18972
+ # MEMFIX: сохранение теперь СТРИМИТСЯ внутри _fetch_date_range_data
18973
+ if rows_saved:
18925
18974
  total_rows_saved += rows_saved
18926
- _rich_print(f" ✅ Got {len(chunk_df):,} rows → saved {rows_saved:,}")
18927
- del chunk_df # Free memory immediately
18975
+ _rich_print(f" ✅ streamed {rows_saved:,} rows to DuckDB")
18928
18976
  else:
18929
18977
  _rich_print(f" ⚠️ No data")
18930
18978
  _rich_print(f" 💾 Total saved: {total_rows_saved:,} rows (streamed)")
@@ -18950,21 +18998,14 @@ def _try_read_from_duckdb_storage(config, cache_config, symbol, extended_start,
18950
18998
  total_rows_saved_fallback = 0
18951
18999
  for name, d_start, d_end, dte_from, dte_to in chunks_to_fetch:
18952
19000
  _rich_print(f" 🔄 Fetching: {name} ({d_start} → {d_end}, DTE {dte_from}-{dte_to})")
18953
- chunk_df = _fetch_date_range_data(
19001
+ rows_saved = _fetch_date_range_data(
18954
19002
  config, cache_config, symbol,
18955
19003
  d_start, d_end, dte_from, dte_to
18956
- )
18957
- if chunk_df is not None and not chunk_df.empty:
18958
- # STREAMING: Save immediately, don't accumulate!
18959
- rows_saved = _save_to_duckdb_storage(
18960
- chunk_df,
18961
- '/equities/eod/stock-opts-by-param',
18962
- cache_config,
18963
- debug=(debuginfo >= 2)
18964
- )
19004
+ ) or 0
19005
+ # MEMFIX: сохранение теперь СТРИМИТСЯ внутри _fetch_date_range_data
19006
+ if rows_saved:
18965
19007
  total_rows_saved_fallback += rows_saved
18966
- _rich_print(f" ✅ Got {len(chunk_df):,} rows → saved {rows_saved:,}")
18967
- del chunk_df # Free memory immediately
19008
+ _rich_print(f" ✅ streamed {rows_saved:,} rows to DuckDB")
18968
19009
  else:
18969
19010
  _rich_print(f" ⚠️ No data")
18970
19011
  if total_rows_saved_fallback > 0:
@@ -19825,6 +19866,14 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
19825
19866
  # Resolve snapshot-aware endpoint ONCE for both fetch and save
19826
19867
  _save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
19827
19868
 
19869
+ # MEMFIX(backpressure): воркеры качают быстрее, чем главный поток успевает
19870
+ # писать чанк в DuckDB (запись ~3с), поэтому готовые, но ещё не обработанные
19871
+ # DataFrame копились в памяти. Семафор держит очередь готовых результатов
19872
+ # ограниченной: воркер не отдаёт результат, пока главный поток не разгрёб
19873
+ # предыдущие. Это ограничивает пик числом воркеров, а не длиной периода.
19874
+ import threading as _mf_thr
19875
+ _mf_inflight = _mf_thr.Semaphore(max(2, max_workers))
19876
+
19828
19877
  # Function to fetch single chunk
19829
19878
  def fetch_chunk(args):
19830
19879
  params, chunk_idx = args
@@ -19845,6 +19894,8 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
19845
19894
  _chunk_info=chunk_info,
19846
19895
  **params
19847
19896
  )
19897
+ if df is not None and not df.empty:
19898
+ _mf_inflight.acquire() # ждём, пока главный поток освободит слот
19848
19899
  return df
19849
19900
  except Exception as e:
19850
19901
  if debuginfo >= 1:
@@ -19942,6 +19993,59 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
19942
19993
  if len(all_data) < 5:
19943
19994
  all_data.append(df)
19944
19995
 
19996
+ if df is not None and not getattr(df, 'empty', True):
19997
+ try:
19998
+ _mf_inflight.release()
19999
+ except Exception:
20000
+ pass
20001
+ # MEMFIX: release the chunk once it is persisted — the Future keeps a
20002
+ # reference to its result until the executor block exits, so without this
20003
+ # the whole fetch accumulates in RAM regardless of DuckDB writes.
20004
+ try:
20005
+ future._result = None
20006
+ except Exception:
20007
+ pass
20008
+ df = None
20009
+ if completed % 25 == 0:
20010
+ # MEMFIX(page-cache): страницы записанного .duckdb оседают в file-кэше
20011
+ # cgroup и подтягивают memory.current к лимиту. Кэш вытесняемый, но
20012
+ # держит счётчик у потолка и сокращает буфер до OOM. Сбрасываем его:
20013
+ # сначала fdatasync (грязные страницы иначе не освободить), затем
20014
+ # POSIX_FADV_DONTNEED на файл БД и его WAL.
20015
+ try:
20016
+ import os as _os4
20017
+ _dbp = None
20018
+ try:
20019
+ _dbp = _get_duckdb_storage_path(cache_config)
20020
+ except Exception:
20021
+ _cd = (cache_config or {}).get('cache_dir')
20022
+ if _cd:
20023
+ _dbp = _os4.path.join(_os4.path.expanduser(_cd), 'market_data.duckdb')
20024
+ for _f in ([_dbp, _dbp + '.wal'] if _dbp else []):
20025
+ try:
20026
+ if not _os4.path.exists(_f):
20027
+ continue
20028
+ _fd = _os4.open(_f, _os4.O_RDONLY)
20029
+ try:
20030
+ try:
20031
+ _os4.fdatasync(_fd)
20032
+ except Exception:
20033
+ pass
20034
+ _os4.posix_fadvise(_fd, 0, 0, _os4.POSIX_FADV_DONTNEED)
20035
+ finally:
20036
+ _os4.close(_fd)
20037
+ except Exception:
20038
+ pass
20039
+ except Exception:
20040
+ pass
20041
+ import gc as _gc
20042
+ _gc.collect()
20043
+ try:
20044
+ import ctypes as _ct
20045
+ _ct.CDLL("libc.so.6").malloc_trim(0)
20046
+ except Exception:
20047
+ pass
20048
+
19945
20049
  _watchdog_completed[0] = completed
19946
20050
  if completed % 10 == 0 or completed == total_requests:
19947
20051
  _wall = _time_mod.time() - _parallel_start
@@ -20932,7 +21036,7 @@ def _resolve_futures_underlying(symbol, region=None, mic=None,
20932
21036
  for attempt in (1, 2):
20933
21037
  try:
20934
21038
  r = _requests.get(
20935
- 'https://restapi.ivolatility.com/futures/eod/fut-underlying-info',
21039
+ f'{_API_BASE}/futures/eod/fut-underlying-info',
20936
21040
  params=params, timeout=60, headers={'User-Agent': _IVB_UA})
20937
21041
  if r.status_code != 200:
20938
21042
  if debuginfo >= 1:
@@ -21150,7 +21254,7 @@ def _load_futures_for_hedge(config, preloaded):
21150
21254
  chunk_end = min(current + _td(days=chunk_days), end_dt)
21151
21255
  try:
21152
21256
  r = _requests.get(
21153
- 'https://restapi.ivolatility.com/futures/eod/prices',
21257
+ f'{_API_BASE}/futures/eod/prices',
21154
21258
  params={'apiKey': _api_key, 'symbol': fut_root,
21155
21259
  'from': current.strftime('%Y-%m-%d'),
21156
21260
  'to': chunk_end.strftime('%Y-%m-%d')},
@@ -21718,7 +21822,7 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
21718
21822
  # Through the SDK (api_call): session-level urllib3 Retry on 429/5xx
21719
21823
  # plus async-CSV overflow handling come for free, same as equity.
21720
21824
  try:
21721
- return _api_call_logged(
21825
+ _df = _api_call_logged(
21722
21826
  '/futures/eod/fut-opts-by-param',
21723
21827
  cache_config,
21724
21828
  skip_parquet_cache=True,
@@ -21728,6 +21832,9 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
21728
21832
  'end': params.get('to', '?')},
21729
21833
  **params,
21730
21834
  )
21835
+ if _df is not None and not _df.empty:
21836
+ _mf_fut_sem.acquire() # MEMFIX: backpressure — ждём разгрузки очереди
21837
+ return _df
21731
21838
  except Exception as e:
21732
21839
  if debuginfo >= 1:
21733
21840
  print(f" ❌ fut-opts-by-param "
@@ -21898,6 +22005,33 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
21898
22005
  duck_conn.execute("CHECKPOINT")
21899
22006
  except Exception:
21900
22007
  pass # best-effort; e.g. if another tx is open
22008
+ # MEMFIX(page-cache): сбросить страницы .duckdb из file-кэша cgroup
22009
+ try:
22010
+ import os as _o5
22011
+ _dbp5 = _get_duckdb_storage_path(cache_config)
22012
+ for _f5 in (_dbp5, _dbp5 + '.wal'):
22013
+ try:
22014
+ if not _o5.path.exists(_f5):
22015
+ continue
22016
+ _fd5 = _o5.open(_f5, _o5.O_RDONLY)
22017
+ try:
22018
+ try:
22019
+ _o5.fdatasync(_fd5)
22020
+ except Exception:
22021
+ pass
22022
+ _o5.posix_fadvise(_fd5, 0, 0, _o5.POSIX_FADV_DONTNEED)
22023
+ finally:
22024
+ _o5.close(_fd5)
22025
+ except Exception:
22026
+ pass
22027
+ except Exception:
22028
+ pass
22029
+
22030
+ # MEMFIX: тот же класс утечки, что в equity-загрузчике — Future держит ссылку
22031
+ # на свой DataFrame до выхода из executor-блока, а воркеры качают быстрее,
22032
+ # чем главный поток пишет в DuckDB. Семафор + сброс f._result.
22033
+ import threading as _mf_thr2
22034
+ _mf_fut_sem = _mf_thr2.Semaphore(max(2, max_workers))
21901
22035
 
21902
22036
  if use_parallel:
21903
22037
  with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as exe:
@@ -21905,7 +22039,18 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
21905
22039
  for i, p in enumerate(all_requests)}
21906
22040
  for f in as_completed(futs):
21907
22041
  completed += 1
21908
- _save_chunk(f.result())
22042
+ _fdf = f.result()
22043
+ _save_chunk(_fdf)
22044
+ if _fdf is not None and not getattr(_fdf, 'empty', True):
22045
+ try:
22046
+ _mf_fut_sem.release()
22047
+ except Exception:
22048
+ pass
22049
+ try:
22050
+ f._result = None
22051
+ except Exception:
22052
+ pass
22053
+ _fdf = None
21909
22054
  _maybe_checkpoint()
21910
22055
  if completed % 20 == 0 or completed == total:
21911
22056
  print(f" ⚡ {completed}/{total} requests, "
@@ -27861,8 +28006,33 @@ def _get_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
27861
28006
  print(f"[DUCKDB] 🔓 Opened: {_rel_to_project_root(db_path_abs)}", flush=True)
27862
28007
 
27863
28008
  # Configure for performance
27864
- _DUCKDB_STORAGE_CONN.execute("SET threads TO 4")
27865
- _DUCKDB_STORAGE_CONN.execute("SET memory_limit = '2GB'")
28009
+ # MEMFIX: size the buffer pool from the cgroup, not a fixed 2GB.
28010
+ # Same formula already used for the indicator connection (~1/6 of the
28011
+ # container limit, floor 256MB): a 3GB pod must not let DuckDB alone
28012
+ # claim 2GB. Threads follow cgroup CPU quota instead of a fixed 4.
28013
+ _st_lim_mb = 512
28014
+ _st_threads = 4
28015
+ try:
28016
+ for _p in ('/sys/fs/cgroup/memory.max',
28017
+ '/sys/fs/cgroup/memory/memory.limit_in_bytes'):
28018
+ try:
28019
+ _raw = open(_p).read().strip()
28020
+ if _raw != 'max' and int(_raw) < (1 << 50):
28021
+ _st_lim_mb = max(192, int(int(_raw) / 10 / 1024 / 1024))
28022
+ break
28023
+ except Exception:
28024
+ continue
28025
+ try:
28026
+ _q, _pd = open('/sys/fs/cgroup/cpu.max').read().split()[:2]
28027
+ if _q != 'max':
28028
+ _st_threads = max(1, min(4, int(int(_q) / int(_pd))))
28029
+ except Exception:
28030
+ pass
28031
+ except Exception:
28032
+ pass
28033
+ _DUCKDB_STORAGE_CONN.execute(f"SET threads TO {_st_threads}")
28034
+ _DUCKDB_STORAGE_CONN.execute(f"SET memory_limit = '{_st_lim_mb}MB'")
28035
+ print(f"[DUCKDB] 🧱 memory_limit={_st_lim_mb}MB threads={_st_threads} (cgroup-aware)", flush=True)
27866
28036
 
27867
28037
  # Handle stale in-memory cache after force_clear
27868
28038
  if just_cleared:
@@ -27971,7 +28141,7 @@ def _get_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
27971
28141
  _DUCKDB_STORAGE_CONN = duckdb.connect(db_path, read_only=True)
27972
28142
  _DUCKDB_READ_ONLY = True
27973
28143
  print(f"[DUCKDB] 🔒 OPENED (read-only) | PID={os.getpid()} | {db_path}", flush=True)
27974
- _DUCKDB_STORAGE_CONN.execute("SET threads TO 4")
28144
+ _apply_duckdb_tuning(_DUCKDB_STORAGE_CONN)
27975
28145
  except:
27976
28146
  return None
27977
28147
  else:
@@ -29473,6 +29643,7 @@ class OptionsChunkManager:
29473
29643
  import duckdb
29474
29644
  self._conn_is_shared = False
29475
29645
  self.conn = duckdb.connect(self.db_path, read_only=False)
29646
+ _apply_duckdb_tuning(self.conn)
29476
29647
 
29477
29648
  def plan_chunks(self, start_date, end_date, trading_days=None, balance_by_rows: bool = False):
29478
29649
  """
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ivolatility_backtesting
3
- Version: 2.139
3
+ Version: 2.141
4
4
  Summary: A universal backtesting framework for financial strategies using the IVolatility API.
5
5
  Author-email: IVolatility <support@ivolatility.com>
6
6
  Project-URL: Homepage, https://ivolatility.com
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "ivolatility_backtesting"
7
- version = "2.139"
7
+ version = "2.141"
8
8
  description = "A universal backtesting framework for financial strategies using the IVolatility API."
9
9
  readme = "README.md"
10
10
  authors = [