ivolatility-backtesting 2.139__tar.gz → 2.141__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/PKG-INFO +1 -1
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/ivolatility_backtesting/ivolatility_backtesting.py +225 -54
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/ivolatility_backtesting.egg-info/PKG-INFO +1 -1
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/pyproject.toml +1 -1
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/README.md +0 -0
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/ivolatility_backtesting/__init__.py +0 -0
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/ivolatility_backtesting.egg-info/SOURCES.txt +0 -0
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/ivolatility_backtesting.egg-info/dependency_links.txt +0 -0
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/ivolatility_backtesting.egg-info/requires.txt +0 -0
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/ivolatility_backtesting.egg-info/top_level.txt +0 -0
- {ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ivolatility_backtesting
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.141
|
|
4
4
|
Summary: A universal backtesting framework for financial strategies using the IVolatility API.
|
|
5
5
|
Author-email: IVolatility <support@ivolatility.com>
|
|
6
6
|
Project-URL: Homepage, https://ivolatility.com
|
|
@@ -56,6 +56,9 @@ def _ivb_user_agent():
|
|
|
56
56
|
|
|
57
57
|
|
|
58
58
|
_IVB_UA = _ivb_user_agent()
|
|
59
|
+
# ivollive/DEV: point every REST call at the internal API when IVOL_API_BASE_URL is
|
|
60
|
+
# set (workspace has an internal key, invalid for prod). Default stays external.
|
|
61
|
+
_API_BASE = __import__('os').getenv('IVOL_API_BASE_URL') or 'https://restapi.ivolatility.com'
|
|
59
62
|
try:
|
|
60
63
|
# _session lives in the ivolatility.ivolatility submodule; the top-level
|
|
61
64
|
# package does not re-export it (same lookup as _thread_safe_session)
|
|
@@ -8142,7 +8145,7 @@ def _api_call_internal(endpoint, cache_config, debug, debug_level, skip_parquet_
|
|
|
8142
8145
|
# TIMING: URL construction (for debug)
|
|
8143
8146
|
if debug_level >= 3 or (debug and APIManager._api_key):
|
|
8144
8147
|
t_url_start = time.perf_counter()
|
|
8145
|
-
base_url =
|
|
8148
|
+
base_url = _API_BASE
|
|
8146
8149
|
url_params = {}
|
|
8147
8150
|
for key, value in kwargs.items():
|
|
8148
8151
|
clean_key = key.rstrip('_') if key.endswith('_') else key
|
|
@@ -9313,7 +9316,7 @@ class StopLossManager:
|
|
|
9313
9316
|
# Log API call
|
|
9314
9317
|
api_key = APIManager._api_key or "API_KEY"
|
|
9315
9318
|
api_key_short = f"{api_key[:10]}...{api_key[-6:]}" if api_key and len(api_key) > 16 else api_key
|
|
9316
|
-
full_url = f"
|
|
9319
|
+
full_url = f"{_API_BASE}{endpoint}?apiKey={api_key_short}&symbol={symbol}&date={date_str}&expDate={exp_date_str}&strike={strike}&optType={api_opt_type}&minuteType={minute_interval}"
|
|
9317
9320
|
_bt_logger.info(f"[INTRADAY-OPTIONS] {full_url}")
|
|
9318
9321
|
|
|
9319
9322
|
result = method(
|
|
@@ -11495,7 +11498,7 @@ class StopLossManager:
|
|
|
11495
11498
|
|
|
11496
11499
|
# Log intraday API call with full URL
|
|
11497
11500
|
api_key = APIManager._api_key or "API_KEY"
|
|
11498
|
-
full_url = f"
|
|
11501
|
+
full_url = f"{_API_BASE}/equities/intraday/stock-prices?apiKey={api_key}&symbol={symbol}&date={date_str}&minuteType={minute_interval}"
|
|
11499
11502
|
if self.debuginfo >= 2:
|
|
11500
11503
|
print(f" 📡 Intraday API: {full_url}")
|
|
11501
11504
|
_bt_logger.info(f"[INTRADAY] {full_url}")
|
|
@@ -17962,6 +17965,7 @@ def _compute_options_chunk_days(symbol, delta_from, delta_to, dte_from, dte_to,
|
|
|
17962
17965
|
if os.path.exists(db_path) and _DUCKDB_AVAILABLE:
|
|
17963
17966
|
import duckdb as _ddb
|
|
17964
17967
|
con = _ddb.connect(db_path, read_only=True)
|
|
17968
|
+
_apply_duckdb_tuning(con)
|
|
17965
17969
|
try:
|
|
17966
17970
|
row = con.execute(f"""
|
|
17967
17971
|
SELECT MAX(cnt) as max_per_day
|
|
@@ -18041,6 +18045,7 @@ def _probe_options_density_if_needed(
|
|
|
18041
18045
|
if os.path.exists(db_path) and _DUCKDB_AVAILABLE:
|
|
18042
18046
|
import duckdb as _ddb
|
|
18043
18047
|
con = _ddb.connect(db_path, read_only=True)
|
|
18048
|
+
_apply_duckdb_tuning(con)
|
|
18044
18049
|
try:
|
|
18045
18050
|
row = con.execute(
|
|
18046
18051
|
f"SELECT COUNT(*) FROM {_opt_tbl} WHERE symbol = ? LIMIT 1",
|
|
@@ -18091,6 +18096,41 @@ def _probe_options_density_if_needed(
|
|
|
18091
18096
|
# ============================================================
|
|
18092
18097
|
# SMART CHUNKING: Fetch specific date/DTE range from API
|
|
18093
18098
|
# ============================================================
|
|
18099
|
+
def _cgroup_duckdb_tuning(divisor: int = 10, floor_mb: int = 192, max_threads: int = 4):
|
|
18100
|
+
"""MEMFIX: (memory_limit_mb, threads) от cgroup контейнера, не от RAM хоста.
|
|
18101
|
+
Без явного лимита DuckDB берёт 80% памяти ХОСТА — на ноде со 125GB это
|
|
18102
|
+
~100GB на буферный пул при поде в 3GB."""
|
|
18103
|
+
lim_mb, threads = 512, max_threads
|
|
18104
|
+
try:
|
|
18105
|
+
for _p in ('/sys/fs/cgroup/memory.max', '/sys/fs/cgroup/memory/memory.limit_in_bytes'):
|
|
18106
|
+
try:
|
|
18107
|
+
_raw = open(_p).read().strip()
|
|
18108
|
+
if _raw != 'max' and int(_raw) < (1 << 50):
|
|
18109
|
+
lim_mb = max(floor_mb, int(int(_raw) / divisor / 1024 / 1024))
|
|
18110
|
+
break
|
|
18111
|
+
except Exception:
|
|
18112
|
+
continue
|
|
18113
|
+
try:
|
|
18114
|
+
_q, _pd = open('/sys/fs/cgroup/cpu.max').read().split()[:2]
|
|
18115
|
+
if _q != 'max':
|
|
18116
|
+
threads = max(1, min(max_threads, int(int(_q) / int(_pd))))
|
|
18117
|
+
except Exception:
|
|
18118
|
+
pass
|
|
18119
|
+
except Exception:
|
|
18120
|
+
pass
|
|
18121
|
+
return lim_mb, threads
|
|
18122
|
+
|
|
18123
|
+
|
|
18124
|
+
def _apply_duckdb_tuning(conn):
|
|
18125
|
+
"""MEMFIX: применить cgroup-лимиты к соединению DuckDB (best-effort)."""
|
|
18126
|
+
try:
|
|
18127
|
+
_lim, _thr = _cgroup_duckdb_tuning()
|
|
18128
|
+
conn.execute(f"SET threads TO {_thr}")
|
|
18129
|
+
conn.execute(f"SET memory_limit = '{_lim}MB'")
|
|
18130
|
+
except Exception:
|
|
18131
|
+
pass
|
|
18132
|
+
|
|
18133
|
+
|
|
18094
18134
|
def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, dte_from, dte_to):
|
|
18095
18135
|
"""
|
|
18096
18136
|
Fetch data for specific date and DTE range from API.
|
|
@@ -18213,8 +18253,11 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
|
|
|
18213
18253
|
'end': params.get('to', params.get('endDate', '?'))
|
|
18214
18254
|
}
|
|
18215
18255
|
try:
|
|
18256
|
+
# MEMFIX(snapshot): в 1545-режиме качаем -1545 endpoint, иначе данные
|
|
18257
|
+
# close-снапшота легли бы в чужую таблицу и бэктест их не увидел
|
|
18258
|
+
_snap_ep_gap = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
|
|
18216
18259
|
df = _api_call_logged(
|
|
18217
|
-
|
|
18260
|
+
_snap_ep_gap,
|
|
18218
18261
|
cache_config,
|
|
18219
18262
|
skip_parquet_cache=True,
|
|
18220
18263
|
debuginfo=api_debuginfo,
|
|
@@ -18227,6 +18270,24 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
|
|
|
18227
18270
|
print(f" ❌ Error: {e}")
|
|
18228
18271
|
return None
|
|
18229
18272
|
|
|
18273
|
+
# MEMFIX(gap-path): раньше весь диапазон копился в all_data и склеивался одним
|
|
18274
|
+
# DataFrame — на 15-летнем gap-fill это весь датасет в RAM и OOM 3GB-пода
|
|
18275
|
+
# (сценарий Fprater39: кэш с 3-летних прогонов есть -> идём этой веткой).
|
|
18276
|
+
# Теперь каждый чанк пишется в DuckDB сразу и освобождается; возвращаем
|
|
18277
|
+
# число сохранённых строк (int), а не DataFrame — оба вызывающих обновлены.
|
|
18278
|
+
rows_streamed = 0
|
|
18279
|
+
_gap_save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
|
|
18280
|
+
|
|
18281
|
+
def _stream_one(df):
|
|
18282
|
+
nonlocal rows_streamed, total_rows
|
|
18283
|
+
if df is None or df.empty:
|
|
18284
|
+
return
|
|
18285
|
+
if 'symbol' not in df.columns or df['symbol'].isna().all():
|
|
18286
|
+
df['symbol'] = symbol
|
|
18287
|
+
rows_streamed += _save_to_duckdb_storage(
|
|
18288
|
+
df, _gap_save_ep, cache_config, debug=(debuginfo >= 2))
|
|
18289
|
+
total_rows += len(df)
|
|
18290
|
+
|
|
18230
18291
|
if use_parallel:
|
|
18231
18292
|
completed = 0
|
|
18232
18293
|
with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
@@ -18239,29 +18300,22 @@ def _fetch_date_range_data(config, cache_config, symbol, start_date, end_date, d
|
|
|
18239
18300
|
except Exception:
|
|
18240
18301
|
df = None
|
|
18241
18302
|
|
|
18242
|
-
|
|
18243
|
-
|
|
18244
|
-
|
|
18303
|
+
_stream_one(df)
|
|
18304
|
+
try:
|
|
18305
|
+
future._result = None
|
|
18306
|
+
except Exception:
|
|
18307
|
+
pass
|
|
18308
|
+
df = None
|
|
18245
18309
|
|
|
18246
18310
|
if debuginfo >= 1 and (completed % 10 == 0 or completed == total_requests):
|
|
18247
|
-
_safe_print(f" ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows")
|
|
18311
|
+
_safe_print(f" ⚡ Progress: {completed}/{total_requests}, {total_rows:,} rows streamed")
|
|
18248
18312
|
else:
|
|
18249
18313
|
for i, params in enumerate(all_requests):
|
|
18250
18314
|
df = fetch_chunk((params, i + 1))
|
|
18251
|
-
|
|
18252
|
-
|
|
18253
|
-
total_rows += len(df)
|
|
18254
|
-
|
|
18255
|
-
if not all_data:
|
|
18256
|
-
return None
|
|
18257
|
-
|
|
18258
|
-
combined_df = pd.concat(all_data, ignore_index=True)
|
|
18259
|
-
|
|
18260
|
-
# Ensure symbol column
|
|
18261
|
-
if 'symbol' not in combined_df.columns or combined_df['symbol'].isna().all():
|
|
18262
|
-
combined_df['symbol'] = symbol
|
|
18315
|
+
_stream_one(df)
|
|
18316
|
+
df = None
|
|
18263
18317
|
|
|
18264
|
-
return
|
|
18318
|
+
return rows_streamed
|
|
18265
18319
|
|
|
18266
18320
|
|
|
18267
18321
|
def _fetch_missing_dte_data(config, cache_config, symbol, start_date, end_date, dte_from, dte_to):
|
|
@@ -18333,7 +18387,7 @@ def _fetch_missing_dte_data(config, cache_config, symbol, start_date, end_date,
|
|
|
18333
18387
|
chunk_idx = 1 if cp == 'C' else 2
|
|
18334
18388
|
chunk_info = {'chunk': chunk_idx, 'total': 2, 'start': start_date, 'end': end_date}
|
|
18335
18389
|
df = _api_call_logged(
|
|
18336
|
-
'
|
|
18390
|
+
_get_options_endpoints(_get_options_snapshot_mode(config))['filtered'],
|
|
18337
18391
|
cache_config,
|
|
18338
18392
|
skip_parquet_cache=True,
|
|
18339
18393
|
debuginfo=debuginfo,
|
|
@@ -18359,7 +18413,7 @@ def _fetch_missing_dte_data(config, cache_config, symbol, start_date, end_date,
|
|
|
18359
18413
|
|
|
18360
18414
|
rows_saved = _save_to_duckdb_storage(
|
|
18361
18415
|
combined_df,
|
|
18362
|
-
'
|
|
18416
|
+
_get_options_endpoints(_get_options_snapshot_mode(config))['filtered'],
|
|
18363
18417
|
cache_config,
|
|
18364
18418
|
debug=False
|
|
18365
18419
|
)
|
|
@@ -18408,6 +18462,7 @@ def _get_duckdb_coverage(config, cache_config, symbol, start_date, end_date, dte
|
|
|
18408
18462
|
try:
|
|
18409
18463
|
_opt_tbl = _get_options_eod_table(config)
|
|
18410
18464
|
conn = duckdb.connect(db_path, read_only=True)
|
|
18465
|
+
_apply_duckdb_tuning(conn)
|
|
18411
18466
|
|
|
18412
18467
|
# Get options coverage
|
|
18413
18468
|
result = conn.execute(f"""
|
|
@@ -18909,22 +18964,15 @@ def _try_read_from_duckdb_storage(config, cache_config, symbol, extended_start,
|
|
|
18909
18964
|
total_rows_saved = 0
|
|
18910
18965
|
for i, (range_start, range_end) in enumerate(date_ranges, 1):
|
|
18911
18966
|
_rich_print(f" 🔄 [{i}/{len(date_ranges)}] {range_start} → {range_end}")
|
|
18912
|
-
|
|
18967
|
+
rows_saved = _fetch_date_range_data(
|
|
18913
18968
|
config, cache_config, symbol,
|
|
18914
18969
|
range_start, range_end,
|
|
18915
18970
|
0, required_max_dte
|
|
18916
|
-
)
|
|
18917
|
-
|
|
18918
|
-
|
|
18919
|
-
rows_saved = _save_to_duckdb_storage(
|
|
18920
|
-
chunk_df,
|
|
18921
|
-
'/equities/eod/stock-opts-by-param',
|
|
18922
|
-
cache_config,
|
|
18923
|
-
debug=(debuginfo >= 2)
|
|
18924
|
-
)
|
|
18971
|
+
) or 0
|
|
18972
|
+
# MEMFIX: сохранение теперь СТРИМИТСЯ внутри _fetch_date_range_data
|
|
18973
|
+
if rows_saved:
|
|
18925
18974
|
total_rows_saved += rows_saved
|
|
18926
|
-
_rich_print(f" ✅
|
|
18927
|
-
del chunk_df # Free memory immediately
|
|
18975
|
+
_rich_print(f" ✅ streamed {rows_saved:,} rows to DuckDB")
|
|
18928
18976
|
else:
|
|
18929
18977
|
_rich_print(f" ⚠️ No data")
|
|
18930
18978
|
_rich_print(f" 💾 Total saved: {total_rows_saved:,} rows (streamed)")
|
|
@@ -18950,21 +18998,14 @@ def _try_read_from_duckdb_storage(config, cache_config, symbol, extended_start,
|
|
|
18950
18998
|
total_rows_saved_fallback = 0
|
|
18951
18999
|
for name, d_start, d_end, dte_from, dte_to in chunks_to_fetch:
|
|
18952
19000
|
_rich_print(f" 🔄 Fetching: {name} ({d_start} → {d_end}, DTE {dte_from}-{dte_to})")
|
|
18953
|
-
|
|
19001
|
+
rows_saved = _fetch_date_range_data(
|
|
18954
19002
|
config, cache_config, symbol,
|
|
18955
19003
|
d_start, d_end, dte_from, dte_to
|
|
18956
|
-
)
|
|
18957
|
-
|
|
18958
|
-
|
|
18959
|
-
rows_saved = _save_to_duckdb_storage(
|
|
18960
|
-
chunk_df,
|
|
18961
|
-
'/equities/eod/stock-opts-by-param',
|
|
18962
|
-
cache_config,
|
|
18963
|
-
debug=(debuginfo >= 2)
|
|
18964
|
-
)
|
|
19004
|
+
) or 0
|
|
19005
|
+
# MEMFIX: сохранение теперь СТРИМИТСЯ внутри _fetch_date_range_data
|
|
19006
|
+
if rows_saved:
|
|
18965
19007
|
total_rows_saved_fallback += rows_saved
|
|
18966
|
-
_rich_print(f" ✅
|
|
18967
|
-
del chunk_df # Free memory immediately
|
|
19008
|
+
_rich_print(f" ✅ streamed {rows_saved:,} rows to DuckDB")
|
|
18968
19009
|
else:
|
|
18969
19010
|
_rich_print(f" ⚠️ No data")
|
|
18970
19011
|
if total_rows_saved_fallback > 0:
|
|
@@ -19825,6 +19866,14 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
|
|
|
19825
19866
|
# Resolve snapshot-aware endpoint ONCE for both fetch and save
|
|
19826
19867
|
_save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
|
|
19827
19868
|
|
|
19869
|
+
# MEMFIX(backpressure): воркеры качают быстрее, чем главный поток успевает
|
|
19870
|
+
# писать чанк в DuckDB (запись ~3с), поэтому готовые, но ещё не обработанные
|
|
19871
|
+
# DataFrame копились в памяти. Семафор держит очередь готовых результатов
|
|
19872
|
+
# ограниченной: воркер не отдаёт результат, пока главный поток не разгрёб
|
|
19873
|
+
# предыдущие. Это ограничивает пик числом воркеров, а не длиной периода.
|
|
19874
|
+
import threading as _mf_thr
|
|
19875
|
+
_mf_inflight = _mf_thr.Semaphore(max(2, max_workers))
|
|
19876
|
+
|
|
19828
19877
|
# Function to fetch single chunk
|
|
19829
19878
|
def fetch_chunk(args):
|
|
19830
19879
|
params, chunk_idx = args
|
|
@@ -19845,6 +19894,8 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
|
|
|
19845
19894
|
_chunk_info=chunk_info,
|
|
19846
19895
|
**params
|
|
19847
19896
|
)
|
|
19897
|
+
if df is not None and not df.empty:
|
|
19898
|
+
_mf_inflight.acquire() # ждём, пока главный поток освободит слот
|
|
19848
19899
|
return df
|
|
19849
19900
|
except Exception as e:
|
|
19850
19901
|
if debuginfo >= 1:
|
|
@@ -19942,6 +19993,59 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
|
|
|
19942
19993
|
if len(all_data) < 5:
|
|
19943
19994
|
all_data.append(df)
|
|
19944
19995
|
|
|
19996
|
+
if df is not None and not getattr(df, 'empty', True):
|
|
19997
|
+
try:
|
|
19998
|
+
_mf_inflight.release()
|
|
19999
|
+
except Exception:
|
|
20000
|
+
pass
|
|
20001
|
+
# MEMFIX: release the chunk once it is persisted — the Future keeps a
|
|
20002
|
+
# reference to its result until the executor block exits, so without this
|
|
20003
|
+
# the whole fetch accumulates in RAM regardless of DuckDB writes.
|
|
20004
|
+
try:
|
|
20005
|
+
future._result = None
|
|
20006
|
+
except Exception:
|
|
20007
|
+
pass
|
|
20008
|
+
df = None
|
|
20009
|
+
if completed % 25 == 0:
|
|
20010
|
+
# MEMFIX(page-cache): страницы записанного .duckdb оседают в file-кэше
|
|
20011
|
+
# cgroup и подтягивают memory.current к лимиту. Кэш вытесняемый, но
|
|
20012
|
+
# держит счётчик у потолка и сокращает буфер до OOM. Сбрасываем его:
|
|
20013
|
+
# сначала fdatasync (грязные страницы иначе не освободить), затем
|
|
20014
|
+
# POSIX_FADV_DONTNEED на файл БД и его WAL.
|
|
20015
|
+
try:
|
|
20016
|
+
import os as _os4
|
|
20017
|
+
_dbp = None
|
|
20018
|
+
try:
|
|
20019
|
+
_dbp = _get_duckdb_storage_path(cache_config)
|
|
20020
|
+
except Exception:
|
|
20021
|
+
_cd = (cache_config or {}).get('cache_dir')
|
|
20022
|
+
if _cd:
|
|
20023
|
+
_dbp = _os4.path.join(_os4.path.expanduser(_cd), 'market_data.duckdb')
|
|
20024
|
+
for _f in ([_dbp, _dbp + '.wal'] if _dbp else []):
|
|
20025
|
+
try:
|
|
20026
|
+
if not _os4.path.exists(_f):
|
|
20027
|
+
continue
|
|
20028
|
+
_fd = _os4.open(_f, _os4.O_RDONLY)
|
|
20029
|
+
try:
|
|
20030
|
+
try:
|
|
20031
|
+
_os4.fdatasync(_fd)
|
|
20032
|
+
except Exception:
|
|
20033
|
+
pass
|
|
20034
|
+
_os4.posix_fadvise(_fd, 0, 0, _os4.POSIX_FADV_DONTNEED)
|
|
20035
|
+
finally:
|
|
20036
|
+
_os4.close(_fd)
|
|
20037
|
+
except Exception:
|
|
20038
|
+
pass
|
|
20039
|
+
except Exception:
|
|
20040
|
+
pass
|
|
20041
|
+
import gc as _gc
|
|
20042
|
+
_gc.collect()
|
|
20043
|
+
try:
|
|
20044
|
+
import ctypes as _ct
|
|
20045
|
+
_ct.CDLL("libc.so.6").malloc_trim(0)
|
|
20046
|
+
except Exception:
|
|
20047
|
+
pass
|
|
20048
|
+
|
|
19945
20049
|
_watchdog_completed[0] = completed
|
|
19946
20050
|
if completed % 10 == 0 or completed == total_requests:
|
|
19947
20051
|
_wall = _time_mod.time() - _parallel_start
|
|
@@ -20932,7 +21036,7 @@ def _resolve_futures_underlying(symbol, region=None, mic=None,
|
|
|
20932
21036
|
for attempt in (1, 2):
|
|
20933
21037
|
try:
|
|
20934
21038
|
r = _requests.get(
|
|
20935
|
-
'
|
|
21039
|
+
f'{_API_BASE}/futures/eod/fut-underlying-info',
|
|
20936
21040
|
params=params, timeout=60, headers={'User-Agent': _IVB_UA})
|
|
20937
21041
|
if r.status_code != 200:
|
|
20938
21042
|
if debuginfo >= 1:
|
|
@@ -21150,7 +21254,7 @@ def _load_futures_for_hedge(config, preloaded):
|
|
|
21150
21254
|
chunk_end = min(current + _td(days=chunk_days), end_dt)
|
|
21151
21255
|
try:
|
|
21152
21256
|
r = _requests.get(
|
|
21153
|
-
'
|
|
21257
|
+
f'{_API_BASE}/futures/eod/prices',
|
|
21154
21258
|
params={'apiKey': _api_key, 'symbol': fut_root,
|
|
21155
21259
|
'from': current.strftime('%Y-%m-%d'),
|
|
21156
21260
|
'to': chunk_end.strftime('%Y-%m-%d')},
|
|
@@ -21718,7 +21822,7 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
|
|
|
21718
21822
|
# Through the SDK (api_call): session-level urllib3 Retry on 429/5xx
|
|
21719
21823
|
# plus async-CSV overflow handling come for free, same as equity.
|
|
21720
21824
|
try:
|
|
21721
|
-
|
|
21825
|
+
_df = _api_call_logged(
|
|
21722
21826
|
'/futures/eod/fut-opts-by-param',
|
|
21723
21827
|
cache_config,
|
|
21724
21828
|
skip_parquet_cache=True,
|
|
@@ -21728,6 +21832,9 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
|
|
|
21728
21832
|
'end': params.get('to', '?')},
|
|
21729
21833
|
**params,
|
|
21730
21834
|
)
|
|
21835
|
+
if _df is not None and not _df.empty:
|
|
21836
|
+
_mf_fut_sem.acquire() # MEMFIX: backpressure — ждём разгрузки очереди
|
|
21837
|
+
return _df
|
|
21731
21838
|
except Exception as e:
|
|
21732
21839
|
if debuginfo >= 1:
|
|
21733
21840
|
print(f" ❌ fut-opts-by-param "
|
|
@@ -21898,6 +22005,33 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
|
|
|
21898
22005
|
duck_conn.execute("CHECKPOINT")
|
|
21899
22006
|
except Exception:
|
|
21900
22007
|
pass # best-effort; e.g. if another tx is open
|
|
22008
|
+
# MEMFIX(page-cache): сбросить страницы .duckdb из file-кэша cgroup
|
|
22009
|
+
try:
|
|
22010
|
+
import os as _o5
|
|
22011
|
+
_dbp5 = _get_duckdb_storage_path(cache_config)
|
|
22012
|
+
for _f5 in (_dbp5, _dbp5 + '.wal'):
|
|
22013
|
+
try:
|
|
22014
|
+
if not _o5.path.exists(_f5):
|
|
22015
|
+
continue
|
|
22016
|
+
_fd5 = _o5.open(_f5, _o5.O_RDONLY)
|
|
22017
|
+
try:
|
|
22018
|
+
try:
|
|
22019
|
+
_o5.fdatasync(_fd5)
|
|
22020
|
+
except Exception:
|
|
22021
|
+
pass
|
|
22022
|
+
_o5.posix_fadvise(_fd5, 0, 0, _o5.POSIX_FADV_DONTNEED)
|
|
22023
|
+
finally:
|
|
22024
|
+
_o5.close(_fd5)
|
|
22025
|
+
except Exception:
|
|
22026
|
+
pass
|
|
22027
|
+
except Exception:
|
|
22028
|
+
pass
|
|
22029
|
+
|
|
22030
|
+
# MEMFIX: тот же класс утечки, что в equity-загрузчике — Future держит ссылку
|
|
22031
|
+
# на свой DataFrame до выхода из executor-блока, а воркеры качают быстрее,
|
|
22032
|
+
# чем главный поток пишет в DuckDB. Семафор + сброс f._result.
|
|
22033
|
+
import threading as _mf_thr2
|
|
22034
|
+
_mf_fut_sem = _mf_thr2.Semaphore(max(2, max_workers))
|
|
21901
22035
|
|
|
21902
22036
|
if use_parallel:
|
|
21903
22037
|
with _thread_safe_session(), ThreadPoolExecutor(max_workers=max_workers) as exe:
|
|
@@ -21905,7 +22039,18 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
|
|
|
21905
22039
|
for i, p in enumerate(all_requests)}
|
|
21906
22040
|
for f in as_completed(futs):
|
|
21907
22041
|
completed += 1
|
|
21908
|
-
|
|
22042
|
+
_fdf = f.result()
|
|
22043
|
+
_save_chunk(_fdf)
|
|
22044
|
+
if _fdf is not None and not getattr(_fdf, 'empty', True):
|
|
22045
|
+
try:
|
|
22046
|
+
_mf_fut_sem.release()
|
|
22047
|
+
except Exception:
|
|
22048
|
+
pass
|
|
22049
|
+
try:
|
|
22050
|
+
f._result = None
|
|
22051
|
+
except Exception:
|
|
22052
|
+
pass
|
|
22053
|
+
_fdf = None
|
|
21909
22054
|
_maybe_checkpoint()
|
|
21910
22055
|
if completed % 20 == 0 or completed == total:
|
|
21911
22056
|
print(f" ⚡ {completed}/{total} requests, "
|
|
@@ -27861,8 +28006,33 @@ def _get_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
|
|
|
27861
28006
|
print(f"[DUCKDB] 🔓 Opened: {_rel_to_project_root(db_path_abs)}", flush=True)
|
|
27862
28007
|
|
|
27863
28008
|
# Configure for performance
|
|
27864
|
-
|
|
27865
|
-
|
|
28009
|
+
# MEMFIX: size the buffer pool from the cgroup, not a fixed 2GB.
|
|
28010
|
+
# Same formula already used for the indicator connection (~1/6 of the
|
|
28011
|
+
# container limit, floor 256MB): a 3GB pod must not let DuckDB alone
|
|
28012
|
+
# claim 2GB. Threads follow cgroup CPU quota instead of a fixed 4.
|
|
28013
|
+
_st_lim_mb = 512
|
|
28014
|
+
_st_threads = 4
|
|
28015
|
+
try:
|
|
28016
|
+
for _p in ('/sys/fs/cgroup/memory.max',
|
|
28017
|
+
'/sys/fs/cgroup/memory/memory.limit_in_bytes'):
|
|
28018
|
+
try:
|
|
28019
|
+
_raw = open(_p).read().strip()
|
|
28020
|
+
if _raw != 'max' and int(_raw) < (1 << 50):
|
|
28021
|
+
_st_lim_mb = max(192, int(int(_raw) / 10 / 1024 / 1024))
|
|
28022
|
+
break
|
|
28023
|
+
except Exception:
|
|
28024
|
+
continue
|
|
28025
|
+
try:
|
|
28026
|
+
_q, _pd = open('/sys/fs/cgroup/cpu.max').read().split()[:2]
|
|
28027
|
+
if _q != 'max':
|
|
28028
|
+
_st_threads = max(1, min(4, int(int(_q) / int(_pd))))
|
|
28029
|
+
except Exception:
|
|
28030
|
+
pass
|
|
28031
|
+
except Exception:
|
|
28032
|
+
pass
|
|
28033
|
+
_DUCKDB_STORAGE_CONN.execute(f"SET threads TO {_st_threads}")
|
|
28034
|
+
_DUCKDB_STORAGE_CONN.execute(f"SET memory_limit = '{_st_lim_mb}MB'")
|
|
28035
|
+
print(f"[DUCKDB] 🧱 memory_limit={_st_lim_mb}MB threads={_st_threads} (cgroup-aware)", flush=True)
|
|
27866
28036
|
|
|
27867
28037
|
# Handle stale in-memory cache after force_clear
|
|
27868
28038
|
if just_cleared:
|
|
@@ -27971,7 +28141,7 @@ def _get_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
|
|
|
27971
28141
|
_DUCKDB_STORAGE_CONN = duckdb.connect(db_path, read_only=True)
|
|
27972
28142
|
_DUCKDB_READ_ONLY = True
|
|
27973
28143
|
print(f"[DUCKDB] 🔒 OPENED (read-only) | PID={os.getpid()} | {db_path}", flush=True)
|
|
27974
|
-
_DUCKDB_STORAGE_CONN
|
|
28144
|
+
_apply_duckdb_tuning(_DUCKDB_STORAGE_CONN)
|
|
27975
28145
|
except:
|
|
27976
28146
|
return None
|
|
27977
28147
|
else:
|
|
@@ -29473,6 +29643,7 @@ class OptionsChunkManager:
|
|
|
29473
29643
|
import duckdb
|
|
29474
29644
|
self._conn_is_shared = False
|
|
29475
29645
|
self.conn = duckdb.connect(self.db_path, read_only=False)
|
|
29646
|
+
_apply_duckdb_tuning(self.conn)
|
|
29476
29647
|
|
|
29477
29648
|
def plan_chunks(self, start_date, end_date, trading_days=None, balance_by_rows: bool = False):
|
|
29478
29649
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ivolatility_backtesting
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.141
|
|
4
4
|
Summary: A universal backtesting framework for financial strategies using the IVolatility API.
|
|
5
5
|
Author-email: IVolatility <support@ivolatility.com>
|
|
6
6
|
Project-URL: Homepage, https://ivolatility.com
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ivolatility_backtesting"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.141"
|
|
8
8
|
description = "A universal backtesting framework for financial strategies using the IVolatility API."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [
|
|
File without changes
|
{ivolatility_backtesting-2.139 → ivolatility_backtesting-2.141}/ivolatility_backtesting/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|