ivolatility-backtesting 2.152__tar.gz → 2.153__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/PKG-INFO +1 -1
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting/__init__.py +2 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting/ivolatility_backtesting.py +123 -36
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/PKG-INFO +1 -1
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/SOURCES.txt +2 -1
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/pyproject.toml +1 -1
- ivolatility_backtesting-2.153/tests/test_2153_cache_lock_kept.py +301 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/README.md +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/dependency_links.txt +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/requires.txt +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/top_level.txt +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/setup.cfg +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2142_fixes.py +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2144_duckdb_dedup.py +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2146_vix_vro.py +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2147_cents_multiplier.py +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2148_loader_memory.py +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2149_client_errors.py +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2150_prompt_patterns.py +0 -0
- {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2152_cache_lock_and_rate_limit.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ivolatility_backtesting
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.153
|
|
4
4
|
Summary: A universal backtesting framework for financial strategies using the IVolatility API.
|
|
5
5
|
Author-email: IVolatility <support@ivolatility.com>
|
|
6
6
|
Project-URL: Homepage, https://ivolatility.com
|
{ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting/__init__.py
RENAMED
|
@@ -29,6 +29,7 @@ from .ivolatility_backtesting import (
|
|
|
29
29
|
set_project_root, get_project_root,
|
|
30
30
|
get_position_multiplier, get_hedge_multiplier,
|
|
31
31
|
parse_earnings_calendar,
|
|
32
|
+
SharedCacheTakenError,
|
|
32
33
|
)
|
|
33
34
|
|
|
34
35
|
__all__ = [
|
|
@@ -62,4 +63,5 @@ __all__ = [
|
|
|
62
63
|
'set_project_root', 'get_project_root',
|
|
63
64
|
'get_position_multiplier', 'get_hedge_multiplier',
|
|
64
65
|
'parse_earnings_calendar',
|
|
66
|
+
'SharedCacheTakenError',
|
|
65
67
|
]
|
|
@@ -153,6 +153,13 @@ def _resolve_path(path: str, base_dir: str = None) -> str:
|
|
|
153
153
|
return os.path.abspath(os.path.join(base_dir, p))
|
|
154
154
|
|
|
155
155
|
_DUCKDB_PRIVATE_CACHE = {} # abs shared cache_dir -> this process's private cache dir
|
|
156
|
+
_SHARED_CACHE_USED = set() # abs shared cache_dirs this process has opened; never cleared
|
|
157
|
+
_SHARED_CACHE_TAKEN = set() # used shared dirs found held after the full wait; cleared at run start, so only opens within one run skip the wait
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class SharedCacheTakenError(BaseException):
|
|
161
|
+
"""BaseException: the library's `except Exception` handlers must not turn it into a silent retry."""
|
|
162
|
+
|
|
156
163
|
|
|
157
164
|
def _effective_cache_dir(cache_config) -> str:
|
|
158
165
|
"""cache_config['cache_dir'] resolved; redirected to this process's private cache
|
|
@@ -561,6 +568,73 @@ def _release_freed_memory():
|
|
|
561
568
|
|
|
562
569
|
import threading
|
|
563
570
|
|
|
571
|
+
_PAGE_DROP_FDS = globals().get('_PAGE_DROP_FDS') or {} # path -> read-only fd, survives module reload
|
|
572
|
+
_PAGE_DROP_LOCK = globals().get('_PAGE_DROP_LOCK') or threading.Lock()
|
|
573
|
+
|
|
574
|
+
|
|
575
|
+
def _stat_or_none(path: str):
|
|
576
|
+
try:
|
|
577
|
+
return os.stat(path)
|
|
578
|
+
except OSError:
|
|
579
|
+
return None
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _live_page_drop_fd(path: str, st) -> Optional[int]:
|
|
583
|
+
"""Caller holds _PAGE_DROP_LOCK. Returns the kept fd if it still points at the file at <path>;
|
|
584
|
+
otherwise (path gone, file deleted or replaced) closes it: it no longer shares an inode with that file."""
|
|
585
|
+
fd = _PAGE_DROP_FDS.get(path)
|
|
586
|
+
if fd is None:
|
|
587
|
+
return None
|
|
588
|
+
try:
|
|
589
|
+
fst = os.fstat(fd)
|
|
590
|
+
live = st is not None and fst.st_nlink > 0 and (fst.st_ino, fst.st_dev) == (st.st_ino, st.st_dev)
|
|
591
|
+
except OSError:
|
|
592
|
+
live = False
|
|
593
|
+
if live:
|
|
594
|
+
return fd
|
|
595
|
+
del _PAGE_DROP_FDS[path]
|
|
596
|
+
try:
|
|
597
|
+
os.close(fd)
|
|
598
|
+
except OSError:
|
|
599
|
+
pass
|
|
600
|
+
return None
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def _close_stale_page_drop_fds() -> None:
|
|
604
|
+
"""Release kept fds of deleted / replaced cache files, so their disk space is freed."""
|
|
605
|
+
with _PAGE_DROP_LOCK:
|
|
606
|
+
for path in list(_PAGE_DROP_FDS):
|
|
607
|
+
_live_page_drop_fd(path, _stat_or_none(path))
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def _drop_file_pages(path: str) -> None:
|
|
611
|
+
"""fdatasync + POSIX_FADV_DONTNEED on a cache file whose DuckDB lock this process may hold.
|
|
612
|
+
The fd stays open while it points at the live file: closing any fd of it drops our fcntl locks.
|
|
613
|
+
Only the .duckdb is passed: the .wal carries the same lock and is deleted at every checkpoint
|
|
614
|
+
and recreated under a new inode, so it gets no open/close calls either."""
|
|
615
|
+
if not hasattr(os, 'posix_fadvise'):
|
|
616
|
+
return
|
|
617
|
+
with _PAGE_DROP_LOCK:
|
|
618
|
+
st = _stat_or_none(path)
|
|
619
|
+
fd = _live_page_drop_fd(path, st)
|
|
620
|
+
if st is None:
|
|
621
|
+
return
|
|
622
|
+
if fd is None:
|
|
623
|
+
try:
|
|
624
|
+
fd = os.open(path, os.O_RDONLY | getattr(os, 'O_CLOEXEC', 0))
|
|
625
|
+
except OSError:
|
|
626
|
+
return
|
|
627
|
+
_PAGE_DROP_FDS[path] = fd
|
|
628
|
+
try:
|
|
629
|
+
getattr(os, 'fdatasync', os.fsync)(fd)
|
|
630
|
+
except OSError:
|
|
631
|
+
pass
|
|
632
|
+
try:
|
|
633
|
+
os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
|
|
634
|
+
except OSError:
|
|
635
|
+
pass
|
|
636
|
+
|
|
637
|
+
|
|
564
638
|
# Admission control by cgroup free memory; the first download always passes so a load never stalls.
|
|
565
639
|
def _env_int(name, default):
|
|
566
640
|
try:
|
|
@@ -4626,10 +4700,12 @@ class DuckDBIndicatorManager:
|
|
|
4626
4700
|
# Only open new connection if none was passed to __init__
|
|
4627
4701
|
if not os.path.exists(_effective_duckdb_path(self.db_path)):
|
|
4628
4702
|
raise FileNotFoundError(f"DuckDB file not found: {self.db_path}")
|
|
4629
|
-
self.conn = _duckdb_open(self.db_path)
|
|
4703
|
+
self.conn = _duckdb_open(self.db_path, wait_sec=_shared_cache_open_wait(self.db_path))
|
|
4630
4704
|
if self.conn is None:
|
|
4705
|
+
if _shared_cache_used(self.db_path):
|
|
4706
|
+
_raise_shared_cache_taken(self.db_path)
|
|
4631
4707
|
raise RuntimeError(f"DuckDB cache is in use by another run: {self.db_path}")
|
|
4632
|
-
|
|
4708
|
+
|
|
4633
4709
|
def _create_all_indicator_tables(self):
|
|
4634
4710
|
"""Create temp tables for ALL indicators."""
|
|
4635
4711
|
import time
|
|
@@ -20825,21 +20901,8 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
|
|
|
20825
20901
|
_cd = (cache_config or {}).get('cache_dir')
|
|
20826
20902
|
if _cd:
|
|
20827
20903
|
_dbp = _os4.path.join(_os4.path.expanduser(_cd), 'market_data.duckdb')
|
|
20828
|
-
|
|
20829
|
-
|
|
20830
|
-
if not _os4.path.exists(_f):
|
|
20831
|
-
continue
|
|
20832
|
-
_fd = _os4.open(_f, _os4.O_RDONLY)
|
|
20833
|
-
try:
|
|
20834
|
-
try:
|
|
20835
|
-
_os4.fdatasync(_fd)
|
|
20836
|
-
except Exception:
|
|
20837
|
-
pass
|
|
20838
|
-
_os4.posix_fadvise(_fd, 0, 0, _os4.POSIX_FADV_DONTNEED)
|
|
20839
|
-
finally:
|
|
20840
|
-
_os4.close(_fd)
|
|
20841
|
-
except Exception:
|
|
20842
|
-
pass
|
|
20904
|
+
if _dbp:
|
|
20905
|
+
_drop_file_pages(_dbp)
|
|
20843
20906
|
except Exception:
|
|
20844
20907
|
pass
|
|
20845
20908
|
_release_freed_memory()
|
|
@@ -22924,23 +22987,7 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
|
|
|
22924
22987
|
pass # best-effort; e.g. if another tx is open
|
|
22925
22988
|
# MEMFIX(page-cache): сбросить страницы .duckdb из file-кэша cgroup
|
|
22926
22989
|
try:
|
|
22927
|
-
|
|
22928
|
-
_dbp5 = _get_duckdb_storage_path(cache_config)
|
|
22929
|
-
for _f5 in (_dbp5, _dbp5 + '.wal'):
|
|
22930
|
-
try:
|
|
22931
|
-
if not _o5.path.exists(_f5):
|
|
22932
|
-
continue
|
|
22933
|
-
_fd5 = _o5.open(_f5, _o5.O_RDONLY)
|
|
22934
|
-
try:
|
|
22935
|
-
try:
|
|
22936
|
-
_o5.fdatasync(_fd5)
|
|
22937
|
-
except Exception:
|
|
22938
|
-
pass
|
|
22939
|
-
_o5.posix_fadvise(_fd5, 0, 0, _o5.POSIX_FADV_DONTNEED)
|
|
22940
|
-
finally:
|
|
22941
|
-
_o5.close(_fd5)
|
|
22942
|
-
except Exception:
|
|
22943
|
-
pass
|
|
22990
|
+
_drop_file_pages(_get_duckdb_storage_path(cache_config))
|
|
22944
22991
|
except Exception:
|
|
22945
22992
|
pass
|
|
22946
22993
|
|
|
@@ -23825,6 +23872,7 @@ def preload_data_universal(config, data_requests=None, debug=False):
|
|
|
23825
23872
|
# auto-switch to 'close' and warn instead of silently returning no data.
|
|
23826
23873
|
_enforce_regional_data_constraints(config)
|
|
23827
23874
|
|
|
23875
|
+
_begin_run_cache_state()
|
|
23828
23876
|
# a run never mixes caches: switching back from a private cache happens only here
|
|
23829
23877
|
if _DUCKDB_PRIVATE_CACHE:
|
|
23830
23878
|
_reclaim_shared_cache(_normalize_cache_config(config.get('cache_config', get_cache_config())))
|
|
@@ -29200,6 +29248,36 @@ def _reclaim_shared_cache(cache_config: Dict[str, Any]) -> bool:
|
|
|
29200
29248
|
return True
|
|
29201
29249
|
|
|
29202
29250
|
|
|
29251
|
+
def _begin_run_cache_state() -> None:
|
|
29252
|
+
"""Run start: a shared cache held by another run gets one full wait per run."""
|
|
29253
|
+
with _DUCKDB_OPEN_LOCK:
|
|
29254
|
+
_SHARED_CACHE_TAKEN.clear()
|
|
29255
|
+
_close_stale_page_drop_fds()
|
|
29256
|
+
|
|
29257
|
+
|
|
29258
|
+
def _shared_cache_used(db_path: str) -> bool:
|
|
29259
|
+
return os.path.dirname(os.path.abspath(db_path)) in _SHARED_CACHE_USED
|
|
29260
|
+
|
|
29261
|
+
|
|
29262
|
+
def _shared_cache_open_wait(db_path: str) -> Optional[float]:
|
|
29263
|
+
return 0 if os.path.dirname(os.path.abspath(db_path)) in _SHARED_CACHE_TAKEN else None
|
|
29264
|
+
|
|
29265
|
+
|
|
29266
|
+
def _raise_shared_cache_taken(db_path: str) -> None:
|
|
29267
|
+
"""A process that has used the shared cache never moves to an empty private one: its data is there."""
|
|
29268
|
+
shared = os.path.dirname(os.path.abspath(db_path))
|
|
29269
|
+
msg = (f"The shared cache {_rel_to_project_root(db_path)} is held by another backtest run, and this "
|
|
29270
|
+
f"process has already worked with it, so it does not switch to an empty private copy. "
|
|
29271
|
+
f"Re-run when the other run finishes, or restart the kernel: a fresh process uses a private "
|
|
29272
|
+
f"cache while the shared one is busy.")
|
|
29273
|
+
with _DUCKDB_OPEN_LOCK:
|
|
29274
|
+
first = shared not in _SHARED_CACHE_TAKEN
|
|
29275
|
+
_SHARED_CACHE_TAKEN.add(shared)
|
|
29276
|
+
if first:
|
|
29277
|
+
print(f"\n{'='*70}\n❌ SHARED CACHE IS HELD BY ANOTHER RUN\n{'='*70}\n {msg}\n", flush=True)
|
|
29278
|
+
raise SharedCacheTakenError(msg)
|
|
29279
|
+
|
|
29280
|
+
|
|
29203
29281
|
def _use_private_duckdb_cache(shared_dir: str) -> str:
|
|
29204
29282
|
"""Another run holds <shared_dir>/market_data.duckdb after the wait: this process uses a
|
|
29205
29283
|
private <shared_dir>/_pid<PID>_<random> until the shared file is free again; says so once."""
|
|
@@ -29269,7 +29347,9 @@ def _open_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
|
|
|
29269
29347
|
if not force_readonly:
|
|
29270
29348
|
_conn = None
|
|
29271
29349
|
try:
|
|
29272
|
-
_conn = _duckdb_open(db_path)
|
|
29350
|
+
_conn = _duckdb_open(db_path, wait_sec=_shared_cache_open_wait(db_path))
|
|
29351
|
+
if _conn is None and os.path.abspath(cache_dir) in _SHARED_CACHE_USED:
|
|
29352
|
+
_raise_shared_cache_taken(db_path)
|
|
29273
29353
|
if _conn is None:
|
|
29274
29354
|
cache_dir = _use_private_duckdb_cache(cache_dir)
|
|
29275
29355
|
db_path = os.path.join(cache_dir, 'market_data.duckdb')
|
|
@@ -29282,6 +29362,9 @@ def _open_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
|
|
|
29282
29362
|
_DUCKDB_LOCK_ERROR_SHOWN = True
|
|
29283
29363
|
print(f"[DUCKDB] ❌ Cache file is in use by another process: {db_path}", flush=True)
|
|
29284
29364
|
return None
|
|
29365
|
+
if os.path.abspath(cache_dir) == os.path.abspath(shared_dir):
|
|
29366
|
+
_SHARED_CACHE_USED.add(os.path.abspath(shared_dir))
|
|
29367
|
+
_SHARED_CACHE_TAKEN.discard(os.path.abspath(shared_dir))
|
|
29285
29368
|
_remove_orphan_private_caches(shared_dir)
|
|
29286
29369
|
try:
|
|
29287
29370
|
db_path_abs = os.path.abspath(db_path)
|
|
@@ -30571,6 +30654,7 @@ def cleanup_duckdb_runtime(config: Dict[str, Any], silent: bool = True) -> None:
|
|
|
30571
30654
|
close_duckdb_storage()
|
|
30572
30655
|
except Exception:
|
|
30573
30656
|
pass
|
|
30657
|
+
_close_stale_page_drop_fds()
|
|
30574
30658
|
|
|
30575
30659
|
|
|
30576
30660
|
def reset_session(silent: bool = False):
|
|
@@ -30963,8 +31047,10 @@ class OptionsChunkManager:
|
|
|
30963
31047
|
"""Ensure we have a valid connection"""
|
|
30964
31048
|
if self.conn is None:
|
|
30965
31049
|
self._conn_is_shared = False
|
|
30966
|
-
self.conn = _duckdb_open(self.db_path)
|
|
31050
|
+
self.conn = _duckdb_open(self.db_path, wait_sec=_shared_cache_open_wait(self.db_path))
|
|
30967
31051
|
if self.conn is None:
|
|
31052
|
+
if _shared_cache_used(self.db_path):
|
|
31053
|
+
_raise_shared_cache_taken(self.db_path)
|
|
30968
31054
|
raise RuntimeError(f"DuckDB cache is in use by another run: {self.db_path}")
|
|
30969
31055
|
_apply_duckdb_tuning(self.conn)
|
|
30970
31056
|
|
|
@@ -33004,6 +33090,7 @@ __all__ = [
|
|
|
33004
33090
|
'query_signals', 'clear_signals', # DEPRECATED: signals recalculated each run
|
|
33005
33091
|
# DuckDB storage (native file with deduplication)
|
|
33006
33092
|
'_get_cache_mode_string', 'migrate_parquet_to_duckdb', 'close_duckdb_storage', 'reset_session', 'is_duckdb_readonly',
|
|
33093
|
+
'SharedCacheTakenError',
|
|
33007
33094
|
# Indicator pre-calculation (v2.27.3+)
|
|
33008
33095
|
'precalculate_indicators_from_config', 'build_indicator_lookup',
|
|
33009
33096
|
'INDICATOR_REGISTRY', 'auto_calculate_lookback_period',
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ivolatility_backtesting
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.153
|
|
4
4
|
Summary: A universal backtesting framework for financial strategies using the IVolatility API.
|
|
5
5
|
Author-email: IVolatility <support@ivolatility.com>
|
|
6
6
|
Project-URL: Homepage, https://ivolatility.com
|
|
@@ -15,4 +15,5 @@ tests/test_2147_cents_multiplier.py
|
|
|
15
15
|
tests/test_2148_loader_memory.py
|
|
16
16
|
tests/test_2149_client_errors.py
|
|
17
17
|
tests/test_2150_prompt_patterns.py
|
|
18
|
-
tests/test_2152_cache_lock_and_rate_limit.py
|
|
18
|
+
tests/test_2152_cache_lock_and_rate_limit.py
|
|
19
|
+
tests/test_2153_cache_lock_kept.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "ivolatility_backtesting"
|
|
7
|
-
version = "2.
|
|
7
|
+
version = "2.153"
|
|
8
8
|
description = "A universal backtesting framework for financial strategies using the IVolatility API."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
# The page-cache drop of the cache file must not release DuckDB's write lock held by this process;
|
|
2
|
+
# a mid-run reopen must not move a run to an empty private cache.
|
|
3
|
+
# Run: python3 tests/test_2153_cache_lock_kept.py (or pytest). No network.
|
|
4
|
+
import ast
|
|
5
|
+
import contextlib
|
|
6
|
+
import io
|
|
7
|
+
import os
|
|
8
|
+
import subprocess
|
|
9
|
+
import sys
|
|
10
|
+
import tempfile
|
|
11
|
+
import time
|
|
12
|
+
import traceback
|
|
13
|
+
|
|
14
|
+
import matplotlib
|
|
15
|
+
matplotlib.use('Agg')
|
|
16
|
+
|
|
17
|
+
os.environ.setdefault('MPLCONFIGDIR', matplotlib.get_configdir())
|
|
18
|
+
os.environ['HOME'] = tempfile.mkdtemp(prefix='ivb_home_')
|
|
19
|
+
|
|
20
|
+
_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
21
|
+
_LIBDIR = os.path.join(_REPO, 'ivolatility_backtesting')
|
|
22
|
+
sys.path.insert(0, _LIBDIR)
|
|
23
|
+
import duckdb # noqa: E402
|
|
24
|
+
import pandas as pd # noqa: E402
|
|
25
|
+
import ivolatility_backtesting as lib # noqa: E402
|
|
26
|
+
|
|
27
|
+
if not hasattr(os, 'posix_fadvise'): # macOS/Windows: exercise the fd bookkeeping with real fcntl locks
|
|
28
|
+
os.posix_fadvise = lambda fd, offset, length, advice: None
|
|
29
|
+
os.POSIX_FADV_DONTNEED = 4
|
|
30
|
+
_FADVISE = True
|
|
31
|
+
|
|
32
|
+
_OTHER_WRITER = (
|
|
33
|
+
"import duckdb, sys\n"
|
|
34
|
+
"try:\n"
|
|
35
|
+
" c = duckdb.connect(sys.argv[1]); c.close(); print('OPENED')\n"
|
|
36
|
+
"except Exception as e:\n"
|
|
37
|
+
" print('LOCKED' if 'lock' in str(e).lower() else 'ERROR ' + str(e)[:80])\n"
|
|
38
|
+
)
|
|
39
|
+
|
|
40
|
+
_HOLDER = (
|
|
41
|
+
"import duckdb, os, sys, time\n"
|
|
42
|
+
"c = duckdb.connect(sys.argv[1])\n"
|
|
43
|
+
"print('HELD', flush=True)\n"
|
|
44
|
+
"while not os.path.exists(sys.argv[2]):\n"
|
|
45
|
+
" time.sleep(0.05)\n"
|
|
46
|
+
"c.close()\n"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _other_process_can_write(db_path: str) -> str:
|
|
51
|
+
return subprocess.run([sys.executable, '-c', _OTHER_WRITER, db_path], capture_output=True, text=True,
|
|
52
|
+
timeout=60).stdout.strip()
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _fresh_db():
|
|
56
|
+
d = tempfile.mkdtemp(prefix='ivb_lock_')
|
|
57
|
+
path = os.path.join(d, 'market_data.duckdb')
|
|
58
|
+
conn = duckdb.connect(path)
|
|
59
|
+
conn.execute('create table t(x int)')
|
|
60
|
+
conn.execute('insert into t values (1)')
|
|
61
|
+
return path, conn
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_old_open_close_pattern_drops_the_lock():
|
|
65
|
+
path, conn = _fresh_db()
|
|
66
|
+
try:
|
|
67
|
+
assert _other_process_can_write(path) == 'LOCKED'
|
|
68
|
+
fd = os.open(path, os.O_RDONLY)
|
|
69
|
+
os.close(fd)
|
|
70
|
+
assert _other_process_can_write(path) == 'OPENED'
|
|
71
|
+
finally:
|
|
72
|
+
conn.close()
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_page_drop_keeps_the_lock():
|
|
76
|
+
path, conn = _fresh_db()
|
|
77
|
+
try:
|
|
78
|
+
for _ in range(3):
|
|
79
|
+
lib._drop_file_pages(path)
|
|
80
|
+
conn.execute('insert into t values (2)')
|
|
81
|
+
assert _other_process_can_write(path) == 'LOCKED'
|
|
82
|
+
assert (path in lib._PAGE_DROP_FDS) == _FADVISE
|
|
83
|
+
finally:
|
|
84
|
+
conn.close()
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_replaced_file_gets_a_new_fd_and_keeps_the_lock():
|
|
88
|
+
if not _FADVISE:
|
|
89
|
+
return
|
|
90
|
+
path, conn = _fresh_db()
|
|
91
|
+
conn.close()
|
|
92
|
+
lib._drop_file_pages(path)
|
|
93
|
+
old_fd = lib._PAGE_DROP_FDS[path]
|
|
94
|
+
old_ino = os.fstat(old_fd).st_ino
|
|
95
|
+
os.remove(path)
|
|
96
|
+
conn = duckdb.connect(path)
|
|
97
|
+
try:
|
|
98
|
+
conn.execute('create table t(x int)')
|
|
99
|
+
lib._drop_file_pages(path)
|
|
100
|
+
new_fd = lib._PAGE_DROP_FDS[path]
|
|
101
|
+
assert os.fstat(new_fd).st_ino == os.stat(path).st_ino != old_ino
|
|
102
|
+
assert _other_process_can_write(path) == 'LOCKED'
|
|
103
|
+
finally:
|
|
104
|
+
conn.close()
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_deleted_file_fd_is_released():
|
|
108
|
+
if not _FADVISE:
|
|
109
|
+
return
|
|
110
|
+
path, conn = _fresh_db()
|
|
111
|
+
conn.close()
|
|
112
|
+
lib._drop_file_pages(path)
|
|
113
|
+
fd = lib._PAGE_DROP_FDS[path]
|
|
114
|
+
os.remove(path)
|
|
115
|
+
lib._drop_file_pages(path)
|
|
116
|
+
assert path not in lib._PAGE_DROP_FDS
|
|
117
|
+
try:
|
|
118
|
+
os.fstat(fd)
|
|
119
|
+
reused = True
|
|
120
|
+
except OSError:
|
|
121
|
+
reused = False
|
|
122
|
+
assert not reused or os.fstat(fd).st_nlink > 0
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_cache_file_is_opened_and_fadvised_only_inside_the_helper():
|
|
126
|
+
tree = ast.parse(open(os.path.join(_LIBDIR, 'ivolatility_backtesting.py'), encoding='utf-8').read())
|
|
127
|
+
offenders = []
|
|
128
|
+
for func in ast.walk(tree):
|
|
129
|
+
if not isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)) or func.name == '_drop_file_pages':
|
|
130
|
+
continue
|
|
131
|
+
for node in ast.walk(func):
|
|
132
|
+
if isinstance(node, ast.Attribute) and node.attr in ('posix_fadvise', 'POSIX_FADV_DONTNEED'):
|
|
133
|
+
offenders.append(f'{func.name}:{node.lineno} {node.attr}')
|
|
134
|
+
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == '_drop_file_pages':
|
|
135
|
+
if any(isinstance(a, ast.BinOp) or (isinstance(a, ast.Constant) and '.wal' in str(a.value))
|
|
136
|
+
for a in node.args):
|
|
137
|
+
offenders.append(f'{func.name}:{node.lineno} _drop_file_pages(<derived path>)')
|
|
138
|
+
assert not offenders, offenders
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def test_process_that_used_shared_cache_stops_instead_of_an_empty_private_cache():
|
|
142
|
+
cache = tempfile.mkdtemp(prefix='ivb_cache_')
|
|
143
|
+
cc = {'cache_dir': cache}
|
|
144
|
+
db = os.path.join(cache, 'market_data.duckdb')
|
|
145
|
+
go = os.path.join(cache, 'go')
|
|
146
|
+
old_wait = os.environ.get('IVOL_CACHE_LOCK_WAIT_SEC')
|
|
147
|
+
os.environ['IVOL_CACHE_LOCK_WAIT_SEC'] = '2'
|
|
148
|
+
lib._SHARED_CACHE_USED.clear()
|
|
149
|
+
lib._SHARED_CACHE_TAKEN.clear()
|
|
150
|
+
holder = None
|
|
151
|
+
try:
|
|
152
|
+
conn = lib._get_duckdb_storage_conn(cc)
|
|
153
|
+
assert conn is not None and lib._effective_cache_dir(cc) == os.path.abspath(cache)
|
|
154
|
+
lib.close_duckdb_storage()
|
|
155
|
+
holder = subprocess.Popen([sys.executable, '-u', '-c', _HOLDER, db, go], stdout=subprocess.PIPE, text=True)
|
|
156
|
+
assert holder.stdout.readline().strip() == 'HELD'
|
|
157
|
+
t0 = time.time()
|
|
158
|
+
try:
|
|
159
|
+
lib._get_duckdb_storage_conn(cc)
|
|
160
|
+
raised = False
|
|
161
|
+
except lib.SharedCacheTakenError:
|
|
162
|
+
raised = True
|
|
163
|
+
assert raised and time.time() - t0 >= 1.5
|
|
164
|
+
assert lib._effective_cache_dir(cc) == os.path.abspath(cache)
|
|
165
|
+
|
|
166
|
+
df = pd.DataFrame({'symbol': ['SPY'], 'date': [pd.Timestamp('2024-01-02')], 'close': [470.0]})
|
|
167
|
+
t0 = time.time()
|
|
168
|
+
try:
|
|
169
|
+
lib._save_to_duckdb_storage(df, '/equities/eod/stock-prices', cc)
|
|
170
|
+
propagated = False
|
|
171
|
+
except lib.SharedCacheTakenError:
|
|
172
|
+
propagated = True
|
|
173
|
+
assert propagated and time.time() - t0 < 1.0
|
|
174
|
+
|
|
175
|
+
cm = lib.OptionsChunkManager(db_path=db, symbol='SPY', conn=None)
|
|
176
|
+
im = object.__new__(lib.DuckDBIndicatorManager)
|
|
177
|
+
im.db_path, im.conn = db, None
|
|
178
|
+
for reconnect in (cm._ensure_connection, im._connect):
|
|
179
|
+
t0 = time.time()
|
|
180
|
+
try:
|
|
181
|
+
reconnect()
|
|
182
|
+
reconnect_raised = False
|
|
183
|
+
except lib.SharedCacheTakenError:
|
|
184
|
+
reconnect_raised = True
|
|
185
|
+
assert reconnect_raised and time.time() - t0 < 1.0, reconnect
|
|
186
|
+
|
|
187
|
+
lib._SHARED_CACHE_USED.clear() # a fresh process: never opened the shared cache
|
|
188
|
+
lib._SHARED_CACHE_TAKEN.clear()
|
|
189
|
+
private = lib._get_duckdb_storage_conn(cc)
|
|
190
|
+
assert private is not None and lib._effective_cache_dir(cc) != os.path.abspath(cache)
|
|
191
|
+
lib.close_duckdb_storage()
|
|
192
|
+
finally:
|
|
193
|
+
open(go, 'w').close()
|
|
194
|
+
if holder is not None:
|
|
195
|
+
holder.wait(30)
|
|
196
|
+
lib._DUCKDB_PRIVATE_CACHE.pop(os.path.abspath(cache), None)
|
|
197
|
+
lib._SHARED_CACHE_USED.clear()
|
|
198
|
+
lib._SHARED_CACHE_TAKEN.clear()
|
|
199
|
+
if old_wait is None:
|
|
200
|
+
os.environ.pop('IVOL_CACHE_LOCK_WAIT_SEC', None)
|
|
201
|
+
else:
|
|
202
|
+
os.environ['IVOL_CACHE_LOCK_WAIT_SEC'] = old_wait
|
|
203
|
+
time.sleep(0.1)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def test_cleanup_releases_fds_of_deleted_cache_files():
|
|
207
|
+
live_path, live_conn = _fresh_db()
|
|
208
|
+
gone_path, gone_conn = _fresh_db()
|
|
209
|
+
gone_conn.close()
|
|
210
|
+
try:
|
|
211
|
+
lib._drop_file_pages(live_path)
|
|
212
|
+
lib._drop_file_pages(gone_path)
|
|
213
|
+
gone_fd = lib._PAGE_DROP_FDS[gone_path]
|
|
214
|
+
os.remove(gone_path)
|
|
215
|
+
lib.cleanup_duckdb_runtime({})
|
|
216
|
+
assert gone_path not in lib._PAGE_DROP_FDS and live_path in lib._PAGE_DROP_FDS
|
|
217
|
+
try:
|
|
218
|
+
assert os.fstat(gone_fd).st_nlink > 0
|
|
219
|
+
except OSError:
|
|
220
|
+
pass
|
|
221
|
+
assert _other_process_can_write(live_path) == 'LOCKED'
|
|
222
|
+
finally:
|
|
223
|
+
live_conn.close()
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def test_preload_starts_each_run_with_a_fresh_wait():
|
|
227
|
+
tree = ast.parse(open(os.path.join(_LIBDIR, 'ivolatility_backtesting.py'), encoding='utf-8').read())
|
|
228
|
+
preload = next(f for f in ast.walk(tree) if isinstance(f, ast.FunctionDef) and f.name == 'preload_data_universal')
|
|
229
|
+
calls = {n.func.id for n in ast.walk(preload) if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)}
|
|
230
|
+
assert '_begin_run_cache_state' in calls
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def test_next_run_waits_again_and_gets_the_shared_cache_when_free():
|
|
234
|
+
cache = tempfile.mkdtemp(prefix='ivb_cache_')
|
|
235
|
+
cc = {'cache_dir': cache}
|
|
236
|
+
db = os.path.join(cache, 'market_data.duckdb')
|
|
237
|
+
go = os.path.join(cache, 'go')
|
|
238
|
+
old_wait = os.environ.get('IVOL_CACHE_LOCK_WAIT_SEC')
|
|
239
|
+
os.environ['IVOL_CACHE_LOCK_WAIT_SEC'] = '2'
|
|
240
|
+
lib._SHARED_CACHE_USED.clear()
|
|
241
|
+
lib._SHARED_CACHE_TAKEN.clear()
|
|
242
|
+
holder = None
|
|
243
|
+
try:
|
|
244
|
+
assert lib._get_duckdb_storage_conn(cc) is not None
|
|
245
|
+
lib.close_duckdb_storage()
|
|
246
|
+
holder = subprocess.Popen([sys.executable, '-u', '-c', _HOLDER, db, go], stdout=subprocess.PIPE, text=True)
|
|
247
|
+
assert holder.stdout.readline().strip() == 'HELD'
|
|
248
|
+
for run in (1, 2):
|
|
249
|
+
lib._begin_run_cache_state()
|
|
250
|
+
out = io.StringIO()
|
|
251
|
+
t0 = time.time()
|
|
252
|
+
with contextlib.redirect_stdout(out):
|
|
253
|
+
try:
|
|
254
|
+
lib._get_duckdb_storage_conn(cc)
|
|
255
|
+
raised = False
|
|
256
|
+
except lib.SharedCacheTakenError:
|
|
257
|
+
raised = True
|
|
258
|
+
assert raised and time.time() - t0 >= 1.5, run
|
|
259
|
+
assert 'SHARED CACHE IS HELD BY ANOTHER RUN' in out.getvalue(), run
|
|
260
|
+
open(go, 'w').close()
|
|
261
|
+
holder.wait(30)
|
|
262
|
+
holder = None
|
|
263
|
+
lib._begin_run_cache_state()
|
|
264
|
+
conn = lib._get_duckdb_storage_conn(cc)
|
|
265
|
+
assert conn is not None and lib._effective_cache_dir(cc) == os.path.abspath(cache)
|
|
266
|
+
lib.close_duckdb_storage()
|
|
267
|
+
finally:
|
|
268
|
+
open(go, 'w').close()
|
|
269
|
+
if holder is not None:
|
|
270
|
+
holder.wait(30)
|
|
271
|
+
lib._DUCKDB_PRIVATE_CACHE.pop(os.path.abspath(cache), None)
|
|
272
|
+
lib._SHARED_CACHE_USED.clear()
|
|
273
|
+
lib._SHARED_CACHE_TAKEN.clear()
|
|
274
|
+
if old_wait is None:
|
|
275
|
+
os.environ.pop('IVOL_CACHE_LOCK_WAIT_SEC', None)
|
|
276
|
+
else:
|
|
277
|
+
os.environ['IVOL_CACHE_LOCK_WAIT_SEC'] = old_wait
|
|
278
|
+
time.sleep(0.1)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def test_error_is_exported_by_the_package():
|
|
282
|
+
out = subprocess.run([sys.executable, '-c',
|
|
283
|
+
'from ivolatility_backtesting import SharedCacheTakenError as E; '
|
|
284
|
+
'print(issubclass(E, BaseException) and not issubclass(E, Exception))'],
|
|
285
|
+
capture_output=True, text=True, timeout=120,
|
|
286
|
+
env=dict(os.environ, PYTHONPATH=_REPO, MPLBACKEND='Agg'))
|
|
287
|
+
assert out.stdout.strip().splitlines()[-1] == 'True', out.stdout[-500:] + out.stderr[-500:]
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
if __name__ == '__main__':
|
|
291
|
+
failed = 0
|
|
292
|
+
for name, fn in list(globals().items()):
|
|
293
|
+
if name.startswith('test_') and callable(fn):
|
|
294
|
+
try:
|
|
295
|
+
fn()
|
|
296
|
+
print(f'PASS {name}')
|
|
297
|
+
except Exception:
|
|
298
|
+
failed += 1
|
|
299
|
+
print(f'FAIL {name}')
|
|
300
|
+
traceback.print_exc()
|
|
301
|
+
sys.exit(1 if failed else 0)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2144_duckdb_dedup.py
RENAMED
|
File without changes
|
|
File without changes
|
{ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2147_cents_multiplier.py
RENAMED
|
File without changes
|
{ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2148_loader_memory.py
RENAMED
|
File without changes
|
{ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2149_client_errors.py
RENAMED
|
File without changes
|
{ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2150_prompt_patterns.py
RENAMED
|
File without changes
|
|
File without changes
|