ivolatility-backtesting 2.152__tar.gz → 2.153__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/PKG-INFO +1 -1
  2. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting/__init__.py +2 -0
  3. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting/ivolatility_backtesting.py +123 -36
  4. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/PKG-INFO +1 -1
  5. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/SOURCES.txt +2 -1
  6. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/pyproject.toml +1 -1
  7. ivolatility_backtesting-2.153/tests/test_2153_cache_lock_kept.py +301 -0
  8. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/README.md +0 -0
  9. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/dependency_links.txt +0 -0
  10. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/requires.txt +0 -0
  11. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/ivolatility_backtesting.egg-info/top_level.txt +0 -0
  12. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/setup.cfg +0 -0
  13. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2142_fixes.py +0 -0
  14. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2144_duckdb_dedup.py +0 -0
  15. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2146_vix_vro.py +0 -0
  16. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2147_cents_multiplier.py +0 -0
  17. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2148_loader_memory.py +0 -0
  18. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2149_client_errors.py +0 -0
  19. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2150_prompt_patterns.py +0 -0
  20. {ivolatility_backtesting-2.152 → ivolatility_backtesting-2.153}/tests/test_2152_cache_lock_and_rate_limit.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ivolatility_backtesting
3
- Version: 2.152
3
+ Version: 2.153
4
4
  Summary: A universal backtesting framework for financial strategies using the IVolatility API.
5
5
  Author-email: IVolatility <support@ivolatility.com>
6
6
  Project-URL: Homepage, https://ivolatility.com
@@ -29,6 +29,7 @@ from .ivolatility_backtesting import (
29
29
  set_project_root, get_project_root,
30
30
  get_position_multiplier, get_hedge_multiplier,
31
31
  parse_earnings_calendar,
32
+ SharedCacheTakenError,
32
33
  )
33
34
 
34
35
  __all__ = [
@@ -62,4 +63,5 @@ __all__ = [
62
63
  'set_project_root', 'get_project_root',
63
64
  'get_position_multiplier', 'get_hedge_multiplier',
64
65
  'parse_earnings_calendar',
66
+ 'SharedCacheTakenError',
65
67
  ]
@@ -153,6 +153,13 @@ def _resolve_path(path: str, base_dir: str = None) -> str:
153
153
  return os.path.abspath(os.path.join(base_dir, p))
154
154
 
155
155
  _DUCKDB_PRIVATE_CACHE = {} # abs shared cache_dir -> this process's private cache dir
156
+ _SHARED_CACHE_USED = set() # abs shared cache_dirs this process has opened; never cleared
157
+ _SHARED_CACHE_TAKEN = set() # used shared dirs found held after the full wait; cleared at run start, so only opens within one run skip the wait
158
+
159
+
160
+ class SharedCacheTakenError(BaseException):
161
+ """BaseException: the library's `except Exception` handlers must not turn it into a silent retry."""
162
+
156
163
 
157
164
  def _effective_cache_dir(cache_config) -> str:
158
165
  """cache_config['cache_dir'] resolved; redirected to this process's private cache
@@ -561,6 +568,73 @@ def _release_freed_memory():
561
568
 
562
569
  import threading
563
570
 
571
+ _PAGE_DROP_FDS = globals().get('_PAGE_DROP_FDS') or {} # path -> read-only fd, survives module reload
572
+ _PAGE_DROP_LOCK = globals().get('_PAGE_DROP_LOCK') or threading.Lock()
573
+
574
+
575
+ def _stat_or_none(path: str):
576
+ try:
577
+ return os.stat(path)
578
+ except OSError:
579
+ return None
580
+
581
+
582
+ def _live_page_drop_fd(path: str, st) -> Optional[int]:
583
+ """Caller holds _PAGE_DROP_LOCK. Returns the kept fd if it still points at the file at <path>;
584
+ otherwise (path gone, file deleted or replaced) closes it: it no longer shares an inode with that file."""
585
+ fd = _PAGE_DROP_FDS.get(path)
586
+ if fd is None:
587
+ return None
588
+ try:
589
+ fst = os.fstat(fd)
590
+ live = st is not None and fst.st_nlink > 0 and (fst.st_ino, fst.st_dev) == (st.st_ino, st.st_dev)
591
+ except OSError:
592
+ live = False
593
+ if live:
594
+ return fd
595
+ del _PAGE_DROP_FDS[path]
596
+ try:
597
+ os.close(fd)
598
+ except OSError:
599
+ pass
600
+ return None
601
+
602
+
603
+ def _close_stale_page_drop_fds() -> None:
604
+ """Release kept fds of deleted / replaced cache files, so their disk space is freed."""
605
+ with _PAGE_DROP_LOCK:
606
+ for path in list(_PAGE_DROP_FDS):
607
+ _live_page_drop_fd(path, _stat_or_none(path))
608
+
609
+
610
+ def _drop_file_pages(path: str) -> None:
611
+ """fdatasync + POSIX_FADV_DONTNEED on a cache file whose DuckDB lock this process may hold.
612
+ The fd stays open while it points at the live file: closing any fd of it drops our fcntl locks.
613
+ Only the .duckdb is passed: the .wal carries the same lock and is deleted at every checkpoint
614
+ and recreated under a new inode, so it gets no open/close calls either."""
615
+ if not hasattr(os, 'posix_fadvise'):
616
+ return
617
+ with _PAGE_DROP_LOCK:
618
+ st = _stat_or_none(path)
619
+ fd = _live_page_drop_fd(path, st)
620
+ if st is None:
621
+ return
622
+ if fd is None:
623
+ try:
624
+ fd = os.open(path, os.O_RDONLY | getattr(os, 'O_CLOEXEC', 0))
625
+ except OSError:
626
+ return
627
+ _PAGE_DROP_FDS[path] = fd
628
+ try:
629
+ getattr(os, 'fdatasync', os.fsync)(fd)
630
+ except OSError:
631
+ pass
632
+ try:
633
+ os.posix_fadvise(fd, 0, 0, os.POSIX_FADV_DONTNEED)
634
+ except OSError:
635
+ pass
636
+
637
+
564
638
  # Admission control by cgroup free memory; the first download always passes so a load never stalls.
565
639
  def _env_int(name, default):
566
640
  try:
@@ -4626,10 +4700,12 @@ class DuckDBIndicatorManager:
4626
4700
  # Only open new connection if none was passed to __init__
4627
4701
  if not os.path.exists(_effective_duckdb_path(self.db_path)):
4628
4702
  raise FileNotFoundError(f"DuckDB file not found: {self.db_path}")
4629
- self.conn = _duckdb_open(self.db_path)
4703
+ self.conn = _duckdb_open(self.db_path, wait_sec=_shared_cache_open_wait(self.db_path))
4630
4704
  if self.conn is None:
4705
+ if _shared_cache_used(self.db_path):
4706
+ _raise_shared_cache_taken(self.db_path)
4631
4707
  raise RuntimeError(f"DuckDB cache is in use by another run: {self.db_path}")
4632
-
4708
+
4633
4709
  def _create_all_indicator_tables(self):
4634
4710
  """Create temp tables for ALL indicators."""
4635
4711
  import time
@@ -20825,21 +20901,8 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
20825
20901
  _cd = (cache_config or {}).get('cache_dir')
20826
20902
  if _cd:
20827
20903
  _dbp = _os4.path.join(_os4.path.expanduser(_cd), 'market_data.duckdb')
20828
- for _f in ([_dbp, _dbp + '.wal'] if _dbp else []):
20829
- try:
20830
- if not _os4.path.exists(_f):
20831
- continue
20832
- _fd = _os4.open(_f, _os4.O_RDONLY)
20833
- try:
20834
- try:
20835
- _os4.fdatasync(_fd)
20836
- except Exception:
20837
- pass
20838
- _os4.posix_fadvise(_fd, 0, 0, _os4.POSIX_FADV_DONTNEED)
20839
- finally:
20840
- _os4.close(_fd)
20841
- except Exception:
20842
- pass
20904
+ if _dbp:
20905
+ _drop_file_pages(_dbp)
20843
20906
  except Exception:
20844
20907
  pass
20845
20908
  _release_freed_memory()
@@ -22924,23 +22987,7 @@ def _load_futures_options_to_duckdb(config, cache_config, symbol,
22924
22987
  pass # best-effort; e.g. if another tx is open
22925
22988
  # MEMFIX(page-cache): сбросить страницы .duckdb из file-кэша cgroup
22926
22989
  try:
22927
- import os as _o5
22928
- _dbp5 = _get_duckdb_storage_path(cache_config)
22929
- for _f5 in (_dbp5, _dbp5 + '.wal'):
22930
- try:
22931
- if not _o5.path.exists(_f5):
22932
- continue
22933
- _fd5 = _o5.open(_f5, _o5.O_RDONLY)
22934
- try:
22935
- try:
22936
- _o5.fdatasync(_fd5)
22937
- except Exception:
22938
- pass
22939
- _o5.posix_fadvise(_fd5, 0, 0, _o5.POSIX_FADV_DONTNEED)
22940
- finally:
22941
- _o5.close(_fd5)
22942
- except Exception:
22943
- pass
22990
+ _drop_file_pages(_get_duckdb_storage_path(cache_config))
22944
22991
  except Exception:
22945
22992
  pass
22946
22993
 
@@ -23825,6 +23872,7 @@ def preload_data_universal(config, data_requests=None, debug=False):
23825
23872
  # auto-switch to 'close' and warn instead of silently returning no data.
23826
23873
  _enforce_regional_data_constraints(config)
23827
23874
 
23875
+ _begin_run_cache_state()
23828
23876
  # a run never mixes caches: switching back from a private cache happens only here
23829
23877
  if _DUCKDB_PRIVATE_CACHE:
23830
23878
  _reclaim_shared_cache(_normalize_cache_config(config.get('cache_config', get_cache_config())))
@@ -29200,6 +29248,36 @@ def _reclaim_shared_cache(cache_config: Dict[str, Any]) -> bool:
29200
29248
  return True
29201
29249
 
29202
29250
 
29251
+ def _begin_run_cache_state() -> None:
29252
+ """Run start: a shared cache held by another run gets one full wait per run."""
29253
+ with _DUCKDB_OPEN_LOCK:
29254
+ _SHARED_CACHE_TAKEN.clear()
29255
+ _close_stale_page_drop_fds()
29256
+
29257
+
29258
+ def _shared_cache_used(db_path: str) -> bool:
29259
+ return os.path.dirname(os.path.abspath(db_path)) in _SHARED_CACHE_USED
29260
+
29261
+
29262
+ def _shared_cache_open_wait(db_path: str) -> Optional[float]:
29263
+ return 0 if os.path.dirname(os.path.abspath(db_path)) in _SHARED_CACHE_TAKEN else None
29264
+
29265
+
29266
+ def _raise_shared_cache_taken(db_path: str) -> None:
29267
+ """A process that has used the shared cache never moves to an empty private one: its data is there."""
29268
+ shared = os.path.dirname(os.path.abspath(db_path))
29269
+ msg = (f"The shared cache {_rel_to_project_root(db_path)} is held by another backtest run, and this "
29270
+ f"process has already worked with it, so it does not switch to an empty private copy. "
29271
+ f"Re-run when the other run finishes, or restart the kernel: a fresh process uses a private "
29272
+ f"cache while the shared one is busy.")
29273
+ with _DUCKDB_OPEN_LOCK:
29274
+ first = shared not in _SHARED_CACHE_TAKEN
29275
+ _SHARED_CACHE_TAKEN.add(shared)
29276
+ if first:
29277
+ print(f"\n{'='*70}\n❌ SHARED CACHE IS HELD BY ANOTHER RUN\n{'='*70}\n {msg}\n", flush=True)
29278
+ raise SharedCacheTakenError(msg)
29279
+
29280
+
29203
29281
  def _use_private_duckdb_cache(shared_dir: str) -> str:
29204
29282
  """Another run holds <shared_dir>/market_data.duckdb after the wait: this process uses a
29205
29283
  private <shared_dir>/_pid<PID>_<random> until the shared file is free again; says so once."""
@@ -29269,7 +29347,9 @@ def _open_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
29269
29347
  if not force_readonly:
29270
29348
  _conn = None
29271
29349
  try:
29272
- _conn = _duckdb_open(db_path)
29350
+ _conn = _duckdb_open(db_path, wait_sec=_shared_cache_open_wait(db_path))
29351
+ if _conn is None and os.path.abspath(cache_dir) in _SHARED_CACHE_USED:
29352
+ _raise_shared_cache_taken(db_path)
29273
29353
  if _conn is None:
29274
29354
  cache_dir = _use_private_duckdb_cache(cache_dir)
29275
29355
  db_path = os.path.join(cache_dir, 'market_data.duckdb')
@@ -29282,6 +29362,9 @@ def _open_duckdb_storage_conn(cache_config: Dict[str, Any], force_readonly: bool
29282
29362
  _DUCKDB_LOCK_ERROR_SHOWN = True
29283
29363
  print(f"[DUCKDB] ❌ Cache file is in use by another process: {db_path}", flush=True)
29284
29364
  return None
29365
+ if os.path.abspath(cache_dir) == os.path.abspath(shared_dir):
29366
+ _SHARED_CACHE_USED.add(os.path.abspath(shared_dir))
29367
+ _SHARED_CACHE_TAKEN.discard(os.path.abspath(shared_dir))
29285
29368
  _remove_orphan_private_caches(shared_dir)
29286
29369
  try:
29287
29370
  db_path_abs = os.path.abspath(db_path)
@@ -30571,6 +30654,7 @@ def cleanup_duckdb_runtime(config: Dict[str, Any], silent: bool = True) -> None:
30571
30654
  close_duckdb_storage()
30572
30655
  except Exception:
30573
30656
  pass
30657
+ _close_stale_page_drop_fds()
30574
30658
 
30575
30659
 
30576
30660
  def reset_session(silent: bool = False):
@@ -30963,8 +31047,10 @@ class OptionsChunkManager:
30963
31047
  """Ensure we have a valid connection"""
30964
31048
  if self.conn is None:
30965
31049
  self._conn_is_shared = False
30966
- self.conn = _duckdb_open(self.db_path)
31050
+ self.conn = _duckdb_open(self.db_path, wait_sec=_shared_cache_open_wait(self.db_path))
30967
31051
  if self.conn is None:
31052
+ if _shared_cache_used(self.db_path):
31053
+ _raise_shared_cache_taken(self.db_path)
30968
31054
  raise RuntimeError(f"DuckDB cache is in use by another run: {self.db_path}")
30969
31055
  _apply_duckdb_tuning(self.conn)
30970
31056
 
@@ -33004,6 +33090,7 @@ __all__ = [
33004
33090
  'query_signals', 'clear_signals', # DEPRECATED: signals recalculated each run
33005
33091
  # DuckDB storage (native file with deduplication)
33006
33092
  '_get_cache_mode_string', 'migrate_parquet_to_duckdb', 'close_duckdb_storage', 'reset_session', 'is_duckdb_readonly',
33093
+ 'SharedCacheTakenError',
33007
33094
  # Indicator pre-calculation (v2.27.3+)
33008
33095
  'precalculate_indicators_from_config', 'build_indicator_lookup',
33009
33096
  'INDICATOR_REGISTRY', 'auto_calculate_lookback_period',
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ivolatility_backtesting
3
- Version: 2.152
3
+ Version: 2.153
4
4
  Summary: A universal backtesting framework for financial strategies using the IVolatility API.
5
5
  Author-email: IVolatility <support@ivolatility.com>
6
6
  Project-URL: Homepage, https://ivolatility.com
@@ -15,4 +15,5 @@ tests/test_2147_cents_multiplier.py
15
15
  tests/test_2148_loader_memory.py
16
16
  tests/test_2149_client_errors.py
17
17
  tests/test_2150_prompt_patterns.py
18
- tests/test_2152_cache_lock_and_rate_limit.py
18
+ tests/test_2152_cache_lock_and_rate_limit.py
19
+ tests/test_2153_cache_lock_kept.py
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "ivolatility_backtesting"
7
- version = "2.152"
7
+ version = "2.153"
8
8
  description = "A universal backtesting framework for financial strategies using the IVolatility API."
9
9
  readme = "README.md"
10
10
  authors = [
@@ -0,0 +1,301 @@
1
+ # The page-cache drop of the cache file must not release DuckDB's write lock held by this process;
2
+ # a mid-run reopen must not move a run to an empty private cache.
3
+ # Run: python3 tests/test_2153_cache_lock_kept.py (or pytest). No network.
4
+ import ast
5
+ import contextlib
6
+ import io
7
+ import os
8
+ import subprocess
9
+ import sys
10
+ import tempfile
11
+ import time
12
+ import traceback
13
+
14
+ import matplotlib
15
+ matplotlib.use('Agg')
16
+
17
+ os.environ.setdefault('MPLCONFIGDIR', matplotlib.get_configdir())
18
+ os.environ['HOME'] = tempfile.mkdtemp(prefix='ivb_home_')
19
+
20
+ _REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
21
+ _LIBDIR = os.path.join(_REPO, 'ivolatility_backtesting')
22
+ sys.path.insert(0, _LIBDIR)
23
+ import duckdb # noqa: E402
24
+ import pandas as pd # noqa: E402
25
+ import ivolatility_backtesting as lib # noqa: E402
26
+
27
+ if not hasattr(os, 'posix_fadvise'): # macOS/Windows: exercise the fd bookkeeping with real fcntl locks
28
+ os.posix_fadvise = lambda fd, offset, length, advice: None
29
+ os.POSIX_FADV_DONTNEED = 4
30
+ _FADVISE = True
31
+
32
+ _OTHER_WRITER = (
33
+ "import duckdb, sys\n"
34
+ "try:\n"
35
+ " c = duckdb.connect(sys.argv[1]); c.close(); print('OPENED')\n"
36
+ "except Exception as e:\n"
37
+ " print('LOCKED' if 'lock' in str(e).lower() else 'ERROR ' + str(e)[:80])\n"
38
+ )
39
+
40
+ _HOLDER = (
41
+ "import duckdb, os, sys, time\n"
42
+ "c = duckdb.connect(sys.argv[1])\n"
43
+ "print('HELD', flush=True)\n"
44
+ "while not os.path.exists(sys.argv[2]):\n"
45
+ " time.sleep(0.05)\n"
46
+ "c.close()\n"
47
+ )
48
+
49
+
50
+ def _other_process_can_write(db_path: str) -> str:
51
+ return subprocess.run([sys.executable, '-c', _OTHER_WRITER, db_path], capture_output=True, text=True,
52
+ timeout=60).stdout.strip()
53
+
54
+
55
+ def _fresh_db():
56
+ d = tempfile.mkdtemp(prefix='ivb_lock_')
57
+ path = os.path.join(d, 'market_data.duckdb')
58
+ conn = duckdb.connect(path)
59
+ conn.execute('create table t(x int)')
60
+ conn.execute('insert into t values (1)')
61
+ return path, conn
62
+
63
+
64
+ def test_old_open_close_pattern_drops_the_lock():
65
+ path, conn = _fresh_db()
66
+ try:
67
+ assert _other_process_can_write(path) == 'LOCKED'
68
+ fd = os.open(path, os.O_RDONLY)
69
+ os.close(fd)
70
+ assert _other_process_can_write(path) == 'OPENED'
71
+ finally:
72
+ conn.close()
73
+
74
+
75
+ def test_page_drop_keeps_the_lock():
76
+ path, conn = _fresh_db()
77
+ try:
78
+ for _ in range(3):
79
+ lib._drop_file_pages(path)
80
+ conn.execute('insert into t values (2)')
81
+ assert _other_process_can_write(path) == 'LOCKED'
82
+ assert (path in lib._PAGE_DROP_FDS) == _FADVISE
83
+ finally:
84
+ conn.close()
85
+
86
+
87
+ def test_replaced_file_gets_a_new_fd_and_keeps_the_lock():
88
+ if not _FADVISE:
89
+ return
90
+ path, conn = _fresh_db()
91
+ conn.close()
92
+ lib._drop_file_pages(path)
93
+ old_fd = lib._PAGE_DROP_FDS[path]
94
+ old_ino = os.fstat(old_fd).st_ino
95
+ os.remove(path)
96
+ conn = duckdb.connect(path)
97
+ try:
98
+ conn.execute('create table t(x int)')
99
+ lib._drop_file_pages(path)
100
+ new_fd = lib._PAGE_DROP_FDS[path]
101
+ assert os.fstat(new_fd).st_ino == os.stat(path).st_ino != old_ino
102
+ assert _other_process_can_write(path) == 'LOCKED'
103
+ finally:
104
+ conn.close()
105
+
106
+
107
+ def test_deleted_file_fd_is_released():
108
+ if not _FADVISE:
109
+ return
110
+ path, conn = _fresh_db()
111
+ conn.close()
112
+ lib._drop_file_pages(path)
113
+ fd = lib._PAGE_DROP_FDS[path]
114
+ os.remove(path)
115
+ lib._drop_file_pages(path)
116
+ assert path not in lib._PAGE_DROP_FDS
117
+ try:
118
+ os.fstat(fd)
119
+ reused = True
120
+ except OSError:
121
+ reused = False
122
+ assert not reused or os.fstat(fd).st_nlink > 0
123
+
124
+
125
+ def test_cache_file_is_opened_and_fadvised_only_inside_the_helper():
126
+ tree = ast.parse(open(os.path.join(_LIBDIR, 'ivolatility_backtesting.py'), encoding='utf-8').read())
127
+ offenders = []
128
+ for func in ast.walk(tree):
129
+ if not isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)) or func.name == '_drop_file_pages':
130
+ continue
131
+ for node in ast.walk(func):
132
+ if isinstance(node, ast.Attribute) and node.attr in ('posix_fadvise', 'POSIX_FADV_DONTNEED'):
133
+ offenders.append(f'{func.name}:{node.lineno} {node.attr}')
134
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == '_drop_file_pages':
135
+ if any(isinstance(a, ast.BinOp) or (isinstance(a, ast.Constant) and '.wal' in str(a.value))
136
+ for a in node.args):
137
+ offenders.append(f'{func.name}:{node.lineno} _drop_file_pages(<derived path>)')
138
+ assert not offenders, offenders
139
+
140
+
141
+ def test_process_that_used_shared_cache_stops_instead_of_an_empty_private_cache():
142
+ cache = tempfile.mkdtemp(prefix='ivb_cache_')
143
+ cc = {'cache_dir': cache}
144
+ db = os.path.join(cache, 'market_data.duckdb')
145
+ go = os.path.join(cache, 'go')
146
+ old_wait = os.environ.get('IVOL_CACHE_LOCK_WAIT_SEC')
147
+ os.environ['IVOL_CACHE_LOCK_WAIT_SEC'] = '2'
148
+ lib._SHARED_CACHE_USED.clear()
149
+ lib._SHARED_CACHE_TAKEN.clear()
150
+ holder = None
151
+ try:
152
+ conn = lib._get_duckdb_storage_conn(cc)
153
+ assert conn is not None and lib._effective_cache_dir(cc) == os.path.abspath(cache)
154
+ lib.close_duckdb_storage()
155
+ holder = subprocess.Popen([sys.executable, '-u', '-c', _HOLDER, db, go], stdout=subprocess.PIPE, text=True)
156
+ assert holder.stdout.readline().strip() == 'HELD'
157
+ t0 = time.time()
158
+ try:
159
+ lib._get_duckdb_storage_conn(cc)
160
+ raised = False
161
+ except lib.SharedCacheTakenError:
162
+ raised = True
163
+ assert raised and time.time() - t0 >= 1.5
164
+ assert lib._effective_cache_dir(cc) == os.path.abspath(cache)
165
+
166
+ df = pd.DataFrame({'symbol': ['SPY'], 'date': [pd.Timestamp('2024-01-02')], 'close': [470.0]})
167
+ t0 = time.time()
168
+ try:
169
+ lib._save_to_duckdb_storage(df, '/equities/eod/stock-prices', cc)
170
+ propagated = False
171
+ except lib.SharedCacheTakenError:
172
+ propagated = True
173
+ assert propagated and time.time() - t0 < 1.0
174
+
175
+ cm = lib.OptionsChunkManager(db_path=db, symbol='SPY', conn=None)
176
+ im = object.__new__(lib.DuckDBIndicatorManager)
177
+ im.db_path, im.conn = db, None
178
+ for reconnect in (cm._ensure_connection, im._connect):
179
+ t0 = time.time()
180
+ try:
181
+ reconnect()
182
+ reconnect_raised = False
183
+ except lib.SharedCacheTakenError:
184
+ reconnect_raised = True
185
+ assert reconnect_raised and time.time() - t0 < 1.0, reconnect
186
+
187
+ lib._SHARED_CACHE_USED.clear() # a fresh process: never opened the shared cache
188
+ lib._SHARED_CACHE_TAKEN.clear()
189
+ private = lib._get_duckdb_storage_conn(cc)
190
+ assert private is not None and lib._effective_cache_dir(cc) != os.path.abspath(cache)
191
+ lib.close_duckdb_storage()
192
+ finally:
193
+ open(go, 'w').close()
194
+ if holder is not None:
195
+ holder.wait(30)
196
+ lib._DUCKDB_PRIVATE_CACHE.pop(os.path.abspath(cache), None)
197
+ lib._SHARED_CACHE_USED.clear()
198
+ lib._SHARED_CACHE_TAKEN.clear()
199
+ if old_wait is None:
200
+ os.environ.pop('IVOL_CACHE_LOCK_WAIT_SEC', None)
201
+ else:
202
+ os.environ['IVOL_CACHE_LOCK_WAIT_SEC'] = old_wait
203
+ time.sleep(0.1)
204
+
205
+
206
+ def test_cleanup_releases_fds_of_deleted_cache_files():
207
+ live_path, live_conn = _fresh_db()
208
+ gone_path, gone_conn = _fresh_db()
209
+ gone_conn.close()
210
+ try:
211
+ lib._drop_file_pages(live_path)
212
+ lib._drop_file_pages(gone_path)
213
+ gone_fd = lib._PAGE_DROP_FDS[gone_path]
214
+ os.remove(gone_path)
215
+ lib.cleanup_duckdb_runtime({})
216
+ assert gone_path not in lib._PAGE_DROP_FDS and live_path in lib._PAGE_DROP_FDS
217
+ try:
218
+ assert os.fstat(gone_fd).st_nlink > 0
219
+ except OSError:
220
+ pass
221
+ assert _other_process_can_write(live_path) == 'LOCKED'
222
+ finally:
223
+ live_conn.close()
224
+
225
+
226
+ def test_preload_starts_each_run_with_a_fresh_wait():
227
+ tree = ast.parse(open(os.path.join(_LIBDIR, 'ivolatility_backtesting.py'), encoding='utf-8').read())
228
+ preload = next(f for f in ast.walk(tree) if isinstance(f, ast.FunctionDef) and f.name == 'preload_data_universal')
229
+ calls = {n.func.id for n in ast.walk(preload) if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)}
230
+ assert '_begin_run_cache_state' in calls
231
+
232
+
233
+ def test_next_run_waits_again_and_gets_the_shared_cache_when_free():
234
+ cache = tempfile.mkdtemp(prefix='ivb_cache_')
235
+ cc = {'cache_dir': cache}
236
+ db = os.path.join(cache, 'market_data.duckdb')
237
+ go = os.path.join(cache, 'go')
238
+ old_wait = os.environ.get('IVOL_CACHE_LOCK_WAIT_SEC')
239
+ os.environ['IVOL_CACHE_LOCK_WAIT_SEC'] = '2'
240
+ lib._SHARED_CACHE_USED.clear()
241
+ lib._SHARED_CACHE_TAKEN.clear()
242
+ holder = None
243
+ try:
244
+ assert lib._get_duckdb_storage_conn(cc) is not None
245
+ lib.close_duckdb_storage()
246
+ holder = subprocess.Popen([sys.executable, '-u', '-c', _HOLDER, db, go], stdout=subprocess.PIPE, text=True)
247
+ assert holder.stdout.readline().strip() == 'HELD'
248
+ for run in (1, 2):
249
+ lib._begin_run_cache_state()
250
+ out = io.StringIO()
251
+ t0 = time.time()
252
+ with contextlib.redirect_stdout(out):
253
+ try:
254
+ lib._get_duckdb_storage_conn(cc)
255
+ raised = False
256
+ except lib.SharedCacheTakenError:
257
+ raised = True
258
+ assert raised and time.time() - t0 >= 1.5, run
259
+ assert 'SHARED CACHE IS HELD BY ANOTHER RUN' in out.getvalue(), run
260
+ open(go, 'w').close()
261
+ holder.wait(30)
262
+ holder = None
263
+ lib._begin_run_cache_state()
264
+ conn = lib._get_duckdb_storage_conn(cc)
265
+ assert conn is not None and lib._effective_cache_dir(cc) == os.path.abspath(cache)
266
+ lib.close_duckdb_storage()
267
+ finally:
268
+ open(go, 'w').close()
269
+ if holder is not None:
270
+ holder.wait(30)
271
+ lib._DUCKDB_PRIVATE_CACHE.pop(os.path.abspath(cache), None)
272
+ lib._SHARED_CACHE_USED.clear()
273
+ lib._SHARED_CACHE_TAKEN.clear()
274
+ if old_wait is None:
275
+ os.environ.pop('IVOL_CACHE_LOCK_WAIT_SEC', None)
276
+ else:
277
+ os.environ['IVOL_CACHE_LOCK_WAIT_SEC'] = old_wait
278
+ time.sleep(0.1)
279
+
280
+
281
+ def test_error_is_exported_by_the_package():
282
+ out = subprocess.run([sys.executable, '-c',
283
+ 'from ivolatility_backtesting import SharedCacheTakenError as E; '
284
+ 'print(issubclass(E, BaseException) and not issubclass(E, Exception))'],
285
+ capture_output=True, text=True, timeout=120,
286
+ env=dict(os.environ, PYTHONPATH=_REPO, MPLBACKEND='Agg'))
287
+ assert out.stdout.strip().splitlines()[-1] == 'True', out.stdout[-500:] + out.stderr[-500:]
288
+
289
+
290
+ if __name__ == '__main__':
291
+ failed = 0
292
+ for name, fn in list(globals().items()):
293
+ if name.startswith('test_') and callable(fn):
294
+ try:
295
+ fn()
296
+ print(f'PASS {name}')
297
+ except Exception:
298
+ failed += 1
299
+ print(f'FAIL {name}')
300
+ traceback.print_exc()
301
+ sys.exit(1 if failed else 0)