ivolatility-backtesting 2.147__tar.gz → 2.148__tar.gz

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.148}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.147
+Version: 2.148
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.148}/ivolatility_backtesting/ivolatility_backtesting.py

@@ -518,9 +518,8 @@ from typing import Callable, Dict, List, Optional, Tuple, Union, Any
 # malloc_trim→250MB). malloc_trim(0) forces glibc to return free pages. Resolved
 # lazily once; a no-op on musl/macOS (returns None / raises → swallowed).
 _libc_malloc_trim = None
-def _release_freed_memory():
-    """gc.collect() + glibc malloc_trim(0) — actually shrinks RSS, unlike gc alone."""
-    gc.collect()
+def _trim_heap():
+    """glibc malloc_trim(0) without a gc pass — cheap enough to call after every chunk."""
     global _libc_malloc_trim
     if _libc_malloc_trim is None:
         try:
@@ -534,6 +533,67 @@ def _release_freed_memory():
         except Exception:
             pass
 
+
+def _release_freed_memory():
+    """gc.collect() + glibc malloc_trim(0) — actually shrinks RSS, unlike gc alone."""
+    gc.collect()
+    _trim_heap()
+
+
+import threading
+
+# Admission control by cgroup free memory; the first download always passes so a load never stalls.
+def _env_int(name, default):
+    try:
+        return int(os.environ.get(name, default))
+    except ValueError:
+        return default
+
+
+_API_MEM_EST_MB = _env_int('IVOL_API_MEM_EST_MB', 150)
+_API_MEM_SAFETY_MB = _env_int('IVOL_API_MEM_SAFETY_MB', 300)
+_api_mem_lock = threading.Lock()
+_api_mem_inflight = 0
+
+
+def _cgroup_free_mb():
+    """Container free memory in MB (limit minus usage excluding inactive file cache); None without a cgroup v2 limit."""
+    try:
+        with open('/sys/fs/cgroup/memory.max') as f:
+            raw = f.read().strip()
+        if raw == 'max':
+            return None
+        with open('/sys/fs/cgroup/memory.current') as f:
+            current = int(f.read())
+        inactive = 0
+        with open('/sys/fs/cgroup/memory.stat') as f:
+            for line in f:
+                if line.startswith('inactive_file '):
+                    inactive = int(line.split()[1])
+                    break
+        return (int(raw) - (current - inactive)) >> 20
+    except (OSError, ValueError):
+        return None
+
+
+def _api_mem_acquire():
+    """Block until the container can fit one more in-flight download."""
+    global _api_mem_inflight
+    while True:
+        with _api_mem_lock:
+            free_mb = _cgroup_free_mb()
+            if (free_mb is None or _api_mem_inflight == 0
+                    or free_mb - _api_mem_inflight * _API_MEM_EST_MB >= _API_MEM_EST_MB + _API_MEM_SAFETY_MB):
+                _api_mem_inflight += 1
+                return
+        time.sleep(0.5)
+
+
+def _api_mem_release():
+    global _api_mem_inflight
+    with _api_mem_lock:
+        _api_mem_inflight = max(0, _api_mem_inflight - 1)
+
 # Trim once at import: if the lib is re-imported into a kernel that already ran
 # heavy work, hand back any glibc arenas before this session allocates.
 _release_freed_memory()
@@ -7415,8 +7475,7 @@ class APIHelper:
             if response.empty:
                 return None
             
-            records = response.to_dict('records')
-            return {'data': records, 'status': 'success'}
+            return {'data': response, 'status': 'success', '_is_df': True}
         
         if debug:
             print(f"[APIHelper] Unexpected type: {type(response)}")
@@ -20193,23 +20252,28 @@ def _api_call_logged(endpoint, cache_config, skip_parquet_cache=False, debuginfo
     
     # Make API call with debug level
     # debuginfo: 0=silent, 1=basic, 2=detailed (URLs), 3=verbose timing
-    response = api_call(
-        endpoint,
-        cache_config,
-        debug=debuginfo,  # Pass integer directly
-        skip_parquet_cache=skip_parquet_cache,
-        _chunk_info=_chunk_info,
-        **normalized_params
-    )
-    
+    _api_mem_acquire()
+    try:
+        response = api_call(
+            endpoint,
+            cache_config,
+            debug=debuginfo,  # Pass integer directly
+            skip_parquet_cache=skip_parquet_cache,
+            _chunk_info=_chunk_info,
+            **normalized_params
+        )
+    finally:
+        _api_mem_release()
+
     # Parse response
     if response is None:
         return None
-    
+
     if isinstance(response, pd.DataFrame):
         return response if not response.empty else None
     elif isinstance(response, dict) and 'data' in response:
-        df = pd.DataFrame(response['data'])
+        data = response['data']
+        df = data if isinstance(data, pd.DataFrame) else pd.DataFrame(data)
         return df if not df.empty else None
     
     return None
@@ -20308,16 +20372,14 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
         print(f"           ⚡ PARALLEL mode: {total_requests} requests with {max_workers} workers")
     
     all_data = []
+    # Legacy (pandas-indicator) mode still returns the sample if reading DuckDB back fails.
+    _keep_sample = not config.get('use_duckdb_indicators', True)
     total_rows = 0
     api_debuginfo = debuginfo
     # Resolve snapshot-aware endpoint ONCE for both fetch and save
     _save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
     
-    # MEMFIX(backpressure): воркеры качают быстрее, чем главный поток успевает
-    # писать чанк в DuckDB (запись ~3с), поэтому готовые, но ещё не обработанные
-    # DataFrame копились в памяти. Семафор держит очередь готовых результатов
-    # ограниченной: воркер не отдаёт результат, пока главный поток не разгрёб
-    # предыдущие. Это ограничивает пик числом воркеров, а не длиной периода.
+    # Caps downloaded-but-not-yet-written chunks; each loop must release after writing a non-empty chunk.
     import threading as _mf_thr
     _mf_inflight = _mf_thr.Semaphore(max(2, max_workers))
 
@@ -20438,29 +20500,27 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                             _safe_print(f"           ⏱️ [SLOW DUCKDB WRITE] {_db_elapsed:.1f}s for {len(df)} rows (chunk {completed})")
                         rows_saved_total += rows_saved
                         total_rows += len(df)
-                        
-                        if len(all_data) < 5:
+
+                        # Sample is only a fallback for a failed DuckDB write.
+                        if rows_saved_total > 0 and not _keep_sample:
+                            all_data.clear()
+                        elif len(all_data) < 5:
                             all_data.append(df)
-                    
+
                     if df is not None and not getattr(df, 'empty', True):
                         try:
                             _mf_inflight.release()
                         except Exception:
                             pass
-                    # MEMFIX: release the chunk once it is persisted — the Future keeps a
-                    # reference to its result until the executor block exits, so without this
-                    # the whole fetch accumulates in RAM regardless of DuckDB writes.
+                    # The Future holds its result until the executor exits; drop it once persisted.
                     try:
                         future._result = None
                     except Exception:
                         pass
                     df = None
+                    _trim_heap()
                     if completed % 25 == 0:
-                        # MEMFIX(page-cache): страницы записанного .duckdb оседают в file-кэше
-                        # cgroup и подтягивают memory.current к лимиту. Кэш вытесняемый, но
-                        # держит счётчик у потолка и сокращает буфер до OOM. Сбрасываем его:
-                        # сначала fdatasync (грязные страницы иначе не освободить), затем
-                        # POSIX_FADV_DONTNEED на файл БД и его WAL.
+                        # Written .duckdb pages count toward the cgroup limit: fdatasync, then drop them from page cache.
                         try:
                             import os as _os4
                             _dbp = None
@@ -20487,13 +20547,7 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                                     pass
                         except Exception:
                             pass
-                        import gc as _gc
-                        _gc.collect()
-                        try:
-                            import ctypes as _ct
-                            _ct.CDLL("libc.so.6").malloc_trim(0)
-                        except Exception:
-                            pass
+                        _release_freed_memory()
 
                     _watchdog_completed[0] = completed
                     if completed % 10 == 0 or completed == total_requests:
@@ -20534,11 +20588,16 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                 )
                 rows_saved_total += rows_saved
                 total_rows += len(df)
-                
-                # Keep small sample for return
-                if len(all_data) < 5:
+
+                # Sample is only a fallback for a failed DuckDB write.
+                if rows_saved_total > 0 and not _keep_sample:
+                    all_data.clear()
+                elif len(all_data) < 5:
                     all_data.append(df)
-            
+                _mf_inflight.release()
+            df = None
+            _trim_heap()
+
             # Progress summary every 10 requests
             if (i + 1) % 10 == 0 or i == len(all_requests) - 1:
                 print(f"           📦 Request {i + 1}/{total_requests}: {total_rows:,} rows so far")

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.148}/ivolatility_backtesting.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.147
+Version: 2.148
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.148}/ivolatility_backtesting.egg-info/SOURCES.txt

@@ -11,4 +11,5 @@ ivolatility_backtesting.egg-info/top_level.txt
 tests/test_2142_fixes.py
 tests/test_2144_duckdb_dedup.py
 tests/test_2146_vix_vro.py
-tests/test_2147_cents_multiplier.py
+tests/test_2147_cents_multiplier.py
+tests/test_2148_loader_memory.py

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.148}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ivolatility_backtesting"
-version = "2.147"
+version = "2.148"
 description = "A universal backtesting framework for financial strategies using the IVolatility API."
 readme = "README.md"
 authors = [

ivolatility_backtesting-2.148/tests/test_2148_loader_memory.py

@@ -0,0 +1,223 @@
+# Loader memory: DataFrame passthrough, download admission by cgroup free memory, heap trim.
+# Run: python3 tests/test_2148_loader_memory.py  (or pytest)
+import os
+import sys
+import threading
+import time
+import traceback
+
+import matplotlib
+matplotlib.use('Agg')
+
+_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(_REPO, 'ivolatility_backtesting'))
+import ivolatility_backtesting as lib  # noqa: E402
+
+import pandas as pd  # noqa: E402
+
+
+def _chain(n=3):
+    return pd.DataFrame({
+        'date': ['2024-01-02'] * n,
+        'expiration': ['2024-01-19'] * n,
+        'strike': [100.0 + i for i in range(n)],
+        'Call/Put': ['C'] * n,
+        'bid': [1.0 + i for i in range(n)],
+        'ask': [1.1 + i for i in range(n)],
+    })
+
+
+def test_normalize_keeps_same_dataframe():
+    df = _chain()
+    out = lib.APIHelper.normalize_response(df)
+    assert out['status'] == 'success'
+    assert out['_is_df'] is True
+    assert out['data'] is df
+
+
+def test_normalize_empty_dataframe_is_none():
+    assert lib.APIHelper.normalize_response(pd.DataFrame()) is None
+
+
+def test_normalize_dict_passthrough_unchanged():
+    resp = {'data': [{'a': 1}], 'status': 'success'}
+    assert lib.APIHelper.normalize_response(resp) is resp
+
+
+def test_api_call_logged_returns_frame_without_copy():
+    df = _chain()
+    orig = lib.api_call
+    try:
+        lib.api_call = lambda *a, **k: {'data': df, 'status': 'success', '_is_df': True}
+        got = lib._api_call_logged('/equities/eod/options-rawiv', {}, symbol='SPY',
+                                   from_='2024-01-02', to='2024-01-02')
+        assert got is df
+        lib.api_call = lambda *a, **k: {'data': df.to_dict('records'), 'status': 'success'}
+        got = lib._api_call_logged('/equities/eod/options-rawiv', {}, symbol='SPY',
+                                   from_='2024-01-02', to='2024-01-02')
+        pd.testing.assert_frame_equal(got, df)
+        lib.api_call = lambda *a, **k: {'data': pd.DataFrame(), 'status': 'success', '_is_df': True}
+        assert lib._api_call_logged('/equities/eod/options-rawiv', {}, symbol='SPY') is None
+    finally:
+        lib.api_call = orig
+    assert lib._api_mem_inflight == 0
+
+
+def test_api_call_logged_releases_slot_on_error():
+    orig = lib.api_call
+
+    def boom(*a, **k):
+        raise RuntimeError('network')
+    try:
+        lib.api_call = boom
+        try:
+            lib._api_call_logged('/equities/eod/options-rawiv', {}, symbol='SPY')
+        except RuntimeError:
+            pass
+    finally:
+        lib.api_call = orig
+    assert lib._api_mem_inflight == 0
+
+
+def test_gate_first_download_always_passes():
+    orig = lib._cgroup_free_mb
+    try:
+        lib._cgroup_free_mb = lambda: 0
+        lib._api_mem_acquire()
+        assert lib._api_mem_inflight == 1
+        lib._api_mem_release()
+        assert lib._api_mem_inflight == 0
+    finally:
+        lib._cgroup_free_mb = orig
+
+
+def test_gate_waits_for_memory_then_admits():
+    free = {'mb': 400}
+    orig = lib._cgroup_free_mb
+    try:
+        lib._cgroup_free_mb = lambda: free['mb']
+        lib._api_mem_acquire()
+        t = threading.Thread(target=lib._api_mem_acquire, daemon=True)
+        t.start()
+        t.join(1.2)
+        assert t.is_alive(), 'second download must wait: 400 - 150 < 150 + 300'
+        assert lib._api_mem_inflight == 1
+        free['mb'] = 2000
+        t.join(2.0)
+        assert not t.is_alive()
+        assert lib._api_mem_inflight == 2
+    finally:
+        lib._cgroup_free_mb = orig
+        lib._api_mem_release()
+        lib._api_mem_release()
+    assert lib._api_mem_inflight == 0
+
+
+def test_gate_reserves_per_inflight_download():
+    orig = lib._cgroup_free_mb
+    try:
+        # 900 free: 1st passes (inflight 0), 2nd 900-150=750 >= 450, 3rd 900-300=600 >= 450,
+        # 4th 900-450=450 >= 450, 5th 900-600=300 < 450 → waits.
+        lib._cgroup_free_mb = lambda: 900
+        for _ in range(4):
+            lib._api_mem_acquire()
+        assert lib._api_mem_inflight == 4
+        t = threading.Thread(target=lib._api_mem_acquire, daemon=True)
+        t.start()
+        t.join(1.2)
+        assert t.is_alive()
+        lib._api_mem_release()
+        t.join(2.0)
+        assert not t.is_alive()
+    finally:
+        lib._cgroup_free_mb = orig
+        for _ in range(4):
+            lib._api_mem_release()
+    assert lib._api_mem_inflight == 0
+
+
+def test_gate_is_noop_without_cgroup_limit():
+    orig = lib._cgroup_free_mb
+    try:
+        lib._cgroup_free_mb = lambda: None
+        start = time.time()
+        for _ in range(8):
+            lib._api_mem_acquire()
+        assert time.time() - start < 0.5
+    finally:
+        lib._cgroup_free_mb = orig
+        for _ in range(8):
+            lib._api_mem_release()
+    assert lib._api_mem_inflight == 0
+
+
+def test_release_never_goes_negative():
+    lib._api_mem_release()
+    assert lib._api_mem_inflight == 0
+
+
+def test_cgroup_free_mb_type():
+    v = lib._cgroup_free_mb()
+    assert v is None or isinstance(v, int)
+
+
+def _run_loader(config, calls, timeout=20):
+    saved = []
+    orig_call, orig_save = lib._api_call_logged, lib._save_to_duckdb_storage
+
+    def fake_call(endpoint, cache_config, **params):
+        calls.append(params['cp'])
+        return _chain(4)
+
+    def fake_save(df, endpoint, cache_config, debug=False):
+        saved.append(len(df))
+        return len(df)
+    box = {}
+    try:
+        lib._api_call_logged, lib._save_to_duckdb_storage = fake_call, fake_save
+        t = threading.Thread(target=lambda: box.setdefault('r', lib._load_options_to_duckdb(
+            config, {}, 'SPY', '2024-01-01', '2024-01-30')), daemon=True)
+        t.start()
+        t.join(timeout)
+        return (not t.is_alive()), saved
+    finally:
+        lib._api_call_logged, lib._save_to_duckdb_storage = orig_call, orig_save
+
+
+def test_sequential_loader_does_not_stall():
+    # 6 chunks x C/P = 12 requests > semaphore size max(2, workers=2)
+    cfg = {'strategy_type': 'STRADDLE', 'chunk_days_options': 5, 'parallel_mode': False,
+           'parallel_max_workers': 2, 'debuginfo': 0}
+    calls = []
+    finished, saved = _run_loader(cfg, calls)
+    assert finished, f'sequential load stalled after {len(calls)} requests'
+    assert len(calls) == 12 and saved == [4] * 12
+
+
+def test_parallel_loader_saves_every_chunk():
+    cfg = {'strategy_type': 'STRADDLE', 'chunk_days_options': 5, 'parallel_mode': True,
+           'parallel_max_workers': 3, 'debuginfo': 0}
+    calls = []
+    finished, saved = _run_loader(cfg, calls)
+    assert finished
+    assert len(calls) == 12 and saved == [4] * 12
+
+
+def test_trim_heap_safe_everywhere():
+    lib._trim_heap()
+    lib._trim_heap()
+    lib._release_freed_memory()
+
+
+if __name__ == '__main__':
+    failed = 0
+    for name, fn in sorted((n, f) for n, f in globals().items() if n.startswith('test_') and callable(f)):
+        try:
+            fn()
+            print(f'PASS {name}')
+        except Exception:
+            failed += 1
+            print(f'FAIL {name}')
+            traceback.print_exc()
+    print(f'{failed} failed')
+    sys.exit(1 if failed else 0)