ivolatility-backtesting 2.147__tar.gz → 2.149__tar.gz

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.149}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.147
+Version: 2.149
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com
@@ -19,7 +19,7 @@ Requires-Dist: pandas<3.0,>=1.5.0
 Requires-Dist: numpy<3.0,>=1.21.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: seaborn>=0.11.0
-Requires-Dist: ivolatility>=1.8.2
+Requires-Dist: ivolatility>=1.9.6
 Requires-Dist: psutil>=7.1.0
 
 # IVolatility Backtesting Framework v1.32

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.149}/ivolatility_backtesting/ivolatility_backtesting.py

@@ -40,13 +40,24 @@ import os
 # Distinctive User-Agent on ALL REST traffic: lets the gateway/Graylog tell
 # framework calls apart from hand-written scripts, and workspace runs from
 # local pip installs (CODER_AGENT_URL is set inside Coder workspaces only).
+def _ivb_client():
+    if os.getenv('IVOL_CLIENT'):
+        return os.getenv('IVOL_CLIENT')
+    if os.getenv('CODER_AGENT_URL'):
+        return 'workspace'
+    # venv created by the IVolAI plugin (ivolai-dist setup_env.sh); pods use /home/coder/.venv
+    if os.path.normpath(__import__('sys').prefix).endswith(os.path.join('ivolatility', '.venv')):
+        return 'plugin'
+    return 'local'
+
+
 def _ivb_user_agent():
     try:
         from importlib.metadata import version as _pkg_version
         _ver = _pkg_version('ivolatility_backtesting')
     except Exception:
         _ver = 'dev'
-    _env = 'workspace' if os.getenv('CODER_AGENT_URL') else 'local'
+    _env = _ivb_client()
     try:
         import requests as _rq
         _rq_ver = _rq.__version__
@@ -518,9 +529,8 @@ from typing import Callable, Dict, List, Optional, Tuple, Union, Any
 # malloc_trim→250MB). malloc_trim(0) forces glibc to return free pages. Resolved
 # lazily once; a no-op on musl/macOS (returns None / raises → swallowed).
 _libc_malloc_trim = None
-def _release_freed_memory():
-    """gc.collect() + glibc malloc_trim(0) — actually shrinks RSS, unlike gc alone."""
-    gc.collect()
+def _trim_heap():
+    """glibc malloc_trim(0) without a gc pass — cheap enough to call after every chunk."""
     global _libc_malloc_trim
     if _libc_malloc_trim is None:
         try:
@@ -534,6 +544,67 @@ def _release_freed_memory():
         except Exception:
             pass
 
+
+def _release_freed_memory():
+    """gc.collect() + glibc malloc_trim(0) — actually shrinks RSS, unlike gc alone."""
+    gc.collect()
+    _trim_heap()
+
+
+import threading
+
+# Admission control by cgroup free memory; the first download always passes so a load never stalls.
+def _env_int(name, default):
+    try:
+        return int(os.environ.get(name, default))
+    except ValueError:
+        return default
+
+
+_API_MEM_EST_MB = _env_int('IVOL_API_MEM_EST_MB', 150)
+_API_MEM_SAFETY_MB = _env_int('IVOL_API_MEM_SAFETY_MB', 300)
+_api_mem_lock = threading.Lock()
+_api_mem_inflight = 0
+
+
+def _cgroup_free_mb():
+    """Container free memory in MB (limit minus usage excluding inactive file cache); None without a cgroup v2 limit."""
+    try:
+        with open('/sys/fs/cgroup/memory.max') as f:
+            raw = f.read().strip()
+        if raw == 'max':
+            return None
+        with open('/sys/fs/cgroup/memory.current') as f:
+            current = int(f.read())
+        inactive = 0
+        with open('/sys/fs/cgroup/memory.stat') as f:
+            for line in f:
+                if line.startswith('inactive_file '):
+                    inactive = int(line.split()[1])
+                    break
+        return (int(raw) - (current - inactive)) >> 20
+    except (OSError, ValueError):
+        return None
+
+
+def _api_mem_acquire():
+    """Block until the container can fit one more in-flight download."""
+    global _api_mem_inflight
+    while True:
+        with _api_mem_lock:
+            free_mb = _cgroup_free_mb()
+            if (free_mb is None or _api_mem_inflight == 0
+                    or free_mb - _api_mem_inflight * _API_MEM_EST_MB >= _API_MEM_EST_MB + _API_MEM_SAFETY_MB):
+                _api_mem_inflight += 1
+                return
+        time.sleep(0.5)
+
+
+def _api_mem_release():
+    global _api_mem_inflight
+    with _api_mem_lock:
+        _api_mem_inflight = max(0, _api_mem_inflight - 1)
+
 # Trim once at import: if the lib is re-imported into a kernel that already ran
 # heavy work, hand back any glibc arenas before this session allocates.
 _release_freed_memory()
@@ -1733,8 +1804,10 @@ def _create_sdk_like_session():
     from urllib3.util.retry import Retry
     from requests.adapters import HTTPAdapter
     s = _req.Session()
-    retry = Retry(total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504],
-                  allowed_methods=["GET", "POST", "DELETE"])
+    # SDK >= 1.9.6 pauses before the first retry; reuse it so thread sessions behave the same
+    retry = getattr(_ivol_inner, '_retry_strategy', None) or Retry(
+        total=3, backoff_factor=1, status_forcelist=[429, 500, 502, 503, 504],
+        allowed_methods=["GET", "POST", "DELETE"])
     adapter = HTTPAdapter(pool_connections=10, pool_maxsize=10, max_retries=retry)
     s.mount("https://", adapter)
     s.mount("http://", adapter)
@@ -7415,8 +7488,7 @@ class APIHelper:
             if response.empty:
                 return None
             
-            records = response.to_dict('records')
-            return {'data': records, 'status': 'success'}
+            return {'data': response, 'status': 'success', '_is_df': True}
         
         if debug:
             print(f"[APIHelper] Unexpected type: {type(response)}")
@@ -7763,8 +7835,9 @@ def _api_call_direct(endpoint: str, max_retries: int = 3, **kwargs) -> Optional[
                 return None
             df = data if isinstance(data, pd.DataFrame) else pd.DataFrame(data)
             return df if not df.empty else None
-        except RETRYABLE:
-            if attempt == max_retries - 1:
+        except RETRYABLE as e:
+            if _is_client_error(e) or attempt == max_retries - 1:
+                _safe_print(f"   ⚠️ {endpoint}: {e}")
                 return None
             continue
         except Exception:
@@ -7772,6 +7845,12 @@ def _api_call_direct(endpoint: str, max_retries: int = 3, **kwargs) -> Optional[
     return None
 
 
+def _is_client_error(exc):
+    """4xx other than 429: the same request cannot succeed on retry."""
+    status = getattr(getattr(exc, 'response', None), 'status_code', None)
+    return status is not None and 400 <= status < 500 and status != 429
+
+
 def api_call(endpoint, cache_config=None, debug=False, max_retries=3, skip_parquet_cache=False, 
              _chunk_info=None, **kwargs):
     """
@@ -7868,6 +7947,8 @@ def api_call(endpoint, cache_config=None, debug=False, max_retries=3, skip_parqu
         
         except RETRYABLE_ERRORS as e:
             last_exception = e
+            if _is_client_error(e):
+                raise
             if debug_level >= 1:
                 print(f"[RETRY] Network error on attempt {attempt + 1}/{max_retries}: {type(e).__name__}: {e}")
             
@@ -20193,23 +20274,28 @@ def _api_call_logged(endpoint, cache_config, skip_parquet_cache=False, debuginfo
     
     # Make API call with debug level
     # debuginfo: 0=silent, 1=basic, 2=detailed (URLs), 3=verbose timing
-    response = api_call(
-        endpoint,
-        cache_config,
-        debug=debuginfo,  # Pass integer directly
-        skip_parquet_cache=skip_parquet_cache,
-        _chunk_info=_chunk_info,
-        **normalized_params
-    )
-    
+    _api_mem_acquire()
+    try:
+        response = api_call(
+            endpoint,
+            cache_config,
+            debug=debuginfo,  # Pass integer directly
+            skip_parquet_cache=skip_parquet_cache,
+            _chunk_info=_chunk_info,
+            **normalized_params
+        )
+    finally:
+        _api_mem_release()
+
     # Parse response
     if response is None:
         return None
-    
+
     if isinstance(response, pd.DataFrame):
         return response if not response.empty else None
     elif isinstance(response, dict) and 'data' in response:
-        df = pd.DataFrame(response['data'])
+        data = response['data']
+        df = data if isinstance(data, pd.DataFrame) else pd.DataFrame(data)
         return df if not df.empty else None
     
     return None
@@ -20308,16 +20394,14 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
         print(f"           ⚡ PARALLEL mode: {total_requests} requests with {max_workers} workers")
     
     all_data = []
+    # Legacy (pandas-indicator) mode still returns the sample if reading DuckDB back fails.
+    _keep_sample = not config.get('use_duckdb_indicators', True)
     total_rows = 0
     api_debuginfo = debuginfo
     # Resolve snapshot-aware endpoint ONCE for both fetch and save
     _save_ep = _get_options_endpoints(_get_options_snapshot_mode(config))['filtered']
     
-    # MEMFIX(backpressure): воркеры качают быстрее, чем главный поток успевает
-    # писать чанк в DuckDB (запись ~3с), поэтому готовые, но ещё не обработанные
-    # DataFrame копились в памяти. Семафор держит очередь готовых результатов
-    # ограниченной: воркер не отдаёт результат, пока главный поток не разгрёб
-    # предыдущие. Это ограничивает пик числом воркеров, а не длиной периода.
+    # Caps downloaded-but-not-yet-written chunks; each loop must release after writing a non-empty chunk.
     import threading as _mf_thr
     _mf_inflight = _mf_thr.Semaphore(max(2, max_workers))
 
@@ -20438,29 +20522,27 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                             _safe_print(f"           ⏱️ [SLOW DUCKDB WRITE] {_db_elapsed:.1f}s for {len(df)} rows (chunk {completed})")
                         rows_saved_total += rows_saved
                         total_rows += len(df)
-                        
-                        if len(all_data) < 5:
+
+                        # Sample is only a fallback for a failed DuckDB write.
+                        if rows_saved_total > 0 and not _keep_sample:
+                            all_data.clear()
+                        elif len(all_data) < 5:
                             all_data.append(df)
-                    
+
                     if df is not None and not getattr(df, 'empty', True):
                         try:
                             _mf_inflight.release()
                         except Exception:
                             pass
-                    # MEMFIX: release the chunk once it is persisted — the Future keeps a
-                    # reference to its result until the executor block exits, so without this
-                    # the whole fetch accumulates in RAM regardless of DuckDB writes.
+                    # The Future holds its result until the executor exits; drop it once persisted.
                     try:
                         future._result = None
                     except Exception:
                         pass
                     df = None
+                    _trim_heap()
                     if completed % 25 == 0:
-                        # MEMFIX(page-cache): страницы записанного .duckdb оседают в file-кэше
-                        # cgroup и подтягивают memory.current к лимиту. Кэш вытесняемый, но
-                        # держит счётчик у потолка и сокращает буфер до OOM. Сбрасываем его:
-                        # сначала fdatasync (грязные страницы иначе не освободить), затем
-                        # POSIX_FADV_DONTNEED на файл БД и его WAL.
+                        # Written .duckdb pages count toward the cgroup limit: fdatasync, then drop them from page cache.
                         try:
                             import os as _os4
                             _dbp = None
@@ -20487,13 +20569,7 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                                     pass
                         except Exception:
                             pass
-                        import gc as _gc
-                        _gc.collect()
-                        try:
-                            import ctypes as _ct
-                            _ct.CDLL("libc.so.6").malloc_trim(0)
-                        except Exception:
-                            pass
+                        _release_freed_memory()
 
                     _watchdog_completed[0] = completed
                     if completed % 10 == 0 or completed == total_requests:
@@ -20534,11 +20610,16 @@ def _load_options_to_duckdb(config, cache_config, symbol, start_date, end_date):
                 )
                 rows_saved_total += rows_saved
                 total_rows += len(df)
-                
-                # Keep small sample for return
-                if len(all_data) < 5:
+
+                # Sample is only a fallback for a failed DuckDB write.
+                if rows_saved_total > 0 and not _keep_sample:
+                    all_data.clear()
+                elif len(all_data) < 5:
                     all_data.append(df)
-            
+                _mf_inflight.release()
+            df = None
+            _trim_heap()
+
             # Progress summary every 10 requests
             if (i + 1) % 10 == 0 or i == len(all_requests) - 1:
                 print(f"           📦 Request {i + 1}/{total_requests}: {total_rows:,} rows so far")
@@ -21794,8 +21875,11 @@ def _load_futures_for_hedge(config, preloaded):
                 if r.status_code == 200:
                     data = r.json().get('data', [])
                     all_rows.extend(data)
-            except Exception:
-                pass
+                else:
+                    print(f"           ⚠️  futures/eod/prices {fut_root} {current:%Y-%m-%d}..{chunk_end:%Y-%m-%d}: "
+                          f"HTTP {r.status_code} {r.text[:300]}")
+            except Exception as e:
+                print(f"           ⚠️  futures/eod/prices {fut_root} {current:%Y-%m-%d}: {e}")
             current = chunk_end + _td(days=1)
 
         if all_rows:

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.149}/ivolatility_backtesting.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ivolatility_backtesting
-Version: 2.147
+Version: 2.149
 Summary: A universal backtesting framework for financial strategies using the IVolatility API.
 Author-email: IVolatility <support@ivolatility.com>
 Project-URL: Homepage, https://ivolatility.com
@@ -19,7 +19,7 @@ Requires-Dist: pandas<3.0,>=1.5.0
 Requires-Dist: numpy<3.0,>=1.21.0
 Requires-Dist: matplotlib>=3.5.0
 Requires-Dist: seaborn>=0.11.0
-Requires-Dist: ivolatility>=1.8.2
+Requires-Dist: ivolatility>=1.9.6
 Requires-Dist: psutil>=7.1.0
 
 # IVolatility Backtesting Framework v1.32

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.149}/ivolatility_backtesting.egg-info/SOURCES.txt

@@ -11,4 +11,6 @@ ivolatility_backtesting.egg-info/top_level.txt
 tests/test_2142_fixes.py
 tests/test_2144_duckdb_dedup.py
 tests/test_2146_vix_vro.py
-tests/test_2147_cents_multiplier.py
+tests/test_2147_cents_multiplier.py
+tests/test_2148_loader_memory.py
+tests/test_2149_client_errors.py

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.149}/ivolatility_backtesting.egg-info/requires.txt

@@ -2,5 +2,5 @@ pandas<3.0,>=1.5.0
 numpy<3.0,>=1.21.0
 matplotlib>=3.5.0
 seaborn>=0.11.0
-ivolatility>=1.8.2
+ivolatility>=1.9.6
 psutil>=7.1.0

{ivolatility_backtesting-2.147 → ivolatility_backtesting-2.149}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "ivolatility_backtesting"
-version = "2.147"
+version = "2.149"
 description = "A universal backtesting framework for financial strategies using the IVolatility API."
 readme = "README.md"
 authors = [
@@ -17,7 +17,7 @@ dependencies = [
     "numpy>=1.21.0,<3.0",
     "matplotlib>=3.5.0",
     "seaborn>=0.11.0",
-    "ivolatility>=1.8.2",
+    "ivolatility>=1.9.6",
     "psutil>=7.1.0",
 ]
 keywords = ["backtesting", "finance", "trading", "ivolatility"]

ivolatility_backtesting-2.149/tests/test_2148_loader_memory.py

@@ -0,0 +1,223 @@
+# Loader memory: DataFrame passthrough, download admission by cgroup free memory, heap trim.
+# Run: python3 tests/test_2148_loader_memory.py  (or pytest)
+import os
+import sys
+import threading
+import time
+import traceback
+
+import matplotlib
+matplotlib.use('Agg')
+
+_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(_REPO, 'ivolatility_backtesting'))
+import ivolatility_backtesting as lib  # noqa: E402
+
+import pandas as pd  # noqa: E402
+
+
+def _chain(n=3):
+    return pd.DataFrame({
+        'date': ['2024-01-02'] * n,
+        'expiration': ['2024-01-19'] * n,
+        'strike': [100.0 + i for i in range(n)],
+        'Call/Put': ['C'] * n,
+        'bid': [1.0 + i for i in range(n)],
+        'ask': [1.1 + i for i in range(n)],
+    })
+
+
+def test_normalize_keeps_same_dataframe():
+    df = _chain()
+    out = lib.APIHelper.normalize_response(df)
+    assert out['status'] == 'success'
+    assert out['_is_df'] is True
+    assert out['data'] is df
+
+
+def test_normalize_empty_dataframe_is_none():
+    assert lib.APIHelper.normalize_response(pd.DataFrame()) is None
+
+
+def test_normalize_dict_passthrough_unchanged():
+    resp = {'data': [{'a': 1}], 'status': 'success'}
+    assert lib.APIHelper.normalize_response(resp) is resp
+
+
+def test_api_call_logged_returns_frame_without_copy():
+    df = _chain()
+    orig = lib.api_call
+    try:
+        lib.api_call = lambda *a, **k: {'data': df, 'status': 'success', '_is_df': True}
+        got = lib._api_call_logged('/equities/eod/options-rawiv', {}, symbol='SPY',
+                                   from_='2024-01-02', to='2024-01-02')
+        assert got is df
+        lib.api_call = lambda *a, **k: {'data': df.to_dict('records'), 'status': 'success'}
+        got = lib._api_call_logged('/equities/eod/options-rawiv', {}, symbol='SPY',
+                                   from_='2024-01-02', to='2024-01-02')
+        pd.testing.assert_frame_equal(got, df)
+        lib.api_call = lambda *a, **k: {'data': pd.DataFrame(), 'status': 'success', '_is_df': True}
+        assert lib._api_call_logged('/equities/eod/options-rawiv', {}, symbol='SPY') is None
+    finally:
+        lib.api_call = orig
+    assert lib._api_mem_inflight == 0
+
+
+def test_api_call_logged_releases_slot_on_error():
+    orig = lib.api_call
+
+    def boom(*a, **k):
+        raise RuntimeError('network')
+    try:
+        lib.api_call = boom
+        try:
+            lib._api_call_logged('/equities/eod/options-rawiv', {}, symbol='SPY')
+        except RuntimeError:
+            pass
+    finally:
+        lib.api_call = orig
+    assert lib._api_mem_inflight == 0
+
+
+def test_gate_first_download_always_passes():
+    orig = lib._cgroup_free_mb
+    try:
+        lib._cgroup_free_mb = lambda: 0
+        lib._api_mem_acquire()
+        assert lib._api_mem_inflight == 1
+        lib._api_mem_release()
+        assert lib._api_mem_inflight == 0
+    finally:
+        lib._cgroup_free_mb = orig
+
+
+def test_gate_waits_for_memory_then_admits():
+    free = {'mb': 400}
+    orig = lib._cgroup_free_mb
+    try:
+        lib._cgroup_free_mb = lambda: free['mb']
+        lib._api_mem_acquire()
+        t = threading.Thread(target=lib._api_mem_acquire, daemon=True)
+        t.start()
+        t.join(1.2)
+        assert t.is_alive(), 'second download must wait: 400 - 150 < 150 + 300'
+        assert lib._api_mem_inflight == 1
+        free['mb'] = 2000
+        t.join(2.0)
+        assert not t.is_alive()
+        assert lib._api_mem_inflight == 2
+    finally:
+        lib._cgroup_free_mb = orig
+        lib._api_mem_release()
+        lib._api_mem_release()
+    assert lib._api_mem_inflight == 0
+
+
+def test_gate_reserves_per_inflight_download():
+    orig = lib._cgroup_free_mb
+    try:
+        # 900 free: 1st passes (inflight 0), 2nd 900-150=750 >= 450, 3rd 900-300=600 >= 450,
+        # 4th 900-450=450 >= 450, 5th 900-600=300 < 450 → waits.
+        lib._cgroup_free_mb = lambda: 900
+        for _ in range(4):
+            lib._api_mem_acquire()
+        assert lib._api_mem_inflight == 4
+        t = threading.Thread(target=lib._api_mem_acquire, daemon=True)
+        t.start()
+        t.join(1.2)
+        assert t.is_alive()
+        lib._api_mem_release()
+        t.join(2.0)
+        assert not t.is_alive()
+    finally:
+        lib._cgroup_free_mb = orig
+        for _ in range(4):
+            lib._api_mem_release()
+    assert lib._api_mem_inflight == 0
+
+
+def test_gate_is_noop_without_cgroup_limit():
+    orig = lib._cgroup_free_mb
+    try:
+        lib._cgroup_free_mb = lambda: None
+        start = time.time()
+        for _ in range(8):
+            lib._api_mem_acquire()
+        assert time.time() - start < 0.5
+    finally:
+        lib._cgroup_free_mb = orig
+        for _ in range(8):
+            lib._api_mem_release()
+    assert lib._api_mem_inflight == 0
+
+
+def test_release_never_goes_negative():
+    lib._api_mem_release()
+    assert lib._api_mem_inflight == 0
+
+
+def test_cgroup_free_mb_type():
+    v = lib._cgroup_free_mb()
+    assert v is None or isinstance(v, int)
+
+
+def _run_loader(config, calls, timeout=20):
+    saved = []
+    orig_call, orig_save = lib._api_call_logged, lib._save_to_duckdb_storage
+
+    def fake_call(endpoint, cache_config, **params):
+        calls.append(params['cp'])
+        return _chain(4)
+
+    def fake_save(df, endpoint, cache_config, debug=False):
+        saved.append(len(df))
+        return len(df)
+    box = {}
+    try:
+        lib._api_call_logged, lib._save_to_duckdb_storage = fake_call, fake_save
+        t = threading.Thread(target=lambda: box.setdefault('r', lib._load_options_to_duckdb(
+            config, {}, 'SPY', '2024-01-01', '2024-01-30')), daemon=True)
+        t.start()
+        t.join(timeout)
+        return (not t.is_alive()), saved
+    finally:
+        lib._api_call_logged, lib._save_to_duckdb_storage = orig_call, orig_save
+
+
+def test_sequential_loader_does_not_stall():
+    # 6 chunks x C/P = 12 requests > semaphore size max(2, workers=2)
+    cfg = {'strategy_type': 'STRADDLE', 'chunk_days_options': 5, 'parallel_mode': False,
+           'parallel_max_workers': 2, 'debuginfo': 0}
+    calls = []
+    finished, saved = _run_loader(cfg, calls)
+    assert finished, f'sequential load stalled after {len(calls)} requests'
+    assert len(calls) == 12 and saved == [4] * 12
+
+
+def test_parallel_loader_saves_every_chunk():
+    cfg = {'strategy_type': 'STRADDLE', 'chunk_days_options': 5, 'parallel_mode': True,
+           'parallel_max_workers': 3, 'debuginfo': 0}
+    calls = []
+    finished, saved = _run_loader(cfg, calls)
+    assert finished
+    assert len(calls) == 12 and saved == [4] * 12
+
+
+def test_trim_heap_safe_everywhere():
+    lib._trim_heap()
+    lib._trim_heap()
+    lib._release_freed_memory()
+
+
+if __name__ == '__main__':
+    failed = 0
+    for name, fn in sorted((n, f) for n, f in globals().items() if n.startswith('test_') and callable(f)):
+        try:
+            fn()
+            print(f'PASS {name}')
+        except Exception:
+            failed += 1
+            print(f'FAIL {name}')
+            traceback.print_exc()
+    print(f'{failed} failed')
+    sys.exit(1 if failed else 0)

ivolatility_backtesting-2.149/tests/test_2149_client_errors.py

@@ -0,0 +1,131 @@
+# No retry on 4xx, pause before retrying a 429, client tag in User-Agent.
+# Run: python3 tests/test_2149_client_errors.py  (or pytest). No network: a local server stands in for the API.
+import json
+import os
+import sys
+import threading
+import time
+import traceback
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import matplotlib
+matplotlib.use('Agg')
+
+HITS = {}
+
+
+class _Api(BaseHTTPRequestHandler):
+    def log_message(self, *args):
+        pass
+
+    def _reply(self, code, body):
+        data = json.dumps(body).encode()
+        self.send_response(code)
+        self.send_header('Content-Type', 'application/json')
+        self.send_header('Content-Length', str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_GET(self):
+        path = self.path.split('?')[0]
+        HITS[path] = HITS.get(path, 0) + 1
+        if path == '/busy' and HITS[path] == 1:
+            self._reply(429, {'code': 429, 'message': 'per-second rate limit'})
+        elif path == '/equities/eod/stock-prices':
+            self._reply(403, {'code': 403, 'name': 'Forbidden',
+                              'message': 'Trial records limit exhausted. To continue, please upgrade to a paid plan'})
+        else:
+            self._reply(200, {'ok': True})
+
+
+# CI runners set HTTP_PROXY; the local stand-in API must bypass it
+for _var in ('NO_PROXY', 'no_proxy'):
+    os.environ[_var] = ','.join(filter(None, [os.environ.get(_var), '127.0.0.1', 'localhost']))
+
+_SERVER = ThreadingHTTPServer(('127.0.0.1', 0), _Api)
+threading.Thread(target=_SERVER.serve_forever, daemon=True).start()
+_BASE = f'http://127.0.0.1:{_SERVER.server_address[1]}'
+os.environ['IVOL_API_BASE_URL'] = _BASE
+os.environ['API_KEY'] = 'TESTKEY000000000'
+
+_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.insert(0, os.path.join(_REPO, 'ivolatility_backtesting'))
+import ivolatility_backtesting as lib  # noqa: E402
+import requests  # noqa: E402
+
+
+def _sdk_version():
+    return tuple(int(x) for x in lib.ivol.__version__.split('.')[:3])
+
+
+def test_client_error_not_retried():
+    HITS.pop('/equities/eod/stock-prices', None)
+    try:
+        lib.api_call('/equities/eod/stock-prices', symbol='AAPL', from_='2024-01-02', to='2024-01-03')
+        raise AssertionError('403 must raise')
+    except requests.HTTPError as e:
+        assert e.response.status_code == 403
+        if _sdk_version() >= (1, 9, 6):
+            assert 'Trial records limit' in str(e)
+    assert HITS['/equities/eod/stock-prices'] == 1
+
+
+def test_direct_call_not_retried_on_client_error():
+    HITS.pop('/equities/eod/stock-prices', None)
+    assert lib._api_call_direct('/equities/eod/stock-prices', symbol='AAPL',
+                                from_='2024-01-02', to='2024-01-03') is None
+    assert HITS['/equities/eod/stock-prices'] == 1
+
+
+def test_thread_session_pauses_before_retrying_429():
+    if _sdk_version() < (1, 9, 6):
+        return
+    HITS.pop('/busy', None)
+    t0 = time.time()
+    r = lib._create_sdk_like_session().get(f'{_BASE}/busy', timeout=10)
+    assert r.status_code == 200 and HITS['/busy'] == 2
+    assert time.time() - t0 >= 1.0
+
+
+def test_is_client_error():
+    class E(Exception):
+        def __init__(self, code):
+            self.response = type('R', (), {'status_code': code})()
+    assert lib._is_client_error(E(403)) and lib._is_client_error(E(400))
+    assert not lib._is_client_error(E(429)) and not lib._is_client_error(E(500))
+    assert not lib._is_client_error(ValueError())
+
+
+def test_client_tag():
+    saved = {k: os.environ.pop(k, None) for k in ('IVOL_CLIENT', 'CODER_AGENT_URL')}
+    saved_prefix = sys.prefix
+    try:
+        sys.prefix = '/opt/venv'
+        assert lib._ivb_client() == 'local'
+        sys.prefix = os.path.join('/Users/x', 'ivolatility', '.venv')
+        assert lib._ivb_client() == 'plugin'
+        os.environ['CODER_AGENT_URL'] = 'http://agent'
+        assert lib._ivb_client() == 'workspace'
+        os.environ['IVOL_CLIENT'] = 'chat'
+        assert lib._ivb_client() == 'chat'
+    finally:
+        sys.prefix = saved_prefix
+        for k, v in saved.items():
+            os.environ.pop(k, None)
+            if v is not None:
+                os.environ[k] = v
+
+
+if __name__ == '__main__':
+    failed = 0
+    for name, fn in list(globals().items()):
+        if name.startswith('test_') and callable(fn):
+            try:
+                fn()
+                print(f'PASS {name}')
+            except Exception:
+                failed += 1
+                print(f'FAIL {name}')
+                traceback.print_exc()
+    _SERVER.shutdown()
+    sys.exit(1 if failed else 0)