lumibot 4.2.7__py3-none-any.whl → 4.2.9__py3-none-any.whl

lumibot/tools/thetadata_helper.py

@@ -26,6 +26,7 @@ CONNECTION_RETRY_SLEEP = 1.0
 CONNECTION_MAX_RETRIES = 60
 BOOT_GRACE_PERIOD = 5.0
 MAX_RESTART_ATTEMPTS = 3
+MAX_TERMINAL_RESTART_CYCLES = 3
 
 
 def _resolve_asset_folder(asset_obj: Asset) -> str:
@@ -43,6 +44,12 @@ THETA_DATA_PROCESS = None
 THETA_DATA_PID = None
 THETA_DATA_LOG_HANDLE = None
 
+
+class ThetaDataConnectionError(RuntimeError):
+    """Raised when ThetaTerminal cannot reconnect to Theta Data after multiple restarts."""
+
+    pass
+
 def reset_connection_diagnostics():
     """Reset ThetaData connection counters (useful for tests)."""
     CONNECTION_DIAGNOSTICS.update({
@@ -50,6 +57,7 @@ def reset_connection_diagnostics():
         "start_terminal_calls": 0,
         "network_requests": 0,
         "placeholder_writes": 0,
+        "terminal_restarts": 0,
     })
 
 
@@ -198,6 +206,7 @@ CONNECTION_DIAGNOSTICS = {
     "start_terminal_calls": 0,
     "network_requests": 0,
     "placeholder_writes": 0,
+    "terminal_restarts": 0,
 }
 
 
@@ -1395,7 +1404,6 @@ def check_connection(username: str, password: str, wait_for_connection: bool = F
 
     max_retries = CONNECTION_MAX_RETRIES
     sleep_interval = CONNECTION_RETRY_SLEEP
-    restart_attempts = 0
     client = None
 
     def probe_status() -> Optional[str]:
@@ -1435,47 +1443,72 @@ def check_connection(username: str, password: str, wait_for_connection: bool = F
         logger.debug("ThetaTerminal running but not yet CONNECTED; waiting for status.")
         return check_connection(username=username, password=password, wait_for_connection=True)
 
-    counter = 0
-    connected = False
-
-    while counter < max_retries:
-        status_text = probe_status()
-        if status_text == "CONNECTED":
-            if counter:
-                logger.info("ThetaTerminal connected after %s attempt(s).", counter + 1)
-            connected = True
-            break
-        elif status_text == "DISCONNECTED":
-            logger.debug("ThetaTerminal reports DISCONNECTED; will retry.")
-        elif status_text is not None:
-            logger.debug(f"ThetaTerminal returned unexpected status: {status_text}")
-
-        if not is_process_alive():
-            if restart_attempts >= MAX_RESTART_ATTEMPTS:
-                logger.error("ThetaTerminal not running after %s restart attempts.", restart_attempts)
-                break
-            restart_attempts += 1
-            logger.warning("ThetaTerminal process is not running (restart #%s).", restart_attempts)
-            client = start_theta_data_client(username=username, password=password)
-            time.sleep(max(BOOT_GRACE_PERIOD, sleep_interval))
-            counter = 0
-            continue
+    total_restart_cycles = 0
 
-        counter += 1
-        if counter % 10 == 0:
-            logger.info("Waiting for ThetaTerminal connection (attempt %s/%s).", counter, max_retries)
-        time.sleep(sleep_interval)
+    while True:
+        counter = 0
+        restart_attempts = 0
+
+        while counter < max_retries:
+            status_text = probe_status()
+            if status_text == "CONNECTED":
+                if counter or total_restart_cycles:
+                    logger.info(
+                        "ThetaTerminal connected after %s attempt(s) (restart cycles=%s).",
+                        counter + 1,
+                        total_restart_cycles,
+                    )
+                return client, True
+            elif status_text == "DISCONNECTED":
+                logger.debug("ThetaTerminal reports DISCONNECTED; will retry.")
+            elif status_text is not None:
+                logger.debug(f"ThetaTerminal returned unexpected status: {status_text}")
+
+            if not is_process_alive():
+                if restart_attempts >= MAX_RESTART_ATTEMPTS:
+                    logger.error("ThetaTerminal not running after %s restart attempts.", restart_attempts)
+                    break
+                restart_attempts += 1
+                logger.warning("ThetaTerminal process is not running (restart #%s).", restart_attempts)
+                client = start_theta_data_client(username=username, password=password)
+                CONNECTION_DIAGNOSTICS["terminal_restarts"] = CONNECTION_DIAGNOSTICS.get("terminal_restarts", 0) + 1
+                time.sleep(max(BOOT_GRACE_PERIOD, sleep_interval))
+                counter = 0
+                continue
 
-    if not connected and counter >= max_retries:
-        logger.error("Cannot connect to Theta Data after %s attempts.", counter)
+            counter += 1
+            if counter % 10 == 0:
+                logger.info("Waiting for ThetaTerminal connection (attempt %s/%s).", counter, max_retries)
+            time.sleep(sleep_interval)
+
+        total_restart_cycles += 1
+        if total_restart_cycles > MAX_TERMINAL_RESTART_CYCLES:
+            logger.error(
+                "Unable to connect to Theta Data after %s restart cycle(s) (%s attempts each).",
+                MAX_TERMINAL_RESTART_CYCLES,
+                max_retries,
+            )
+            raise ThetaDataConnectionError(
+                f"Unable to connect to Theta Data after {MAX_TERMINAL_RESTART_CYCLES} restart cycle(s)."
+            )
 
-    return client, connected
+        logger.warning(
+            "ThetaTerminal still disconnected after %s attempts; restarting (cycle %s/%s).",
+            max_retries,
+            total_restart_cycles,
+            MAX_TERMINAL_RESTART_CYCLES,
+        )
+        client = start_theta_data_client(username=username, password=password)
+        CONNECTION_DIAGNOSTICS["terminal_restarts"] = CONNECTION_DIAGNOSTICS.get("terminal_restarts", 0) + 1
+        time.sleep(max(BOOT_GRACE_PERIOD, sleep_interval))
 
 
 def get_request(url: str, headers: dict, querystring: dict, username: str, password: str):
     all_responses = []
     next_page_url = None
     page_count = 0
+    consecutive_disconnects = 0
+    restart_budget = 3
 
     # Lightweight liveness probe before issuing the request
     check_connection(username=username, password=password, wait_for_connection=False)
@@ -1498,25 +1531,52 @@ def get_request(url: str, headers: dict, querystring: dict, username: str, passw
                 )
 
                 response = requests.get(request_url, headers=headers, params=request_params)
+                status_code = response.status_code
                 # Status code 472 means "No data" - this is valid, return None
-                if response.status_code == 472:
+                if status_code == 472:
                     logger.warning(f"No data available for request: {response.text[:200]}")
                     # DEBUG-LOG: API response - no data
                     logger.debug(
                         "[THETA][DEBUG][API][RESPONSE] status=472 result=NO_DATA"
                     )
+                    consecutive_disconnects = 0
                     return None
+                elif status_code == 474:
+                    consecutive_disconnects += 1
+                    logger.warning("Received 474 from Theta Data (attempt %s): %s", counter + 1, response.text[:200])
+                    if consecutive_disconnects >= 2:
+                        if restart_budget <= 0:
+                            logger.error("Restart budget exhausted after repeated 474 responses.")
+                            raise ValueError("Cannot connect to Theta Data!")
+                        logger.warning(
+                            "Restarting ThetaTerminal after %s consecutive 474 responses (restart budget remaining %s).",
+                            consecutive_disconnects,
+                            restart_budget - 1,
+                        )
+                        restart_budget -= 1
+                        start_theta_data_client(username=username, password=password)
+                        CONNECTION_DIAGNOSTICS["terminal_restarts"] = CONNECTION_DIAGNOSTICS.get("terminal_restarts", 0) + 1
+                        check_connection(username=username, password=password, wait_for_connection=True)
+                        time.sleep(max(BOOT_GRACE_PERIOD, CONNECTION_RETRY_SLEEP))
+                        consecutive_disconnects = 0
+                        counter = 0
+                    else:
+                        check_connection(username=username, password=password, wait_for_connection=True)
+                        time.sleep(CONNECTION_RETRY_SLEEP)
+                    continue
                 # If status code is not 200, then we are not connected
-                elif response.status_code != 200:
-                    logger.warning(f"Non-200 status code {response.status_code}: {response.text[:200]}")
+                elif status_code != 200:
+                    logger.warning(f"Non-200 status code {status_code}: {response.text[:200]}")
                     # DEBUG-LOG: API response - error
                     logger.debug(
                         "[THETA][DEBUG][API][RESPONSE] status=%d result=ERROR",
-                        response.status_code
+                        status_code
                     )
                     check_connection(username=username, password=password, wait_for_connection=True)
+                    consecutive_disconnects = 0
                 else:
                     json_resp = response.json()
+                    consecutive_disconnects = 0
 
                     # DEBUG-LOG: API response - success
                     response_rows = len(json_resp.get("response", [])) if isinstance(json_resp.get("response"), list) else 0
@@ -1542,6 +1602,9 @@ def get_request(url: str, headers: dict, querystring: dict, username: str, passw
                     else:
                         break
 
+            except ThetaDataConnectionError as exc:
+                logger.error("Theta Data connection failed after supervised restarts: %s", exc)
+                raise
             except Exception as e:
                 logger.warning(f"Exception during request (attempt {counter + 1}): {e}")
                 check_connection(username=username, password=password, wait_for_connection=True)
@@ -1551,7 +1614,7 @@ def get_request(url: str, headers: dict, querystring: dict, username: str, passw
 
             counter += 1
             if counter > 1:
-                raise ValueError("Cannot connect to Theta Data!")
+                raise ThetaDataConnectionError("Unable to connect to Theta Data after repeated retries.")
 
         # Store this page's response data
         page_count += 1

{lumibot-4.2.7.dist-info → lumibot-4.2.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lumibot
-Version: 4.2.7
+Version: 4.2.9
 Summary: Backtesting and Trading Library, Made by Lumiwealth
 Home-page: https://github.com/Lumiwealth/lumibot
 Author: Robert Grzesik

{lumibot-4.2.7.dist-info → lumibot-4.2.9.dist-info}/RECORD

@@ -132,7 +132,7 @@ lumibot/tools/polygon_helper_async.py,sha256=YHDXa9kmkkn8jh7hToY6GP5etyXS9Tj-uky
 lumibot/tools/polygon_helper_polars_optimized.py,sha256=NaIZ-5Av-G2McPEKHyJ-x65W72W_Agnz4lRgvXfQp8c,30415
 lumibot/tools/projectx_helpers.py,sha256=EIemLfbG923T_RBV_i6s6A9xgs7dt0et0oCnhFwdWfA,58299
 lumibot/tools/schwab_helper.py,sha256=CXnYhgsXOIb5MgmIYOp86aLxsBF9oeVrMGrjwl_GEv0,11768
-lumibot/tools/thetadata_helper.py,sha256=QXuFzfHi22L9sqzwL_mXZGhU2m9ortbE9U1-Kt7jSew,86743
+lumibot/tools/thetadata_helper.py,sha256=F5CdaWZkfJYFrParvN8KhowRWc5cRmPL4TnI9SH-VpI,90111
 lumibot/tools/types.py,sha256=x-aQBeC6ZTN2-pUyxyo69Q0j5e0c_swdfe06kfrWSVc,1978
 lumibot/tools/yahoo_helper.py,sha256=htcKKkuktatIckVKfLc_ms0X75mXColysQhrZW244z8,19497
 lumibot/tools/yahoo_helper_polars_optimized.py,sha256=g9xBN-ReHSW4Aj9EMU_OncBXVS1HpfL8LTHit9ZxFY4,7417
@@ -142,7 +142,7 @@ lumibot/traders/trader.py,sha256=KMif3WoZtnSxA0BzoK3kvkTITNELrDFIortx1BYBv8s,967
 lumibot/trading_builtins/__init__.py,sha256=vH2QL5zLjL3slfEV1YW-BvQHtEYLCFkIWTZDfh3y8LE,87
 lumibot/trading_builtins/custom_stream.py,sha256=8_XiPT0JzyXrgnXCXoovGGUrWEfnG4ohIYMPfB_Nook,5264
 lumibot/trading_builtins/safe_list.py,sha256=IIjZOHSiZYK25A4WBts0oJaZNOJDsjZL65MOSHhE3Ig,1975
-lumibot-4.2.7.dist-info/licenses/LICENSE,sha256=fYhGIyxjyNXACgpNQS3xxpxDOaVOWRVxZMCRbsDv8k0,35130
+lumibot-4.2.9.dist-info/licenses/LICENSE,sha256=fYhGIyxjyNXACgpNQS3xxpxDOaVOWRVxZMCRbsDv8k0,35130
 tests/__init__.py,sha256=3-VoT-nAuqMfwufd4ceN6fXaHl_zCfDCSXJOTp1ywYQ,393
 tests/conftest.py,sha256=UBw_2fx7r6TZPKus2b1Qxrzmd4bg8EEBnX1vCHUuSVA,3311
 tests/fixtures.py,sha256=wOHQsh1SGHnXe_PGi6kDWI30CS_Righi7Ig7vwSEKT4,9082
@@ -236,7 +236,7 @@ tests/test_session_manager.py,sha256=1qygN3aQ2Xe2uh4BMPm0E3V8KXLFNGq5qdL8KkZjef4
 tests/test_strategy_methods.py,sha256=j9Mhr6nnG1fkiVQXnx7gLjzGbeQmwt0UbJr_4plD36o,12539
 tests/test_strategy_price_guard.py,sha256=3GJdlfROwx6-adsSi8ZBrWaLOy9e-0N6V1eqpikj8e4,1540
 tests/test_thetadata_backwards_compat.py,sha256=RzNLhNZNJZ2hPkEDyG-T_4mRRXh5XqavK6r-OjfRASQ,3306
-tests/test_thetadata_helper.py,sha256=RQ-q7OfKc_2aO7-dInaVRTUrrI5iTh9oteYsdmfn4Lg,63735
+tests/test_thetadata_helper.py,sha256=4EdFHoCTTKELU216ZNYGjfo6Uvq-29kK-TVpFLPWT3M,67263
 tests/test_thetadata_pandas_verification.py,sha256=MWUecqBY6FGFslWLRo_C5blGbom_unmXCZikAfZXLks,6553
 tests/test_tradier.py,sha256=iCEM2FTxJSzJ2oLNaRqSx05XaX_DCiMzLx1aEYPANko,33280
 tests/test_tradier_data.py,sha256=1jTxDzQtzaC42CQJVXMRMElBwExy1mVci3NFfKjjVH0,13363
@@ -282,7 +282,7 @@ tests/backtest/test_thetadata.py,sha256=xWYfC9C4EhbMDb29qyZWHO3sSWaLIPzzvcMbHCt5
 tests/backtest/test_thetadata_comprehensive.py,sha256=-gN3xLJcJtlB-k4vlaK82DCZDGDmr0LNZZDzn-aN3l4,26120
 tests/backtest/test_thetadata_vs_polygon.py,sha256=dZqsrOx3u3cz-1onIO6o5BDRjI1ey7U9vIkZupfXoig,22831
 tests/backtest/test_yahoo.py,sha256=2FguUTUMC9_A20eqxnZ17rN3tT9n6hyvJHaL98QKpqY,3443
-lumibot-4.2.7.dist-info/METADATA,sha256=cCEnxjhYBqtO5Uvd2pEJ90ULy2OlCiX1FHSaJPEpXAI,12093
-lumibot-4.2.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-lumibot-4.2.7.dist-info/top_level.txt,sha256=otUnUjDFVASauEDiTiAzNgMyqQ1B6jjS3QqqP-WSx38,14
-lumibot-4.2.7.dist-info/RECORD,,
+lumibot-4.2.9.dist-info/METADATA,sha256=Bis7WU6RY3Wp7YXHsNa7-xklQzQcXRi5zcJzZhC2j0w,12093
+lumibot-4.2.9.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+lumibot-4.2.9.dist-info/top_level.txt,sha256=otUnUjDFVASauEDiTiAzNgMyqQ1B6jjS3QqqP-WSx38,14
+lumibot-4.2.9.dist-info/RECORD,,

tests/test_thetadata_helper.py

@@ -1426,8 +1426,10 @@ class TestThetaDataChainsCaching:
 
         monkeypatch.setattr(thetadata_helper, "build_historical_chain", lambda **kwargs: None)
         monkeypatch.setattr(thetadata_helper, "LUMIBOT_CACHE_FOLDER", str(tmp_path))
+        monkeypatch.delenv("BACKTESTING_QUIET_LOGS", raising=False)
+        caplog.set_level(logging.WARNING, logger="lumibot.tools.thetadata_helper")
 
-        with caplog.at_level(logging.WARNING):
+        with caplog.at_level(logging.WARNING, logger="lumibot.tools.thetadata_helper"):
             result = thetadata_helper.get_chains_cached("user", "pass", asset, test_date)
 
         cache_folder = Path(tmp_path) / "thetadata" / "stock" / "option_chains"
@@ -1508,8 +1510,10 @@ def test_build_historical_chain_returns_none_when_no_dates(monkeypatch, caplog):
         raise AssertionError(f"Unexpected URL {url}")
 
     monkeypatch.setattr(thetadata_helper, "get_request", fake_get_request)
+    monkeypatch.delenv("BACKTESTING_QUIET_LOGS", raising=False)
+    caplog.set_level(logging.WARNING, logger="lumibot.tools.thetadata_helper")
 
-    with caplog.at_level(logging.WARNING):
+    with caplog.at_level(logging.WARNING, logger="lumibot.tools.thetadata_helper"):
         result = thetadata_helper.build_historical_chain("user", "pass", asset, as_of_date)
 
     assert result is None
@@ -1525,14 +1529,83 @@ def test_build_historical_chain_empty_response(monkeypatch, caplog):
         raise AssertionError("Unexpected call after empty expirations")
 
     monkeypatch.setattr(thetadata_helper, "get_request", fake_get_request)
+    monkeypatch.delenv("BACKTESTING_QUIET_LOGS", raising=False)
+    caplog.set_level(logging.WARNING, logger="lumibot.tools.thetadata_helper")
 
-    with caplog.at_level(logging.WARNING):
+    with caplog.at_level(logging.WARNING, logger="lumibot.tools.thetadata_helper"):
         result = thetadata_helper.build_historical_chain("user", "pass", asset, as_of_date)
 
     assert result is None
     assert "returned no expirations" in caplog.text
 
 
+class TestThetaDataConnectionSupervision:
+
+    def setup_method(self):
+        thetadata_helper.reset_connection_diagnostics()
+
+    def test_check_connection_recovers_after_restart(self, monkeypatch):
+        statuses = iter(["DISCONNECTED", "DISCONNECTED", "CONNECTED"])
+
+        class FakeResponse:
+            def __init__(self, text):
+                self.text = text
+
+        def fake_get(url, timeout):
+            try:
+                text = next(statuses)
+            except StopIteration:
+                text = "CONNECTED"
+            return FakeResponse(text)
+
+        start_calls = []
+
+        def fake_start(username, password):
+            start_calls.append((username, password))
+            return object()
+
+        monkeypatch.setattr(thetadata_helper.requests, "get", fake_get)
+        monkeypatch.setattr(thetadata_helper, "start_theta_data_client", fake_start)
+        monkeypatch.setattr(thetadata_helper, "is_process_alive", lambda: True)
+        monkeypatch.setattr(thetadata_helper, "CONNECTION_MAX_RETRIES", 2, raising=False)
+        monkeypatch.setattr(thetadata_helper, "MAX_TERMINAL_RESTART_CYCLES", 2, raising=False)
+        monkeypatch.setattr(thetadata_helper, "BOOT_GRACE_PERIOD", 0, raising=False)
+        monkeypatch.setattr(thetadata_helper, "CONNECTION_RETRY_SLEEP", 0, raising=False)
+        monkeypatch.setattr(thetadata_helper.time, "sleep", lambda *args, **kwargs: None)
+
+        client, connected = thetadata_helper.check_connection("user", "pass", wait_for_connection=True)
+
+        assert connected is True
+        assert len(start_calls) == 1
+        assert thetadata_helper.CONNECTION_DIAGNOSTICS["terminal_restarts"] >= 1
+
+    def test_check_connection_raises_after_restart_cycles(self, monkeypatch):
+        statuses = iter(["DISCONNECTED"] * 10)
+
+        class FakeResponse:
+            def __init__(self, text):
+                self.text = text
+
+        def fake_get(url, timeout):
+            try:
+                text = next(statuses)
+            except StopIteration:
+                text = "DISCONNECTED"
+            return FakeResponse(text)
+
+        monkeypatch.setattr(thetadata_helper.requests, "get", fake_get)
+        monkeypatch.setattr(thetadata_helper, "start_theta_data_client", lambda *args, **kwargs: object())
+        monkeypatch.setattr(thetadata_helper, "is_process_alive", lambda: True)
+        monkeypatch.setattr(thetadata_helper, "CONNECTION_MAX_RETRIES", 1, raising=False)
+        monkeypatch.setattr(thetadata_helper, "MAX_TERMINAL_RESTART_CYCLES", 1, raising=False)
+        monkeypatch.setattr(thetadata_helper, "BOOT_GRACE_PERIOD", 0, raising=False)
+        monkeypatch.setattr(thetadata_helper, "CONNECTION_RETRY_SLEEP", 0, raising=False)
+        monkeypatch.setattr(thetadata_helper.time, "sleep", lambda *args, **kwargs: None)
+
+        with pytest.raises(thetadata_helper.ThetaDataConnectionError):
+            thetadata_helper.check_connection("user", "pass", wait_for_connection=True)
+
+
 def test_finalize_day_frame_handles_dst_fallback():
     tz = pytz.timezone("America/New_York")
     utc = pytz.UTC