OctoBot-Trading 2.4.127__tar.gz → 2.4.128__tar.gz

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.4.128] - 2023-12-03
+### Updated
+[OHLCVUpdater] prevent missing candles spam
+
 ## [2.4.127] - 2023-11-28
 ### Updated
 [MarketStatus] fix min cost overriding

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128/OctoBot_Trading.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: OctoBot-Trading
-Version: 2.4.127
+Version: 2.4.128
 Summary: OctoBot project trading package
 Home-page: https://github.com/Drakkar-Software/OctoBot-Trading
 Author: Drakkar-Software
@@ -16,7 +16,7 @@ Classifier: Operating System :: POSIX
 Requires-Python: >=3.8
 License-File: LICENSE
 
-# OctoBot-Trading [2.4.127](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
+# OctoBot-Trading [2.4.128](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/903b6b22bceb4661b608a86fea655f69)](https://app.codacy.com/gh/Drakkar-Software/OctoBot-Trading?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot-Trading&utm_campaign=Badge_Grade_Dashboard)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot-Trading.svg)](https://pypi.python.org/pypi/OctoBot-Trading/)
 [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot-Trading/badge.svg?branch=master)](https://coveralls.io/github/Drakkar-Software/OctoBot-Trading?branch=master)

{OctoBot-Trading-2.4.127/OctoBot_Trading.egg-info → OctoBot-Trading-2.4.128}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: OctoBot-Trading
-Version: 2.4.127
+Version: 2.4.128
 Summary: OctoBot project trading package
 Home-page: https://github.com/Drakkar-Software/OctoBot-Trading
 Author: Drakkar-Software
@@ -16,7 +16,7 @@ Classifier: Operating System :: POSIX
 Requires-Python: >=3.8
 License-File: LICENSE
 
-# OctoBot-Trading [2.4.127](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
+# OctoBot-Trading [2.4.128](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/903b6b22bceb4661b608a86fea655f69)](https://app.codacy.com/gh/Drakkar-Software/OctoBot-Trading?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot-Trading&utm_campaign=Badge_Grade_Dashboard)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot-Trading.svg)](https://pypi.python.org/pypi/OctoBot-Trading/)
 [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot-Trading/badge.svg?branch=master)](https://coveralls.io/github/Drakkar-Software/OctoBot-Trading?branch=master)

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/README.md

@@ -1,4 +1,4 @@
-# OctoBot-Trading [2.4.127](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
+# OctoBot-Trading [2.4.128](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/903b6b22bceb4661b608a86fea655f69)](https://app.codacy.com/gh/Drakkar-Software/OctoBot-Trading?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot-Trading&utm_campaign=Badge_Grade_Dashboard)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot-Trading.svg)](https://pypi.python.org/pypi/OctoBot-Trading/)
 [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot-Trading/badge.svg?branch=master)](https://coveralls.io/github/Drakkar-Software/OctoBot-Trading?branch=master)

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/octobot_trading/__init__.py

@@ -15,4 +15,4 @@
 #  License along with this library.
 
 PROJECT_NAME = "OctoBot-Trading"
-VERSION = "2.4.127"  # major.minor.revision
+VERSION = "2.4.128"  # major.minor.revision

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/octobot_trading/exchange_data/kline/channel/kline_updater.py

@@ -78,7 +78,7 @@ class KlineUpdater(kline_channel.KlineProducer):
             except errors.FailedRequest as e:
                 self.logger.warning(str(e))
                 # avoid spamming on disconnected situation
-                await asyncio.sleep(constants.DEFAULT_FAILED_REQUEST_RETRY_TIME)
+                await asyncio.sleep(constants.FAILED_NETWORK_REQUEST_RETRY_ATTEMPTS)
             except errors.NotSupported:
                 self.logger.warning(f"{self.channel.exchange_manager.exchange_name} is not supporting updates")
                 await self.pause()

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/octobot_trading/exchange_data/ohlcv/channel/ohlcv_updater.py

@@ -35,7 +35,14 @@ class OHLCVUpdater(ohlcv_channel.OHLCVProducer):
     OHLCV_OLD_LIMIT = constants.DEFAULT_CANDLE_HISTORY_SIZE  # should be <= to candle manager's MAX_CANDLES_COUNT
     OHLCV_ON_ERROR_TIME = 5
     OHLCV_MIN_REFRESH_TIME = 1
-    OHLCV_REFRESH_TIME_THRESHOLD = 1.5  # to prevent spamming at candle closing
+    OHLCV_REFRESH_TIME_THRESHOLD_BY_RETRY_ATTEMPT = [
+        2,      # retry 1: t+2
+        5,      # retry 2: t+7
+        8 ,     # retry 3: t+15
+        15,     # retry 4: t+30
+        15,     # retry 5: t+45
+        60      # retry N: every 60s
+    ] # to prevent spamming when missing closed candles
     OHLCV_MISSING_DATA_REFRESH_RETRY_MAX_DELAY = 30 * common_constants.MINUTE_TO_SECONDS
 
     OHLCV_INITIALIZATION_TIMEOUT = 60
@@ -206,11 +213,18 @@ class OHLCVUpdater(ohlcv_channel.OHLCVProducer):
         time_frame_seconds: int = common_enums.TimeFramesMinutes[time_frame] * common_constants.MINUTE_TO_SECONDS
         time_frame_sleep: int = time_frame_seconds
         last_candle_timestamp: float = 0
-        missing_data_sleep_time = min(int(time_frame_seconds / 6), self.OHLCV_MISSING_DATA_REFRESH_RETRY_MAX_DELAY)
 
+        current_candle_start_time = 0
+        attempt = 1
         while not self.should_stop and not self.channel.is_paused:
+            start_update_time = time.time()
+            iteration_candle_start_time = start_update_time - (start_update_time % time_frame_seconds)
             try:
-                start_update_time = time.time()
+                if iteration_candle_start_time == current_candle_start_time:
+                    attempt += 1    # not working?
+                else:
+                    current_candle_start_time = iteration_candle_start_time
+                    attempt = 1
                 await self._ensure_candles_initialization(pair)
                 # skip uninitialized candles
                 if self.initialized_candles_by_tf_by_symbol[pair][time_frame]:
@@ -225,21 +239,37 @@ class OHLCVUpdater(ohlcv_channel.OHLCVProducer):
 
                     if last_candle and len(candles) > 1:
                         last_candle_timestamp, sleep_time = await self._refresh_current_candle(
-                            time_frame, pair, candles, last_candle, last_candle_timestamp, time_frame_sleep
+                            time_frame, pair, candles, last_candle, last_candle_timestamp,
+                            iteration_candle_start_time, time_frame_seconds, attempt
+                        )
+                        updated_sleep_time = self._ensure_correct_sleep_time(
+                            sleep_time, iteration_candle_start_time, time_frame_seconds
                         )
-                        await asyncio.sleep(self._ensure_correct_sleep_time(sleep_time, time_frame_sleep))
+                        await asyncio.sleep(updated_sleep_time)
                     else:
                         # not enough candles: retry soon
+                        sleep_time = self._ensure_correct_sleep_time(
+                            self._get_missing_candle_retry_sleep_time(attempt),
+                            iteration_candle_start_time, time_frame_seconds
+                        )
                         self.logger.debug(f"Missing candles in request results for {pair} on {time_frame}, refreshing "
-                                          f"in {missing_data_sleep_time} seconds (available candles: {candles}).")
-                        await asyncio.sleep(missing_data_sleep_time)
+                                          f"in {sleep_time} seconds (available candles: {candles}).")
+                        await asyncio.sleep(sleep_time)
                 else:
                     # candles on this time frame have not been initialized: sleep until the next candle update
                     await asyncio.sleep(max(0.0, time_frame_sleep - (time.time() - start_update_time)))
-            except errors.FailedRequest as e:
-                self.logger.warning(str(e))
+            except errors.FailedRequest as err:
                 # avoid spamming on disconnected situation
-                await asyncio.sleep(constants.DEFAULT_FAILED_REQUEST_RETRY_TIME)
+                sleep_time = self._ensure_correct_sleep_time(
+                    self._get_missing_candle_retry_sleep_time(attempt),
+                    iteration_candle_start_time, time_frame_seconds
+                )
+                sleep_time = max(sleep_time, constants.DEFAULT_FAILED_REQUEST_RETRY_TIME)
+                self.logger.warning(
+                    f"Impossible to fetch {time_frame.value} {pair} candles. Retry in {sleep_time} seconds: "
+                    f"{html_util.get_html_summary_if_relevant(err)}"
+                )
+                await asyncio.sleep(sleep_time)
             except errors.NotSupported:
                 self.logger.warning(
                     f"{self.channel.exchange_manager.exchange_name} is not supporting updates")
@@ -253,36 +283,51 @@ class OHLCVUpdater(ohlcv_channel.OHLCVProducer):
                 )
                 await asyncio.sleep(self.OHLCV_ON_ERROR_TIME)
 
-    async def _refresh_current_candle(self, time_frame, pair, candles, last_candle,
-                                      last_candle_timestamp, time_frame_sleep):
+    async def _refresh_current_candle(
+        self, time_frame, pair, candles, last_candle, last_candle_timestamp,
+        iteration_candle_start_time, time_frame_seconds, attempt
+    ):
         current_candle_timestamp: float = last_candle[common_enums.PriceIndexes.IND_PRICE_TIME.value]
-        should_sleep_time: float = current_candle_timestamp + time_frame_sleep - time.time()
+        should_sleep_time: float = current_candle_timestamp + time_frame_seconds - time.time()
 
         # if we're trying to refresh the current candle => useless
         if last_candle_timestamp == current_candle_timestamp:
             if should_sleep_time < 0:
-                # up to date candle is not yet available on exchange: retry in a few seconds
+                # up-to-date candle is not yet available on exchange: retry in a few seconds
                 should_sleep_time = self._ensure_correct_sleep_time(
-                    self.OHLCV_REFRESH_TIME_THRESHOLD,
-                    time_frame_sleep
+                    self._get_missing_candle_retry_sleep_time(attempt),
+                    iteration_candle_start_time,
+                    time_frame_seconds
                 )
             else:
                 should_sleep_time = self._ensure_correct_sleep_time(
-                    should_sleep_time + time_frame_sleep + self.OHLCV_REFRESH_TIME_THRESHOLD,
-                    time_frame_sleep
+                    should_sleep_time + time_frame_seconds + self._get_missing_candle_retry_sleep_time(attempt),
+                    iteration_candle_start_time,
+                    time_frame_seconds
                 )
+            self.logger.debug(
+                f"Failed to fetch up-to-date candle for {pair} on {time_frame.value}. "
+                f"Retrying in {round(should_sleep_time, 2)} seconds"
+            )
+
         else:
             # A fresh candle happened
             last_candle_timestamp = current_candle_timestamp
             await self._push_complete_candles(time_frame, pair, candles)
         return last_candle_timestamp, should_sleep_time
 
-    def _ensure_correct_sleep_time(self, sleep_time_candidate, time_frame_sleep):
+    def _get_missing_candle_retry_sleep_time(self, attempt):
+        if attempt < len(self.OHLCV_REFRESH_TIME_THRESHOLD_BY_RETRY_ATTEMPT):
+            return self.OHLCV_REFRESH_TIME_THRESHOLD_BY_RETRY_ATTEMPT[attempt - 1]
+        return self.OHLCV_REFRESH_TIME_THRESHOLD_BY_RETRY_ATTEMPT[-1]
+
+    def _ensure_correct_sleep_time(self, sleep_time_candidate, iteration_candle_start_time, time_frame_seconds):
         if sleep_time_candidate < OHLCVUpdater.OHLCV_MIN_REFRESH_TIME:
             return OHLCVUpdater.OHLCV_MIN_REFRESH_TIME
-        elif sleep_time_candidate > time_frame_sleep:
-            return time_frame_sleep
-        return sleep_time_candidate
+        else:
+            max_sleep_time = max(iteration_candle_start_time + time_frame_seconds - time.time(), 0)
+            # ensure does not sleep more than missing time before next candle
+            return min(sleep_time_candidate, max_sleep_time)
 
     def _set_initialized(self, pair, time_frame, initialized):
         if pair not in self.initialized_candles_by_tf_by_symbol:

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/octobot_trading/exchange_data/recent_trades/channel/recent_trade_updater.py

@@ -74,10 +74,14 @@ class RecentTradeUpdater(recent_trade_channel.RecentTradeProducer):
                         except TypeError:
                             pass
                 await asyncio.sleep(self.refresh_time)
-            except errors.FailedRequest as e:
-                self.logger.warning(str(e))
+            except errors.FailedRequest as err:
+                sleep_time = constants.FAILED_NETWORK_REQUEST_RETRY_ATTEMPTS
+                self.logger.warning(
+                    f"Impossible to fetch recent trades. Retry in {sleep_time} seconds: "
+                    f"{html_util.get_html_summary_if_relevant(err)}"
+                )
                 # avoid spamming on disconnected situation
-                await asyncio.sleep(constants.DEFAULT_FAILED_REQUEST_RETRY_TIME)
+                await asyncio.sleep(sleep_time)
             except errors.NotSupported:
                 self.logger.warning(f"{self.channel.exchange_manager.exchange_name} is not supporting updates")
                 await self.pause()

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/octobot_trading/exchange_data/ticker/channel/ticker_updater.py

@@ -74,7 +74,7 @@ class TickerUpdater(ticker_channel.TickerProducer):
         except errors.FailedRequest as e:
             self.logger.warning(html_util.get_html_summary_if_relevant(e))
             # avoid spamming on disconnected situation
-            await asyncio.sleep(constants.DEFAULT_FAILED_REQUEST_RETRY_TIME)
+            await asyncio.sleep(constants.FAILED_NETWORK_REQUEST_RETRY_ATTEMPTS)
 
     async def fetch_and_push_pair(self, pair: str):
         with self._single_pair_update(pair) as can_update:

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py

@@ -664,7 +664,10 @@ class CCXTConnector(abstract_exchange.AbstractExchange):
                 )
                 if cancelled_order is None or personal_data.parse_is_cancelled(cancelled_order):
                     return enums.OrderStatus.CANCELED
-                elif personal_data.parse_is_open(cancelled_order):
+                elif (
+                    personal_data.parse_is_open(cancelled_order)
+                    or personal_data.parse_is_pending_cancel(cancelled_order)
+                ):
                     return enums.OrderStatus.PENDING_CANCEL
                 # cancel command worked but order is still existing and is not open or canceled. unhandled case
                 # log error and consider it canceling. order states will manage the

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/octobot_trading/personal_data/__init__.py

@@ -37,6 +37,7 @@ from octobot_trading.personal_data.orders import (
     parse_raw_fees,
     parse_order_status,
     parse_is_cancelled,
+    parse_is_pending_cancel,
     parse_is_open,
     get_pnl_transaction_source_from_order,
     is_stop_order,
@@ -253,6 +254,7 @@ __all__ = [
     "parse_raw_fees",
     "parse_order_status",
     "parse_is_cancelled",
+    "parse_is_pending_cancel",
     "parse_is_open",
     "get_pnl_transaction_source_from_order",
     "is_stop_order",

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/octobot_trading/personal_data/orders/__init__.py

@@ -87,6 +87,7 @@ from octobot_trading.personal_data.orders.order_util import (
     parse_raw_fees,
     parse_order_status,
     parse_is_cancelled,
+    parse_is_pending_cancel,
     parse_is_open,
     get_up_to_date_price,
     get_pre_order_data,
@@ -160,6 +161,7 @@ __all__ = [
     "parse_raw_fees",
     "parse_order_status",
     "parse_is_cancelled",
+    "parse_is_pending_cancel",
     "parse_is_open",
     "get_up_to_date_price",
     "get_pre_order_data",

{OctoBot-Trading-2.4.127 → OctoBot-Trading-2.4.128}/octobot_trading/personal_data/orders/order_util.py

@@ -395,6 +395,10 @@ def parse_is_cancelled(raw_order):
     return parse_order_status(raw_order) in {enums.OrderStatus.CANCELED, enums.OrderStatus.CLOSED}
 
 
+def parse_is_pending_cancel(raw_order):
+    return parse_order_status(raw_order) is enums.OrderStatus.PENDING_CANCEL
+
+
 def parse_is_open(raw_order):
     return parse_order_status(raw_order) is enums.OrderStatus.OPEN