OctoBot-Trading 2.4.112__tar.gz → 2.4.113__tar.gz

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.4.113] - 2024-10-02
+### Added
+[Orders] improve canceled & filled order error recovery 
+
 ## [2.4.112] - 2024-09-18
 ### Added
 [API] compute_base_and_quote_volume

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113/OctoBot_Trading.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: OctoBot-Trading
-Version: 2.4.112
+Version: 2.4.113
 Summary: OctoBot project trading package
 Home-page: https://github.com/Drakkar-Software/OctoBot-Trading
 Author: Drakkar-Software
@@ -16,7 +16,7 @@ Classifier: Operating System :: POSIX
 Requires-Python: >=3.8
 License-File: LICENSE
 
-# OctoBot-Trading [2.4.112](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
+# OctoBot-Trading [2.4.113](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/903b6b22bceb4661b608a86fea655f69)](https://app.codacy.com/gh/Drakkar-Software/OctoBot-Trading?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot-Trading&utm_campaign=Badge_Grade_Dashboard)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot-Trading.svg)](https://pypi.python.org/pypi/OctoBot-Trading/)
 [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot-Trading/badge.svg?branch=master)](https://coveralls.io/github/Drakkar-Software/OctoBot-Trading?branch=master)

{OctoBot-Trading-2.4.112/OctoBot_Trading.egg-info → OctoBot-Trading-2.4.113}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: OctoBot-Trading
-Version: 2.4.112
+Version: 2.4.113
 Summary: OctoBot project trading package
 Home-page: https://github.com/Drakkar-Software/OctoBot-Trading
 Author: Drakkar-Software
@@ -16,7 +16,7 @@ Classifier: Operating System :: POSIX
 Requires-Python: >=3.8
 License-File: LICENSE
 
-# OctoBot-Trading [2.4.112](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
+# OctoBot-Trading [2.4.113](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/903b6b22bceb4661b608a86fea655f69)](https://app.codacy.com/gh/Drakkar-Software/OctoBot-Trading?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot-Trading&utm_campaign=Badge_Grade_Dashboard)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot-Trading.svg)](https://pypi.python.org/pypi/OctoBot-Trading/)
 [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot-Trading/badge.svg?branch=master)](https://coveralls.io/github/Drakkar-Software/OctoBot-Trading?branch=master)

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/README.md

@@ -1,4 +1,4 @@
-# OctoBot-Trading [2.4.112](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
+# OctoBot-Trading [2.4.113](https://github.com/Drakkar-Software/OctoBot-Trading/blob/master/CHANGELOG.md)
 [![Codacy Badge](https://api.codacy.com/project/badge/Grade/903b6b22bceb4661b608a86fea655f69)](https://app.codacy.com/gh/Drakkar-Software/OctoBot-Trading?utm_source=github.com&utm_medium=referral&utm_content=Drakkar-Software/OctoBot-Trading&utm_campaign=Badge_Grade_Dashboard)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot-Trading.svg)](https://pypi.python.org/pypi/OctoBot-Trading/)
 [![Coverage Status](https://coveralls.io/repos/github/Drakkar-Software/OctoBot-Trading/badge.svg?branch=master)](https://coveralls.io/github/Drakkar-Software/OctoBot-Trading?branch=master)

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/__init__.py

@@ -15,4 +15,4 @@
 #  License along with this library.
 
 PROJECT_NAME = "OctoBot-Trading"
-VERSION = "2.4.112"  # major.minor.revision
+VERSION = "2.4.113"  # major.minor.revision

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/constants.py

@@ -189,7 +189,7 @@ BALANCE_CHANNEL = "Balance"
 BALANCE_PROFITABILITY_CHANNEL = "BalanceProfitability"
 POSITIONS_CHANNEL = "Positions"
 INDIVIDUAL_ORDER_SYNC_TIMEOUT = 1 * commons_constants.MINUTE_TO_SECONDS
-MAX_TRADES_COUNT = int(os.getenv("MAX_TRADES_COUNT", "6000"))    # larger values can use a large part of ram
+MAX_TRADES_COUNT = int(os.getenv("MAX_TRADES_COUNT", "10000"))    # larger values can use a large part of ram
 
 # History
 DEFAULT_SAVED_HISTORICAL_TIMEFRAMES = [commons_enums.TimeFrames.ONE_DAY]

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/errors.py

@@ -51,6 +51,12 @@ class OrderCancelError(Exception):
     """
 
 
+class OrderNotFoundOnCancelError(OrderCancelError):
+    """
+    Raised upon a failed order cancel because order is not found
+    """
+
+
 class UnexpectedExchangeSideOrderStateError(Exception):
     """
     Raised when an order is in an unexpected state when fetched from exchange

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py

@@ -654,8 +654,11 @@ class CCXTConnector(abstract_exchange.AbstractExchange):
                 # get a cancelled order).
                 return enums.OrderStatus.CANCELED
         except ccxt.OrderNotFound as e:
-            self.logger.debug(f"Trying to cancel order with id {exchange_order_id} but order was not found")
-            raise octobot_trading.errors.OrderCancelError(e) from e
+            self.logger.debug(
+                f"Trying to cancel order with id {exchange_order_id} but order was not found. It might have "
+                f"already been cancelled or be filled."
+            )
+            raise octobot_trading.errors.OrderNotFoundOnCancelError(e) from e
         except (ccxt.NotSupported, octobot_trading.errors.NotSupported) as e:
             raise octobot_trading.errors.NotSupported(e) from e
         except Exception as e:

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/exchanges/traders/trader.py

@@ -369,8 +369,9 @@ class Trader(util.Initializable):
                                                          wait_for_cancelling, cancelling_timeout)
         return False
 
-    async def _handle_order_cancellation(self, order, ignored_order,
-                                         wait_for_cancelling: bool, cancelling_timeout: float) -> bool:
+    async def _handle_order_cancellation(
+        self, order, ignored_order, wait_for_cancelling: bool, cancelling_timeout: float
+    ) -> bool:
         success = True
         if order.is_waiting_for_chained_trigger:
             # order will just never get created
@@ -386,17 +387,19 @@ class Trader(util.Initializable):
                         )
                     except errors.NotSupported:
                         raise
-                    except (errors.OrderCancelError, Exception) as err:
+                    except (errors.OrderCancelError, Exception) as inner_err:
                         # retry to cancel order
-                        self.logger.debug(f"Failed to cancel order ({err} {err.__class__.__name__}), retrying")
+                        self.logger.info(
+                            f"Failed to cancel order ({inner_err} {inner_err.__class__.__name__}), retrying"
+                        )
                         order_status = await self.exchange_manager.exchange.cancel_order(
                             order.exchange_order_id, order.symbol, order.order_type
                         )
             except errors.OrderCancelError as err:
                 if await self._handle_order_cancel_error(order, err, wait_for_cancelling, cancelling_timeout):
                     return True
-            except Exception as e:
-                self.logger.exception(e, True, f"Failed to cancel order {order}")
+            except Exception as err:
+                self.logger.exception(err, True, f"Failed to cancel order {order}")
                 return False
             if order_status is enums.OrderStatus.CANCELED:
                 order.status = octobot_trading.enums.OrderStatus.CANCELED
@@ -446,6 +449,11 @@ class Trader(util.Initializable):
                 await self._wait_for_order_cancel(order, cancelling_timeout)
             return True
         elif order.is_open():
+            if isinstance(err, errors.OrderNotFoundOnCancelError):
+                raise errors.OrderNotFoundOnCancelError(
+                    f"Tried to cancel an order that can't be found, it might be cancelled or filled already ({err}). "
+                    f"Order: {order}"
+                ) from err
             raise errors.OpenOrderError(
                 f"Order is open, but can't be cancelled. This is unexpected. Order: {order}"
             ) from err

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/orders/channel/orders.py

@@ -29,7 +29,7 @@ class OrdersProducer(exchanges_channel.ExchangeChannelProducer):
 
     async def perform(self, orders, is_from_bot=False, are_closed=False):
         try:
-            self.logger.debug(f"Received order update for {len(orders)} {'closed' if are_closed else ''}orders.")
+            self.logger.debug(f"Received order update for {len(orders)} {'closed' if are_closed else ''} orders.")
             has_new_order = False
             waiting_complete_init_orders = []
             symbols = set()
@@ -245,8 +245,10 @@ class OrdersProducer(exchanges_channel.ExchangeChannelProducer):
             )
         )
         if missing_exchange_order_ids:
-            self.logger.debug(f"{len(missing_exchange_order_ids)} open orders are missing on exchange, "
-                              f"synchronizing with exchange (exchange ids: {missing_exchange_order_ids})...")
+            self.logger.info(
+                f"{len(missing_exchange_order_ids)} open orders are missing on exchange, "
+                f"synchronizing with exchange (exchange ids: {missing_exchange_order_ids})..."
+            )
             synchronize_tasks = []
             for missing_order_id in missing_exchange_order_ids:
                 try:
@@ -254,11 +256,13 @@ class OrdersProducer(exchanges_channel.ExchangeChannelProducer):
                         get_order(None, exchange_order_id=missing_order_id)
                     if order_to_update.state is not None:
                         # catch exception not to prevent multiple synchronize to be cancelled in asyncio.gather
-                        synchronize_tasks.append(order_to_update.state.synchronize(force_synchronization=True,
-                                                                                   catch_exception=True))
+                        synchronize_tasks.append(
+                            order_to_update.state.synchronize(force_synchronization=True, catch_exception=True)
+                        )
                 except KeyError:
-                    self.logger.error(f"Order with id {missing_order_id} could not be synchronized: "
-                                      f"missing from order manager")
+                    self.logger.error(
+                        f"Order with id {missing_order_id} could not be synchronized: missing from order manager"
+                    )
             await asyncio.gather(*synchronize_tasks)
 
     async def send(

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/orders/channel/orders_updater.py

@@ -194,17 +194,17 @@ class OrdersUpdater(orders_channel.OrdersProducer):
         :return: True if the order was updated
         """
         exchange_name = self.channel.exchange_manager.exchange_name
-        self.logger.debug(f"Requested update for {order} on {exchange_name}")
+        self.logger.info(f"Requested update for {order} on {exchange_name}")
         raw_order = await self.channel.exchange_manager.exchange.get_order(order.exchange_order_id, order.symbol)
 
         if raw_order is not None:
-            self.logger.debug(f"Received update for {order} on {exchange_name}: {raw_order}")
+            self.logger.info(f"Received update for {order} on {exchange_name}: {raw_order}")
 
             await self.channel.exchange_manager.exchange_personal_data.handle_order_update_from_raw(
                 order.exchange_order_id, raw_order, should_notify=should_notify
             )
         else:
-            self.logger.debug(f"Can't received update for {order} on {exchange_name}: received order is None")
+            self.logger.info(f"Can't received update for {order} on {exchange_name}: received order is None")
 
     async def stop(self) -> None:
         """

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/orders/order_state.py

@@ -13,6 +13,7 @@
 #
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library.
+import asyncio
 
 import octobot_commons.logging as logging
 
@@ -24,6 +25,8 @@ import octobot_trading.personal_data.state as state_class
 
 
 class OrderState(state_class.State):
+    MAX_SYNCHRONIZATION_ATTEMPTS = -1   # implement self._force_final_state() when this is enabled
+
     def __init__(self, order, is_from_exchange_data, enable_associated_orders_creation=True):
         super().__init__(is_from_exchange_data)
 
@@ -59,6 +62,12 @@ class OrderState(state_class.State):
         """
         return False
 
+    def is_max_synchronization_attempts_reached(self) -> bool:
+        """
+        :return: True if the MAX_SYNCHRONIZATION_ATTEMPTS is set and has been reached
+        """
+        return -1 < self.__class__.MAX_SYNCHRONIZATION_ATTEMPTS <= self.synchronization_attempts
+
     def get_logger(self):
         """
         :return: the order logger
@@ -98,27 +107,40 @@ class OrderState(state_class.State):
             self.order = new_order
             self.order.state = self
 
+    def _force_final_state(self):
+        raise NotImplementedError("_force_final_state is not implemented")
+
     async def _synchronize_with_exchange(self, force_synchronization: bool = False) -> None:
         """
         Ask OrdersChannel Internal producer to refresh the order from the exchange
         :param force_synchronization: When True, for the update of the order from the exchange
         :return: the result of OrdersProducer.update_order_from_exchange()
         """
-        try:
-            self.ensure_not_cleared(self.order)
+        if self.is_max_synchronization_attempts_reached():
             self.get_logger().info(
-                f"Synchronizing order {self.order} with {self.order.exchange_manager.exchange_name} exchange"
-            )
-            await exchange_channel.get_chan(
-                octobot_trading.constants.ORDERS_CHANNEL,
-                self.order.exchange_manager.id
-            ).get_internal_producer().update_order_from_exchange(
-                order=self.order,
-                wait_for_refresh=True,
-                force_job_execution=force_synchronization,
+                f"Forcing {self.__class__.__name__} final state after {self.MAX_SYNCHRONIZATION_ATTEMPTS} "
+                f"synchronization attempts for order {self.order}"
             )
-        except octobot_trading.errors.InvalidOrderState:
-            self.get_logger().debug(f"Skipping exchange synchronisation as order has already been closed.")
+            self._force_final_state()
+            asyncio.create_task(self.on_refresh_successful())
+        else:
+            try:
+                self.ensure_not_cleared(self.order)
+                self.get_logger().info(
+                    f"Synchronizing [{self._underlying_refreshed_state.value}] order {self.order} "
+                    f"with {self.order.exchange_manager.exchange_name} exchange"
+                )
+                self.synchronization_attempts += 1
+                await exchange_channel.get_chan(
+                    octobot_trading.constants.ORDERS_CHANNEL,
+                    self.order.exchange_manager.id
+                ).get_internal_producer().update_order_from_exchange(
+                    order=self.order,
+                    wait_for_refresh=True,
+                    force_job_execution=force_synchronization,
+                )
+            except octobot_trading.errors.InvalidOrderState:
+                self.get_logger().debug(f"Skipping exchange synchronisation as order has already been closed.")
 
     async def wait_for_next_state(self, timeout) -> None:
         # terminate is called at the end of the state for most order states

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/orders/states/cancel_order_state.py

@@ -22,6 +22,8 @@ import octobot_trading.personal_data.orders.states.order_state_factory as order_
 
 
 class CancelOrderState(order_state.OrderState):
+    MAX_SYNCHRONIZATION_ATTEMPTS = 5
+
     def __init__(self, order, is_from_exchange_data, enable_associated_orders_creation=True):
         super().__init__(
             order, is_from_exchange_data, enable_associated_orders_creation=enable_associated_orders_creation
@@ -32,8 +34,7 @@ class CancelOrderState(order_state.OrderState):
 
     async def initialize_impl(self, forced=False, ignored_order=None) -> None:
         if forced:
-            self.state = enums.OrderStates.CANCELED
-            self.order.status = enums.OrderStatus.CANCELED
+            self._force_final_state()
 
         if self.order.order_group and self.enable_associated_orders_creation:
             await self.order.order_group.on_cancel(self.order, ignored_orders=[ignored_order])
@@ -52,6 +53,10 @@ class CancelOrderState(order_state.OrderState):
     def is_status_cancelled(self) -> bool:
         return not self.is_status_pending() and self.order.status in constants.CANCEL_ORDER_STATUS_SCOPE
 
+    def _force_final_state(self):
+        self.state = enums.OrderStates.CANCELED
+        self.order.status = enums.OrderStatus.CANCELED
+
     def allows_new_status(self, status) -> bool:
         """
         Don't allow going from canceling to open
@@ -68,8 +73,9 @@ class CancelOrderState(order_state.OrderState):
         if not self.has_already_been_synchronized_once:
             # If we want to sync this state, it means the order is being canceled by the exchange but is not
             # fully canceled yet. Giving some time to the exchange before re-requesting it.
-            self.get_logger().debug(f"{self.__class__.__name__} still pending, "
-                                    f"synchronizing in {self.PENDING_REFRESH_INTERVAL}s")
+            self.get_logger().info(
+                f"{self.__class__.__name__} still pending, synchronizing in {self.PENDING_REFRESH_INTERVAL}s"
+            )
             await asyncio.sleep(self.PENDING_REFRESH_INTERVAL)
         await super()._synchronize_with_exchange(force_synchronization=force_synchronization)
 

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/orders/states/fill_order_state.py

@@ -21,6 +21,8 @@ import octobot_trading.personal_data.orders.states.order_state_factory as order_
 
 
 class FillOrderState(order_state.OrderState):
+    MAX_SYNCHRONIZATION_ATTEMPTS = 5
+
     def __init__(self, order, is_from_exchange_data, enable_associated_orders_creation=True):
         super().__init__(
             order, is_from_exchange_data, enable_associated_orders_creation=enable_associated_orders_creation
@@ -34,10 +36,13 @@ class FillOrderState(order_state.OrderState):
 
     async def initialize_impl(self, forced=False) -> None:
         if forced:
-            self.state = enums.OrderStates.FILLED
-            self.order.status = enums.OrderStatus.FILLED
+            self._force_final_state()
         return await super().initialize_impl()
 
+    def _force_final_state(self):
+        self.state = enums.OrderStates.FILLED
+        self.order.status = enums.OrderStatus.FILLED
+
     def is_pending(self) -> bool:
         return self.state is enums.OrderStates.FILLING
 

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/orders/states/pending_creation_order_state.py

@@ -47,11 +47,13 @@ class PendingCreationOrderState(order_state.OrderState):
             # open orders refresh.
             self.ensure_not_cleared(self.order)
             self.get_logger().info(
-                f"Synchronizing order {self.order} with {self.order.exchange_manager.exchange_name} exchange"
+                f"Synchronizing [{self._underlying_refreshed_state.value}] order {self.order} "
+                f"with {self.order.exchange_manager.exchange_name} exchange"
             )
             while self.order.is_pending_creation() \
                     and time.time() - t0 < octobot_trading.constants.INDIVIDUAL_ORDER_SYNC_TIMEOUT:
                 iteration += 1
+                self.synchronization_attempts += 1
                 await exchange_channel.get_chan(
                     octobot_trading.constants.ORDERS_CHANNEL,
                     self.order.exchange_manager.id

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/portfolios/history/historical_portfolio_value_manager.py

@@ -192,6 +192,10 @@ class HistoricalPortfolioValueManager(util.Initializable):
 
     def _add_historical_portfolio_value(self, timestamp, value_by_currency):
         if len(self.historical_portfolio_value) >= self.max_history_size:
+            self.logger.info(
+                f"Clearing oldest historical portfolio value as the maximum historical values ({self.max_history_size}) "
+                f"has been reached"
+            )
             # remove the oldest element
             self.historical_portfolio_value.popitem(0)
         self.historical_portfolio_value[timestamp] = \

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/positions/position_state.py

@@ -61,6 +61,7 @@ class PositionState(state_class.State):
         :param force_synchronization: When True, for the update of the position from the exchange
         :return: the result of PositionsProducer.update_position_from_exchange()
         """
+        self.synchronization_attempts += 1
         return self.position.exchange_manager.exchange_personal_data.positions_manager.\
             refresh_real_trader_position(self.position, force_job_execution=force_synchronization)
 

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/state.py

@@ -29,7 +29,10 @@ class State(util.Initializable):
         super().__init__()
 
         # default state
-        self.state = enums.States.UNKNOWN
+        self.state: enums.States = enums.States.UNKNOWN
+
+        # state when refreshing, is only indicative and can be outdated du to avoid concurrency issues
+        self._underlying_refreshed_state: enums.States = enums.States.UNKNOWN
 
         # if this state has been created from exchange data or OctoBot internal mechanism
         self.is_from_exchange_data = is_from_exchange_data
@@ -43,6 +46,9 @@ class State(util.Initializable):
         # set at True after synchronize has been called
         self.has_already_been_synchronized_once = False
 
+        # how many times this state has been synchronized
+        self.synchronization_attempts = 0
+
     def is_pending(self) -> bool:
         """
         :return: True if the state is pending for update
@@ -179,6 +185,8 @@ class State(util.Initializable):
     async def refresh_operation(self):
         self.get_logger().debug("Starting refresh_operation")
         previous_state = self.state
+        # don't rely on self._underlying_refreshed_state only to avoid concurrency issues (use local variable instead)
+        self._underlying_refreshed_state = previous_state
         async with self.lock:
             self.state = enums.States.REFRESHING
         try:
@@ -187,6 +195,7 @@ class State(util.Initializable):
             async with self.lock:
                 if self.state is enums.States.REFRESHING:
                     self.state = previous_state
+                self._underlying_refreshed_state = enums.States.UNKNOWN
             self.get_logger().debug("Completed refresh_operation")
 
     async def _synchronize_with_exchange(self, force_synchronization: bool = False) -> None:

{OctoBot-Trading-2.4.112 → OctoBot-Trading-2.4.113}/octobot_trading/personal_data/trades/trades_manager.py

@@ -184,6 +184,10 @@ class TradesManager(util.Initializable):
             self.logger.exception(err, True, f"Error when loading local trade history {err}")
 
     def _remove_oldest_trades(self, nb_to_remove):
+        self.logger.info(
+            f"Clearing the {nb_to_remove} oldest historical trades as the maximum count of trades ("
+            f"{self.MAX_TRADES_COUNT}) has been reached"
+        )
         for _ in range(nb_to_remove):
             self.trades.popitem(last=False)