inorbit-edge-executor 3.2.0__tar.gz → 3.2.4__tar.gz

{inorbit_edge_executor-3.2.0 → inorbit_edge_executor-3.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: inorbit_edge_executor
-Version: 3.2.0
+Version: 3.2.4
 Summary: InOrbit Edge Missions Executor
 Author-email: Mariano Cereda <mariano.cereda@inorbit.ai>, Hernan Badenes <herchu@inorbit.ai>, Miguel Garcia <mike@inorbit.ai>
 License: MIT

{inorbit_edge_executor-3.2.0 → inorbit_edge_executor-3.2.4}/inorbit_edge_executor/behavior_tree.py

@@ -557,6 +557,8 @@ class RunActionNode(BehaviorTree):
         action_id,
         arguments,
         target: Target = None,
+        max_retries: int = 3,
+        retry_wait_seconds: float = 5.0,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -564,6 +566,8 @@ class RunActionNode(BehaviorTree):
         self.action_id = action_id
         self.arguments = arguments
         self.target = target
+        self.max_retries = max_retries
+        self.retry_wait_seconds = retry_wait_seconds
         if self.target is None:
             self.robot = context.robot_api
         else:
@@ -571,9 +575,29 @@ class RunActionNode(BehaviorTree):
 
     async def _execute(self):
         arguments = await self.mt.resolve_arguments(self.arguments)
-        resp = await self.robot.execute_action(self.action_id, arguments=arguments)
-        # TODO track action execution, as done in the app. This JSON response only guarantees
-        # the action was *started*.
+
+        last_exception = None
+        for attempt in range(self.max_retries + 1):
+            try:
+                resp = await self.robot.execute_action(self.action_id, arguments=arguments)
+                # TODO track action execution, as done in the app. This JSON response only guarantees
+                # the action was *started*.
+                return  # Success, exit retry loop
+            except Exception as e:
+                last_exception = e
+                if attempt < self.max_retries:
+                    logger.warning(
+                        f"Action execution failed (attempt {attempt + 1}/{self.max_retries + 1}): {e}. "
+                        f"Retrying in {self.retry_wait_seconds} seconds..."
+                    )
+                    await asyncio.sleep(self.retry_wait_seconds)
+                else:
+                    logger.error(
+                        f"Action execution failed after {self.max_retries + 1} attempts: {e}"
+                    )
+
+        # All retries exhausted, raise the last exception
+        raise last_exception
 
     async def on_pause(self):
         # TODO (Elvio): Here goes the logic to stop an action when a Mission is paused
@@ -584,15 +608,28 @@ class RunActionNode(BehaviorTree):
         object = super().dump_object()
         object["action_id"] = self.action_id
         object["arguments"] = self.arguments
+        object["max_retries"] = self.max_retries
+        object["retry_wait_seconds"] = self.retry_wait_seconds
         if self.target is not None:
             object["target"] = self.target.dump_object()
         return object
 
     @classmethod
-    def from_object(cls, context, action_id, arguments, target=None, **kwargs):
+    def from_object(
+        cls,
+        context,
+        action_id,
+        arguments,
+        target=None,
+        max_retries=3,
+        retry_wait_seconds=5.0,
+        **kwargs,
+    ):
         if target is not None:
             target = Target.from_object(**target)
-        return RunActionNode(context, action_id, arguments, target, **kwargs)
+        return RunActionNode(
+            context, action_id, arguments, target, max_retries, retry_wait_seconds, **kwargs
+        )
 
 
 class WaitExpressionNode(BehaviorTree):

{inorbit_edge_executor-3.2.0 → inorbit_edge_executor-3.2.4}/inorbit_edge_executor/worker.py

@@ -38,6 +38,7 @@ class Worker(Observable):
         self._robot_api_factory = None
         self._behavior_tree = None
         self._shared_memory = shared_memory
+        self._task: asyncio.Task = None
 
     def id(self):
         return self._mission.id
@@ -133,6 +134,10 @@ class Worker(Observable):
             self.set_paused(True)
             self._task.cancel(CANCEL_TASK_PAUSE_MESSAGE)
 
+    def get_task(self) -> asyncio.Task | None:
+        """Get the task associated with the worker. This is used to wait for the task to complete during shutdown."""
+        return self._task
+
     async def resume(self):
         """
         Resumes the worker if it was paused. Returns True if the worker was paused and resumed, False otherwise.

{inorbit_edge_executor-3.2.0 → inorbit_edge_executor-3.2.4}/inorbit_edge_executor/worker_pool.py

@@ -103,12 +103,53 @@ class WorkerPool:
         for worker_state in serialized_workers:
             await self.execute_serialized_worker(worker_state)
 
-    async def shutdown(self):
+    async def shutdown(self, timeout: float = 10.0):
         """
-        Stops the worker pool. It's actually not doing anything more than preventing new jobs
-        to be submitted.
+        Stops the worker pool gracefully by cancelling all running tasks.
+
+        Args:
+            timeout: Maximum time to wait for graceful shutdown before forcing cancellation
         """
+        if not self._running:
+            return
+
         self._running = False
+        logger.info("Starting worker pool shutdown...")
+
+        # Get all running workers and cancel them
+        async with self._mutex:
+            workers_to_cancel = list(self._workers.values())
+
+        if workers_to_cancel:
+            logger.info(f"Cancelling {len(workers_to_cancel)} running workers...")
+
+            # Cancel all workers and collect their tasks
+            cancelled_tasks = []
+            for worker in workers_to_cancel:
+                try:
+                    if worker.cancel():
+                        cancelled_tasks.append(worker.get_task())
+                except Exception as e:
+                    logger.error(f"Error cancelling worker {worker.id()}: {e}")
+
+            if cancelled_tasks:
+                # Wait for tasks to complete cancellation with timeout
+                try:
+                    await asyncio.wait_for(
+                        asyncio.gather(*cancelled_tasks, return_exceptions=True), timeout=timeout
+                    )
+                    logger.info("All workers cancelled gracefully")
+                except asyncio.TimeoutError:
+                    logger.warning(f"Timeout waiting for graceful shutdown after {timeout}s")
+        else:
+            logger.info("No running workers to cancel")
+
+        # Clean up workers dictionary
+        async with self._mutex:
+            self._workers.clear()
+
+        # Shutdown the database connection
+        await self._db.shutdown()
 
     async def notify(self, worker: Worker):
         """Notified when a worker changed its state. Persist it"""

{inorbit_edge_executor-3.2.0 → inorbit_edge_executor-3.2.4}/inorbit_edge_executor.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: inorbit_edge_executor
-Version: 3.2.0
+Version: 3.2.4
 Summary: InOrbit Edge Missions Executor
 Author-email: Mariano Cereda <mariano.cereda@inorbit.ai>, Hernan Badenes <herchu@inorbit.ai>, Miguel Garcia <mike@inorbit.ai>
 License: MIT

{inorbit_edge_executor-3.2.0 → inorbit_edge_executor-3.2.4}/inorbit_edge_executor.egg-info/SOURCES.txt

@@ -21,5 +21,6 @@ inorbit_edge_executor.egg-info/requires.txt
 inorbit_edge_executor.egg-info/top_level.txt
 tests/test_behavior_tree.py
 tests/test_behavior_tree_error_handler.py
+tests/test_run_action_node.py
 tests/test_set_data_node.py
 tests/test_wait_expression_node.py

{inorbit_edge_executor-3.2.0 → inorbit_edge_executor-3.2.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "inorbit_edge_executor"
-version = "3.2.0"
+version = "3.2.4"
 description = "InOrbit Edge Missions Executor"
 readme = "README.md"
 authors = [
@@ -47,7 +47,7 @@ target-version = ['py310']
 line-length = 100
 
 [tool.bumpversion]
-current_version = "3.2.0"
+current_version = "3.2.4"
 commit = true
 tag = true
 tag_name = "v{new_version}"

inorbit_edge_executor-3.2.4/tests/test_run_action_node.py

@@ -0,0 +1,288 @@
+import pytest
+from unittest.mock import AsyncMock, Mock
+
+from inorbit_edge_executor.behavior_tree import RunActionNode, BehaviorTreeBuilderContext
+from inorbit_edge_executor.datatypes import Target
+from inorbit_edge_executor.inorbit import RobotApi, RobotApiFactory
+
+
+class TestRunActionNodeRetry:
+    @pytest.fixture
+    def mock_robot_api(self):
+        """Mock RobotApi for testing"""
+        mock_api = Mock(spec=RobotApi)
+        mock_api.robot_id = "test_robot"
+        mock_api.execute_action = AsyncMock()
+        return mock_api
+
+    @pytest.fixture
+    def mock_mission_tracking(self):
+        """Mock mission tracking for testing"""
+        mock_mt = Mock()
+        mock_mt.resolve_arguments = AsyncMock(return_value={"param": "value"})
+        return mock_mt
+
+    @pytest.fixture
+    def context(self, mock_robot_api, mock_mission_tracking):
+        """Create a test context with mocked dependencies"""
+        context = BehaviorTreeBuilderContext()
+        context.robot_api = mock_robot_api
+        context.mt = mock_mission_tracking
+        return context
+
+    @pytest.mark.asyncio
+    async def test_execute_action_success_first_try(self, context, mock_robot_api):
+        """Test successful action execution on first attempt"""
+        # Setup
+        mock_robot_api.execute_action.return_value = {"status": "success"}
+
+        node = RunActionNode(
+            context=context,
+            action_id="test_action",
+            arguments={"param": "value"},
+            max_retries=3,
+            retry_wait_seconds=0.01,
+        )
+
+        # Execute
+        await node._execute()
+
+        # Verify
+        mock_robot_api.execute_action.assert_called_once_with(
+            "test_action", arguments={"param": "value"}
+        )
+
+    @pytest.mark.asyncio
+    async def test_execute_action_success_after_retries(self, context, mock_robot_api):
+        """Test successful action execution after some failures"""
+        # Setup - fail first 2 attempts, succeed on 3rd
+        mock_robot_api.execute_action.side_effect = [
+            Exception("Network timeout"),
+            Exception("Connection refused"),
+            {"status": "success"},  # Success on third try
+        ]
+
+        node = RunActionNode(
+            context=context,
+            action_id="test_action",
+            arguments={"param": "value"},
+            max_retries=3,
+            retry_wait_seconds=0.01,
+        )
+
+        # Execute
+        await node._execute()
+
+        # Verify
+        assert mock_robot_api.execute_action.call_count == 3
+
+    @pytest.mark.asyncio
+    async def test_execute_action_fail_all_retries(self, context, mock_robot_api):
+        """Test action execution failure after exhausting all retries"""
+        # Setup - all attempts fail
+        test_exception = Exception("Persistent network error")
+        mock_robot_api.execute_action.side_effect = test_exception
+
+        node = RunActionNode(
+            context=context,
+            action_id="test_action",
+            arguments={"param": "value"},
+            max_retries=2,
+            retry_wait_seconds=0.01,
+        )
+
+        # Execute and verify exception is raised
+        with pytest.raises(Exception, match="Persistent network error"):
+            await node._execute()
+
+        # Verify all retry attempts were made (max_retries + 1 = 3 attempts)
+        assert mock_robot_api.execute_action.call_count == 3
+
+    @pytest.mark.asyncio
+    async def test_execute_action_different_exception_types(self, context, mock_robot_api):
+        """Test that different types of exceptions are handled consistently"""
+        exceptions_to_test = [
+            ValueError("Invalid parameter"),
+            ConnectionError("Network unreachable"),
+            TimeoutError("Request timeout"),
+            RuntimeError("Service unavailable"),
+        ]
+
+        for exception in exceptions_to_test:
+            # Reset mock
+            mock_robot_api.execute_action.reset_mock()
+            mock_robot_api.execute_action.side_effect = exception
+
+            node = RunActionNode(
+                context=context,
+                action_id="test_action",
+                arguments={"param": "value"},
+                max_retries=1,
+                retry_wait_seconds=0.01,
+            )
+
+            # Execute and verify exception is raised
+            with pytest.raises(type(exception)):
+                await node._execute()
+
+            # Verify retries were attempted
+            assert mock_robot_api.execute_action.call_count == 2
+
+    @pytest.mark.asyncio
+    async def test_execute_action_with_custom_max_retries(self, context, mock_robot_api):
+        """Test action execution with different max_retries values"""
+        test_cases = [0, 1, 5, 10]
+
+        for max_retries in test_cases:
+            # Reset mock
+            mock_robot_api.execute_action.reset_mock()
+            mock_robot_api.execute_action.side_effect = Exception("Always fail")
+
+            node = RunActionNode(
+                context=context,
+                action_id="test_action",
+                arguments={"param": "value"},
+                max_retries=max_retries,
+                retry_wait_seconds=0.01,
+            )
+
+            # Execute and verify exception is raised
+            with pytest.raises(Exception):
+                await node._execute()
+
+            # Verify correct number of attempts (max_retries + 1)
+            expected_attempts = max_retries + 1
+            assert mock_robot_api.execute_action.call_count == expected_attempts
+
+    @pytest.mark.asyncio
+    async def test_retry_delay_timing(self, context, mock_robot_api):
+        """Test that retry delays are working (approximate timing)"""
+        # Setup
+        mock_robot_api.execute_action.side_effect = [
+            Exception("Fail 1"),
+            Exception("Fail 2"),
+            {"status": "success"},
+        ]
+
+        node = RunActionNode(
+            context=context,
+            action_id="test_action",
+            arguments={"param": "value"},
+            max_retries=2,
+            retry_wait_seconds=0.01,
+        )
+
+        # Execute with timing
+        import time
+
+        start_time = time.time()
+        await node._execute()
+        end_time = time.time()
+
+        # Should take at least 0.02 seconds (2 retries * 0.01 seconds each)
+        # but allow some tolerance for test execution overhead
+        assert end_time - start_time >= 0.015
+
+    @pytest.mark.asyncio
+    async def test_execute_action_with_target_robot(self, mock_mission_tracking):
+        """Test action execution with a different target robot"""
+        # Setup target robot
+        target = Target(robotId="target_robot")
+
+        # Create mock robot API factory
+        mock_target_robot_api = Mock(spec=RobotApi)
+        mock_target_robot_api.execute_action = AsyncMock(return_value={"status": "success"})
+
+        mock_factory = Mock(spec=RobotApiFactory)
+        mock_factory.build.return_value = mock_target_robot_api
+
+        context = BehaviorTreeBuilderContext()
+        context.robot_api_factory = mock_factory
+        context.mt = mock_mission_tracking
+
+        node = RunActionNode(
+            context=context,
+            action_id="test_action",
+            arguments={"param": "value"},
+            target=target,
+            max_retries=2,
+            retry_wait_seconds=0.01,
+        )
+
+        # Execute
+        await node._execute()
+
+        # Verify
+        mock_factory.build.assert_called_once_with("target_robot")
+        mock_target_robot_api.execute_action.assert_called_once_with(
+            "test_action", arguments={"param": "value"}
+        )
+
+    def test_dump_object_includes_max_retries(self, context):
+        """Test that dump_object includes max_retries parameter"""
+        node = RunActionNode(
+            context=context,
+            action_id="test_action",
+            arguments={"param": "value"},
+            max_retries=5,
+            retry_wait_seconds=0.01,
+        )
+
+        dumped = node.dump_object()
+
+        assert dumped["max_retries"] == 5
+        assert dumped["retry_wait_seconds"] == 0.01
+        assert dumped["action_id"] == "test_action"
+        assert dumped["arguments"] == {"param": "value"}
+
+    def test_from_object_creates_node_with_max_retries(self, context):
+        """Test that from_object correctly handles max_retries parameter"""
+        node = RunActionNode.from_object(
+            context=context,
+            action_id="test_action",
+            arguments={"param": "value"},
+            max_retries=7,
+            retry_wait_seconds=0.01,
+        )
+
+        assert node.max_retries == 7
+        assert node.retry_wait_seconds == 0.01
+        assert node.action_id == "test_action"
+        assert node.arguments == {"param": "value"}
+
+    def test_from_object_defaults_max_retries(self, context):
+        """Test that from_object uses default max_retries when not specified"""
+        node = RunActionNode.from_object(
+            context=context, action_id="test_action", arguments={"param": "value"}
+        )
+
+        assert node.max_retries == 3  # Default value
+        assert node.retry_wait_seconds == 5.0  # Default value
+
+    def test_serialization_round_trip(self, context):
+        """Test that serialization and deserialization preserve max_retries"""
+        original_node = RunActionNode(
+            context=context,
+            action_id="test_action",
+            arguments={"param": "value"},
+            max_retries=4,
+            retry_wait_seconds=0.01,
+            label="test_label",
+        )
+
+        # Serialize
+        dumped = original_node.dump_object()
+
+        # Remove type field that's used by the serialization system
+        dumped_for_restore = dumped.copy()
+        del dumped_for_restore["type"]
+
+        # Deserialize
+        restored_node = RunActionNode.from_object(context=context, **dumped_for_restore)
+
+        # Verify all properties are preserved
+        assert restored_node.max_retries == 4
+        assert restored_node.retry_wait_seconds == 0.01
+        assert restored_node.action_id == "test_action"
+        assert restored_node.arguments == {"param": "value"}
+        assert restored_node.label == "test_label"