manta-node 0.5b0.dev91__py3-none-any.whl → 0.5b0.dev117__py3-none-any.whl

manta_node/infrastructure/mqtt/command_handler.py

@@ -10,7 +10,7 @@ import aiomqtt
 from manta_common.base_mqtt import MqttBase
 from manta_common.build.common.mqtt import MqMetricsSnapshot, MqTaskUpdate
 from manta_common.build.common.system import Logs
-from manta_common.build.common.tasks import TaskStatus, TaskUpdate
+from manta_common.build.common.tasks import MqttTask, TaskStatus, TaskUpdate
 from manta_common.const import CHUNK_SIZE
 from manta_common.conversions import ID
 from manta_common.errors import (
@@ -121,52 +121,83 @@ class CommandHandler(TaskManager, Collector, MqttBase):
         except Exception:
             self.tracer.exception("Error when closing Docker client")
 
+    async def main(self):
+        """
+        Override MqttBase.main to run task checker as a background task
+        alongside the MQTT message loop.
+        """
+        self.tracer.debug("Starting background task checker")
+        checker = asyncio.create_task(self._ping_loop())
+        try:
+            await MqttBase.main(self)
+        finally:
+            checker.cancel()
+            try:
+                await checker
+            except asyncio.CancelledError:
+                pass
+
     async def loop_tasks(self):
         """
-        Processes messages or check tasks to ping
+        MQTT message loop (subscribe + publish).
+        Task checking runs independently in the background via main().
+
+        Overrides base class to fix a race condition: the background
+        _ping_loop can queue messages during subscribe processing,
+        causing publish_task (queue.get()) to consume and remove them
+        from the queue. If the base class processes subscribe first
+        without also handling the completed publish_task, the message
+        is silently dropped. See issue #285.
         """
         while True:
-            # Wait until we either (1) receive a message or (2) publish a message
             try:
                 subscribe_task = asyncio.create_task(self.client.messages.__anext__())
                 publish_task = asyncio.create_task(self.queue.get())
-                ping_task = asyncio.create_task(self._ping())
                 done, _ = await asyncio.wait(
-                    (subscribe_task, publish_task, ping_task),
+                    (subscribe_task, publish_task),
                     return_when=asyncio.FIRST_COMPLETED,
                 )
-            # If the asyncio.wait is cancelled, we must also cancel the queue task
             except asyncio.CancelledError:
                 subscribe_task.cancel()
                 publish_task.cancel()
-                ping_task.cancel()
+                await asyncio.gather(
+                    subscribe_task, publish_task, return_exceptions=True
+                )
                 raise
 
-            # When we receive a message, process it
-            if subscribe_task in done and publish_task in done:
-                await self._subscribe_from_task(subscribe_task)
-                await self._publish_from_task(publish_task)
-                ping_task.cancel()
-            elif subscribe_task in done:
-                await self._subscribe_from_task(subscribe_task)
-                publish_task.cancel()
-                ping_task.cancel()
-            # Publish the message to its topic with specific QoS
-            elif publish_task in done:
-                await self._publish_from_task(publish_task)
-                subscribe_task.cancel()
-                ping_task.cancel()
-            # If we disconnect from the broker, stop the loop with an exception
-            elif ping_task in done:
-                ping_task.cancel()
-                publish_task.cancel()
-                subscribe_task.cancel()
-            else:
-                self.tracer.warning("Disconnected during message iteration")
-                publish_task.cancel()
-                subscribe_task.cancel()
-                ping_task.cancel()
-                raise aiomqtt.MqttError("Disconnected during message iteration")
+            try:
+                if subscribe_task in done and publish_task in done:
+                    try:
+                        await self._subscribe_from_task(subscribe_task)
+                    finally:
+                        await self._publish_from_task(publish_task)
+                elif subscribe_task in done:
+                    await self._subscribe_from_task(subscribe_task)
+                    if publish_task.done():
+                        await self._publish_from_task(publish_task)
+                elif publish_task in done:
+                    await self._publish_from_task(publish_task)
+                else:
+                    self.tracer.warning("Disconnected during message iteration")
+                    raise aiomqtt.MqttError("Disconnected during message iteration")
+            finally:
+                if not subscribe_task.done():
+                    subscribe_task.cancel()
+                if not publish_task.done():
+                    publish_task.cancel()
+
+    async def _ping_loop(self):
+        """
+        Background loop that periodically checks task statuses and
+        sends heartbeats. Runs independently of the MQTT connection state.
+        """
+        while True:
+            try:
+                await self._ping()
+            except asyncio.CancelledError:
+                break
+            except Exception:
+                self.tracer.exception("Unhandled error in ping loop, continuing...")
 
     async def _ping(self):
         """
@@ -313,3 +344,32 @@ class CommandHandler(TaskManager, Collector, MqttBase):
             Node ID to stop
         """
         await self.stop_callback()
+
+    async def on_task_setup_failed(self, task: MqttTask, exc: Exception):
+        """
+        Publish a FAILED task update when task setup fails.
+
+        Parameters
+        ----------
+        task : MqttTask
+            Task that failed to setup
+        exc : Exception
+            The exception that caused the failure
+        """
+        task_update = TaskUpdate(
+            swarm_id=task.swarm_id,
+            node_id=self.tracer.metadata.node_id,
+            task_id=task.task_id,
+            iteration=task.iteration,
+            circular=task.circular,
+            task_status=TaskStatus.FAILED,
+        )
+        try:
+            await self.publish_task_update(task_update)
+            self.tracer.info(
+                f"Published FAILED status for task {task.task_id} after setup failure"
+            )
+        except Exception as publish_exc:
+            self.tracer.exception(
+                f"Failed to publish FAILED status for task {task.task_id}: {repr(publish_exc)}"
+            )

manta_node/task_manager.py

@@ -352,12 +352,31 @@ class TaskManager(ContainerManager, ImageExecutor):
         swarm_id = ID(task.swarm_id)
         self.tracer.info(f"Deploying task {task_id}")
 
-        if (swarm_id.tid, task_id.tid) in self.tasks:
-            await self.restart_task(self.tasks[(swarm_id.tid, task_id.tid)], task)
-        else:
-            await self.create_task(task)
+        try:
+            if (swarm_id.tid, task_id.tid) in self.tasks:
+                await self.restart_task(self.tasks[(swarm_id.tid, task_id.tid)], task)
+            else:
+                await self.create_task(task)
+        except Exception as exc:
+            self.tracer.exception(f"Task {task_id} setup failed: {repr(exc)}")
+            await self.on_task_setup_failed(task, exc)
+            raise
         return task_id
 
+    async def on_task_setup_failed(self, task: MqttTask, exc: Exception):
+        """
+        Hook called when task setup fails. Override in subclasses to handle
+        failure notifications (e.g., publish MQTT update).
+
+        Parameters
+        ----------
+        task : MqttTask
+            Task that failed to setup
+        exc : Exception
+            The exception that caused the failure
+        """
+        pass
+
     async def restart_task(self, task_progression: TaskProgression, task: MqttTask):
         """
         Restart a task

{manta_node-0.5b0.dev91.dist-info → manta_node-0.5b0.dev117.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: manta_node
-Version: 0.5b0.dev91
+Version: 0.5b0.dev117
 Summary: Node side software of the Manta platform. It is responsible for the communication with the Manta Core and the execution of the tasks.
 Author-email: Hugo Miralles <hugomiralles@manta-tech.io>, Matthew Thompson <matthew.thompson@manta-tech.io>
 License:                     GNU AFFERO GENERAL PUBLIC LICENSE

{manta_node-0.5b0.dev91.dist-info → manta_node-0.5b0.dev117.dist-info}/RECORD

@@ -1,7 +1,7 @@
 manta_node/__init__.py,sha256=bgGWXH3yQVhNf0v1ZGUBAy5Y51fqb0Y-EaLnocbcino,153
 manta_node/__main__.py,sha256=0VzbY1KfezaARZDuCBc7Th1GO_-BnKTK8eDEXhgP5J4,1731
 manta_node/node_orchestrator.py,sha256=0VMBsZSMQiIe9L-Bd7Jn4avNpNyDCMa1kyPSTgRiOes,17336
-manta_node/task_manager.py,sha256=B_h6Mc7to8CTzLLgqrINgEALTYWX1Aek3Tio34ICXag,19887
+manta_node/task_manager.py,sha256=cusrPrctHC4NY6Ryb_hu9BCEVi2DUceu99H-56ztVTA,20523
 manta_node/tasks.py,sha256=W__XjU17AX8VqOtgy8HRBmxWtVufTkFvri8s8Y3R394,6721
 manta_node/utils.py,sha256=gkYRB7fylk1jy1sm7nKU7o8eadGxn0AX4IjCHT6Wzuw,1179
 manta_node/cli/__init__.py,sha256=arzO3mXDJ-E2yeeytbZfN7mCswwxdQ43_QjyeL2E5yE,134
@@ -34,10 +34,10 @@ manta_node/infrastructure/metrics/__init__.py,sha256=ODGAj9s63F5jba2c3dQRMfNK6I4
 manta_node/infrastructure/metrics/collector.py,sha256=nhFPAbjA04ye3UaGN95_zdpf9hLs2lrMXwOwKlFKYsI,3070
 manta_node/infrastructure/metrics/metrics_collector.py,sha256=jhEocAPWS6xgbNvcBBiEB9cAN_9hEGssZJXoysQzWOk,10518
 manta_node/infrastructure/mqtt/__init__.py,sha256=riXo1sXgO4YV6DHCmWRqK61BAmuGLdHiVdQFunAJadA,140
-manta_node/infrastructure/mqtt/command_handler.py,sha256=bDH8k3Vkbbb7q3P68VamhMbtLJ4vSSivqZP3IfMBqIQ,11833
-manta_node-0.5b0.dev91.dist-info/licenses/LICENSE,sha256=P5HojupjKoAusNtHqWOvO-YFolwLFWQ7Uq0XVKK9nFQ,35291
-manta_node-0.5b0.dev91.dist-info/METADATA,sha256=A0_n5jnU92mC1l6gTncaA5r-PGfkTC6W-rjDogDBXdA,44653
-manta_node-0.5b0.dev91.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-manta_node-0.5b0.dev91.dist-info/entry_points.txt,sha256=C_zcw6mdmztNxVUZGLZ83UH5TwcSuqb2C-rB4e2CQoc,56
-manta_node-0.5b0.dev91.dist-info/top_level.txt,sha256=nbT5J5WETuXM9UPoQckl29uUOq8wMYdRb9QDx5kAFeo,11
-manta_node-0.5b0.dev91.dist-info/RECORD,,
+manta_node/infrastructure/mqtt/command_handler.py,sha256=Q-QYtjJVD2UByEzbbVpwRt67BAws81XaJTZlkNw7nY0,13888
+manta_node-0.5b0.dev117.dist-info/licenses/LICENSE,sha256=P5HojupjKoAusNtHqWOvO-YFolwLFWQ7Uq0XVKK9nFQ,35291
+manta_node-0.5b0.dev117.dist-info/METADATA,sha256=L1BilnSrL844ItSAKHZNZmwCaUXKnIiLC7V4w2cl3r8,44654
+manta_node-0.5b0.dev117.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+manta_node-0.5b0.dev117.dist-info/entry_points.txt,sha256=C_zcw6mdmztNxVUZGLZ83UH5TwcSuqb2C-rB4e2CQoc,56
+manta_node-0.5b0.dev117.dist-info/top_level.txt,sha256=nbT5J5WETuXM9UPoQckl29uUOq8wMYdRb9QDx5kAFeo,11
+manta_node-0.5b0.dev117.dist-info/RECORD,,