boilermaker-servicebus 1.0.0.dev4__tar.gz → 1.0.0.dev5__tar.gz

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boilermaker-servicebus
-Version: 1.0.0.dev4
+Version: 1.0.0.dev5
 Summary: An async python Background task system using Azure Service Bus Queues
 Author-email: Erik Aker <eaker@mulliganfunding.com>
 License:                                  Apache License

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/app.py

@@ -322,9 +322,7 @@ class Boilermaker:
             await app.apply_async(cleanup_temp_files, delay=300)
         """
         task = self.create_task(fn, *args, policy=policy, **kwargs)
-        return await self.publish_task(
-            task, delay=delay, publish_attempts=publish_attempts
-        )
+        return await self.publish_task(task, delay=delay, publish_attempts=publish_attempts)
 
     @tracer.start_as_current_span("boilermaker.publish-task")
     async def publish_task(
@@ -366,9 +364,7 @@ class Boilermaker:
                 if results and len(results) == 1:
                     sequence_number = results[0]
                     task.mark_published(sequence_number)
-                    logger.debug(
-                        f"Published task {task.task_id} to queue with sequence_number={sequence_number}"
-                    )
+                    logger.debug(f"Published task {task.task_id} to queue with sequence_number={sequence_number}")
                 return task
 
             except (
@@ -407,6 +403,13 @@ class Boilermaker:
                     [f"Expected graph_id={graph.graph_id}, found {task.graph_id=}"],
                 )
 
+        for task in graph.fail_children.values():
+            if task.graph_id != graph.graph_id:
+                raise BoilermakerAppException(
+                    "All failure callback tasks must have graph_id matching graph",
+                    [f"Expected graph_id={graph.graph_id}, found {task.graph_id=}"],
+                )
+
         # Store the graph definition and all pending task results
         # If this graph has already been stored, this should fail.
         try:
@@ -414,8 +417,34 @@ class Boilermaker:
         except BoilermakerStorageError as exc:
             raise BoilermakerAppException("Error storing TaskGraph to storage", [str(exc)]) from exc
 
-        # Publish all ready tasks (should be root nodes with no dependencies)
-        for task in graph.generate_ready_tasks():
+        # Reload to get ETags on pending result blobs (required for ETag-guarded Scheduled writes)
+        try:
+            loaded_graph = await self.results_storage.load_graph(graph.graph_id)
+        except BoilermakerStorageError as exc:
+            raise BoilermakerAppException("Error loading TaskGraph after storage", [str(exc)]) from exc
+
+        if not loaded_graph:
+            raise BoilermakerAppException("TaskGraph not found after storing — this should not happen", [])
+
+        # Publish root tasks with ETag-guarded Scheduled write (same protocol as continue_graph)
+        for task in loaded_graph.generate_ready_tasks():
+            try:
+                result = loaded_graph.schedule_task(task.task_id)
+                await self.results_storage.store_task_result(result, etag=result.etag)
+            except BoilermakerStorageError:
+                logger.error(
+                    f"Failed to write Scheduled status for root task {task.task_id} in graph "
+                    f"{graph.graph_id}. Not publishing to avoid inconsistent state.",
+                    exc_info=True,
+                )
+                continue
+            except ValueError:
+                logger.error(
+                    f"schedule_task raised ValueError for root task {task.task_id} in graph "
+                    f"{graph.graph_id}. Skipping.",
+                    exc_info=True,
+                )
+                continue
             await self.publish_task(task)
 
         return graph

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/evaluators/common.py

@@ -50,9 +50,7 @@ class MessageActions:
     """
 
     @classmethod
-    async def task_decoder(
-        cls, msg: ServiceBusReceivedMessage, receiver: ServiceBusReceiver
-    ) -> Task | None:
+    async def task_decoder(cls, msg: ServiceBusReceivedMessage, receiver: ServiceBusReceiver) -> Task | None:
         """Decode a ServiceBusReceivedMessage into a Task."""
         try:
             task = Task.model_validate_json(str(msg))
@@ -60,10 +58,7 @@ class MessageActions:
             return task
         except (JSONDecodeError, ValidationError):
             # This task is not parseable
-            log_err_msg = (
-                f"Invalid task sequence_number={msg.sequence_number} "
-                f"exc_info={traceback.format_exc()}"
-            )
+            log_err_msg = f"Invalid task sequence_number={msg.sequence_number} exc_info={traceback.format_exc()}"
             logger.error(log_err_msg)
             await receiver.dead_letter_message(
                 msg,
@@ -73,17 +68,13 @@ class MessageActions:
             return None
 
     @staticmethod
-    async def abandon_message(
-        msg: ServiceBusReceivedMessage, receiver: ServiceBusReceiver
-    ) -> None:
+    async def abandon_message(msg: ServiceBusReceivedMessage, receiver: ServiceBusReceiver) -> None:
         """Abandon the message being processed."""
         if msg is not None:
             sequence_number = msg.sequence_number
             try:
                 await receiver.abandon_message(msg)
-                logger.warning(
-                    f"Abandoning message: returned to queue {sequence_number=}"
-                )
+                logger.warning(f"Abandoning message: returned to queue {sequence_number=}")
             except (
                 MessageAlreadySettled,
                 MessageLockLostError,
@@ -93,18 +84,13 @@ class MessageActions:
                 logger.error(err_msg)
                 raise exc.BoilermakerTaskLeaseLost(err_msg) from None
             except ServiceBusError as sb_exc:
-                err_msg = (
-                    f"ServiceBusError requeuing message {sequence_number=} "
-                    f"exc_info={traceback.format_exc()}"
-                )
+                err_msg = f"ServiceBusError requeuing message {sequence_number=} exc_info={traceback.format_exc()}"
                 logger.error(err_msg)
                 raise exc.BoilermakerServiceBusError(err_msg) from sb_exc
         return None
 
     @staticmethod
-    async def complete_message(
-        msg: ServiceBusReceivedMessage, receiver: ServiceBusReceiver
-    ):
+    async def complete_message(msg: ServiceBusReceivedMessage, receiver: ServiceBusReceiver):
         """Complete the current message being processed."""
         try:
             await receiver.complete_message(msg)
@@ -113,10 +99,7 @@ class MessageActions:
             MessageLockLostError,
             SessionLockLostError,
         ):
-            logmsg = (
-                f"Failed to settle message sequence_number={msg.sequence_number} "
-                f"exc_info={traceback.format_exc()}"
-            )
+            logmsg = f"Failed to settle message sequence_number={msg.sequence_number} exc_info={traceback.format_exc()}"
             logger.error(logmsg)
             raise exc.BoilermakerTaskLeaseLost(msg) from None
         except ServiceBusError as sb_exc:
@@ -148,8 +131,7 @@ class MessageActions:
             SessionLockLostError,
         ):
             logmsg = (
-                f"Failed to deadletter message sequence_number={msg.sequence_number} "
-                f"exc_info={traceback.format_exc()}"
+                f"Failed to deadletter message sequence_number={msg.sequence_number} exc_info={traceback.format_exc()}"
             )
             logger.error(logmsg)
             raise exc.BoilermakerTaskLeaseLost(logmsg) from None
@@ -178,8 +160,7 @@ class MessageActions:
             return await receiver.renew_message_lock(msg)
         except (MessageLockLostError, MessageAlreadySettled, SessionLockLostError):
             logmsg = (
-                f"Failed to renew message lock sequence_number={msg.sequence_number} "
-                f"exc_info={traceback.format_exc()}"
+                f"Failed to renew message lock sequence_number={msg.sequence_number} exc_info={traceback.format_exc()}"
             )
             logger.error(logmsg)
             raise exc.BoilermakerTaskLeaseLost(logmsg) from None
@@ -201,9 +182,7 @@ class MessageActions:
     ):
         """Deadletter or complete the current task based on its configuration."""
         if task.msg is None:
-            logger.warning(
-                "No current message to settle for deadletter_or_complete_task"
-            )
+            logger.warning("No current message to settle for deadletter_or_complete_task")
             return None
 
         if task.should_dead_letter:
@@ -270,22 +249,16 @@ class TaskEvaluatorBase:
     # Message handling actions
     async def abandon_current_message(self) -> None:
         if self.current_msg is not None:
-            return await MessageActions.abandon_message(
-                self.current_msg, self._receiver
-            )
+            return await MessageActions.abandon_message(self.current_msg, self._receiver)
 
     async def complete_message(self) -> None:
         if self.current_msg is not None:
-            return await MessageActions.complete_message(
-                self.current_msg, self._receiver
-            )
+            return await MessageActions.complete_message(self.current_msg, self._receiver)
 
     async def renew_message_lock(self) -> datetime.datetime | None:
         """Renew the lock on the current message being processed."""
         if self.current_msg is not None:
-            return await MessageActions.renew_message_lock(
-                self.current_msg, self._receiver
-            )
+            return await MessageActions.renew_message_lock(self.current_msg, self._receiver)
         return None
 
     async def deadletter_or_complete_task(
@@ -293,9 +266,7 @@ class TaskEvaluatorBase:
         reason: str,
         detail: Exception | str | None = None,
     ) -> None:
-        return await MessageActions.deadletter_or_complete_task(
-            self.task, self._receiver, reason, detail=detail
-        )
+        return await MessageActions.deadletter_or_complete_task(self.task, self._receiver, reason, detail=detail)
 
     async def __call__(self) -> TaskResult | None:
         """Call pre-processing hook and then `message_handler`."""
@@ -304,9 +275,7 @@ class TaskEvaluatorBase:
             if not await self.pre_process():
                 return None
         except exc.BoilermakerUnregisteredFunction:
-            await self.deadletter_or_complete_task(
-                "ExpectationFailed", detail="Pre-processing expectation failed"
-            )
+            await self.deadletter_or_complete_task("ExpectationFailed", detail="Pre-processing expectation failed")
             return TaskResult(
                 task_id=self.task.task_id,
                 graph_id=self.task.graph_id,
@@ -316,9 +285,7 @@ class TaskEvaluatorBase:
             )
         except Exception:
             logger.error("Exception in pre_process", exc_info=True)
-            await self.deadletter_or_complete_task(
-                "ProcessingError", detail="Pre-processing exception"
-            )
+            await self.deadletter_or_complete_task("ProcessingError", detail="Pre-processing exception")
             return TaskResult(
                 task_id=self.task.task_id,
                 graph_id=self.task.graph_id,
@@ -349,7 +316,5 @@ class TaskEvaluatorBase:
             return False
 
         if not self.function_registry.get(self.task.function_name):
-            raise exc.BoilermakerUnregisteredFunction(
-                f"Missing registered function {self.task.function_name}"
-            )
+            raise exc.BoilermakerUnregisteredFunction(f"Missing registered function {self.task.function_name}")
         return True

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/evaluators/eval.py

@@ -44,9 +44,7 @@ async def eval_task(
     logger.info(f"[{task.function_name}] Begin Task {task.sequence_number=}")
 
     if function is None:
-        raise BoilermakerUnregisteredFunction(
-            f"Function {task.function_name} not found in registry"
-        )
+        raise BoilermakerUnregisteredFunction(f"Function {task.function_name} not found in registry")
 
     try:
         result = await function(
@@ -77,8 +75,7 @@ async def eval_task(
                 status=TaskStatus.Success,
             )
             logger.info(
-                f"[{task.function_name}] Completed Task {task.sequence_number=} "
-                f"in {time.monotonic() - start:.3f}s"
+                f"[{task.function_name}] Completed Task {task.sequence_number=} in {time.monotonic() - start:.3f}s"
             )
     except RetryException as retry:
         # A retry has been requested:

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/evaluators/results_store.py

@@ -62,9 +62,7 @@ class ResultsStorageTaskEvaluator(TaskEvaluatorBase):
                 await self.complete_message()
                 message_settled = True
             except exc.BoilermakerTaskLeaseLost:
-                logger.error(
-                    f"Lost message lease when trying to complete early for task {self.task.function_name}"
-                )
+                logger.error(f"Lost message lease when trying to complete early for task {self.task.function_name}")
                 return TaskResult(
                     task_id=self.task.task_id,
                     graph_id=self.task.graph_id,
@@ -89,8 +87,7 @@ class ResultsStorageTaskEvaluator(TaskEvaluatorBase):
                     message_settled = True
                 except exc.BoilermakerTaskLeaseLost:
                     logger.error(
-                        f"Lost message lease when trying to deadletter/complete "
-                        f" for task {self.task.function_name}"
+                        f"Lost message lease when trying to deadletter/complete  for task {self.task.function_name}"
                     )
                     return TaskResult(
                         task_id=self.task.task_id,

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/evaluators/simple.py

@@ -45,9 +45,7 @@ class NoStorageEvaluator(TaskEvaluatorBase):
                 await self.complete_message()
                 message_settled = True
             except exc.BoilermakerTaskLeaseLost:
-                logger.error(
-                    f"Lost message lease when trying to complete early for task {self.task.function_name}"
-                )
+                logger.error(f"Lost message lease when trying to complete early for task {self.task.function_name}")
                 return TaskResult(
                     task_id=self.task.task_id,
                     graph_id=self.task.graph_id,
@@ -71,8 +69,7 @@ class NoStorageEvaluator(TaskEvaluatorBase):
                     message_settled = True
                 except exc.BoilermakerTaskLeaseLost:
                     logger.error(
-                        f"Lost message lease when trying to deadletter/complete "
-                        f" for task {self.task.function_name}"
+                        f"Lost message lease when trying to deadletter/complete  for task {self.task.function_name}"
                     )
                     return TaskResult(
                         task_id=self.task.task_id,

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/evaluators/task_graph.py

@@ -45,6 +45,14 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
         if storage_interface is None:
             raise ValueError("Storage interface is required for TaskGraphEvaluator")
 
+        if task.acks_early:
+            logger.warning(
+                f"Task {task.task_id} ({task.function_name}) uses acks_early=True in a "
+                "TaskGraph context. If the worker crashes after message settlement but before "
+                "the task result is written, this task will be permanently stuck in Started "
+                "status with no recovery path. Use acks_late=True (the default) for graph tasks."
+            )
+
         super().__init__(
             receiver,
             task,
@@ -61,6 +69,51 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
         """Individual message handler"""
         message_settled = False
 
+        # Idempotent redelivery guard — if this task already reached a terminal state
+        # (e.g. a prior execution succeeded before the SB lock expired and redelivered), skip
+        # re-execution entirely.  Writing Started on top of Success/Failure would regress the
+        # blob status and corrupt graph state.
+        if self.task.graph_id:
+            try:
+                _existing = await self.storage_interface.load_task_result(self.task.task_id, self.task.graph_id)
+            except exc.BoilermakerStorageError:
+                # Transient read failure — proceed normally; do NOT skip execution on a read
+                # error, as that would permanently stall the graph.
+                logger.warning(
+                    f"Failed to read current status for task {self.task.task_id} before "
+                    "writing Started; proceeding with execution",
+                    exc_info=True,
+                )
+                _existing = None
+
+            if _existing is not None and _existing.status.finished:
+                logger.info(
+                    f"Task {self.task.task_id} already in terminal state {_existing.status!r} "
+                    "(SB redelivery); skipping re-execution"
+                )
+                _terminal_result = TaskResult(
+                    task_id=self.task.task_id,
+                    graph_id=self.task.graph_id,
+                    status=_existing.status,
+                )
+                try:
+                    await self.continue_graph(_terminal_result)
+                except exc.ContinueGraphError:
+                    logger.error(
+                        f"continue_graph failed on redelivery for task {self.task.task_id}; "
+                        "suppressing settlement to allow redelivery",
+                        exc_info=True,
+                    )
+                    return _terminal_result
+                try:
+                    await self.complete_message()
+                except (exc.BoilermakerTaskLeaseLost, exc.BoilermakerServiceBusError):
+                    logger.warning(
+                        f"Failed to complete message on redelivery for task {self.task.task_id}; SB will redeliver",
+                        exc_info=True,
+                    )
+                return _terminal_result
+
         start_result = TaskResult(
             task_id=self.task.task_id,
             graph_id=self.task.graph_id,
@@ -78,9 +131,7 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
                 await self.complete_message()
                 message_settled = True
             except exc.BoilermakerTaskLeaseLost:
-                logger.error(
-                    f"Lost message lease when trying to complete early for task {self.task.function_name}"
-                )
+                logger.error(f"Lost message lease when trying to complete early for task {self.task.function_name}")
                 return TaskResult(
                     task_id=self.task.task_id,
                     graph_id=self.task.graph_id,
@@ -104,8 +155,7 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
                     message_settled = True
                 except exc.BoilermakerTaskLeaseLost:
                     logger.error(
-                        f"Lost message lease when trying to deadletter/complete "
-                        f" for task {self.task.function_name}"
+                        f"Lost message lease when trying to deadletter/complete  for task {self.task.function_name}"
                     )
                     return TaskResult(
                         task_id=self.task.task_id,
@@ -266,8 +316,7 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
                         exc_info=True,
                     )
                     raise exc.ContinueGraphError(
-                        f"load_graph failed for graph {graph_id} after "
-                        f"{_LOAD_GRAPH_RETRY_POLICY.max_tries} attempts"
+                        f"load_graph failed for graph {graph_id} after {_LOAD_GRAPH_RETRY_POLICY.max_tries} attempts"
                     ) from last_exc
         else:
             # Should only be reached if max_tries == 0 (not expected).
@@ -289,9 +338,12 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
             logger.error(
                 f"Task status mismatch in continue_graph for graph {graph_id}: "
                 f"expected {completed_task_result.task_id} to be {completed_task_result.status}, "
-                f"but got {loaded_task_status}"
+                f"but got {loaded_task_status}. Suppressing settlement to allow redelivery."
+            )
+            raise exc.ContinueGraphError(
+                f"Status mismatch for task {completed_task_result.task_id} in graph {graph_id}: "
+                f"expected {completed_task_result.status}, got {loaded_task_status}"
             )
-            return None
 
         # Snapshot tasks already in Scheduled status BEFORE the first pass.
         # The second pass uses this snapshot so that tasks freshly scheduled
@@ -322,14 +374,19 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
                 )
                 continue
 
-            ready_count += 1
-            await self.publish_task(ready_task)
-            logger.info(f"Publishing ready task {ready_task.task_id} in graph {graph_id} total={ready_count}")
+            try:
+                await self.publish_task(ready_task)
+                ready_count += 1
+                logger.info(f"Publishing ready task {ready_task.task_id} in graph {graph_id} total={ready_count}")
+            except Exception:
+                logger.error(
+                    f"Failed to publish ready task {ready_task.task_id} in graph {graph_id}; "
+                    "task is in Scheduled status in blob and will be recovered by crash-recovery pass on redelivery.",
+                    exc_info=True,
+                )
 
         if ready_count == 0:
-            logger.info(
-                f"No new tasks ready in graph {graph_id} after task {completed_task_result.task_id}"
-            )
+            logger.info(f"No new tasks ready in graph {graph_id} after task {completed_task_result.task_id}")
 
         # Second pass: re-publish tasks that were ALREADY in Scheduled status when the
         # graph was loaded (crash-recovery).

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/exc.py

@@ -33,9 +33,7 @@ class BoilermakerStorageError(Exception):
     def __getattr__(self, item):
         if self.details.get(item, None):
             return self.details[item]
-        raise AttributeError(
-            f"BoilermakerStorageError object has no attribute '{item}'"
-        )
+        raise AttributeError(f"BoilermakerStorageError object has no attribute '{item}'")
 
 
 class BoilermakerUnregisteredFunction(ValueError):

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/retries.py

@@ -138,7 +138,7 @@ class RetryPolicy(BaseModel):
             case RetryMode.Fixed:
                 return min(self.delay, self.delay_max)
             case RetryMode.Linear:
-                return min(self.delay * attempts_so_far, self.delay_max)
+                return min(self.delay * (attempts_so_far + 1), self.delay_max)
             case RetryMode.Exponential:
                 # Jitter is added so that a lot of work doesn't get bunched up at the
                 # end and eventually hurt throughput.

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/storage/base.py

@@ -1,7 +1,7 @@
 import logging
 from abc import ABC, abstractmethod
 
-from boilermaker.task import GraphId, TaskGraph, TaskResult, TaskResultSlim
+from boilermaker.task import GraphId, TaskGraph, TaskId, TaskResult, TaskResultSlim
 
 logger = logging.getLogger(__name__)
 
@@ -40,3 +40,22 @@ class StorageInterface(ABC):
             task_result: The TaskResult instance to storage.
         """
         raise NotImplementedError
+
+    @abstractmethod
+    async def load_task_result(self, task_id: TaskId, graph_id: GraphId) -> TaskResultSlim | None:
+        """Load a single task result from storage.
+
+        Used by the idempotent redelivery guard in TaskGraphEvaluator to check whether
+        a task has already reached a terminal state before writing Started on redelivery.
+
+        Args:
+            task_id: The TaskId of the task result to load.
+            graph_id: The GraphId the task belongs to.
+
+        Returns:
+            The TaskResultSlim instance, or None if not found.
+
+        Raises:
+            BoilermakerStorageError: If the result cannot be loaded for reasons other than not found.
+        """
+        raise NotImplementedError

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/storage/blob_storage.py

@@ -16,7 +16,7 @@ from pydantic import ValidationError
 
 from boilermaker.exc import BoilermakerStorageError
 from boilermaker.storage import StorageInterface
-from boilermaker.task import GraphId, TaskGraph, TaskResult, TaskResultSlim
+from boilermaker.task import GraphId, TaskGraph, TaskId, TaskResult, TaskResultSlim
 
 logger = logging.getLogger(__name__)
 
@@ -89,7 +89,10 @@ class BlobClientStorage(AzureBlobStorageClient, StorageInterface):
             except ValidationError as e:
                 raise BoilermakerStorageError(
                     f"Failed to deserialize task result in graph {graph_id}: {e}",
+                    name=blob.name,
+                    graph_id=graph_id,
                     status_code=None,
+                    reason="DeserializationError",
                 ) from e
             tr.etag = blob.etag
             if tr.graph_id == graph_id:
@@ -161,6 +164,50 @@ class BlobClientStorage(AzureBlobStorageClient, StorageInterface):
                     await lease.release()
         return graph
 
+    async def load_task_result(self, task_id: TaskId, graph_id: GraphId) -> TaskResultSlim | None:
+        """Load a single task result from Azure Blob Storage.
+
+        Returns None if the blob does not exist (404). Raises BoilermakerStorageError
+        for any other failure. Used by the idempotent redelivery guard in
+        TaskGraphEvaluator to check terminal status before writing Started.
+
+        Args:
+            task_id: The TaskId of the task result to load.
+            graph_id: The GraphId the task belongs to.
+        """
+        fname = f"{self.task_result_prefix}/{graph_id}/{task_id}.json"
+        blob_etag = None
+        try:
+            # We need to make sure we load the etag
+            async with self.get_blob_client(fname) as blob_client:
+                blob_properties = await blob_client.get_blob_properties()
+                blob_etag = blob_properties.etag if blob_properties and blob_properties.etag is not None else None
+            contents = await self.download_blob(fname)
+        except AzureBlobError as exc:
+            if exc.status_code == 404:
+                return None
+            raise BoilermakerStorageError(
+                f"Failed to load task result {task_id}",
+                task_id=task_id,
+                graph_id=graph_id,
+                status_code=exc.status_code,
+                reason=exc.reason,
+            ) from exc
+        if contents is None:
+            return None
+        try:
+            result = TaskResultSlim.model_validate_json(contents)
+            result.etag = blob_etag
+            return result
+        except ValidationError as e:
+            raise BoilermakerStorageError(
+                f"Failed to deserialize task result {task_id}: {e}",
+                task_id=task_id,
+                graph_id=graph_id,
+                status_code=None,
+                reason="DeserializationError",
+            ) from e
+
     async def store_task_result(self, task_result: TaskResult | TaskResultSlim, etag: str | None = None) -> None:
         """Stores a TaskResult to Azure Blob Storage.
 

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/task/graph.py

@@ -1,4 +1,5 @@
 import itertools
+import logging
 import typing
 from collections import defaultdict
 from collections.abc import Generator
@@ -10,6 +11,8 @@ from .result import TaskResult, TaskResultSlim, TaskStatus
 from .task import Task
 from .task_id import GraphId, ident_field, TaskId
 
+logger = logging.getLogger("boilermaker.app")
+
 
 class TaskGraph(BaseModel):
     """
@@ -171,7 +174,9 @@ class TaskGraph(BaseModel):
                     if not self.edges[parent_id]:  # Remove empty set
                         del self.edges[parent_id]
                 del self.children[task.task_id]
-                raise ValueError(f"Adding task {task.task_id} with parent {parent_id} would create a cycle in the DAG")
+                raise ValueError(
+                    f"Adding task {task.task_id} with parents {parent_ids} would create a cycle in the DAG"
+                )
 
         # If we leave `on_success` and `on_failure` it's potentially confusing for both callers
         # and our own evaluation. It also has the potential to create cycles inadvertently, so we
@@ -294,12 +299,14 @@ class TaskGraph(BaseModel):
     def generate_ready_tasks(self) -> Generator[Task]:
         """Get a list of tasks that are ready to be executed (not started and all antecedents succeeded)."""
         for task_id in self.children.keys():
-            # Task is ready if:
-            # 1. It has no result yet (never started) OR it has Pending status
-            # 2. All its antecedents have succeeded
             task_result = self.results.get(task_id)
-            is_not_started = task_result is None or task_result.status == TaskStatus.Pending
-            if is_not_started and self.task_is_ready(task_id):
+            if task_result is None:
+                logger.warning(
+                    f"Task {task_id} has no result blob in graph {self.graph_id}; "
+                    "skipping. This may indicate a partial store_graph failure."
+                )
+                continue
+            if task_result.status == TaskStatus.Pending and self.task_is_ready(task_id):
                 yield self.children[task_id]
 
     def generate_failure_ready_tasks(self) -> Generator[Task]:

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/task/result.py

@@ -19,7 +19,6 @@ class TaskStatus(enum.StrEnum):
     RetriesExhausted = "retries_exhausted"
     Deadlettered = "deadlettered"
 
-
     @classmethod
     def default(cls) -> "TaskStatus":
         """Get the default task status.

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker/tracing.py

@@ -44,9 +44,7 @@ async def start_span_from_parent_event_async(
     """
     if otel_enabled:
         tracectx = extract(get_traceparent_context(event))
-        with tracer.start_as_current_span(
-            name, context=tracectx, kind=trace.SpanKind.CONSUMER
-        ) as current_span:
+        with tracer.start_as_current_span(name, context=tracectx, kind=trace.SpanKind.CONSUMER) as current_span:
             yield current_span
     else:
         yield None

{boilermaker_servicebus-1.0.0.dev4 → boilermaker_servicebus-1.0.0.dev5}/boilermaker_servicebus.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boilermaker-servicebus
-Version: 1.0.0.dev4
+Version: 1.0.0.dev5
 Summary: An async python Background task system using Azure Service Bus Queues
 Author-email: Erik Aker <eaker@mulliganfunding.com>
 License:                                  Apache License