boilermaker-servicebus 1.0.0.dev2__tar.gz → 1.0.0.dev4__tar.gz

{boilermaker_servicebus-1.0.0.dev2 → boilermaker_servicebus-1.0.0.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: boilermaker-servicebus
-Version: 1.0.0.dev2
+Version: 1.0.0.dev4
 Summary: An async python Background task system using Azure Service Bus Queues
 Author-email: Erik Aker <eaker@mulliganfunding.com>
 License:                                  Apache License
@@ -210,7 +210,7 @@ Project-URL: Issues, https://github.com/MulliganFunding/boilermaker-servicebus/i
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: aio-azure-clients-toolbox>=1.0.4
+Requires-Dist: aio-azure-clients-toolbox>=1.1.0
 Requires-Dist: anyio>=4.11.0
 Requires-Dist: azure-core-tracing-opentelemetry>=1.0.0b12
 Requires-Dist: azure-servicebus>=7.14.2
@@ -219,6 +219,8 @@ Requires-Dist: opentelemetry-api>=1.34.0
 Requires-Dist: pydantic>=2.12.2
 Requires-Dist: pydantic-settings>=2.11.0
 Requires-Dist: uuid-utils>=0.11.1
+Provides-Extra: repl
+Requires-Dist: ipython; extra == "repl"
 Dynamic: license-file
 
 # Boilermaker

{boilermaker_servicebus-1.0.0.dev2 → boilermaker_servicebus-1.0.0.dev4}/boilermaker/app.py

@@ -12,7 +12,7 @@ import typing
 import weakref
 from functools import wraps
 
-from aio_azure_clients_toolbox import AzureServiceBus, ManagedAzureServiceBusSender  # type: ignore
+from aio_azure_clients_toolbox import AzureServiceBus, ManagedAzureServiceBusSender
 from anyio import create_task_group, open_signal_receiver
 from anyio.abc import CancelScope
 from azure.servicebus import ServiceBusReceivedMessage
@@ -163,7 +163,7 @@ class Boilermaker:
             raise ValueError(f"Function must be async: {fn_name}")
 
         task = Task.default(fn_name, **options)
-        self.function_registry[fn_name] = fn
+        self.function_registry[fn_name] = typing.cast(TaskHandler, fn)  # why must cast here
         self.task_registry[fn_name] = task
         logger.info(f"Registered background function fn={fn_name}")
         return self
@@ -361,6 +361,7 @@ class Boilermaker:
                 results: list[int] = await self.service_bus_client.send_message(
                     task.model_dump_json(),
                     delay=delay,
+                    unique_msg_id=str(task.task_id),
                 )
                 if results and len(results) == 1:
                     sequence_number = results[0]

{boilermaker_servicebus-1.0.0.dev2 → boilermaker_servicebus-1.0.0.dev4}/boilermaker/evaluators/__init__.py

@@ -4,12 +4,11 @@ import typing
 from azure.servicebus.aio import ServiceBusReceiver
 
 from boilermaker.storage.base import StorageInterface
-from boilermaker.task import Task
+from boilermaker.task import Task, TaskHandler
 
 from .common import (
     MessageActions,
     TaskEvaluatorBase,
-    TaskHandler,
     TaskHandlerRegistry,
     TaskPublisher,
 )

{boilermaker_servicebus-1.0.0.dev2 → boilermaker_servicebus-1.0.0.dev4}/boilermaker/evaluators/common.py

@@ -3,7 +3,7 @@ import logging
 import traceback
 import typing
 from abc import abstractmethod
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable
 from functools import cached_property
 from json.decoder import JSONDecodeError
 
@@ -21,14 +21,12 @@ from pydantic import ValidationError
 from boilermaker import exc, sample
 from boilermaker.storage import StorageInterface
 from boilermaker.task import Task, TaskResult, TaskStatus
+from boilermaker.task import types as task_types
 
 tracer: trace.Tracer = trace.get_tracer(__name__)
 logger = logging.getLogger("boilermaker.app")
 
-
-# Common Types used when evaluating tasks
-TaskHandler: typing.TypeAlias = Callable[..., Awaitable[typing.Any]]
-TaskHandlerRegistry: typing.TypeAlias = dict[str, TaskHandler]
+TaskHandlerRegistry: typing.TypeAlias = dict[str, task_types.TaskHandler]
 
 
 class TaskPublisher(typing.Protocol):

{boilermaker_servicebus-1.0.0.dev2 → boilermaker_servicebus-1.0.0.dev4}/boilermaker/evaluators/eval.py

@@ -7,8 +7,7 @@ from boilermaker.exc import BoilermakerUnregisteredFunction
 from boilermaker.failure import TaskFailureResult
 from boilermaker.retries import RetryException
 from boilermaker.task import Task, TaskResult, TaskStatus
-
-from .common import TaskHandler
+from boilermaker.task.types import TaskHandler
 
 logger = logging.getLogger("boilermaker.app")
 

{boilermaker_servicebus-1.0.0.dev2 → boilermaker_servicebus-1.0.0.dev4}/boilermaker/evaluators/task_graph.py

@@ -1,10 +1,11 @@
+import asyncio
 import itertools
 import logging
 import typing
 
 from azure.servicebus.aio import ServiceBusReceiver
 
-from boilermaker import exc
+from boilermaker import exc, retries
 from boilermaker.storage import StorageInterface
 from boilermaker.task import Task, TaskResult, TaskStatus
 
@@ -13,9 +14,24 @@ from .eval import eval_task
 
 logger = logging.getLogger("boilermaker.app")
 
+# Retry policy used when load_graph raises a transient exception.
+# Up to 3 attempts total (initial + 2 retries) with exponential backoff.
+_LOAD_GRAPH_RETRY_POLICY = retries.RetryPolicy(
+    max_tries=3,
+    delay=1,
+    delay_max=16,
+    retry_mode=retries.RetryMode.Exponential,
+)
+
 
 class TaskGraphEvaluator(TaskEvaluatorBase):
-    """Evaluator for tasks that are part of a TaskGraph workflow."""
+    """Evaluator for tasks that are part of a TaskGraph workflow.
+
+    At-least-once delivery contract: any task in ``Scheduled`` status may be
+    published more than once.  Workers must tolerate at-least-once delivery.
+    Re-publication on Service Bus redelivery is the intentional recovery
+    mechanism for the store-before-publish crash gap in ``continue_graph``.
+    """
 
     def __init__(
         self,
@@ -114,8 +130,17 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
                 result=None,
             )
             await self.storage_interface.store_task_result(task_result)
-            # Publish failure tasks which may be ready now
-            await self.continue_graph(task_result)
+            # Publish failure tasks which may be ready now.
+            # The message is already deadlettered at this point, so suppressing settlement
+            # is not possible — log and return gracefully if continue_graph fails.
+            try:
+                await self.continue_graph(task_result)
+            except exc.ContinueGraphError:
+                logger.error(
+                    f"continue_graph failed after retries exhausted for task {self.task.task_id}; "
+                    "failure callbacks may not be dispatched (message already deadlettered)",
+                    exc_info=True,
+                )
             return task_result
 
         # Actually invoke the task here
@@ -128,7 +153,17 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
         await self.storage_interface.store_task_result(result)
 
         if result.status.finished:
-            await self.continue_graph(result)
+            try:
+                await self.continue_graph(result)
+            except exc.ContinueGraphError:
+                # Transient load_graph failure — do NOT settle the message.
+                # Allow Service Bus redelivery so downstream dispatch can be retried.
+                logger.error(
+                    f"continue_graph failed for task {self.task.task_id}; "
+                    "suppressing message settlement to allow redelivery",
+                    exc_info=True,
+                )
+                return result
         elif result.status == TaskStatus.Retry:
             # Retry requested: republish the same task with delay
             delay = self.task.get_next_delay()
@@ -171,20 +206,81 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
         Continue evaluating TaskGraph workflow after a task completes successfully.
 
         We always reload the graph from storage to get the latest state.
+
+        Transient ``load_graph`` failures (``BoilermakerStorageError`` with a
+        non-404 status code) are retried with exponential backoff up to
+        ``_LOAD_GRAPH_RETRY_POLICY.max_tries`` attempts.  If all attempts fail,
+        ``ContinueGraphError`` is raised so that ``message_handler`` can suppress
+        message settlement and allow Service Bus redelivery.
+
+        Permanent failures (``BoilermakerStorageError`` with ``status_code=404``)
+        are logged at CRITICAL severity and ``None`` is returned.  Settling the
+        message is correct in this case because redelivery will not help — the
+        graph blob is gone and downstream tasks cannot be dispatched.
+
+        Note: in practice ``load_graph`` never returns ``None`` for a missing
+        blob; the underlying library re-raises all ``HttpResponseError``s
+        (including 404) as ``AzureBlobError``, which ``load_graph`` wraps as
+        ``BoilermakerStorageError(status_code=404)``.  The ``if not graph`` guard
+        below is retained as a defensive fallback only.
+
+        At-least-once delivery: any task already in ``Scheduled`` status is
+        re-published without a second blob write (second pass below).  This is
+        the crash-recovery path for the store-before-publish gap.
         """
         graph_id = completed_task_result.graph_id
         if not graph_id:
             return None
 
-        try:
-            # Reload graph with latest results
-            graph = await self.storage_interface.load_graph(graph_id)
-        except Exception:
-            logger.error(f"Exception in continue_graph for graph {graph_id}", exc_info=True)
-            return None
+        # Attempt to load the graph, retrying on transient errors.
+        last_exc: Exception | None = None
+        for attempt in range(_LOAD_GRAPH_RETRY_POLICY.max_tries):
+            try:
+                graph = await self.storage_interface.load_graph(graph_id)
+                break  # success
+            except exc.BoilermakerStorageError as e:
+                if getattr(e, "status_code", None) == 404:
+                    # Permanent: graph blob does not exist. Redelivery will not help.
+                    logger.critical(
+                        f"Graph {graph_id} not found in storage (404); downstream tasks will not be dispatched. "
+                        "This graph may have been deleted.",
+                        exc_info=True,
+                    )
+                    return None
+                # Transient error — will retry or raise ContinueGraphError after max_tries
+                last_exc = e
+                if attempt < _LOAD_GRAPH_RETRY_POLICY.max_tries - 1:
+                    delay = _LOAD_GRAPH_RETRY_POLICY.get_delay_interval(attempt)
+                    logger.warning(
+                        f"load_graph failed for graph {graph_id} "
+                        f"(attempt {attempt + 1}/{_LOAD_GRAPH_RETRY_POLICY.max_tries}); "
+                        f"retrying in {delay}s",
+                        exc_info=True,
+                    )
+                    await asyncio.sleep(delay)
+                else:
+                    logger.error(
+                        f"load_graph failed for graph {graph_id} after "
+                        f"{_LOAD_GRAPH_RETRY_POLICY.max_tries} attempts; "
+                        "raising ContinueGraphError to suppress message settlement",
+                        exc_info=True,
+                    )
+                    raise exc.ContinueGraphError(
+                        f"load_graph failed for graph {graph_id} after "
+                        f"{_LOAD_GRAPH_RETRY_POLICY.max_tries} attempts"
+                    ) from last_exc
+        else:
+            # Should only be reached if max_tries == 0 (not expected).
+            raise exc.ContinueGraphError(f"load_graph not attempted for graph {graph_id}")
 
         if not graph:
-            logger.error(f"Graph {graph_id} not found after task completion")
+            # Permanent failure: graph blob does not exist.  Redelivery will not help.
+            # Settling the upstream message is intentional here.
+            logger.critical(
+                f"Graph {graph_id} not found after task completion — "
+                "downstream tasks will never be dispatched. "
+                "This is a permanent data loss; redelivery will not recover it."
+            )
             return None
 
         # Sanity check: did we load the result that was *just* stored?
@@ -197,14 +293,19 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
             )
             return None
 
+        # Snapshot tasks already in Scheduled status BEFORE the first pass.
+        # The second pass uses this snapshot so that tasks freshly scheduled
+        # in the first pass are not double-published.
+        already_scheduled_tasks = list(graph.generate_scheduled_tasks())
+
         # Find and publish newly ready tasks
         ready_count = 0
         for ready_task in itertools.chain.from_iterable(
             (graph.generate_ready_tasks(), graph.generate_failure_ready_tasks())
         ):
             # Write that the task was *scheduled* back to Blob Storage with blob etag and then publish the task!
-            result = graph.schedule_task(ready_task.task_id)
             try:
+                result = graph.schedule_task(ready_task.task_id)
                 await self.storage_interface.store_task_result(result, etag=result.etag)
             except exc.BoilermakerStorageError:
                 logger.error(
@@ -213,6 +314,13 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
                     exc_info=True,
                 )
                 continue
+            except ValueError:
+                logger.error(
+                    f"schedule_task raised ValueError for task {ready_task.task_id} in graph {graph_id}. "
+                    "Skipping to avoid double-scheduling.",
+                    exc_info=True,
+                )
+                continue
 
             ready_count += 1
             await self.publish_task(ready_task)
@@ -223,4 +331,29 @@ class TaskGraphEvaluator(TaskEvaluatorBase):
                 f"No new tasks ready in graph {graph_id} after task {completed_task_result.task_id}"
             )
 
+        # Second pass: re-publish tasks that were ALREADY in Scheduled status when the
+        # graph was loaded (crash-recovery).
+        #
+        # If a previous invocation of continue_graph wrote Scheduled to blob storage
+        # (store_task_result) but crashed before publishing the Service Bus message
+        # (publish_task), the task blob shows Scheduled but there is no SB message.
+        # generate_ready_tasks() skips Scheduled tasks (is_not_started == False), so
+        # without this pass the task would never be dispatched.
+        #
+        # We snapshot already-scheduled tasks BEFORE the first pass so that tasks
+        # scheduled in the first pass are not double-published here.
+        #
+        # On Service Bus redelivery, we detect these orphaned-Scheduled tasks here and
+        # re-publish them without a second blob write (the blob is already correct).
+        #
+        # NOTE: Workers must tolerate at-least-once delivery.  A task in Scheduled
+        # status may be published more than once.  This is the intentional recovery
+        # mechanism for the store-before-publish crash gap.
+        for scheduled_task in already_scheduled_tasks:
+            logger.info(
+                f"Re-publishing already-scheduled task {scheduled_task.task_id} "
+                f"in graph {graph_id} (crash recovery: blob written but message not published)"
+            )
+            await self.publish_task(scheduled_task)
+
         return ready_count

{boilermaker_servicebus-1.0.0.dev2 → boilermaker_servicebus-1.0.0.dev4}/boilermaker/exc.py

@@ -1,6 +1,22 @@
 from azure.servicebus.exceptions import ServiceBusError
 
 
+class BoilermakerError(Exception):
+    """Base class for Boilermaker-specific exceptions."""
+
+    pass
+
+
+class ContinueGraphError(BoilermakerError):
+    """Raised when continue_graph cannot load the graph after retries.
+
+    Signals message_handler that settlement must be suppressed so that
+    Service Bus will redeliver the message and downstream dispatch can be retried.
+    """
+
+    pass
+
+
 class BoilermakerAppException(Exception):
     def __init__(self, message: str, errors: list):
         super().__init__(message + str(errors))

{boilermaker_servicebus-1.0.0.dev2 → boilermaker_servicebus-1.0.0.dev4}/boilermaker/service_bus.py

@@ -7,7 +7,7 @@ allows sending messages or subscribing to a queue.
 
 from aio_azure_clients_toolbox import (
     CredentialFactory,
-    ManagedAzureServiceBusSender,  # type: ignore
+    ManagedAzureServiceBusSender,
 )
 
 from .config import Config

{boilermaker_servicebus-1.0.0.dev2 → boilermaker_servicebus-1.0.0.dev4}/boilermaker/storage/blob_storage.py

@@ -1,4 +1,3 @@
-import datetime
 import logging
 import traceback
 from functools import partial
@@ -13,7 +12,7 @@ from azure.core.exceptions import (
     ResourceNotFoundError,
 )
 from azure.identity.aio import DefaultAzureCredential
-from azure.storage.blob import ImmutabilityPolicy
+from pydantic import ValidationError
 
 from boilermaker.exc import BoilermakerStorageError
 from boilermaker.storage import StorageInterface
@@ -50,7 +49,8 @@ class BlobClientStorage(AzureBlobStorageClient, StorageInterface):
         Returns:
             The loaded TaskGraph instance, or None if not found.
         Raises:
-            ValidationError: If TaskGraph or TaskResultSlim data cannot be validated.
+            BoilermakerStorageError: If the blob cannot be loaded or if TaskGraph/TaskResultSlim
+                data cannot be validated.
         """
         if not graph_id:
             raise ValueError("`graph_id` must be provided to load a TaskGraph.")
@@ -70,28 +70,38 @@ class BlobClientStorage(AzureBlobStorageClient, StorageInterface):
         if graph_contents is None:
             return None
 
-        graph = TaskGraph.model_validate_json(graph_contents)
+        try:
+            graph = TaskGraph.model_validate_json(graph_contents)
+        except ValidationError as e:
+            raise BoilermakerStorageError(
+                f"Failed to deserialize graph {graph_id}: {e}",
+                status_code=None,
+            ) from e
 
         # Load all TaskResultSlim instances associated with this graph
         # We don't want to load *all* return values into memory. Just the statuses.
         async for blob in self.list_blobs(prefix=graph_dir):
-            tr = TaskResultSlim.model_validate_json(await self.download_blob(blob.name))
+            # DO NOT REDOWNLOAD GRAPH
+            if blob.name == graph_path:
+                continue
+            try:
+                tr = TaskResultSlim.model_validate_json(await self.download_blob(blob.name))
+            except ValidationError as e:
+                raise BoilermakerStorageError(
+                    f"Failed to deserialize task result in graph {graph_id}: {e}",
+                    status_code=None,
+                ) from e
             tr.etag = blob.etag
             if tr.graph_id == graph_id:
                 graph.results[tr.task_id] = tr
             else:
-                logger.warning(
-                    f"TaskResult {tr.task_id} in graph {graph_dir} with wrong graph_id {tr.graph_id}!"
-                )
+                logger.warning(f"TaskResult {tr.task_id} in graph {graph_dir} with wrong graph_id {tr.graph_id}!")
         return graph
 
     async def store_graph(self, graph: TaskGraph) -> TaskGraph:
         """
         Stores a TaskGraph to Azure Blob Storage and stores all children as pending tasks as well.
 
-        We use a lease on the container to make sure *only* one task is writing! This means
-        that we don't have to worry about concurrent writes causing data corruption.
-
         We expect the *written graph* to be **immutable** (see the ImmutabilityPolicy below).
 
         Args:
@@ -99,35 +109,21 @@ class BlobClientStorage(AzureBlobStorageClient, StorageInterface):
         """
         lease = None
         async with self.get_blob_service_client() as blob_service_client:
-            container_client = blob_service_client.get_container_client(
-                self.container_name
-            )
-            lease = await container_client.acquire_lease()
-            upload_kwargs = {
-                "lease": lease,
-                "blob_type": "BlockBlob",
-                "immutability_policy": ImmutabilityPolicy(
-                    expiry_time=datetime.datetime.now(tz=datetime.UTC) + datetime.timedelta(hours=4),
-                    policy_mode="LOCKED",
-                ),
-            }
-
+            container_client = blob_service_client.get_container_client(self.container_name)
             # Store the graph itself first
             fname = f"{self.task_result_prefix}/{graph.storage_path}"
             try:
                 _result = await container_client.upload_blob(
                     fname,
                     graph.model_dump_json(),
-                    **upload_kwargs,
+                    blob_type="BlockBlob",
                 )
             except (
                 ResourceNotFoundError,
                 HttpResponseError,
                 ResourceExistsError,
             ) as exc:
-                logger.error(
-                    f"Error occurred while storing TaskGraph {graph.graph_id}: {exc}"
-                )
+                logger.error(f"Error occurred while storing TaskGraph {graph.graph_id}: {exc}")
                 raise BoilermakerStorageError(
                     f"Failed to store TaskGraph {graph.graph_id}",
                     task_id=None,
@@ -139,15 +135,15 @@ class BlobClientStorage(AzureBlobStorageClient, StorageInterface):
             pending_result = None
             try:
                 async with create_task_group() as tg:
+                    # don't let any tasks that get ahead accidentally clobber us
                     for pending_result in graph.generate_pending_results():
-                        fname_pr = (
-                            f"{self.task_result_prefix}/{pending_result.storage_path}"
-                        )
+                        fname_pr = f"{self.task_result_prefix}/{pending_result.storage_path}"
+
                         uploader = partial(
                             container_client.upload_blob,
                             fname_pr,
                             pending_result.model_dump_json(),
-                            **upload_kwargs,
+                            blob_type="BlockBlob",
                         )
                         tg.start_soon(uploader)
             except* Exception as excgroup:
@@ -180,15 +176,21 @@ class BlobClientStorage(AzureBlobStorageClient, StorageInterface):
             "graph_id": task_result.graph_id or "none",
             "status": task_result.status,
         }
-        concurrency_kwargs: dict[str, str | int] = {}
+        concurrency_kwargs: dict[str, str | int | MatchConditions] = {}
         if etag:
             concurrency_kwargs["etag"] = etag
-            concurrency_kwargs["if_match"] = MatchConditions.IfNotModified.value
+            concurrency_kwargs["match_condition"] = MatchConditions.IfNotModified
 
         try:
             await self.upload_blob(
                 fname, task_result.model_dump_json(), tags=blob_tags, overwrite=True, **concurrency_kwargs
             )
+        # SAFETY: This catch assumes aio_azure_clients_toolbox raises AzureBlobError
+        # (wrapping HTTP 412 Precondition Failed) when an ETag mismatch occurs.
+        # This is the primary guard against concurrent double-scheduling of downstream
+        # tasks. Verified against aio-azure-clients-toolbox v1.0.4 (see uv.lock):
+        # get_blob_client() catches all HttpResponseError (including 412) and re-raises
+        # as AzureBlobError. If the library behavior changes, this guard will silently break.
         except AzureBlobError as exc:
             raise BoilermakerStorageError(
                 f"Failed to store TaskResult {task_result.task_id}",

boilermaker_servicebus-1.0.0.dev4/boilermaker/task/__init__.py

@@ -0,0 +1,20 @@
+from .graph import LAST_ADDED, LastAddedSingleton, TaskChain, TaskGraph, TaskGraphBuilder
+from .result import TaskResult, TaskResultSlim, TaskStatus
+from .task import Task
+from .task_id import GraphId, TaskId
+from .types import TaskHandler
+
+__all__ = [
+    "LAST_ADDED",
+    "LastAddedSingleton",
+    "TaskChain",
+    "TaskGraph",
+    "TaskGraphBuilder",
+    "TaskResult",
+    "TaskResultSlim",
+    "TaskStatus",
+    "Task",
+    "TaskId",
+    "GraphId",
+    "TaskHandler",
+]