durable-workflow 2.10.1__tar.gz → 2.11.0__tar.gz

{durable_workflow-2.10.1/src/durable_workflow.egg-info → durable_workflow-2.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.10.1
+Version: 2.11.0
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.10.1 → durable_workflow-2.11.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "durable-workflow"
-version = "2.10.1"
+version = "2.11.0"
 description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -76,8 +76,8 @@ durable-workflow-replay-conformance = "durable_workflow.replay_conformance:main"
 durable-workflow-workflow-updates-conformance = "durable_workflow.workflow_updates_conformance:main"
 
 [tool.durable-workflow]
-product-train = "2.10.1"
-registry-version = "2.10.1"
+product-train = "2.11.0"
+registry-version = "2.11.0"
 supported-server-versions = "2.8.0"
 worker-protocol-version = "1.20"
 control-plane-version = "2"

{durable_workflow-2.10.1 → durable_workflow-2.11.0}/src/durable_workflow/__init__.py

@@ -81,6 +81,7 @@ from .errors import (
     ExternalPayloadUnauthorized,
     ExternalPayloadUnavailable,
     ExternalPayloadUnsupported,
+    FailureReportTooLarge,
     InvalidArgument,
     InvalidWorkflowDefinition,
     NamespaceNotFound,
@@ -334,6 +335,7 @@ __all__ = [
     "testing",
     "workflow",
     "DurableWorkflowError",
+    "FailureReportTooLarge",
     "EXTERNAL_PAYLOAD_REFERENCE_SCHEMA",
     "RUNTIME_EXTERNAL_PAYLOAD_REFERENCE_SCHEMA",
     "AzureBlobExternalStorage",

durable_workflow-2.11.0/src/durable_workflow/_failure_reporting.py

@@ -0,0 +1,299 @@
+"""Bound diagnostic text using the same JSON encoder as the HTTP transport.
+
+Only failure diagnostics may be reduced. Claim identity, classification and
+retry policy remain exact, even when that makes the complete report unfit.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from .errors import FailureReportTooLarge
+
+MESSAGE_BYTES = 62 * 1024
+STACK_BYTES = 128 * 1024
+DETAIL_BYTES = 64 * 1024
+REQUEST_BYTES = 1024 * 1024
+UNENCODABLE_DETAILS = object()
+_STACK_MARKER = "\n[truncated]"
+_OPTIONAL = ("details", "diagnostics", "runtime_diagnostics", "validation_errors")
+_EXCEPTION_IDENTITY = frozenset({
+    "type", "class", "code", "reason", "non_retryable", "retryable", "retry_after_seconds",
+    "next_retry_delay", "activity_type", "activity_attempt_id", "activity_execution_id",
+    "failed_step_sequence", "failed_activity_execution_id",
+})
+
+
+def encoded_json(value: Any) -> bytes:
+    """Use HTTPX's actual JSON settings, including escaping and NaN refusal."""
+    return httpx.Request("POST", "https://failure.invalid", json=value).content
+
+
+def _size(value: Any) -> int | None:
+    try:
+        return len(encoded_json(value))
+    except (TypeError, ValueError, RecursionError):
+        return None
+
+
+def _utf8(value: str) -> bytes:
+    return value.encode("utf-8", errors="replace")
+
+
+def _prefix(value: str, size: int) -> str:
+    return _utf8(value)[:max(0, size)].decode("utf-8", errors="ignore")
+
+
+def _marker(omitted: dict[str, int | None]) -> str:
+    return "\n[truncated " + encoded_json({"omitted_fields": omitted}).decode("utf-8") + "]"
+
+
+class _ObservedLocalFailure(str):
+    """A local message already delivered to workflow code.
+
+    JSON serializes this as ordinary text. Keep the in-memory distinction until
+    encoding so final batch fitting cannot change first-run/replay behavior.
+    """
+
+
+@dataclass
+class _Report:
+    original: dict[str, Any]
+    terminal: bool = False
+    structured: bool = False
+
+    def _copy(self) -> dict[str, Any]:
+        result = dict(self.original)
+        if self.terminal and isinstance(result.get("exception"), dict):
+            result["exception"] = dict(result["exception"])
+        return result
+
+    def prepare(self) -> dict[str, Any]:
+        result = self._copy()
+        omitted: dict[str, int | None] = {}
+        self._bound_metadata(result, omitted, DETAIL_BYTES)
+        if omitted:
+            self._attach_diagnostics(result, omitted)
+        return result
+
+    def render(self, budget: int) -> dict[str, Any]:
+        result = self._copy()
+        if isinstance(result.get("message"), _ObservedLocalFailure):
+            return result
+        previous: dict[str, int | None] = {}
+        if self.structured:
+            self._previous_sizes(previous)
+        omitted = dict(previous)
+        message = result.get("message", "")
+        message_limit = min(MESSAGE_BYTES, budget)
+        if len(_utf8(message)) > message_limit or _prefix(message, len(_utf8(message))) != message:
+            omitted.setdefault("message", len(_utf8(message)))
+        self._bound_stack(result, omitted, min(STACK_BYTES, budget))
+        self._bound_metadata(result, omitted, min(DETAIL_BYTES, budget))
+        if not omitted:
+            return result
+        if self._unchanged(previous, omitted, budget):
+            return result
+        # Inserting the marker can itself shorten an otherwise small message,
+        # and inserting diagnostics can require omitting another optional field.
+        # Recompute until the finite set of omitted fields is stable.
+        while True:
+            before = dict(omitted)
+            self._attach_diagnostics(result, omitted)
+            marker = _marker(omitted)
+            if len(_utf8(message)) + len(_utf8(marker)) > message_limit:
+                omitted.setdefault("message", len(_utf8(message)))
+            if omitted == before:
+                break
+        retained = max(0, message_limit - len(_utf8(marker)))
+        result["message"] = _prefix(message, retained) + marker
+        return result
+
+    @staticmethod
+    def _bound_stack(result: dict[str, Any], omitted: dict[str, int | None], stack_limit: int) -> None:
+        stack = result.get("stack_trace")
+        if isinstance(stack, str) and len(_utf8(stack)) > stack_limit:
+            omitted.setdefault("stack_trace", len(_utf8(stack)))
+            if stack_limit >= len(_STACK_MARKER):
+                result["stack_trace"] = _prefix(stack, stack_limit - len(_STACK_MARKER)) + _STACK_MARKER
+            else:
+                result.pop("stack_trace", None)
+
+    def _unchanged(
+        self, previous: dict[str, int | None], omitted: dict[str, int | None], budget: int,
+    ) -> bool:
+        message = self.original.get("message", "")
+        stack = self.original.get("stack_trace")
+        return bool(
+            previous and previous == omitted and message.endswith(_marker(previous))
+            and len(_utf8(message)) <= min(MESSAGE_BYTES, budget)
+            and (not isinstance(stack, str) or len(_utf8(stack)) <= min(STACK_BYTES, budget))
+        )
+
+    def _bound_metadata(self, result: dict[str, Any], omitted: dict[str, int | None], budget: int) -> None:
+        fields = ("exception",) if self.terminal else _OPTIONAL
+        values = {key: result[key] for key in fields if key in result}
+        size = _size(values)
+        if size is not None and size <= budget:
+            return
+        for key, value in values.items():
+            omitted.setdefault(key, _size(value))
+            if key == "exception" and isinstance(value, dict):
+                result[key] = {k: v for k, v in value.items() if k in _EXCEPTION_IDENTITY}
+            else:
+                result.pop(key, None)
+
+    def _previous_sizes(self, omitted: dict[str, int | None]) -> None:
+        original = self.original.get("exception", {}) if self.terminal else self.original
+        diagnostics = original.get("diagnostics") if isinstance(original, dict) else None
+        previous = diagnostics.get("failure_report") if isinstance(diagnostics, dict) else None
+        sizes = previous.get("omitted_fields") if isinstance(previous, dict) else None
+        if not isinstance(sizes, dict):
+            return
+        for key, size in sizes.items():
+            if (key in {*_OPTIONAL, "exception", "message", "stack_trace"}
+                    and (size is None or type(size) is int and size >= 0)):
+                omitted[key] = size
+
+    def _attach_diagnostics(self, result: dict[str, Any], omitted: dict[str, int | None]) -> None:
+        if not self.structured:
+            return
+        container = result.setdefault("exception", {}) if self.terminal else result
+        self._previous_sizes(omitted)
+        original = self.original.get("exception", {}) if self.terminal else self.original
+        existing = original.get("diagnostics") if isinstance(original, dict) else None
+        diagnostics = dict(existing) if isinstance(existing, dict) and "diagnostics" not in omitted else {}
+        diagnostics["failure_report"] = {"omitted_fields": omitted}
+        container["diagnostics"] = diagnostics
+        optional = {key: container[key] for key in _OPTIONAL if key in container}
+        size = _size(container if self.terminal else optional)
+        if size is not None and size <= DETAIL_BYTES:
+            return
+        self._omit_optional_metadata(container, omitted, existing)
+
+    def _omit_optional_metadata(
+        self, container: dict[str, Any], omitted: dict[str, int | None], existing: Any,
+    ) -> None:
+        # Count inserted diagnostics too. Omit optional fields whole so an
+        # Avro envelope is either valid and intact or explicitly absent.
+        if existing is not None:
+            omitted.setdefault("diagnostics", _size(existing))
+        # Snapshot the keys because the loop removes entries from this dictionary.
+        for key in list(container):
+            removable = key not in _EXCEPTION_IDENTITY if self.terminal else key in _OPTIONAL
+            if removable and key != "diagnostics":
+                omitted.setdefault(key if key in _OPTIONAL else "exception", _size(container.pop(key)))
+        container["diagnostics"] = {"failure_report": {"omitted_fields": omitted}}
+
+
+def _command_reports(command: dict[str, Any]) -> list[tuple[tuple[str | int, ...], _Report]]:
+    kind = command.get("type")
+    terminal = kind == "fail_workflow"
+    result: list[tuple[tuple[str | int, ...], _Report]] = []
+    if kind in {"fail_workflow", "fail_update"} or kind == "record_local_activity" and "message" in command:
+        result.append(((), _Report(command, terminal=terminal, structured=terminal)))
+    if kind == "record_local_activity":
+        for index, attempt in enumerate(command.get("attempts", [])):
+            if isinstance(attempt, dict) and "message" in attempt:
+                result.append((("attempts", index), _Report(attempt)))
+    return result
+
+
+def _reports(path: str, body: dict[str, Any]) -> list[tuple[tuple[str | int, ...], _Report]]:
+    failure = body.get("failure")
+    if isinstance(failure, dict) and (path.endswith(("/fail", "/reject"))):
+        return [(("failure",), _Report(failure, structured="/activity-tasks/" in path))]
+    if "/local-activities/" in path and path.endswith("/outcome"):
+        report = body.get("report")
+        if isinstance(report, dict) and "message" in report:
+            return [(("report",), _Report(report))]
+    commands = body.get("commands")
+    if not isinstance(commands, list) or not path.endswith("/complete"):
+        return []
+    return [
+        (("commands", index, *location), report)
+        for index, command in enumerate(commands) if isinstance(command, dict)
+        for location, report in _command_reports(command)
+    ]
+
+
+def _replace_report(body: Any, location: tuple[str | int, ...], report: dict[str, Any]) -> Any:
+    if not location:
+        return report
+    result: Any = dict(body) if isinstance(body, dict) else list(body)
+    key = location[0]
+    result[key] = _replace_report(body[key], location[1:], report)
+    return result
+
+
+def prepare_failure_request(path: str, body: Any) -> Any:
+    """Omit oversized diagnostic envelopes before optional payload uploads.
+
+    Keep original message/stack text until the final fit so its original sizes
+    survive normal externalization of unrelated business payloads in a batch.
+    """
+    if not path.startswith("/worker/") or not isinstance(body, dict):
+        return body
+    result = body
+    for location, report in _reports(path, body):
+        if report.structured:
+            result = _replace_report(result, location, report.prepare())
+    return result
+
+
+def _render_reports(
+    body: dict[str, Any], reports: list[tuple[tuple[str | int, ...], _Report]], budget: int, limit: int,
+) -> dict[str, Any]:
+    result = body
+    for location, report in reports:
+        rendered = report.render(budget)
+        metadata = rendered.get("exception", {}) if report.terminal else {
+            key: rendered[key] for key in _OPTIONAL if key in rendered
+        }
+        metadata_size = _size(metadata)
+        if metadata_size is None or metadata_size > DETAIL_BYTES:
+            raise FailureReportTooLarge(limit)
+        result = _replace_report(result, location, rendered)
+    return result
+
+
+def bound_failure_request(path: str, body: Any, limit: int = REQUEST_BYTES) -> Any:
+    if not path.startswith("/worker/") or not isinstance(body, dict):
+        return body
+    reports = _reports(path, body)
+    if not reports:
+        return body
+    limit = min(limit, REQUEST_BYTES)
+    budget = STACK_BYTES
+    while True:
+        result = _render_reports(body, reports, budget, limit)
+        size = _size(result)
+        if size is not None and size <= limit:
+            return result
+        if budget == 0:
+            raise FailureReportTooLarge(limit)
+        budget //= 2
+
+
+def normalize_local_failure(
+    command: dict[str, Any], *, lease_owner: str, workflow_task_attempt: int, limit: int = REQUEST_BYTES,
+) -> dict[str, Any]:
+    """Fit local diagnostics before the workflow sees its durable failure.
+
+    Business arguments can still be externalized by Client. Include all claim,
+    retry and attempt data here; the final complete request is measured again.
+    An already observed message is then immutable, including on a lower limit
+    discovered later or when another command makes the batch unfit.
+    """
+    frame = {key: value for key, value in command.items() if key != "arguments"}
+    body = {"lease_owner": lease_owner, "workflow_task_attempt": workflow_task_attempt, "commands": [frame]}
+    normalized = bound_failure_request("/worker/workflow-tasks/local/complete", body, limit)["commands"][0]
+    normalized = {**command, **normalized}
+    if isinstance(normalized.get("message"), str):
+        normalized["message"] = _ObservedLocalFailure(normalized["message"])
+    for attempt in normalized.get("attempts", []):
+        if isinstance(attempt, dict) and isinstance(attempt.get("message"), str):
+            attempt["message"] = _ObservedLocalFailure(attempt["message"])
+    return normalized

{durable_workflow-2.10.1 → durable_workflow-2.11.0}/src/durable_workflow/client.py

@@ -48,6 +48,14 @@ from ._cancellation_scope_history import (
     scope_timestamp,
 )
 from ._cooperative_cancellation import CancellationDelivery, CancellationRequest
+from ._failure_reporting import (
+    DETAIL_BYTES,
+    REQUEST_BYTES,
+    UNENCODABLE_DETAILS,
+    bound_failure_request,
+    encoded_json,
+    prepare_failure_request,
+)
 from ._metric_routes import route_for_metrics as _route_for_metrics
 from ._query_task_authority import QueryTaskAuthority, current_query_authority
 from .errors import (
@@ -1622,6 +1630,7 @@ class Client:
         payload_size_limit_bytes: int = serializer.DEFAULT_PAYLOAD_SIZE_BYTES,
         payload_size_warning_threshold_percent: int = serializer.DEFAULT_WARNING_THRESHOLD_PERCENT,
         payload_size_warnings: bool = True,
+        failure_report_limit_bytes: int = REQUEST_BYTES,
         external_storage: ExternalStorageDriver | None = None,
         external_storage_threshold_bytes: int | None = None,
         external_storage_cache: ExternalPayloadCache | None = None,
@@ -1643,6 +1652,9 @@ class Client:
         self._tls_context = tls_context
         self.retry_policy = retry_policy or TransportRetryPolicy()
         self.metrics = metrics if metrics is not None else NOOP_METRICS
+        if type(failure_report_limit_bytes) is not int or failure_report_limit_bytes < 1:
+            raise ValueError("failure_report_limit_bytes must be a positive integer")
+        self.failure_report_limit_bytes = min(REQUEST_BYTES, failure_report_limit_bytes)
         self.payload_size_warning_config = (
             serializer.PayloadSizeWarningConfig(
                 limit_bytes=payload_size_limit_bytes,
@@ -1843,6 +1855,14 @@ class Client:
             ),
         )
 
+    def _failure_report_limit(self) -> int:
+        limit = self.failure_report_limit_bytes
+        limits = (self._cluster_info or {}).get("limits")
+        advertised = limits.get("max_payload_bytes") if isinstance(limits, dict) else None
+        if type(advertised) is int and advertised > 0:
+            limit = min(limit, advertised)
+        return limit
+
     async def _request(
         self,
         method: str,
@@ -1869,22 +1889,22 @@ class Client:
         activity_authority = current_activity_authority.get()
         if worker and activity_authority is not None and activity_authority.governs(self, self.namespace, path):
             activity_authority.remaining()
-        request_json = json
+        request_json = prepare_failure_request(path, json)
         if (
             not discovery
             and json is not None
-            and _contains_direct_external_storage_envelope(json)
+            and _contains_direct_external_storage_envelope(request_json)
         ):
             await self._require_direct_external_storage_support()
         if (
             not discovery
             and json is not None
-            and _contains_inline_payload_envelope(json)
+            and _contains_inline_payload_envelope(request_json)
         ):
             transport = await self._runtime_external_payload_transport()
             if transport is not None:
                 request_json = await self._externalize_runtime_payloads(
-                    json,
+                    request_json,
                     worker=worker,
                     transport=transport,
                     uploaded={},
@@ -1894,6 +1914,9 @@ class Client:
                     ),
                 )
 
+        # Discovery/external payload references may change the known limit or
+        # final envelope. Measure again after those transformations.
+        request_json = bound_failure_request(path, request_json, self._failure_report_limit())
         start = time.perf_counter()
         route = _route_for_metrics(path)
         discovery_uses_worker_token = discovery and bool(self.worker_token)
@@ -6208,11 +6231,20 @@ class Client:
         if non_retryable:
             failure["non_retryable"] = True
         if details is not None:
-            failure["details"] = self._payload_envelope(
-                details,
-                codec=codec,
-                kind="activity_failure_details",
-                activity_name=activity_name,
+            # Inspect the official inline encoding before an optional storage
+            # driver uploads diagnostics that the failure budget will omit.
+            try:
+                inline_details: Any = serializer.envelope(details, codec=codec, size_warning=None)
+            except (TypeError, ValueError, RecursionError):
+                if codec != serializer.AVRO_CODEC:
+                    raise
+                inline_details = UNENCODABLE_DETAILS
+            failure["details"] = (
+                self._payload_envelope(
+                    details, codec=codec, kind="activity_failure_details", activity_name=activity_name,
+                )
+                if isinstance(inline_details, dict) and len(encoded_json(inline_details)) <= DETAIL_BYTES
+                else inline_details
             )
         body: dict[str, Any] = {
             "activity_attempt_id": activity_attempt_id,

{durable_workflow-2.10.1 → durable_workflow-2.11.0}/src/durable_workflow/errors.py

@@ -27,6 +27,20 @@ class DurableWorkflowError(Exception):
     """Base class for every exception raised by the SDK."""
 
 
+class FailureReportTooLarge(DurableWorkflowError, ValueError):
+    """Required failure identity or semantics cannot fit the reporting budget.
+
+    This is a local reporting refusal, not the application's failure. No claim
+    identity, failure type or retry decision is shortened to make it fit.
+    """
+
+    def __init__(self, limit_bytes: int) -> None:
+        self.limit_bytes = limit_bytes
+        super().__init__(
+            f"failure report identity or semantics cannot fit the {limit_bytes}-byte encoded request budget"
+        )
+
+
 class QueryTaskAuthorityLost(DurableWorkflowError):
     """A query delivery cannot safely load or invoke its handler."""
 

{durable_workflow-2.10.1 → durable_workflow-2.11.0}/src/durable_workflow/worker.py

@@ -44,6 +44,7 @@ from ._activity_lease import activity_lease_renewal_supported, current_activity_
 from ._activity_process import CallbackFailure, CallbackInvocation, CallbackProcessLost, SupervisedCallback
 from ._cancellation_scope_history import CancellationScopeBudget, CancellationScopeDeliveryReceipt
 from ._cooperative_cancellation import CancellationRequest, read_cancellation_history
+from ._failure_reporting import REQUEST_BYTES, normalize_local_failure
 from ._prepared_local_activity import PreparedAttempt, PreparedCancellationObserved, PreparedLocalRunner
 from ._query_task_authority import QueryTaskAuthority, current_query_authority
 from ._sticky_workflow_cache import CacheKey, StickyWorkflowCache, complete_history
@@ -76,6 +77,7 @@ from .errors import (
     DuplicateRegistrationError,
     DurableWorkflowError,
     ExternalPayloadError,
+    FailureReportTooLarge,
     InvalidArgument,
     NonDeterministicReplayError,
     NonRetryableError,
@@ -2328,8 +2330,22 @@ class Worker:
             }
             if timeout_kind is not None:
                 command.outcome["timeout_kind"] = timeout_kind
+            known_limit = getattr(self.client, "_failure_report_limit", None)
+            limit = known_limit() if callable(known_limit) else REQUEST_BYTES
+            if type(limit) is not int:
+                limit = REQUEST_BYTES
+            try:
+                normalized = normalize_local_failure(
+                    command.to_server_command(self.task_queue, payload_codec=payload_codec),
+                    lease_owner=self.worker_id, workflow_task_attempt=task["workflow_task_attempt"], limit=limit,
+                )
+            except FailureReportTooLarge as error:
+                raise LocalActivityExecutionAborted(
+                    "local failure report cannot fit before workflow observation",
+                ) from error
+            command.outcome = {key: normalized[key] for key in command.outcome}
             return ActivityFailed(
-                message,
+                command.outcome["message"],
                 activity_type=command.activity_type,
                 exception_type=exception_type,
                 non_retryable=non_retryable,
@@ -2469,18 +2485,15 @@ class Worker:
                 cancelled = isinstance(exc, ActivityCancelled)
                 exception_type = type(exc).__name__
                 message = str(exc).strip() or f"local activity failed with {exception_type}"
-                invalid_report = len(exception_type.encode("utf-8")) > 255
                 try:
-                    json.dumps({"message": message, "exception_type": exception_type}, allow_nan=False).encode("utf-8")
-                except (TypeError, ValueError, UnicodeError):
-                    invalid_report = True
-                if invalid_report:
-                    exception_type = "InvalidLocalActivityReport"
-                    message = "local activity failure metadata cannot cross the HTTP JSON boundary"
-                    timed_out = False
-                    cancelled = False
+                    if len(exception_type.encode("utf-8")) > 255:
+                        raise ValueError("local activity failure type exceeds the wire limit")
+                except ValueError as error:
+                    raise LocalActivityExecutionAborted(
+                        "local activity failure type cannot cross the HTTP boundary",
+                    ) from error
                 non_retryable = (
-                    cancelled or invalid_report or isinstance(exc, NonRetryableError)
+                    cancelled or isinstance(exc, NonRetryableError)
                     or exception_type in non_retryable_types
                 )
                 retry = not non_retryable and attempt_number < max_attempts
@@ -4159,6 +4172,7 @@ class Worker:
                 else serializer.DEFAULT_WARNING_THRESHOLD_PERCENT
             ),
             payload_size_warnings=warning_config is not None,
+            failure_report_limit_bytes=self.client.failure_report_limit_bytes,
             external_storage=self.external_storage,
             external_storage_threshold_bytes=self.external_storage_threshold_bytes,
             external_storage_cache=self.external_storage_cache,

{durable_workflow-2.10.1 → durable_workflow-2.11.0/src/durable_workflow.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: durable-workflow
-Version: 2.10.1
+Version: 2.11.0
 Summary: Python client and worker SDK for Durable Workflow Cloud and self-hosted Server
 Author: Durable Workflow Contributors
 License-Expression: MIT

{durable_workflow-2.10.1 → durable_workflow-2.11.0}/src/durable_workflow.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ src/durable_workflow/_avro.py
 src/durable_workflow/_cancellation_scope.py
 src/durable_workflow/_cancellation_scope_history.py
 src/durable_workflow/_cooperative_cancellation.py
+src/durable_workflow/_failure_reporting.py
 src/durable_workflow/_metric_routes.py
 src/durable_workflow/_prepared_local_activity.py
 src/durable_workflow/_query_task_authority.py
@@ -84,11 +85,13 @@ tests/test_errors.py
 tests/test_external_storage.py
 tests/test_external_task_input.py
 tests/test_external_task_result.py
+tests/test_failure_reporting.py
 tests/test_golden_history_replay.py
 tests/test_history_bundle_schema2.py
 tests/test_history_bundle_verify.py
 tests/test_history_event_contract.py
 tests/test_invocable.py
+tests/test_local_failure_reporting.py
 tests/test_message_streams.py
 tests/test_metric_routes.py
 tests/test_metrics.py