aip-agents-binary 0.0.0__py3-none-any.whl

aip_agents/resilience/tool.py

@@ -0,0 +1,1661 @@
+"""Tool-specific resilience policies and execution adapter.
+
+Author:
+    Saul Sayers (saul.sayers@gdplabs.id)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass
+from typing import Any
+
+from langchain_core.tools import BaseTool
+from pybreaker import CircuitBreaker
+from tenacity import wait_exponential
+
+from aip_agents.resilience.base import BaseResilienceExecutor, ResilienceRequest
+
+DEFAULT_TOOL_TIMEOUT_SECONDS = 60.0
+DEFAULT_RETRY_ENABLED = False
+DEFAULT_RETRY_MAX_ATTEMPTS = 3
+DEFAULT_RETRY_BACKOFF_MIN_SECONDS = 1.0
+DEFAULT_RETRY_BACKOFF_MAX_SECONDS = 8.0
+DEFAULT_RETRYABLE_ERROR_KINDS = (
+    "timeout",
+    "connection_error",
+    "endpoint_unreachable",
+    "upstream_5xx",
+    "rate_limited",
+)
+DEFAULT_CIRCUIT_BREAKER_ENABLED = False
+DEFAULT_CIRCUIT_FAIL_MAX = 5
+DEFAULT_CIRCUIT_RESET_TIMEOUT_SECONDS = 60.0
+DEFAULT_CIRCUIT_HALF_OPEN_MAX_CALLS = 1
+
+TOOL_TIMEOUT_SECONDS_KEY = "tool_timeout_seconds"
+TOOL_SOURCE_KEY = "tool_source"
+RESILIENCE_KEY = "resilience"
+TIMEOUT_SECONDS_KEY = "timeout_seconds"
+RETRY_KEY = "retry"
+RETRY_ENABLED_KEY = "enabled"
+RETRY_MAX_ATTEMPTS_KEY = "max_attempts"
+RETRY_BACKOFF_MIN_SECONDS_KEY = "backoff_min_seconds"
+RETRY_BACKOFF_MAX_SECONDS_KEY = "backoff_max_seconds"
+RETRYABLE_ERROR_KINDS_KEY = "retryable_error_kinds"
+CIRCUIT_BREAKER_KEY = "circuit_breaker"
+CIRCUIT_BREAKER_ENABLED_KEY = "enabled"
+CIRCUIT_BREAKER_FAIL_MAX_KEY = "fail_max"
+CIRCUIT_BREAKER_RESET_TIMEOUT_SECONDS_KEY = "reset_timeout_seconds"
+CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS_KEY = "half_open_max_calls"
+
+CATEGORY_TIMEOUT = "timeout"
+CATEGORY_INVALID_TIMEOUT_CONFIG = "invalid_timeout_config"
+CATEGORY_INVALID_RETRY_CONFIG = "invalid_retry_config"
+CATEGORY_INVALID_CIRCUIT_CONFIG = "invalid_circuit_breaker_config"
+CATEGORY_UNAUTHORIZED = "unauthorized"
+CATEGORY_FORBIDDEN = "forbidden"
+CATEGORY_RATE_LIMITED = "rate_limited"
+CATEGORY_ENDPOINT_UNREACHABLE = "endpoint_unreachable"
+CATEGORY_UPSTREAM_FAILURE = "upstream_failure"
+CATEGORY_RETRY_EXHAUSTED = "retry_exhausted"
+CATEGORY_CIRCUIT_OPEN = "circuit_open"
+
+KIND_TIMEOUT = "timeout"
+KIND_CONNECTION_ERROR = "connection_error"
+KIND_ENDPOINT_UNREACHABLE = "endpoint_unreachable"
+KIND_UPSTREAM_5XX = "upstream_5xx"
+KIND_RATE_LIMITED = "rate_limited"
+KIND_UNAUTHORIZED = "unauthorized"
+KIND_FORBIDDEN = "forbidden"
+KIND_UPSTREAM_FAILURE = "upstream_failure"
+KIND_CIRCUIT_OPEN = "circuit_open"
+
+_KNOWN_TOOL_SOURCES = {"gl_connector", "mcp", "custom", "ptc", "unknown"}
+_NON_RETRYABLE_KINDS = {KIND_UNAUTHORIZED, KIND_FORBIDDEN}
+
+
+@dataclass(frozen=True)
+class RetryPolicy:
+    """Resolved retry policy for one tool invocation."""
+
+    enabled: bool
+    max_attempts: int
+    backoff_min_seconds: float
+    backoff_max_seconds: float
+    retryable_error_kinds: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class CircuitBreakerPolicy:
+    """Resolved circuit-breaker policy for one tool invocation."""
+
+    enabled: bool
+    fail_max: int
+    reset_timeout_seconds: float
+    half_open_max_calls: int
+
+
+@dataclass(frozen=True)
+class CircuitBreakerLease:
+    """Lease returned by circuit registry before one tool attempt."""
+
+    tool_key: str
+    policy: CircuitBreakerPolicy
+    state: str
+    allowed: bool
+    half_open_acquired: bool
+    last_error_kind: str | None
+
+
+@dataclass(frozen=True)
+class CircuitBreakerTransition:
+    """Circuit breaker transition/result snapshot for metadata emission."""
+
+    tool_key: str
+    state: str
+    previous_state: str
+    transitioned: bool
+    consecutive_failures: int
+    inflight_half_open_calls: int
+    opened_at_unix_seconds: float | None
+    last_error_kind: str | None
+
+
+@dataclass(frozen=True)
+class ToolFailureDetail:
+    """Classified tool failure detail used for retry and output mapping."""
+
+    category: str
+    kind: str
+    message: str
+    code: str
+    http_status: int | None
+    is_retryable_default: bool
+    cancelled: bool | None = None
+
+
+@dataclass(frozen=True)
+class _ResolvedToolPolicies:
+    timeout_seconds: float
+    retry: RetryPolicy
+    circuit_breaker: CircuitBreakerPolicy
+
+
+class ToolResilienceExecutor(BaseResilienceExecutor):
+    """Resilience executor for tool invocations.
+
+    This executor keeps timeout validation and timeout-error shaping outside the
+    LangGraph agent implementation, so the agent only needs to provide the
+    invocation callback.
+    """
+
+    def __init__(self, *, circuit_breakers: ToolCircuitBreakerRegistry | None = None) -> None:
+        """Initialize executor with optional shared circuit-breaker registry.
+
+        Args:
+            circuit_breakers: Optional shared registry instance for circuit state.
+
+        Returns:
+            None.
+        """
+        self._circuit_breakers = circuit_breakers or ToolCircuitBreakerRegistry()
+
+    async def execute(self, request: ResilienceRequest) -> Any:
+        """Execute one tool request with timeout and retry handling.
+
+        Args:
+            request: Tool resilience request with metadata and invocation callback.
+
+        Returns:
+            Tool output or structured resilience error payload.
+        """
+        policies_or_error = self._resolve_policies_or_error(request)
+        if isinstance(policies_or_error, dict):
+            return policies_or_error
+        policies = policies_or_error
+        return await self._execute_with_policies(request, policies)
+
+    async def _execute_with_policies(self, request: ResilienceRequest, policies: _ResolvedToolPolicies) -> Any:
+        start_time = time.monotonic()
+        attempts_used = 0
+        circuit_tool_key = f"{request.operation_source}:{request.operation_name}"
+        last_failure: ToolFailureDetail | None = None
+        last_timeout_failure: ToolFailureDetail | None = None
+
+        while True:
+            attempts_used += 1
+            remaining_timeout_seconds = policies.timeout_seconds - (time.monotonic() - start_time)
+            caught_error: Exception | None = None
+
+            if remaining_timeout_seconds <= 0:
+                return self._build_timeout_budget_output(request, policies, attempts_used, last_failure=last_failure)
+
+            circuit_lease = await self._acquire_circuit_lease(request, policies, circuit_tool_key)
+            if isinstance(circuit_lease, dict):
+                return circuit_lease
+
+            try:
+                return await self._invoke_attempt(
+                    request,
+                    policies,
+                    attempts_used=attempts_used,
+                    remaining_timeout_seconds=remaining_timeout_seconds,
+                    circuit_lease=circuit_lease,
+                    last_timeout_failure=last_timeout_failure,
+                )
+            except TimeoutError as timeout_error:
+                failure = classify_tool_exception(timeout_error)
+            except Exception as execution_error:  # noqa: BLE001
+                caught_error = execution_error
+                failure = classify_tool_exception(execution_error)
+
+            last_failure = failure
+            if failure.category == CATEGORY_TIMEOUT:
+                last_timeout_failure = failure
+
+            circuit_transition = await self._record_circuit_failure(policies, circuit_lease, failure)
+            retry_output = await self._handle_retry_candidate(
+                request,
+                policies,
+                attempts_used=attempts_used,
+                failure=failure,
+                start_time=start_time,
+                circuit_transition=circuit_transition,
+            )
+            route = self._route_post_attempt_result(
+                request,
+                policies,
+                attempts_used=attempts_used,
+                caught_error=caught_error,
+                failure=failure,
+                retry_output=retry_output,
+                circuit_transition=circuit_transition,
+            )
+            if route == "continue":
+                continue
+            if route is not None:
+                return route
+
+    def _route_post_attempt_result(
+        self,
+        request: ResilienceRequest,
+        policies: _ResolvedToolPolicies,
+        *,
+        attempts_used: int,
+        caught_error: Exception | None,
+        failure: Any,
+        retry_output: str | dict[str, Any] | None,
+        circuit_transition: Any,
+    ) -> str | dict[str, Any] | None:
+        if retry_output == "continue":
+            return "continue"
+        if retry_output is not None:
+            return retry_output
+        if caught_error is not None and not policies.retry.enabled and not policies.circuit_breaker.enabled:
+            raise caught_error
+        return self._build_final_failure_output(
+            request,
+            policies,
+            attempts_used=attempts_used,
+            failure=failure,
+            circuit_transition=circuit_transition,
+        )
+
+    def _resolve_policies_or_error(self, request: ResilienceRequest) -> _ResolvedToolPolicies | dict[str, Any]:
+        try:
+            return _ResolvedToolPolicies(
+                timeout_seconds=resolve_tool_timeout_seconds(request.metadata),
+                retry=resolve_retry_policy(request.metadata),
+                circuit_breaker=resolve_circuit_breaker_policy(request.metadata),
+            )
+        except ValueError as exc:
+            return self._build_invalid_policy_output(request, str(exc))
+
+    @staticmethod
+    def _build_invalid_policy_output(request: ResilienceRequest, error_text: str) -> dict[str, Any]:
+        if error_text.startswith("retry"):
+            return build_invalid_retry_config_output(
+                tool_name=request.operation_name,
+                tool_source=request.operation_source,
+                error=error_text,
+            )
+        if error_text.startswith(("circuit_breaker", "circuit breaker")):
+            return build_invalid_circuit_breaker_config_output(
+                tool_name=request.operation_name,
+                tool_source=request.operation_source,
+                error=error_text,
+            )
+        return build_invalid_timeout_config_output(
+            tool_name=request.operation_name,
+            tool_source=request.operation_source,
+            error=error_text,
+        )
+
+    @staticmethod
+    def _build_timeout_budget_output(
+        request: ResilienceRequest,
+        policies: _ResolvedToolPolicies,
+        attempts_used: int,
+        last_failure: ToolFailureDetail | None = None,
+    ) -> dict[str, Any]:
+        if policies.retry.enabled and attempts_used > 1:
+            return build_retry_exhausted_error_output(
+                tool_name=request.operation_name,
+                tool_source=request.operation_source,
+                timeout_seconds=policies.timeout_seconds,
+                attempts_used=attempts_used,
+                max_attempts=policies.retry.max_attempts,
+                last_failure=last_failure or classify_tool_exception(TimeoutError("tool retry budget exhausted")),
+                retry_budget_exhausted=True,
+            )
+        return build_timeout_error_output(
+            tool_name=request.operation_name,
+            timeout_seconds=policies.timeout_seconds,
+            tool_source=request.operation_source,
+            cancelled=last_failure.cancelled if last_failure is not None else None,
+        )
+
+    async def _acquire_circuit_lease(
+        self,
+        request: ResilienceRequest,
+        policies: _ResolvedToolPolicies,
+        circuit_tool_key: str,
+    ) -> CircuitBreakerLease | dict[str, Any] | None:
+        if not policies.circuit_breaker.enabled:
+            return None
+        lease = await self._circuit_breakers.acquire(tool_key=circuit_tool_key, policy=policies.circuit_breaker)
+        if lease.allowed:
+            return lease
+        return build_circuit_open_error_output(
+            tool_name=request.operation_name,
+            tool_source=request.operation_source,
+            reset_timeout_seconds=policies.circuit_breaker.reset_timeout_seconds,
+            circuit_state=lease.state,
+            failure_kind=lease.last_error_kind,
+        )
+
+    async def _invoke_attempt(
+        self,
+        request: ResilienceRequest,
+        policies: _ResolvedToolPolicies,
+        *,
+        attempts_used: int,
+        remaining_timeout_seconds: float,
+        circuit_lease: CircuitBreakerLease | None,
+        last_timeout_failure: ToolFailureDetail | None,
+    ) -> Any:
+        tool_output = await request.invoke_with_timeout(remaining_timeout_seconds)
+        circuit_transition = None
+        if policies.circuit_breaker.enabled and circuit_lease:
+            circuit_transition = await self._circuit_breakers.record_success(circuit_lease)
+        if policies.retry.enabled and attempts_used > 1 and isinstance(tool_output, dict):
+            tool_output = add_retry_attempt_metadata(
+                tool_output,
+                attempts_used=attempts_used,
+                max_attempts=policies.retry.max_attempts,
+                retry_budget_exhausted=False,
+            )
+        if (
+            isinstance(tool_output, dict)
+            and last_timeout_failure is not None
+            and last_timeout_failure.cancelled is not None
+        ):
+            metadata = tool_output.setdefault("metadata", {})
+            if not isinstance(metadata, dict):
+                metadata = {}
+                tool_output["metadata"] = metadata
+            execution = metadata.setdefault("tool_execution", {})
+            if not isinstance(execution, dict):
+                execution = {}
+                metadata["tool_execution"] = execution
+            execution["previous_timeout_cancelled"] = last_timeout_failure.cancelled
+        return self._with_circuit_metadata(tool_output, circuit_transition)
+
+    async def _record_circuit_failure(
+        self,
+        policies: _ResolvedToolPolicies,
+        circuit_lease: CircuitBreakerLease | None,
+        failure: ToolFailureDetail,
+    ) -> CircuitBreakerTransition | None:
+        if not policies.circuit_breaker.enabled or not circuit_lease:
+            return None
+        return await self._circuit_breakers.record_failure(circuit_lease, error_kind=failure.kind)
+
+    async def _handle_retry_candidate(
+        self,
+        request: ResilienceRequest,
+        policies: _ResolvedToolPolicies,
+        *,
+        attempts_used: int,
+        failure: ToolFailureDetail,
+        start_time: float,
+        circuit_transition: CircuitBreakerTransition | None,
+    ) -> dict[str, Any] | str | None:
+        retry_candidate = should_retry_failure(failure, policies.retry)
+        if retry_candidate and attempts_used < policies.retry.max_attempts:
+            retry_delay_seconds = compute_retry_backoff_seconds(attempts_used, policies.retry)
+            remaining_budget = policies.timeout_seconds - (time.monotonic() - start_time)
+            if retry_delay_seconds >= remaining_budget:
+                return self._build_retry_exhausted_output(
+                    request, policies, attempts_used, failure, True, circuit_transition
+                )
+            await asyncio.sleep(retry_delay_seconds)
+            return "continue"
+        if retry_candidate and policies.retry.enabled:
+            return self._build_retry_exhausted_output(
+                request, policies, attempts_used, failure, False, circuit_transition
+            )
+        return None
+
+    @staticmethod
+    def _build_retry_exhausted_output(
+        request: ResilienceRequest,
+        policies: _ResolvedToolPolicies,
+        attempts_used: int,
+        failure: ToolFailureDetail,
+        retry_budget_exhausted: bool,
+        circuit_transition: CircuitBreakerTransition | None,
+    ) -> dict[str, Any]:
+        tool_output = build_retry_exhausted_error_output(
+            tool_name=request.operation_name,
+            tool_source=request.operation_source,
+            timeout_seconds=policies.timeout_seconds,
+            attempts_used=attempts_used,
+            max_attempts=policies.retry.max_attempts,
+            last_failure=failure,
+            retry_budget_exhausted=retry_budget_exhausted,
+        )
+        return ToolResilienceExecutor._with_circuit_metadata(tool_output, circuit_transition)
+
+    def _build_final_failure_output(
+        self,
+        request: ResilienceRequest,
+        policies: _ResolvedToolPolicies,
+        *,
+        attempts_used: int,
+        failure: ToolFailureDetail,
+        circuit_transition: CircuitBreakerTransition | None,
+    ) -> dict[str, Any]:
+        if failure.category == CATEGORY_TIMEOUT:
+            tool_output = build_timeout_error_output(
+                tool_name=request.operation_name,
+                timeout_seconds=policies.timeout_seconds,
+                tool_source=request.operation_source,
+                cancelled=failure.cancelled,
+            )
+            if policies.retry.enabled:
+                tool_output = add_retry_attempt_metadata(
+                    tool_output,
+                    attempts_used=attempts_used,
+                    max_attempts=policies.retry.max_attempts,
+                    retry_budget_exhausted=False,
+                )
+            return self._with_circuit_metadata(tool_output, circuit_transition)
+
+        tool_output = build_classified_error_output(
+            tool_name=request.operation_name,
+            tool_source=request.operation_source,
+            attempts_used=attempts_used,
+            max_attempts=policies.retry.max_attempts,
+            failure=failure,
+        )
+        return self._with_circuit_metadata(tool_output, circuit_transition)
+
+    @staticmethod
+    def _with_circuit_metadata(tool_output: Any, transition: CircuitBreakerTransition | None) -> Any:
+        if isinstance(tool_output, dict) and transition:
+            return add_circuit_breaker_metadata(tool_output, transition=transition)
+        return tool_output
+
+
+@dataclass
+class _CircuitState:
+    """Mutable state container for one circuit key."""
+
+    breaker: CircuitBreaker
+    policy: CircuitBreakerPolicy
+    inflight_half_open_calls: int = 0
+    opened_at_unix_seconds: float | None = None
+    last_error_kind: str | None = None
+
+
+@dataclass(frozen=True)
+class _TenacityWaitState:
+    """Minimal wait state used by tenacity wait strategies."""
+
+    attempt_number: int
+
+
+def _circuit_success_probe() -> bool:
+    """Return a successful probe value for circuit breaker transitions.
+
+    Args:
+        None.
+
+    Returns:
+        ``True`` to mark a successful probe call.
+    """
+    return True
+
+
+def _raise_circuit_failure() -> None:
+    """Raise a sentinel failure used to trip circuit state transitions.
+
+    Args:
+        None.
+
+    Returns:
+        None.
+    """
+    raise RuntimeError("tool failure")
+
+
+class ToolCircuitBreakerRegistry:
+    """In-memory registry for per-tool circuit breaker transitions.
+
+    Uses ``pybreaker`` to drive circuit state transitions while preserving
+    per-tool metadata snapshots for observability.
+    """
+
+    def __init__(self) -> None:
+        """Initialize an empty registry.
+
+        Args:
+            None.
+
+        Returns:
+            None.
+        """
+        self._states: dict[str, _CircuitState] = {}
+        self._lock = asyncio.Lock()
+
+    async def acquire(self, *, tool_key: str, policy: CircuitBreakerPolicy) -> CircuitBreakerLease:
+        """Attempt to acquire permission for one tool attempt.
+
+        Args:
+            tool_key: Stable circuit identity (for example ``"custom:search"``).
+            policy: Resolved per-tool circuit breaker policy.
+
+        Returns:
+            CircuitBreakerLease with allow/deny decision and acquisition metadata.
+        """
+        async with self._lock:
+            now = time.time()
+            state = self._get_or_create_state(tool_key=tool_key, policy=policy)
+            circuit_state = self._normalize_circuit_state(state.breaker.current_state)
+
+            if circuit_state == "open":
+                opened_at = state.opened_at_unix_seconds
+                if opened_at is not None and now - opened_at >= policy.reset_timeout_seconds:
+                    state.breaker.half_open()
+                    state.inflight_half_open_calls = 0
+                    circuit_state = self._normalize_circuit_state(state.breaker.current_state)
+                else:
+                    return CircuitBreakerLease(
+                        tool_key=tool_key,
+                        policy=policy,
+                        state=circuit_state,
+                        allowed=False,
+                        half_open_acquired=False,
+                        last_error_kind=state.last_error_kind,
+                    )
+
+            if circuit_state == "half_open":
+                if state.inflight_half_open_calls >= policy.half_open_max_calls:
+                    return CircuitBreakerLease(
+                        tool_key=tool_key,
+                        policy=policy,
+                        state=circuit_state,
+                        allowed=False,
+                        half_open_acquired=False,
+                        last_error_kind=state.last_error_kind,
+                    )
+
+                state.inflight_half_open_calls += 1
+                return CircuitBreakerLease(
+                    tool_key=tool_key,
+                    policy=policy,
+                    state=circuit_state,
+                    allowed=True,
+                    half_open_acquired=True,
+                    last_error_kind=state.last_error_kind,
+                )
+
+            return CircuitBreakerLease(
+                tool_key=tool_key,
+                policy=policy,
+                state=circuit_state,
+                allowed=True,
+                half_open_acquired=False,
+                last_error_kind=state.last_error_kind,
+            )
+
+    async def record_success(self, lease: CircuitBreakerLease) -> CircuitBreakerTransition:
+        """Record successful attempt and update breaker state.
+
+        Args:
+            lease: Lease returned by ``acquire`` for the completed attempt.
+
+        Returns:
+            CircuitBreakerTransition snapshot after state mutation.
+        """
+        async with self._lock:
+            state = self._get_or_create_state(tool_key=lease.tool_key, policy=lease.policy)
+            previous_state = self._normalize_circuit_state(state.breaker.current_state)
+
+            if lease.half_open_acquired:
+                state.inflight_half_open_calls = max(0, state.inflight_half_open_calls - 1)
+
+            state.breaker.call(_circuit_success_probe)
+
+            state_name = self._normalize_circuit_state(state.breaker.current_state)
+            if state_name == "closed":
+                state.opened_at_unix_seconds = None
+            state.last_error_kind = None
+            transitioned = previous_state != state_name
+
+            return self._snapshot(lease.tool_key, previous_state, transitioned)
+
+    async def record_failure(self, lease: CircuitBreakerLease, *, error_kind: str) -> CircuitBreakerTransition:
+        """Record failed attempt and update breaker state.
+
+        Args:
+            lease: Lease returned by ``acquire`` for the completed attempt.
+            error_kind: Classified failure kind for observability metadata.
+
+        Returns:
+            CircuitBreakerTransition snapshot after state mutation.
+        """
+        async with self._lock:
+            now = time.time()
+            state = self._get_or_create_state(tool_key=lease.tool_key, policy=lease.policy)
+            previous_state = self._normalize_circuit_state(state.breaker.current_state)
+            state.last_error_kind = error_kind
+
+            if lease.half_open_acquired:
+                state.inflight_half_open_calls = max(0, state.inflight_half_open_calls - 1)
+
+            try:
+                state.breaker.call(_raise_circuit_failure)
+            except Exception:
+                pass
+
+            state_name = self._normalize_circuit_state(state.breaker.current_state)
+            if state_name == "open" and previous_state != "open":
+                state.opened_at_unix_seconds = now
+            transitioned = previous_state != state_name
+
+            return self._snapshot(lease.tool_key, previous_state, transitioned)
+
+    def _snapshot(self, tool_key: str, previous_state: str, transitioned: bool) -> CircuitBreakerTransition:
+        """Create a transition snapshot for metadata emission.
+
+        Args:
+            tool_key: Circuit key associated with the transition.
+            previous_state: Circuit state before mutation.
+            transitioned: Whether the state changed in this mutation.
+
+        Returns:
+            CircuitBreakerTransition with current breaker counters and state.
+        """
+        state = self._states[tool_key]
+        current_state = self._normalize_circuit_state(state.breaker.current_state)
+        return CircuitBreakerTransition(
+            tool_key=tool_key,
+            state=current_state,
+            previous_state=previous_state,
+            transitioned=transitioned,
+            consecutive_failures=state.breaker.fail_counter,
+            inflight_half_open_calls=state.inflight_half_open_calls,
+            opened_at_unix_seconds=state.opened_at_unix_seconds,
+            last_error_kind=state.last_error_kind,
+        )
+
+    def _get_or_create_state(self, *, tool_key: str, policy: CircuitBreakerPolicy) -> _CircuitState:
+        """Get an existing circuit state or create one from policy.
+
+        Args:
+            tool_key: Stable key used to store breaker state.
+            policy: Policy used to build a new ``pybreaker.CircuitBreaker``.
+
+        Returns:
+            Mutable state holder for the given tool key.
+        """
+        state = self._states.get(tool_key)
+        if state and state.policy == policy:
+            return state
+
+        breaker = CircuitBreaker(
+            fail_max=policy.fail_max,
+            reset_timeout=policy.reset_timeout_seconds,
+            success_threshold=1,
+            throw_new_error_on_trip=False,
+        )
+        state = _CircuitState(breaker=breaker, policy=policy)
+        self._states[tool_key] = state
+        return state
+
+    @staticmethod
+    def _normalize_circuit_state(raw_state: str) -> str:
+        """Normalize pybreaker state labels to metadata format.
+
+        Args:
+            raw_state: Raw pybreaker state label.
+
+        Returns:
+            Normalized state label using underscores.
+        """
+        return str(raw_state).replace("-", "_")
+
+
+def resolve_tool_timeout_seconds(
+    metadata: dict[str, Any] | None,
+    *,
+    default_timeout_seconds: float = DEFAULT_TOOL_TIMEOUT_SECONDS,
+) -> float:
+    """Resolve effective timeout for a tool invocation.
+
+    Args:
+        metadata: Optional operation metadata containing resilience overrides.
+        default_timeout_seconds: Fallback timeout when metadata has no override.
+
+    Returns:
+        Effective timeout in seconds.
+
+    Metadata precedence:
+    1) ``metadata["resilience"]["timeout_seconds"]``
+    2) ``metadata["timeout_seconds"]``
+    3) ``metadata["tool_timeout_seconds"]``
+    4) provided default
+    """
+    raw_value = _extract_timeout_value(metadata)
+    if raw_value is None:
+        return float(default_timeout_seconds)
+    return _validate_positive_timeout(raw_value)
+
+
+def resolve_retry_policy(metadata: dict[str, Any] | None) -> RetryPolicy:
+    """Resolve retry policy from tool metadata with defaults and validation.
+
+    Args:
+        metadata: Optional operation metadata containing retry configuration.
+
+    Returns:
+        Normalized retry policy for the current tool invocation.
+    """
+    raw_retry = _extract_retry_value(metadata)
+    if raw_retry is None:
+        return RetryPolicy(
+            enabled=DEFAULT_RETRY_ENABLED,
+            max_attempts=DEFAULT_RETRY_MAX_ATTEMPTS,
+            backoff_min_seconds=DEFAULT_RETRY_BACKOFF_MIN_SECONDS,
+            backoff_max_seconds=DEFAULT_RETRY_BACKOFF_MAX_SECONDS,
+            retryable_error_kinds=DEFAULT_RETRYABLE_ERROR_KINDS,
+        )
+
+    if not isinstance(raw_retry, dict):
+        raise ValueError("retry must be a dictionary")
+
+    enabled_raw = raw_retry.get(RETRY_ENABLED_KEY, DEFAULT_RETRY_ENABLED)
+    enabled = bool(enabled_raw)
+
+    max_attempts = _validate_positive_int(
+        raw_retry.get(RETRY_MAX_ATTEMPTS_KEY, DEFAULT_RETRY_MAX_ATTEMPTS),
+        field_name="retry.max_attempts",
+    )
+    backoff_min_seconds = _validate_positive_timeout(
+        raw_retry.get(RETRY_BACKOFF_MIN_SECONDS_KEY, DEFAULT_RETRY_BACKOFF_MIN_SECONDS)
+    )
+    backoff_max_seconds = _validate_positive_timeout(
+        raw_retry.get(RETRY_BACKOFF_MAX_SECONDS_KEY, DEFAULT_RETRY_BACKOFF_MAX_SECONDS)
+    )
+
+    if backoff_min_seconds > backoff_max_seconds:
+        raise ValueError("retry.backoff_min_seconds must be less than or equal to retry.backoff_max_seconds")
+
+    retryable_error_kinds = _validate_retryable_error_kinds(
+        raw_retry.get(RETRYABLE_ERROR_KINDS_KEY, list(DEFAULT_RETRYABLE_ERROR_KINDS))
+    )
+
+    return RetryPolicy(
+        enabled=enabled,
+        max_attempts=max_attempts,
+        backoff_min_seconds=backoff_min_seconds,
+        backoff_max_seconds=backoff_max_seconds,
+        retryable_error_kinds=retryable_error_kinds,
+    )
+
+
+def resolve_circuit_breaker_policy(metadata: dict[str, Any] | None) -> CircuitBreakerPolicy:
+    """Resolve circuit-breaker policy from tool metadata.
+
+    Args:
+        metadata: Optional tool metadata dictionary.
+
+    Returns:
+        CircuitBreakerPolicy with validated values and defaults applied.
+
+    Raises:
+        ValueError: If the configured policy contains invalid values.
+    """
+    raw_circuit_breaker = _extract_circuit_breaker_value(metadata)
+    if raw_circuit_breaker is None:
+        return CircuitBreakerPolicy(
+            enabled=DEFAULT_CIRCUIT_BREAKER_ENABLED,
+            fail_max=DEFAULT_CIRCUIT_FAIL_MAX,
+            reset_timeout_seconds=DEFAULT_CIRCUIT_RESET_TIMEOUT_SECONDS,
+            half_open_max_calls=DEFAULT_CIRCUIT_HALF_OPEN_MAX_CALLS,
+        )
+
+    if not isinstance(raw_circuit_breaker, dict):
+        raise ValueError("circuit_breaker must be a dictionary")
+
+    enabled = bool(raw_circuit_breaker.get(CIRCUIT_BREAKER_ENABLED_KEY, DEFAULT_CIRCUIT_BREAKER_ENABLED))
+    fail_max = _validate_positive_int(
+        raw_circuit_breaker.get(CIRCUIT_BREAKER_FAIL_MAX_KEY, DEFAULT_CIRCUIT_FAIL_MAX),
+        field_name="circuit_breaker.fail_max",
+    )
+    reset_timeout_seconds = _validate_positive_timeout(
+        raw_circuit_breaker.get(
+            CIRCUIT_BREAKER_RESET_TIMEOUT_SECONDS_KEY,
+            DEFAULT_CIRCUIT_RESET_TIMEOUT_SECONDS,
+        )
+    )
+    half_open_max_calls = _validate_positive_int(
+        raw_circuit_breaker.get(
+            CIRCUIT_BREAKER_HALF_OPEN_MAX_CALLS_KEY,
+            DEFAULT_CIRCUIT_HALF_OPEN_MAX_CALLS,
+        ),
+        field_name="circuit_breaker.half_open_max_calls",
+    )
+
+    return CircuitBreakerPolicy(
+        enabled=enabled,
+        fail_max=fail_max,
+        reset_timeout_seconds=reset_timeout_seconds,
+        half_open_max_calls=half_open_max_calls,
+    )
+
+
+def infer_tool_source(tool: BaseTool | None, metadata: dict[str, Any] | None) -> str:
+    """Infer tool source label for diagnostics and metadata.
+
+    Args:
+        tool: Tool instance used to infer source from module namespace.
+        metadata: Optional metadata that can override source classification.
+
+    Returns:
+        Normalized source label (``gl_connector``, ``mcp``, ``custom``, ``ptc``, ``unknown``).
+    """
+    metadata_source = _extract_source_override(metadata)
+    if metadata_source:
+        return metadata_source
+
+    if tool is None:
+        return "unknown"
+
+    module_name = getattr(tool.__class__, "__module__", "").lower()
+    if module_name == "aip_agents.tools.execute_ptc_code":
+        return "ptc"
+    if "mcp" in module_name:
+        return "mcp"
+    if "connector" in module_name or "gl_connectors" in module_name:
+        return "gl_connector"
+    return "custom"
+
+
+def build_timeout_error_output(
+    *,
+    tool_name: str,
+    timeout_seconds: float,
+    tool_source: str,
+    cancelled: bool | None = None,
+) -> dict[str, Any]:
+    """Build a structured timeout output for tool execution.
+
+    Args:
+        tool_name: Name of the tool that timed out.
+        timeout_seconds: Effective timeout budget used by this invocation.
+        tool_source: Source classification for the tool.
+        cancelled: Whether the timed-out attempt was confirmed cancelled.
+
+    Returns:
+        Structured timeout payload containing user-facing text and metadata.
+    """
+    timeout_display = _format_timeout(timeout_seconds)
+    content = (
+        f"Tool '{tool_name}' timed out after {timeout_display}s "
+        f"(source: {tool_source}). The upstream endpoint may be unavailable. "
+        "Retry later or increase timeout via metadata.tool_configs.<tool>.resilience.timeout_seconds."
+    )
+    if cancelled is False:
+        content += " Cancellation could not be confirmed, so do not assume the previous attempt was safely aborted."
+
+    tool_execution: dict[str, Any] = {
+        "status": "timeout",
+        "category": CATEGORY_TIMEOUT,
+        "tool_name": tool_name,
+        "tool_source": tool_source,
+        "timeout_seconds": timeout_seconds,
+        "is_retryable": cancelled is not False,
+    }
+    if cancelled is not None:
+        tool_execution["cancelled"] = cancelled
+
+    return {"result": content, "metadata": {"tool_execution": tool_execution}}
+
+
+def build_invalid_timeout_config_output(
+    *,
+    tool_name: str,
+    tool_source: str,
+    error: str,
+) -> dict[str, Any]:
+    """Build a structured invalid-timeout-config output.
+
+    Args:
+        tool_name: Name of the tool with invalid timeout config.
+        tool_source: Source classification for the tool.
+        error: Validation error message describing the invalid value.
+
+    Returns:
+        Structured payload describing invalid timeout configuration.
+    """
+    content = (
+        f"Invalid timeout configuration for tool '{tool_name}' (source: {tool_source}): {error}. "
+        "Use a positive numeric timeout value in metadata.tool_configs.<tool>.resilience.timeout_seconds."
+    )
+
+    return {
+        "result": content,
+        "metadata": {
+            "tool_execution": {
+                "status": "error",
+                "category": CATEGORY_INVALID_TIMEOUT_CONFIG,
+                "tool_name": tool_name,
+                "tool_source": tool_source,
+                "is_retryable": False,
+            }
+        },
+    }
+
+
+def build_invalid_retry_config_output(
+    *,
+    tool_name: str,
+    tool_source: str,
+    error: str,
+) -> dict[str, Any]:
+    """Build a structured invalid-retry-config output.
+
+    Args:
+        tool_name: Name of the tool with invalid retry config.
+        tool_source: Source classification for the tool.
+        error: Validation error message describing the invalid value.
+
+    Returns:
+        Structured payload describing invalid retry configuration.
+    """
+    content = (
+        f"Invalid retry configuration for tool '{tool_name}' (source: {tool_source}): {error}. "
+        "Use retry.enabled, retry.max_attempts >= 1, and positive backoff values."
+    )
+
+    return {
+        "result": content,
+        "metadata": {
+            "tool_execution": {
+                "status": "error",
+                "category": CATEGORY_INVALID_RETRY_CONFIG,
+                "tool_name": tool_name,
+                "tool_source": tool_source,
+                "is_retryable": False,
+            }
+        },
+    }
+
+
+def build_invalid_circuit_breaker_config_output(
+    *,
+    tool_name: str,
+    tool_source: str,
+    error: str,
+) -> dict[str, Any]:
+    """Build structured output for invalid circuit-breaker configuration.
+
+    Args:
+        tool_name: Tool name from the active tool call.
+        tool_source: Source label for the tool.
+        error: Validation error message.
+
+    Returns:
+        Tool output payload with categorized metadata.
+    """
+    content = (
+        f"Invalid circuit breaker configuration for tool '{tool_name}' (source: {tool_source}): {error}. "
+        "Use circuit_breaker.enabled, fail_max >= 1, reset_timeout_seconds > 0, and half_open_max_calls >= 1."
+    )
+
+    return {
+        "result": content,
+        "metadata": {
+            "tool_execution": {
+                "status": "error",
+                "category": CATEGORY_INVALID_CIRCUIT_CONFIG,
+                "tool_name": tool_name,
+                "tool_source": tool_source,
+                "is_retryable": False,
+            }
+        },
+    }
+
+
+def build_circuit_open_error_output(
+    *,
+    tool_name: str,
+    tool_source: str,
+    reset_timeout_seconds: float,
+    circuit_state: str,
+    failure_kind: str | None,
+) -> dict[str, Any]:
+    """Build structured output for circuit-open fail-fast behavior.
+
+    Args:
+        tool_name: Tool name from the active tool call.
+        tool_source: Source label for the tool.
+        reset_timeout_seconds: Cooldown duration before half-open probe.
+        circuit_state: Current circuit state label.
+        failure_kind: Last known failure kind, when available.
+
+    Returns:
+        Tool output payload with circuit-open metadata.
+    """
+    content = (
+        f"Tool '{tool_name}' is temporarily unavailable because the circuit is {circuit_state} "
+        f"(source: {tool_source}). Retry after about {_format_timeout(reset_timeout_seconds)}s cooldown."
+    )
+
+    metadata: dict[str, Any] = {
+        "tool_execution": {
+            "status": CATEGORY_CIRCUIT_OPEN,
+            "category": CATEGORY_CIRCUIT_OPEN,
+            "error_kind": KIND_CIRCUIT_OPEN,
+            "tool_name": tool_name,
+            "tool_source": tool_source,
+            "circuit_state": circuit_state,
+            "circuit_reset_timeout_seconds": reset_timeout_seconds,
+            "is_retryable": False,
+        }
+    }
+    if failure_kind:
+        metadata["tool_execution"]["last_error_kind"] = failure_kind
+
+    return {"result": content, "metadata": metadata}
+
+
+def build_retry_exhausted_error_output(
+    *,
+    tool_name: str,
+    tool_source: str,
+    timeout_seconds: float,
+    attempts_used: int,
+    max_attempts: int,
+    last_failure: ToolFailureDetail,
+    retry_budget_exhausted: bool,
+) -> dict[str, Any]:
+    """Build a terminal retry-exhausted output with attempt summary metadata.
+
+    Args:
+        tool_name: Name of the tool that exhausted retries.
+        tool_source: Source classification for the tool.
+        timeout_seconds: Timeout budget enforced for this execution.
+        attempts_used: Total attempts consumed.
+        max_attempts: Maximum attempts allowed by policy.
+        last_failure: Classified failure detail from the final attempt.
+        retry_budget_exhausted: Whether timeout budget prevented additional retry.
+
+    Returns:
+        Structured payload describing retry exhaustion.
+    """
+    budget_note = "retry budget exhausted" if retry_budget_exhausted else "attempt cap reached"
+    content = (
+        f"Tool '{tool_name}' failed after {attempts_used}/{max_attempts} attempt(s) "
+        f"within {_format_timeout(timeout_seconds)}s timeout budget (source: {tool_source}, {budget_note}). "
+        f"Last failure category: {last_failure.category}. {last_failure.message}"
+    )
+
+    metadata: dict[str, Any] = {
+        "tool_execution": {
+            "status": CATEGORY_RETRY_EXHAUSTED,
+            "category": CATEGORY_RETRY_EXHAUSTED,
+            "tool_name": tool_name,
+            "tool_source": tool_source,
+            "timeout_seconds": timeout_seconds,
+            "attempts_used": attempts_used,
+            "max_attempts": max_attempts,
+            "retry_budget_exhausted": retry_budget_exhausted,
+            "last_error_category": last_failure.category,
+            "last_error_kind": last_failure.kind,
+            "is_retryable": False,
+        }
+    }
+    if last_failure.http_status is not None:
+        metadata["tool_execution"]["http_status"] = last_failure.http_status
+    if last_failure.cancelled is not None:
+        metadata["tool_execution"]["cancelled"] = last_failure.cancelled
+        if last_failure.cancelled is False:
+            content += " Cancellation could not be confirmed for the timed-out attempt."
+
+    return {
+        "result": content,
+        "metadata": metadata,
+    }
+
+
+def build_classified_error_output(
+    *,
+    tool_name: str,
+    tool_source: str,
+    attempts_used: int,
+    max_attempts: int,
+    failure: ToolFailureDetail,
+) -> dict[str, Any]:
+    """Build structured classified error output for non-retried failures.
+
+    Args:
+        tool_name: Name of the tool that failed.
+        tool_source: Source classification for the tool.
+        attempts_used: Attempts consumed before terminal failure.
+        max_attempts: Maximum attempts allowed by policy.
+        failure: Classified failure detail to encode in response metadata.
+
+    Returns:
+        Structured payload describing the classified terminal failure.
+    """
+    recovery_hint = _recovery_hint_for_category(failure.category)
+    content = (
+        f"Tool '{tool_name}' failed ({failure.category}) (source: {tool_source}): {failure.message}. {recovery_hint}"
+    )
+
+    metadata: dict[str, Any] = {
+        "tool_execution": {
+            "status": "error",
+            "category": failure.category,
+            "error_kind": failure.kind,
+            "code": failure.code,
+            "tool_name": tool_name,
+            "tool_source": tool_source,
+            "attempts_used": attempts_used,
+            "max_attempts": max_attempts,
+            "is_retryable": failure.is_retryable_default,
+        }
+    }
+    if failure.http_status is not None:
+        metadata["tool_execution"]["http_status"] = failure.http_status
+
+    return {
+        "result": content,
+        "metadata": metadata,
+    }
+
+
+def add_retry_attempt_metadata(
+    tool_output: dict[str, Any],
+    *,
+    attempts_used: int,
+    max_attempts: int,
+    retry_budget_exhausted: bool,
+) -> dict[str, Any]:
+    """Attach retry attempt summary to existing tool metadata.
+
+    Args:
+        tool_output: Structured tool output dictionary to enrich.
+        attempts_used: Total attempts consumed.
+        max_attempts: Maximum attempts allowed by policy.
+        retry_budget_exhausted: Whether timeout budget prevented additional retry.
+
+    Returns:
+        The same tool output dictionary with retry attempt metadata attached.
+    """
+    metadata = tool_output.setdefault("metadata", {})
+    if not isinstance(metadata, dict):
+        metadata = {}
+        tool_output["metadata"] = metadata
+
+    execution = metadata.setdefault("tool_execution", {})
+    if not isinstance(execution, dict):
+        execution = {}
+        metadata["tool_execution"] = execution
+
+    execution.update(
+        {
+            "attempts_used": attempts_used,
+            "max_attempts": max_attempts,
+            "retry_budget_exhausted": retry_budget_exhausted,
+        }
+    )
+    return tool_output
+
+
+def add_timeout_cancellation_metadata(
+    tool_output: dict[str, Any],
+    *,
+    cancelled: bool | None,
+) -> dict[str, Any]:
+    """Attach timeout cancellation truthfulness to existing tool metadata."""
+    if cancelled is None:
+        return tool_output
+
+    metadata = tool_output.setdefault("metadata", {})
+    if not isinstance(metadata, dict):
+        metadata = {}
+        tool_output["metadata"] = metadata
+
+    execution = metadata.setdefault("tool_execution", {})
+    if not isinstance(execution, dict):
+        execution = {}
+        metadata["tool_execution"] = execution
+
+    execution["cancelled"] = cancelled
+    return tool_output
+
+
+def add_circuit_breaker_metadata(
+    tool_output: dict[str, Any],
+    *,
+    transition: CircuitBreakerTransition,
+) -> dict[str, Any]:
+    """Attach circuit-breaker transition metadata to tool output.
+
+    Args:
+        tool_output: Existing tool output payload.
+        transition: Circuit transition snapshot for this attempt.
+
+    Returns:
+        Tool output payload with merged circuit metadata.
+    """
+    metadata = tool_output.setdefault("metadata", {})
+    if not isinstance(metadata, dict):
+        metadata = {}
+        tool_output["metadata"] = metadata
+
+    execution = metadata.setdefault("tool_execution", {})
+    if not isinstance(execution, dict):
+        execution = {}
+        metadata["tool_execution"] = execution
+
+    execution.update(
+        {
+            "circuit_tool_key": transition.tool_key,
+            "circuit_state": transition.state,
+            "circuit_previous_state": transition.previous_state,
+            "circuit_transitioned": transition.transitioned,
+            "circuit_consecutive_failures": transition.consecutive_failures,
+            "circuit_inflight_half_open_calls": transition.inflight_half_open_calls,
+        }
+    )
+
+    if transition.opened_at_unix_seconds is not None:
+        execution["circuit_opened_at_unix_seconds"] = transition.opened_at_unix_seconds
+
+    if transition.last_error_kind:
+        execution["circuit_last_error_kind"] = transition.last_error_kind
+
+    return tool_output
+
+
+def classify_tool_exception(error: Exception) -> ToolFailureDetail:
+    """Classify tool exception into descriptive category and retry metadata.
+
+    Args:
+        error: Raised exception from tool execution.
+
+    Returns:
+        Classified failure detail used by retry and output mapping logic.
+    """
+    message = _normalize_error_message(error)
+    lowered_message = message.lower()
+    type_name = type(error).__name__.lower()
+
+    status_code = _extract_http_status(error)
+    if status_code == 401:
+        return ToolFailureDetail(
+            category=CATEGORY_UNAUTHORIZED,
+            kind=KIND_UNAUTHORIZED,
+            code="tool.unauthorized",
+            message=message,
+            http_status=status_code,
+            is_retryable_default=False,
+        )
+    if status_code == 403:
+        return ToolFailureDetail(
+            category=CATEGORY_FORBIDDEN,
+            kind=KIND_FORBIDDEN,
+            code="tool.forbidden",
+            message=message,
+            http_status=status_code,
+            is_retryable_default=False,
+        )
+    if status_code == 429:
+        return ToolFailureDetail(
+            category=CATEGORY_RATE_LIMITED,
+            kind=KIND_RATE_LIMITED,
+            code="tool.rate_limited",
+            message=message,
+            http_status=status_code,
+            is_retryable_default=True,
+        )
+    if status_code is not None and 500 <= status_code < 600:
+        return ToolFailureDetail(
+            category=CATEGORY_UPSTREAM_FAILURE,
+            kind=KIND_UPSTREAM_5XX,
+            code="tool.upstream_5xx",
+            message=message,
+            http_status=status_code,
+            is_retryable_default=True,
+        )
+
+    if isinstance(error, TimeoutError) or "timeout" in lowered_message or "timed out" in lowered_message:
+        return ToolFailureDetail(
+            category=CATEGORY_TIMEOUT,
+            kind=KIND_TIMEOUT,
+            code="tool.timeout",
+            message=message or "Tool call timed out",
+            http_status=status_code,
+            is_retryable_default=True,
+            cancelled=getattr(error, "cancelled", None),
+        )
+
+    endpoint_keywords = (
+        "connection refused",
+        "name or service not known",
+        "temporary failure in name resolution",
+        "dns",
+        "network is unreachable",
+        "failed to establish a new connection",
+        "nodename nor servname",
+    )
+    if any(keyword in lowered_message for keyword in endpoint_keywords):
+        return ToolFailureDetail(
+            category=CATEGORY_ENDPOINT_UNREACHABLE,
+            kind=KIND_ENDPOINT_UNREACHABLE,
+            code="tool.endpoint_unreachable",
+            message=message,
+            http_status=status_code,
+            is_retryable_default=True,
+        )
+
+    if isinstance(error, ConnectionError) or "connection" in type_name or "connection" in lowered_message:
+        return ToolFailureDetail(
+            category=CATEGORY_ENDPOINT_UNREACHABLE,
+            kind=KIND_CONNECTION_ERROR,
+            code="tool.connection_error",
+            message=message,
+            http_status=status_code,
+            is_retryable_default=True,
+        )
+
+    return ToolFailureDetail(
+        category=CATEGORY_UPSTREAM_FAILURE,
+        kind=KIND_UPSTREAM_FAILURE,
+        code="tool.upstream_failure",
+        message=message,
+        http_status=status_code,
+        is_retryable_default=False,
+    )
+
+
+def should_retry_failure(failure: ToolFailureDetail, retry_policy: RetryPolicy) -> bool:
+    """Return True when failure is retryable under policy rules.
+
+    Args:
+        failure: Classified failure detail for the attempt.
+        retry_policy: Resolved retry policy for current invocation.
+
+    Returns:
+        ``True`` when the failure should be retried, otherwise ``False``.
+    """
+    if not retry_policy.enabled:
+        return False
+    if failure.kind in _NON_RETRYABLE_KINDS:
+        return False
+    if failure.kind == KIND_TIMEOUT and failure.cancelled is False:
+        return False
+    return failure.kind in retry_policy.retryable_error_kinds
+
+
+def compute_retry_backoff_seconds(attempt_number: int, retry_policy: RetryPolicy) -> float:
+    """Compute bounded exponential backoff for the next retry attempt.
+
+    Uses ``tenacity.wait_exponential`` so retry timing logic stays aligned with
+    the same strategy used by runtime retry orchestration.
+
+    Args:
+        attempt_number: 1-based retry attempt number.
+        retry_policy: Resolved retry policy.
+
+    Returns:
+        Delay in seconds for the next retry attempt.
+    """
+    wait_strategy = wait_exponential(
+        multiplier=retry_policy.backoff_min_seconds,
+        min=retry_policy.backoff_min_seconds,
+        max=retry_policy.backoff_max_seconds,
+    )
+    return float(wait_strategy(_TenacityWaitState(attempt_number=attempt_number)))
+
+
+def _extract_timeout_value(metadata: dict[str, Any] | None) -> Any:
+    """Extract timeout override from metadata.
+
+    Args:
+        metadata: Optional tool metadata dictionary.
+
+    Returns:
+        Raw timeout override value, or ``None`` when absent.
+    """
+    if not isinstance(metadata, dict):
+        return None
+
+    resilience = metadata.get(RESILIENCE_KEY)
+    if isinstance(resilience, dict) and TIMEOUT_SECONDS_KEY in resilience:
+        return resilience[TIMEOUT_SECONDS_KEY]
+
+    if TIMEOUT_SECONDS_KEY in metadata:
+        return metadata[TIMEOUT_SECONDS_KEY]
+
+    if TOOL_TIMEOUT_SECONDS_KEY in metadata:
+        return metadata[TOOL_TIMEOUT_SECONDS_KEY]
+
+    return None
+
+
+def _extract_retry_value(metadata: dict[str, Any] | None) -> Any:
+    """Extract retry override from metadata.
+
+    Args:
+        metadata: Optional tool metadata dictionary.
+
+    Returns:
+        Raw retry configuration value, or ``None`` when absent.
+    """
+    if not isinstance(metadata, dict):
+        return None
+
+    resilience = metadata.get(RESILIENCE_KEY)
+    if isinstance(resilience, dict) and RETRY_KEY in resilience:
+        return resilience[RETRY_KEY]
+
+    if RETRY_KEY in metadata:
+        return metadata[RETRY_KEY]
+
+    return None
+
+
+def _extract_circuit_breaker_value(metadata: dict[str, Any] | None) -> Any:
+    """Extract raw circuit-breaker metadata from supported locations.
+
+    Args:
+        metadata: Optional tool metadata dictionary.
+
+    Returns:
+        Raw circuit breaker config value or ``None`` when absent.
+    """
+    if not isinstance(metadata, dict):
+        return None
+
+    resilience = metadata.get(RESILIENCE_KEY)
+    if isinstance(resilience, dict) and CIRCUIT_BREAKER_KEY in resilience:
+        return resilience[CIRCUIT_BREAKER_KEY]
+
+    if CIRCUIT_BREAKER_KEY in metadata:
+        return metadata[CIRCUIT_BREAKER_KEY]
+
+    return None
+
+
+def _extract_source_override(metadata: dict[str, Any] | None) -> str | None:
+    """Extract source override from metadata.
+
+    Args:
+        metadata: Optional tool metadata dictionary.
+
+    Returns:
+        Normalized source override value, or ``None`` when absent/invalid.
+    """
+    if not isinstance(metadata, dict):
+        return None
+
+    direct = _normalize_source(metadata.get(TOOL_SOURCE_KEY))
+    if direct:
+        return direct
+
+    resilience = metadata.get(RESILIENCE_KEY)
+    if isinstance(resilience, dict):
+        nested = _normalize_source(resilience.get(TOOL_SOURCE_KEY))
+        if nested:
+            return nested
+
+    return None
+
+
+def _normalize_source(value: Any) -> str | None:
+    """Normalize a source override into a known source label.
+
+    Args:
+        value: Candidate source value.
+
+    Returns:
+        Normalized source label when valid, otherwise ``None``.
+    """
+    if not isinstance(value, str):
+        return None
+
+    normalized = value.strip().lower()
+    if normalized in _KNOWN_TOOL_SOURCES:
+        return normalized
+
+    return None
+
+
+def _validate_positive_timeout(value: Any) -> float:
+    """Validate and parse a positive timeout value.
+
+    Args:
+        value: Candidate timeout value.
+
+    Returns:
+        Parsed positive timeout in seconds.
+    """
+    try:
+        timeout = float(value)
+    except (TypeError, ValueError) as exc:
+        raise ValueError("timeout must be a numeric value") from exc
+
+    if timeout <= 0:
+        raise ValueError("timeout must be greater than 0")
+
+    return timeout
+
+
+def _validate_positive_int(value: Any, *, field_name: str) -> int:
+    """Validate and parse a positive integer field.
+
+    Args:
+        value: Candidate integer value.
+        field_name: Field name used in validation error messages.
+
+    Returns:
+        Parsed integer value guaranteed to be greater than or equal to ``1``.
+    """
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError) as exc:
+        raise ValueError(f"{field_name} must be an integer") from exc
+
+    if parsed < 1:
+        raise ValueError(f"{field_name} must be greater than or equal to 1")
+
+    return parsed
+
+
+def _validate_retryable_error_kinds(value: Any) -> tuple[str, ...]:
+    """Validate and normalize configured retryable error kinds.
+
+    Args:
+        value: Candidate list of retryable error-kind strings.
+
+    Returns:
+        Normalized, deduplicated tuple of retryable error kinds.
+    """
+    if value is None:
+        return DEFAULT_RETRYABLE_ERROR_KINDS
+
+    if not isinstance(value, list):
+        raise ValueError("retry.retryable_error_kinds must be a list of strings")
+
+    normalized: list[str] = []
+    for item in value:
+        if not isinstance(item, str):
+            raise ValueError("retry.retryable_error_kinds must contain only strings")
+        kind = item.strip().lower()
+        if kind:
+            normalized.append(kind)
+
+    if not normalized:
+        raise ValueError("retry.retryable_error_kinds cannot be empty")
+
+    return tuple(dict.fromkeys(normalized))
+
+
+def _extract_http_status(error: Exception) -> int | None:
+    """Extract HTTP status code from exception objects.
+
+    Args:
+        error: Exception raised during tool execution.
+
+    Returns:
+        HTTP status code when available, otherwise ``None``.
+    """
+    response = getattr(error, "response", None)
+    if response is not None:
+        status = getattr(response, "status_code", None)
+        if isinstance(status, int):
+            return status
+
+    status = getattr(error, "status_code", None)
+    if isinstance(status, int):
+        return status
+
+    return None
+
+
+def _normalize_error_message(error: Exception) -> str:
+    """Normalize an exception into a non-empty message string.
+
+    Args:
+        error: Exception raised during tool execution.
+
+    Returns:
+        Normalized human-readable error message.
+    """
+    message = str(error).strip()
+    if message:
+        return message
+    return f"{type(error).__name__} occurred during tool execution"
+
+
+def _recovery_hint_for_category(category: str) -> str:
+    """Return a user-facing recovery hint for a failure category.
+
+    Args:
+        category: Failure category identifier.
+
+    Returns:
+        Recovery hint string tailored to the given category.
+    """
+    if category == CATEGORY_UNAUTHORIZED:
+        return "Verify authentication credentials and token scope."
+    if category == CATEGORY_FORBIDDEN:
+        return "Verify authorization permissions for this endpoint."
+    if category == CATEGORY_RATE_LIMITED:
+        return "Retry later or reduce request rate."
+    if category == CATEGORY_ENDPOINT_UNREACHABLE:
+        return "Verify endpoint reachability, DNS, and network path."
+    if category == CATEGORY_TIMEOUT:
+        return "Retry later or increase timeout configuration for this tool."
+    return "Check upstream service health and tool configuration."
+
+
+def _format_timeout(timeout_seconds: float) -> str:
+    """Format timeout seconds for user-facing messages.
+
+    Args:
+        timeout_seconds: Timeout value in seconds.
+
+    Returns:
+        String representation without trailing zero-only decimals.
+    """
+    return f"{timeout_seconds:.3f}".rstrip("0").rstrip(".")