fred-runtime 4.0.0__tar.gz → 4.1.0__tar.gz

{fred_runtime-4.0.0 → fred_runtime-4.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-runtime
-Version: 4.0.0
+Version: 4.1.0
 Summary: Runtime adapters and infrastructure wiring for Fred v2 agents.
 Author-email: Thales <noreply@thalesgroup.com>
 License: Apache-2.0
@@ -12,8 +12,8 @@ Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Operating System :: OS Independent
 Requires-Python: <3.13,>=3.12
 Description-Content-Type: text/markdown
-Requires-Dist: fred-core>=4.0.0
-Requires-Dist: fred-sdk>=4.0.0
+Requires-Dist: fred-core>=4.1.0
+Requires-Dist: fred-sdk[agents]>=4.1.0
 Requires-Dist: alembic>=1.18.4
 Requires-Dist: deepagents>=0.6.12
 Requires-Dist: httpx>=0.28.1
@@ -230,6 +230,10 @@ pip install fred-runtime
 pip install fred-runtime[app]
 ```
 
+`fred-runtime` depends on `fred-sdk[agents]`, so an agent pod installs nothing
+else to get the authoring surface. A Knowledge Base pod installs
+`fred-sdk[knowledge-base]` instead and never needs `fred-runtime`.
+
 Requires Python 3.12.
 
 ---
@@ -288,6 +292,7 @@ See [fred-samples](https://github.com/ThalesGroup/fred-samples) for a working re
 
 | Package        | PyPI                                           | Role                                                                          |
 | -------------- | ---------------------------------------------- | ----------------------------------------------------------------------------- |
+| `fred-pod`     | [pypi](https://pypi.org/project/fred-pod/)     | Pod floor — configuration, identity, naming                                   |
 | `fred-core`    | [pypi](https://pypi.org/project/fred-core/)    | Pure utilities — logging, model factories, embeddings, portable observability |
 | `fred-sdk`     | [pypi](https://pypi.org/project/fred-sdk/)     | Agent authoring — ReAct, Graph, tool contracts                                |
 | `fred-runtime` | [pypi](https://pypi.org/project/fred-runtime/) | This package                                                                  |

{fred_runtime-4.0.0 → fred_runtime-4.1.0}/README.md

@@ -190,6 +190,10 @@ pip install fred-runtime
 pip install fred-runtime[app]
 ```
 
+`fred-runtime` depends on `fred-sdk[agents]`, so an agent pod installs nothing
+else to get the authoring surface. A Knowledge Base pod installs
+`fred-sdk[knowledge-base]` instead and never needs `fred-runtime`.
+
 Requires Python 3.12.
 
 ---
@@ -248,6 +252,7 @@ See [fred-samples](https://github.com/ThalesGroup/fred-samples) for a working re
 
 | Package        | PyPI                                           | Role                                                                          |
 | -------------- | ---------------------------------------------- | ----------------------------------------------------------------------------- |
+| `fred-pod`     | [pypi](https://pypi.org/project/fred-pod/)     | Pod floor — configuration, identity, naming                                   |
 | `fred-core`    | [pypi](https://pypi.org/project/fred-core/)    | Pure utilities — logging, model factories, embeddings, portable observability |
 | `fred-sdk`     | [pypi](https://pypi.org/project/fred-sdk/)     | Agent authoring — ReAct, Graph, tool contracts                                |
 | `fred-runtime` | [pypi](https://pypi.org/project/fred-runtime/) | This package                                                                  |

{fred_runtime-4.0.0 → fred_runtime-4.1.0}/fred_runtime/deep/deep_runtime.py

@@ -41,10 +41,13 @@ from langchain_core.tools import BaseTool
 from langgraph.types import Checkpointer
 
 from fred_runtime.capabilities.assembly import CapabilityAgentBlock
+from fred_runtime.react.middleware.checkpoint_hygiene import CheckpointHygieneMiddleware
 from fred_runtime.react.middleware.hitl import (
     CapabilityHitlBinding,
+    DeepChildHitlMiddleware,
     FredHitlMiddleware,
 )
+from fred_runtime.react.middleware.rate_limit_retry import RateLimitRetryMiddleware
 from fred_runtime.react.middleware.tool_observability import (
     ToolObservabilityMiddleware,
 )
@@ -75,6 +78,16 @@ from fred_runtime.react.react_tool_resolution import ReActRuntimeToolResolver
 
 logger = logging.getLogger(__name__)
 
+_SUBAGENT_FRAMING = (
+    "You are running as a sub-agent: another instance of yourself delegated one "
+    "task to you. There is no user in this conversation and no one to ask. "
+    "Instructions about greeting a user, asking what to do next or offering "
+    "options do not apply here. Do not ask clarifying questions or delegate "
+    "further. Carry out the task as described, preserve its arguments, and make "
+    "reasonable assumptions where it is silent. Report any action requiring "
+    "human approval to the parent agent instead of attempting to bypass it."
+)
+
 _FILESYSTEM_TOOL_NAMES: tuple[str, ...] = (
     "ls",
     "read_file",
@@ -144,6 +157,12 @@ class DeepAgentRuntime(ReActRuntime):
             available_tool_names.update(
                 tool.name for tool in capability_block.tools if tool.name
             )
+            available_tool_names.update(
+                tool.name
+                for entry in capability_block.middleware
+                for tool in getattr(entry, "tools", ())
+                if tool.name
+            )
         system_prompt = _render_prompt_template(
             policy.system_prompt_template,
             binding=binding,
@@ -190,6 +209,15 @@ class DeepAgentRuntime(ReActRuntime):
                 available_tool_names=available_tool_names,
                 capability_block=capability_block,
             ),
+            subagent_middleware=_build_deepagent_runtime_middleware(
+                tracer=self.services.tracer,
+                kpi=self.services.kpi_writer,
+                binding=binding,
+                approval_policy=policy.tool_approval,
+                available_tool_names=available_tool_names,
+                capability_block=capability_block,
+                child=True,
+            ),
         )
         return _TransportBackedReActExecutor(
             compiled_agent=compiled_agent,
@@ -206,6 +234,7 @@ def _create_compiled_deep_agent(
     system_prompt: str,
     checkpointer: Checkpointer,
     middleware: Sequence[AgentMiddleware],
+    subagent_middleware: Sequence[AgentMiddleware],
 ) -> _CompiledReActAgent:
     try:
         from deepagents import create_deep_agent
@@ -214,17 +243,23 @@ def _create_compiled_deep_agent(
             "DeepAgentRuntime requires the optional `deepagents` package."
         ) from exc
 
-    if checkpointer is None:
-        return cast(
-            _CompiledReActAgent,
-            create_deep_agent(
-                model=model,
-                tools=list(tools),
-                system_prompt=system_prompt,
-                middleware=list(middleware),
-            ),
-        )
+    from deepagents.middleware.subagents import (
+        DEFAULT_SUBAGENT_PROMPT,
+        GENERAL_PURPOSE_SUBAGENT,
+        SubAgent,
+    )
 
+    # Explicitly replace the library's general-purpose child, which otherwise
+    # has neither Fred's composed prompt nor its middleware frame.
+    subagent: SubAgent = {
+        "name": GENERAL_PURPOSE_SUBAGENT["name"],
+        "description": GENERAL_PURPOSE_SUBAGENT["description"],
+        "system_prompt": "\n\n".join(
+            (system_prompt, _SUBAGENT_FRAMING, DEFAULT_SUBAGENT_PROMPT)
+        ),
+        "tools": list(tools),
+        "middleware": list(subagent_middleware),
+    }
     return cast(
         _CompiledReActAgent,
         create_deep_agent(
@@ -232,6 +267,7 @@ def _create_compiled_deep_agent(
             tools=list(tools),
             system_prompt=system_prompt,
             middleware=list(middleware),
+            subagents=[subagent],
             checkpointer=checkpointer,
         ),
     )
@@ -265,27 +301,30 @@ def _build_deepagent_runtime_middleware(
     approval_policy: ToolApprovalPolicy,
     available_tool_names: set[str] | frozenset[str],
     capability_block: CapabilityAgentBlock | None = None,
+    child: bool = False,
 ) -> list[AgentMiddleware]:
-    """
-    Assemble Deep's middleware list: capability stack, platform observability,
-    the HITL gate (capability-declared and operator-configured approval
-    alike), then the filesystem-tool guard — same relative order as
-    `build_react_platform_middleware_frame` (`after_model` hooks run in
-    REVERSE list order, so the filesystem guard still blocks a disabled call
-    before the human gate ever sees it). RUNTIME-EXECUTION-CONTRACT.md §8.77.
+    """Keep hygiene outermost and guard disabled tools before HITL runs.
+
+    After-model hooks run in reverse order; see the execution contract §8.77.
     """
     capability_hitl: Mapping[str, CapabilityHitlBinding] | None = (
         capability_block.hitl if capability_block is not None else None
     )
     middleware: list[AgentMiddleware] = [
+        CheckpointHygieneMiddleware(
+            max_history_messages=None,
+            binding=binding,
+            kpi=kpi,
+        ),
         *(capability_block.middleware if capability_block is not None else ()),
+        RateLimitRetryMiddleware(kpi=kpi, binding=binding),
         TracingKpiMiddleware(
             tracer=tracer,
             kpi=kpi,
             binding=binding,
         ),
-        ToolObservabilityMiddleware(kpi=kpi, binding=binding),
-        FredHitlMiddleware(
+        ToolObservabilityMiddleware(kpi=kpi, binding=binding, tracer=tracer),
+        (DeepChildHitlMiddleware if child else FredHitlMiddleware)(
             binding=binding,
             approval_policy=approval_policy,
             available_tool_names=available_tool_names,

{fred_runtime-4.0.0 → fred_runtime-4.1.0}/fred_runtime/integrations/v2_runtime/adapters.py

@@ -119,6 +119,7 @@ from fred_runtime.common.kf_workspace_client import (
 from fred_runtime.common.mcp_runtime import MCPRuntime
 from fred_runtime.common.structures import AgentSettingsLike
 from fred_runtime.common.table_hits import repair_table_hits
+from fred_runtime.react.react_tracing import RUNTIME_TOOL_SPAN_NAME
 from fred_runtime.runtime_context import get_runtime_context
 from fred_runtime.runtime_support import (
     get_attachment_uids,
@@ -135,7 +136,7 @@ logger = logging.getLogger(__name__)
 _TRACE_MODEL_SPAN_NAMES = frozenset({"v2.graph.model", "v2.react.model"})
 _TRACE_AWAIT_HUMAN_SPAN_NAMES = frozenset({"v2.graph.await_human"})
 _TRACE_TOOL_SPAN_NAMES = frozenset(
-    {"v2.graph.tool", "v2.graph.runtime_tool", "tool.invoke"}
+    {"v2.graph.tool", "v2.graph.runtime_tool", RUNTIME_TOOL_SPAN_NAME, "tool.invoke"}
 )
 _TRACE_AGENT_SPAN_NAMES = frozenset({"agent.invoke", "agent.stream"})
 # Tool-shaped spans that are not the generic tool invoker.

{fred_runtime-4.0.0 → fred_runtime-4.1.0}/fred_runtime/react/middleware/__init__.py

@@ -36,24 +36,30 @@ The frame, in `create_agent` middleware list order:
        Capability middleware stacks are inserted here, sorted by capability id
        (RFC §5.3). Their `wrap_model_call` nests inside the platform prompt and
        outside tracing, so observability always records the final request.
-    4. TracingKpiMiddleware          — innermost `wrap_model_call`: the
+    4. RateLimitRetryMiddleware      — retries a model call the provider
+       rate-limited (HTTP 429), with backoff + jitter, bounded by both an
+       attempt count and a wall-clock budget. Sits just OUTSIDE tracing so
+       each attempt is its own span / `llm.call_latency_ms` sample, and
+       INSIDE the capability block so hygiene, prompt and capability
+       middleware are computed once for the whole retried call.
+    5. TracingKpiMiddleware          — innermost `wrap_model_call`: the
        `v2.react.model` span, `llm.call_latency_ms` KPI timer, and the
        `[LLM][CALL]`/`[LLM][RESPONSE]` logs measure/describe the bare model
        call, exactly as the legacy `reasoner` node did. The model itself is
        resolved once per turn at runtime activation (`ChatModelFactoryPort.build`)
        — no per-call routing middleware sits in this frame.
-    5. ToolObservabilityMiddleware   — `wrap_tool_call`: `agent.tool_latency_ms`
+    6. ToolObservabilityMiddleware   — `wrap_tool_call`: `agent.tool_latency_ms`
        / `agent.tool_failed_total` KPI and `agent.tool.invocation.
        {started,completed}` audit events for EVERY tool call the tool node
        executes — MCP-catalog tools and capability-native tools alike (#2011).
        Placed next to TracingKpiMiddleware (same job, the other axis: model
        calls vs tool calls).
-    6. FredHitlMiddleware            — `after_model`: filesystem tool-argument
+    7. FredHitlMiddleware            — `after_model`: filesystem tool-argument
        rewrite + the human tool-approval gate (RFC §5.4). ONE combined
        `interrupt()` per turn covering every gated call at once (#2177
        batching) with the `HumanInputRequest` payload; cancel jumps back to
        the model without executing any tool of the batch.
-    7. ToolCallLimitMiddleware       — LangChain prebuilt, appended only when
+    8. ToolCallLimitMiddleware       — LangChain prebuilt, appended only when
        `max_tool_calls_per_turn` is set. Listed AFTER FredHitl on purpose:
        `after_model` hooks run in REVERSE list order, so the limit blocks
        over-limit calls BEFORE a human is asked to approve them.
@@ -83,6 +89,7 @@ from .hitl import (
     FredHitlMiddleware,
     build_tool_approval_request,
 )
+from .rate_limit_retry import ProviderRateLimitError, RateLimitRetryMiddleware
 from .tool_observability import ToolObservabilityMiddleware
 from .tracing_kpi import TracingKpiMiddleware
 
@@ -91,6 +98,8 @@ __all__ = [
     "CheckpointHygieneMiddleware",
     "DynamicPromptMiddleware",
     "FredHitlMiddleware",
+    "ProviderRateLimitError",
+    "RateLimitRetryMiddleware",
     "ToolObservabilityMiddleware",
     "TracingKpiMiddleware",
     "build_react_platform_middleware_frame",

{fred_runtime-4.0.0 → fred_runtime-4.1.0}/fred_runtime/react/middleware/checkpoint_hygiene.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""CheckpointHygieneMiddleware — request-scoped model-input hygiene (#1972)."""
+"""CheckpointHygieneMiddleware — request-scoped model-input hygiene."""
 
 from __future__ import annotations
 
@@ -31,6 +31,7 @@ from fred_runtime.support.tool_loop import (
     ChatTurnTooLargeError,
     collect_tool_outputs,
     sanitize_dangling_tool_calls,
+    strip_message_names,
     total_char_len,
     trim_to_char_budget,
     trim_to_human_boundary,
@@ -80,7 +81,9 @@ class CheckpointHygieneMiddleware(AgentMiddleware):
         request: ModelRequest,
         handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
     ) -> ModelResponse:
-        messages = sanitize_dangling_tool_calls(list(request.messages))
+        messages = strip_message_names(
+            sanitize_dangling_tool_calls(list(request.messages))
+        )
         if self._max_history_messages is not None:
             trimmed = trim_to_human_boundary(messages, self._max_history_messages)
             logger.debug(

{fred_runtime-4.0.0 → fred_runtime-4.1.0}/fred_runtime/react/middleware/frame.py

@@ -34,6 +34,7 @@ from langchain.agents.middleware import AgentMiddleware, ToolCallLimitMiddleware
 from .checkpoint_hygiene import CheckpointHygieneMiddleware
 from .dynamic_prompt import DynamicPromptMiddleware
 from .hitl import CapabilityHitlBinding, FredHitlMiddleware
+from .rate_limit_retry import RateLimitRetryMiddleware
 from .tool_observability import ToolObservabilityMiddleware
 from .tracing_kpi import TracingKpiMiddleware
 
@@ -81,6 +82,10 @@ def build_react_platform_middleware_frame(
         DynamicPromptMiddleware(available_tool_names=available_tool_names),
         # --- CAPABILITY BLOCK INSERTION SLOT (#1973, RFC §5.3) ---
         *capability_middleware,
+        # Just outside tracing on purpose: every retried attempt is then a real
+        # span and a real latency sample, while hygiene, prompt and capability
+        # middleware are computed once for the whole retried call.
+        RateLimitRetryMiddleware(kpi=kpi, binding=binding),
         TracingKpiMiddleware(
             tracer=tracer,
             kpi=kpi,
@@ -95,7 +100,7 @@ def build_react_platform_middleware_frame(
         # actually executes, i.e. strictly after FredHitl's `after_model`
         # gate has already let the call through (a HITL-refused proposal
         # never reaches here, so it never produces a "started" event).
-        ToolObservabilityMiddleware(kpi=kpi, binding=binding),
+        ToolObservabilityMiddleware(kpi=kpi, binding=binding, tracer=tracer),
         FredHitlMiddleware(
             binding=binding,
             approval_policy=approval_policy,

{fred_runtime-4.0.0 → fred_runtime-4.1.0}/fred_runtime/react/middleware/hitl.py

@@ -18,7 +18,7 @@ from __future__ import annotations
 
 import json
 import logging
-from collections.abc import Mapping, Sequence
+from collections.abc import Awaitable, Callable, Mapping, Sequence
 from dataclasses import dataclass
 from typing import Any
 
@@ -35,7 +35,9 @@ from fred_sdk.contracts.runtime import (
     PendingToolCall,
 )
 from langchain.agents.middleware import AgentMiddleware, AgentState
-from langchain.agents.middleware.types import hook_config
+from langchain.agents.middleware.types import ModelRequest, ModelResponse, hook_config
+from langchain_core.messages import ToolMessage
+from langchain_core.tools import BaseTool
 from langgraph.runtime import Runtime
 from langgraph.types import interrupt
 
@@ -478,6 +480,11 @@ class FredHitlMiddleware(AgentMiddleware):
         # so a partial per-call answer was never meaningful even before this.
         if not gated:
             return None
+        return self._resolve_approval(gated)
+
+    def _resolve_approval(
+        self, gated: Sequence[GatedToolCall]
+    ) -> dict[str, Any] | None:
         request = build_tool_approval_request(binding=self._binding, calls=gated)
         decision = interrupt(request.model_dump(mode="json"))
         if _is_cancelled_human_decision(decision):
@@ -485,3 +492,55 @@ class FredHitlMiddleware(AgentMiddleware):
             # let the model replan.
             return {"jump_to": "model"}
         return None
+
+
+class DeepChildHitlMiddleware(FredHitlMiddleware):
+    """Native children reuse the gate but cannot open a human approval wait."""
+
+    async def awrap_model_call(
+        self,
+        request: ModelRequest,
+        handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
+    ) -> ModelResponse:
+        def visible(tool: BaseTool | dict[str, Any]) -> bool:
+            if isinstance(tool, BaseTool):
+                name = tool.name
+            else:
+                function = tool.get("function")
+                name = (
+                    function.get("name")
+                    if isinstance(function, dict)
+                    else tool.get("name")
+                )
+            if not name:
+                return True
+            bound = self._capability_hitl.get(name)
+            return not (
+                self._requires_human_approval(name)
+                or (bound is not None and bound.spec.require)
+            )
+
+        kept = [tool for tool in request.tools if visible(tool)]
+        if len(kept) != len(request.tools):
+            request = request.override(tools=kept)
+        return await handler(request)
+
+    def _resolve_approval(self, gated: Sequence[GatedToolCall]) -> dict[str, Any]:
+        # A result prevents execution of that call while allowing ungated calls
+        # in the same batch. Without a call ID, fail closed on the whole batch.
+        if not all(call.tool_call_id for call in gated):
+            return {"jump_to": "model"}
+        return {
+            "messages": [
+                ToolMessage(
+                    content=(
+                        "This tool requires human approval, which is unavailable in a "
+                        "delegated task. Ask the parent agent to run it directly."
+                    ),
+                    tool_call_id=call.tool_call_id or "",
+                    name=call.tool_name,
+                    status="error",
+                )
+                for call in gated
+            ]
+        }

fred_runtime-4.1.0/fred_runtime/react/middleware/rate_limit_retry.py

@@ -0,0 +1,159 @@
+# Copyright Thales 2026
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""RateLimitRetryMiddleware — absorb provider 429s instead of failing the turn."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+import time
+from collections.abc import Awaitable, Callable
+
+from fred_core import is_rate_limit
+from fred_core.kpi import BaseKPIWriter, KPIActor
+from fred_core.kpi.kpi_writer_structures import MetricNames
+from fred_sdk.contracts.context import BoundRuntimeContext
+from langchain.agents.middleware import AgentMiddleware
+from langchain.agents.middleware.types import ModelRequest, ModelResponse
+
+from ..react_model_adapter import extract_model_name_from_object
+from .shared import identity_kpi_dims
+
+logger = logging.getLogger(__name__)
+
+# The scheduling budget complements the model factory transport timeouts.
+_MAX_ATTEMPTS = 4
+_RETRY_BUDGET_S = 60.0
+_BACKOFF_BASE_S = 2.0
+_BACKOFF_CAP_S = 30.0
+# Below this much budget left, waiting is not worth the turn's remaining time.
+_MIN_USEFUL_WAIT_S = 1.0
+
+
+class ProviderRateLimitError(RuntimeError):
+    """Raised when a model call stays rate-limited after every retry.
+
+    Its message reaches the user as the turn's `execution_error`, so it names
+    the condition in plain language instead of leaking provider JSON.
+    """
+
+    def __init__(self, *, attempts: int, elapsed_s: float) -> None:
+        plural = "" if attempts == 1 else "s"
+        super().__init__(
+            "The model provider is rate-limiting this deployment. Fred gave up "
+            f"after {attempts} attempt{plural} over {elapsed_s:.0f}s. "
+            "Please send your message again in a moment."
+        )
+        self.attempts = attempts
+        self.elapsed_s = elapsed_s
+
+
+class RateLimitRetryMiddleware(AgentMiddleware):
+    """Retry rejected model calls without replaying the agent graph or tools."""
+
+    def __init__(
+        self,
+        *,
+        kpi: BaseKPIWriter | None,
+        binding: BoundRuntimeContext,
+    ) -> None:
+        super().__init__()
+        self._kpi = kpi
+        self._binding = binding
+
+    async def awrap_model_call(
+        self,
+        request: ModelRequest,
+        handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
+    ) -> ModelResponse:
+        started = time.monotonic()
+
+        for attempt in range(_MAX_ATTEMPTS):
+            try:
+                return await handler(request)
+            except Exception as exc:
+                throttled, retry_after = is_rate_limit(exc)
+                if not throttled:
+                    raise
+
+                # Resolved here, not before the call: the happy path runs on
+                # every model call of every turn and must stay free of work
+                # only the throttled path needs.
+                model_name = extract_model_name_from_object(request.model)
+                elapsed = time.monotonic() - started
+                remaining = _RETRY_BUDGET_S - elapsed
+                delay = self._delay_for(attempt, retry_after)
+                exhausted = (
+                    attempt == _MAX_ATTEMPTS - 1
+                    or remaining < _MIN_USEFUL_WAIT_S
+                    or delay >= remaining
+                )
+                self._record(model_name, exhausted=exhausted)
+
+                if exhausted:
+                    logger.error(
+                        "model=%s rate limit persisted after %d attempts in %.1fs; "
+                        "failing the turn",
+                        model_name,
+                        attempt + 1,
+                        elapsed,
+                    )
+                    raise ProviderRateLimitError(
+                        attempts=attempt + 1, elapsed_s=elapsed
+                    ) from exc
+
+                logger.warning(
+                    "model=%s rate-limited (attempt %d/%d); backing off %.1fs",
+                    model_name,
+                    attempt + 1,
+                    _MAX_ATTEMPTS,
+                    delay,
+                )
+                await asyncio.sleep(delay)
+                # Scheduling delays can overshoot: do not start a late attempt.
+                elapsed = time.monotonic() - started
+                if elapsed >= _RETRY_BUDGET_S:
+                    raise ProviderRateLimitError(
+                        attempts=attempt + 1, elapsed_s=elapsed
+                    ) from exc
+
+        raise AssertionError("unreachable: the loop returns or raises")
+
+    def _delay_for(self, attempt: int, retry_after: float | None) -> float:
+        """Honor provider minimum delays and spread fallback retries."""
+
+        if retry_after is not None and retry_after >= 0:
+            return retry_after + random.uniform(0, 1.0)  # nosec B311 — retry jitter, not security-sensitive
+        ceiling = min(_BACKOFF_CAP_S, _BACKOFF_BASE_S * (2**attempt))
+        return random.uniform(ceiling / 2, ceiling)  # nosec B311 — retry jitter, not security-sensitive
+
+    def _record(self, model_name: str | None, *, exhausted: bool) -> None:
+        if self._kpi is None:
+            return
+        # `model_name` is set unconditionally: PrometheusKPIStore freezes a
+        # metric's label set from its FIRST event, so a dim omitted on the first
+        # 429 of a pod's life is dropped from the counter for good.
+        dims: dict[str, str | None] = {
+            **identity_kpi_dims(self._binding),
+            "status": "error" if exhausted else "ok",
+            "model_name": model_name or "unknown",
+        }
+        self._kpi.count(
+            MetricNames.LLM_RATE_LIMITS,
+            1,
+            dims=dims,
+            actor=KPIActor(type="system"),
+        )

{fred_runtime-4.0.0 → fred_runtime-4.1.0}/fred_runtime/react/middleware/tool_observability.py

@@ -83,6 +83,7 @@ from fred_core.logs.audit_log import emit_audit_log
 from fred_core.security.models import Resource
 from fred_core.security.rebac.rebac_engine import RebacReference, TeamPermission
 from fred_sdk.contracts.context import BoundRuntimeContext
+from fred_sdk.contracts.runtime import SpanPort, TracerPort
 from langchain.agents.middleware import AgentMiddleware
 from langchain_core.messages.tool import ToolMessage
 from langgraph.prebuilt.tool_node import ToolCallRequest
@@ -91,6 +92,9 @@ from langgraph.types import Command
 from fred_runtime.common.context_aware_tool import ContextAwareTool
 from fred_runtime.runtime_context import get_runtime_context
 
+from ..react_tool_binding import SELF_TRACED_TOOL_METADATA_KEY
+from ..react_tracing import RUNTIME_TOOL_SPAN_NAME, tool_span
+
 logger = logging.getLogger(__name__)
 
 
@@ -115,10 +119,12 @@ class ToolObservabilityMiddleware(AgentMiddleware):
         *,
         kpi: BaseKPIWriter | None,
         binding: BoundRuntimeContext,
+        tracer: TracerPort | None = None,
     ) -> None:
         super().__init__()
         self._kpi = kpi
         self._binding = binding
+        self._tracer = tracer
 
     def _base_dims(self, *, tool_name: str, source: str) -> dict[str, Optional[str]]:
         """
@@ -211,10 +217,51 @@ class ToolObservabilityMiddleware(AgentMiddleware):
             RebacReference(Resource.TEAM, team_id),
         )
 
+    def _span_tracer(self, request: ToolCallRequest) -> TracerPort | None:
+        """Skip binder-owned spans; middleware tools need their own span."""
+
+        metadata = getattr(request.tool, "metadata", None) or {}
+        if metadata.get(SELF_TRACED_TOOL_METADATA_KEY):
+            return None
+        return self._tracer
+
     async def awrap_tool_call(
         self,
         request: ToolCallRequest,
         handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command[Any]]],
+    ) -> ToolMessage | Command[Any]:
+        tracer = self._span_tracer(request)
+        tool_call = request.tool_call
+        async with tool_span(
+            tracer,
+            name=RUNTIME_TOOL_SPAN_NAME,
+            context=self._binding.portable_context,
+            attributes={"tool_name": self._tool_name(request)},
+            input_payload=tool_call.get("args")
+            if isinstance(tool_call, dict)
+            else None,
+        ) as span:
+            result = await self._observe_tool_call(request, handler, span=span)
+            if span is not None and tracer is not None and tracer.captures_content:
+                output = getattr(result, "content", None)
+                if isinstance(result, Command) and isinstance(result.update, dict):
+                    messages = result.update.get("messages", [])
+                    if isinstance(messages, (list, tuple)):
+                        output = [
+                            message.content
+                            for message in messages
+                            if isinstance(message, ToolMessage)
+                            and message.tool_call_id == tool_call.get("id")
+                        ]
+                span.set_io(output=output)
+            return result
+
+    async def _observe_tool_call(
+        self,
+        request: ToolCallRequest,
+        handler: Callable[[ToolCallRequest], Awaitable[ToolMessage | Command[Any]]],
+        *,
+        span: SpanPort | None,
     ) -> ToolMessage | Command[Any]:
         tool_name = self._tool_name(request)
         # `ContextAwareTool` wraps every MCP-catalog tool (`mcp_toolkit.py`);
@@ -341,6 +388,9 @@ class ToolObservabilityMiddleware(AgentMiddleware):
                         if status_is_error
                         else "tool_error_artifact"
                     )
+                    if span is not None:
+                        span.set_attribute("status", "error")
+                        span.set_attribute("error_type", error_code)
                     if kpi_dims is not None:
                         # `status` only — see the timer comment above.
                         kpi_dims["status"] = "error"