fred-runtime 3.4.0__tar.gz → 3.4.1__tar.gz

{fred_runtime-3.4.0 → fred_runtime-3.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-runtime
-Version: 3.4.0
+Version: 3.4.1
 Summary: Runtime adapters and infrastructure wiring for Fred v2 agents.
 Author-email: Thales <noreply@thalesgroup.com>
 License: Apache-2.0

{fred_runtime-3.4.0 → fred_runtime-3.4.1}/fred_runtime/app/agent_app.py

@@ -56,6 +56,7 @@ from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, status
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import StreamingResponse
 from fred_core.common.config_loader import get_config
+from fred_core.diagnostics import install_gc_diagnostics
 from fred_core.history.history_schema import ChatMessage
 from fred_core.kpi import KPIMiddleware
 from fred_core.kpi.kpi_runtime_stage_metric import runtime_stage_timer
@@ -793,6 +794,7 @@ def _build_runtime_services(
     return RuntimeServices(
         tracer=build_default_tracer(),
         metrics=KPIWriterMetricsAdapter(runtime_config.kpi_writer),
+        kpi_writer=runtime_config.kpi_writer,
         chat_model_factory=runtime_config.chat_model_factory,
         tool_invoker=tool_invoker,
         tool_provider=tool_provider,
@@ -983,6 +985,19 @@ class _ModelCatalogEntry(BaseModel):
     team-settings picker needs the reverse (which profile_ids this enabled
     capability actually offers). Declaration order in the source YAML,
     first-seen first — same order `default_profile_by_capability` favors."""
+    thinking_profile_ids: list[str] = Field(default_factory=list)
+    """The subset of `profile_ids` above declaring `supports_thinking`
+    (`MODEL-REASONING-ENABLEMENT-RFC.md` §5.3, REASON-01). DERIVED here,
+    never authored twice: aptitude is declared per profile (§4, where the
+    behaviour actually differs), while the admin toggle is keyed per model
+    because that is what the capability id space and the admin UI both are
+    (§5.2 — the id space is not negotiable). This field is the projection
+    between the two.
+
+    Empty means this model has NO reasoning-capable profile, and the admin row
+    shows no reasoning control at all — aptitude is not a choice, an
+    administrator cannot make a model reason. Non-empty means the row carries
+    the toggle. Same established join as `profile_ids` above."""
 
 
 class _ModelCatalogResponse(BaseModel):
@@ -1017,15 +1032,20 @@ def _project_model_catalog_entries(catalog: Any) -> list[_ModelCatalogEntry]:
         key = (provider, name)
         existing = seen.get(key)
         if existing is None:
-            seen[key] = _ModelCatalogEntry(
+            existing = _ModelCatalogEntry(
                 id=model_capability_id(provider, name),
                 provider=provider,
                 name=name,
                 description=profile.description,
                 profile_ids=[profile.profile_id],
             )
+            seen[key] = existing
         else:
             existing.profile_ids.append(profile.profile_id)
+        # REASON-01 §5.3: aptitude is per profile, the toggle is per model —
+        # this one condition is the whole projection between them.
+        if profile.supports_thinking:
+            existing.thinking_profile_ids.append(profile.profile_id)
     return list(seen.values())
 
 
@@ -1358,12 +1378,21 @@ async def _authorize_execution_or_raise(
       the owner's `team_editor` tuple self-heals on first touch inside
       `RebacEngine.check_user_team_permission_or_raise`, and `add_relation`'s
       write-guard refuses any tuple naming a personal team except that one, so
-      the plain `CAN_READ` check below already authorizes the owner and denies
-      everyone else (another user's space, or the bare `"personal"` alias, for
-      which no tuple is ever provisioned) — no special-casing needed here.
-    - otherwise require the caller to hold `CAN_READ` on the requested team — the
-      same relation the control-plane required before it would mint a grant. The
-      team is caller-supplied but safe: OpenFGA only authorizes teams the user
+      the `CAN_USE_TEAM_AGENTS` check below already authorizes the owner
+      (`team_editor` implies `team_member`) and denies everyone else (another
+      user's space, or the bare `"personal"` alias, for which no tuple is
+      ever provisioned) — no special-casing needed here.
+    - otherwise require the caller to hold `CAN_USE_TEAM_AGENTS` on the
+      requested team — `team_member`-only, unlike `CAN_READ` (which also
+      admits the `public` marketplace-discovery relation, TEAM-09/TEAM-10).
+      Execution is real content access, not discovery, so a public-team
+      visitor must not pass here even though they can see the team profile.
+      This used to check `CAN_READ` (the same relation the control-plane
+      required before it would mint a grant) — that predates TEAM-09's
+      `public` relation and was never revisited; `product/api.py`'s
+      `post_prepare_execution` made the identical correction for the same
+      reason (returns prompt content, not just config). The team is
+      caller-supplied but safe: OpenFGA only authorizes teams the user
       actually holds a relation to. Authorization and denial are both audited;
       any OpenFGA denial fails closed (403).
 
@@ -1446,7 +1475,7 @@ async def _authorize_execution_or_raise(
     # since no tuple is ever provisioned for that literal string).
     try:
         await rebac.check_user_team_permission_or_raise(
-            authenticated_user, TeamPermission.CAN_READ, team_id
+            authenticated_user, TeamPermission.CAN_USE_TEAM_AGENTS, team_id
         )
     except AuthorizationError as exc:
         _emit_audit_event(
@@ -1730,6 +1759,27 @@ async def _validate_session_checkpoint_access(
         )
 
 
+@dataclass
+class _OpenThought:
+    """
+    One reasoning block being accumulated while mapping a turn to history rows.
+
+    Why the rank is reserved at THOUGHT_START rather than at THOUGHT_END:
+    - a model-native block opens on the first reasoning token and closes only at
+      the first answer delta (RFC AGENT-THINKING-API §7.3), so it brackets every
+      tool call of the turn
+    - the live UI assigns the row's rank at `thought_start` (`useChatSse.ts`);
+      ranking it at close instead would file the reasoning *after* the tools it
+      actually preceded, so a reloaded trace would not match what the user saw
+    """
+
+    rank: int
+    phase: str
+    title: str | None
+    source: str | None
+    text: list[str] = field(default_factory=list)
+
+
 async def _write_turn_history(
     *,
     session_id: str,
@@ -1765,6 +1815,7 @@ async def _write_turn_history(
     Event-to-message mapping:
     - ``tool_call``      → ``Role.assistant / Channel.tool_call``
     - ``tool_result``    → ``Role.tool    / Channel.tool_result``
+    - ``thought_*``      → ``Role.assistant / Channel.thought`` (one row per block)
     - ``awaiting_human`` → ``Role.system  / Channel.hitl_request`` (full choices)
     - ``node_error``     → ``Role.system  / Channel.error``
     - ``final``          → ``Role.assistant / Channel.final`` (answer + sources)
@@ -1774,6 +1825,7 @@ async def _write_turn_history(
     from fred_core.history.history_schema import (
         Channel,
         ChatMessage,
+        ChatMetadata,
         ChatTokenUsage,
         Role,
         TextPart,
@@ -1817,6 +1869,55 @@ async def _write_turn_history(
         messages.append(make_user_text(session_id, exchange_id, rank, request_message))
         rank += 1
 
+    # Reasoning blocks still accumulating, keyed by thought_id. A block reserves
+    # its rank when it opens but only becomes a row once its text is complete.
+    open_thoughts: dict[str, _OpenThought] = {}
+
+    def _thought_row(
+        block: _OpenThought,
+        *,
+        thought_id: str,
+        conclusion: str | None = None,
+        duration_ms: int | None = None,
+    ) -> ChatMessage:
+        """
+        Build one ``Channel.thought`` row from an accumulated reasoning block.
+
+        The envelope must match what the live stream emits client-side
+        (`useChatSse.ts`) — same role, same channel, same ``metadata.extras``
+        keys — because the chat UI feeds streamed and reloaded messages through
+        a single rendering path (`traceUtils.thoughtExtras`).
+
+        ``streaming_delta`` is deliberately absent: a persisted block is
+        complete, and carrying that flag would render it as still running.
+        """
+        return ChatMessage(
+            session_id=session_id,
+            exchange_id=exchange_id,
+            rank=block.rank,
+            timestamp=datetime.now(timezone.utc),
+            role=Role.assistant,
+            channel=Channel.thought,
+            parts=[TextPart(text="".join(block.text))],
+            # `extras` is not a declared field: `ChatMetadata` sets
+            # `extra="allow"` precisely so callers can attach structured payloads
+            # without every consumer of the model growing a dependency on them.
+            # `model_validate` is how that is reached type-safely — a keyword
+            # argument would not type-check against the declared fields.
+            metadata=ChatMetadata.model_validate(
+                {
+                    "extras": {
+                        "thought_id": thought_id,
+                        "phase": block.phase,
+                        "title": block.title,
+                        "source": block.source,
+                        "conclusion": conclusion,
+                        "duration_ms": duration_ms,
+                    }
+                }
+            ),
+        )
+
     # 2. Map runtime events to messages
     final_content = ""
     final_sources: list[VectorSearchHit] = []
@@ -1854,6 +1955,48 @@ async def _write_turn_history(
             )
             rank += 1
 
+        elif kind == "thought_start":
+            # `tool_use` blocks are synthetic bookkeeping the runtime opens around
+            # every tool call (RFC Amendment A, Layer 1). The chat UI has hidden
+            # them since Amendment B — the tool call/result combo row already
+            # carries what ran, how it went and how long it took. Persisting them
+            # would file up to `max_tool_calls_per_turn` content-free rows a turn.
+            if payload.get("phase") == "tool_use":
+                continue
+            thought_id = str(payload.get("thought_id") or "")
+            if not thought_id:
+                continue
+            open_thoughts[thought_id] = _OpenThought(
+                rank=rank,
+                phase=str(payload.get("phase") or "planning"),
+                title=payload.get("title"),
+                source=payload.get("source"),
+            )
+            rank += 1
+
+        elif kind == "thought_delta":
+            block = open_thoughts.get(str(payload.get("thought_id") or ""))
+            if block is not None:
+                block.text.append(str(payload.get("delta") or ""))
+
+        elif kind == "thought_end":
+            thought_id = str(payload.get("thought_id") or "")
+            block = open_thoughts.pop(thought_id, None)
+            # A block with neither text nor conclusion carries nothing to show;
+            # persisting it would render an empty reasoning card on reload. Its
+            # reserved rank is simply left unused — ranks must increase, not be
+            # contiguous.
+            conclusion = payload.get("conclusion")
+            if block is not None and ("".join(block.text) or conclusion):
+                messages.append(
+                    _thought_row(
+                        block,
+                        thought_id=thought_id,
+                        conclusion=conclusion,
+                        duration_ms=payload.get("duration_ms"),
+                    )
+                )
+
         elif kind == "awaiting_human":
             # Store the full HITL gate definition — question and all choices —
             # so audit logs and UI replay have the complete structured record.
@@ -1912,6 +2055,14 @@ async def _write_turn_history(
             final_model = payload.get("model_name")
             final_finish_reason = payload.get("finish_reason")
 
+    # 2b. Blocks the stream never closed (truncated turn, crashed node). The live
+    # UI closes them itself on `final` so the accordion does not pulse forever
+    # (`useChatSse.ts`); history keeps the same parity — reasoning the user was
+    # shown must not disappear on reload just because the block was left open.
+    for thought_id, block in open_thoughts.items():
+        if "".join(block.text):
+            messages.append(_thought_row(block, thought_id=thought_id))
+
     # 3. Terminal assistant message (from FinalRuntimeEvent)
     if final_content or final_model:
         messages.append(
@@ -2646,6 +2797,35 @@ async def _iterate_runtime_event_payloads(
             for rule in (ctx.get("operation_route_rules") or [])
         ]
         or None,
+        # Platform reasoning activation snapshot (REASON-01,
+        # `MODEL-REASONING-ENABLEMENT-RFC.md` §5.5) — same channel and same
+        # session-prep lifecycle as the two fields above, and forwarded here
+        # for the same hard-won reason: this mapping is field-by-field, so a
+        # field that is not named is silently dropped. Both comments above
+        # record a production bug caused by exactly that. `RoutedChatModelFactory`
+        # strips the reasoning settings for every model absent from this list
+        # (§5.6.2), so dropping it here would pin reasoning permanently OFF and
+        # make the admin toggle decorative — the failure §5.6.2 exists to
+        # prevent, one layer up from where it is enforced.
+        #
+        # Intersected with level 3 (the agent's own switch) rather than taken
+        # as-is, and that is the whole point of doing it HERE: this list rides
+        # the request, `tuning` is resolved server-side from the managed
+        # instance. An agent whose author did not enable reasoning must not
+        # reason whatever the request claims the platform allows — all four
+        # levels must hold (§3), so the ceiling is `level 2 AND level 3`.
+        # Absent tuning (agent-to-agent invocation, no managed instance) means
+        # no author ever enabled it: off, the default everywhere in REASON-01.
+        reasoning_enabled_model_ids=(
+            ctx.get("reasoning_enabled_model_ids")
+            if tuning is not None and tuning.reasoning_enabled
+            else []
+        ),
+        # The user's per-question reasoning choice (REASON-01 level 4). Same
+        # trap as every field above: unnamed here means silently dropped. Kept
+        # tri-state on purpose — `ctx.get` yielding None means "the agent never
+        # offered the choice", which is NOT the same as the user answering no.
+        reasoning=ctx.get("reasoning"),
     )
 
     binding = BoundRuntimeContext(
@@ -4027,14 +4207,19 @@ def create_agent_app(
     async def lifespan(app: FastAPI) -> AsyncIterator[None]:
         # Boot order (must be preserved — each step depends on the previous):
         # 1. log_setup         — formatter/handlers ready for all subsequent logs
-        # 2. initialize_kpi_writer — needed by bootstrap_observability
-        # 3. initialize_control_plane_client — sync, no network until first call
-        # 4. bootstrap_observability — global tracer + metrics provider
-        # 5. attach_pod_container — container in app.state before any request
-        # 6. initialize_sql    — async, may take time
-        # 7. start_metrics_exporter — prometheus thread, after KPI writer exists
-        # 8. start_kpi_tasks   — asyncio tasks, after SQL engine is known
-        # 9. set_runtime_context — wires all built parts into the global config
+        # 2. install_gc_diagnostics — independent of everything below; installed
+        #    early so SIGUSR1/SIGUSR2 (fred_core.diagnostics) are live for the
+        #    pod's whole lifetime, protecting every agent pod (ours and third
+        #    parties' alike) against the reference-cycle class of leak found in
+        #    ISSUE-010 without any per-pod code
+        # 3. initialize_kpi_writer — needed by bootstrap_observability
+        # 4. initialize_control_plane_client — sync, no network until first call
+        # 5. bootstrap_observability — global tracer + metrics provider
+        # 6. attach_pod_container — container in app.state before any request
+        # 7. initialize_sql    — async, may take time
+        # 8. start_metrics_exporter — prometheus thread, after KPI writer exists
+        # 9. start_kpi_tasks   — asyncio tasks, after SQL engine is known
+        # 10. set_runtime_context — wires all built parts into the global config
         log_setup(
             service_name=config.app.name,
             log_level=config.app.log_level,
@@ -4043,6 +4228,7 @@ def create_agent_app(
                 opensearch_config=config.storage.opensearch,
             ),
         )
+        gc_diagnostics = install_gc_diagnostics()
         container = build_pod_container(config)
         container.initialize_kpi_writer()
         container.initialize_control_plane_client()
@@ -4111,6 +4297,7 @@ def create_agent_app(
             list(registry.keys()),
         )
         yield
+        await gc_diagnostics.stop()
         await container.shutdown()
 
     app = FastAPI(

{fred_runtime-3.4.0 → fred_runtime-3.4.1}/fred_runtime/app/context.py

@@ -47,6 +47,11 @@ _CONTROL_PLANE_CLIENT_LIMITS = httpx.Limits(
     max_connections=50, max_keepalive_connections=20, keepalive_expiry=10.0
 )
 
+# Bounds the one-time connectivity check in initialize_sql() — asyncpg's own
+# connect timeout defaults to 60s, which would otherwise let a single boot
+# attempt hang well past what a readiness-driven operator expects.
+_SQL_BOOT_PING_TIMEOUT_S = 5.0
+
 
 # ---------------------------------------------------------------------------
 # TypedDicts for ring buffer entries
@@ -141,31 +146,64 @@ class PodApplicationContext:
         )
 
     async def initialize_sql(self) -> None:
-        """Build SQL engine, checkpointer, and history store (SQL-backed only)."""
+        """Build SQL engine, checkpointer, and history store.
+
+        Durable storage is mandatory for every fred-runtime pod — dev uses a
+        local SQLite file (`sqlite_path`), production uses Postgres, but both
+        are "durable", never "no storage". Any failure here (bad config,
+        unreachable/misconfigured Postgres) must abort pod startup so the
+        FastAPI lifespan never completes and Kubernetes never routes traffic
+        to this replica — see RUNTIME-EXECUTION-CONTRACT.md §8 (dated entry)
+        for the incident this closes.
+        """
         from fred_core.history.postgres_history_store import PostgresHistoryStore
         from fred_core.sql.base_sql import create_async_engine_from_config
         from fred_core.users.store.postgres_user_store import init_user_store
+        from sqlalchemy import text
 
         from fred_runtime.runtime_support.sql_checkpointer import FredSqlCheckpointer
 
+        postgres_config = self.configuration.storage.postgres
+        engine = create_async_engine_from_config(postgres_config)
+
+        async def _ping() -> None:
+            # SQLAlchemy async engines are lazy — nothing above opened a real
+            # connection. Without this, an unreachable/misconfigured Postgres
+            # would pass initialize_sql() silently and only fail mid-turn.
+            async with engine.connect() as conn:
+                await conn.execute(text("SELECT 1"))
+
         try:
-            engine = create_async_engine_from_config(
-                self.configuration.storage.postgres
-            )
-            init_user_store(engine)
-            checkpointer = FredSqlCheckpointer(engine, kpi=self.get_kpi_writer())
-            history_store = PostgresHistoryStore(engine, kpi=self.get_kpi_writer())
-            self._sql_engine = engine
-            self._checkpointer = checkpointer
-            self._history_store = history_store
-            logger.info(
-                "[fred-runtime] SQL checkpointer and history store ready (dialect=%s)",
+            await asyncio.wait_for(_ping(), timeout=_SQL_BOOT_PING_TIMEOUT_S)
+        except Exception as exc:
+            logger.error(
+                "[fred-runtime] SQL storage unreachable at startup — "
+                "dialect=%s host=%s port=%s database=%s error_type=%s",
                 engine.dialect.name,
+                postgres_config.host,
+                postgres_config.port,
+                postgres_config.database,
+                type(exc).__name__,
             )
-        except Exception:
-            logger.exception(
-                "[fred-runtime] Failed to initialize SQL storage — running stateless"
-            )
+            # This engine never reaches self._sql_engine on this path, so
+            # shutdown()'s own dispose() (below) can never reach it either —
+            # dispose it here or it leaks. Harmless today (the sole caller,
+            # agent_app.py's lifespan, has no retry and the process exits on
+            # this raise) but becomes a real leak the moment any retry/backoff
+            # wrapper is added around initialize_sql() or the lifespan.
+            await engine.dispose()
+            raise
+
+        init_user_store(engine)
+        checkpointer = FredSqlCheckpointer(engine, kpi=self.get_kpi_writer())
+        history_store = PostgresHistoryStore(engine, kpi=self.get_kpi_writer())
+        self._sql_engine = engine
+        self._checkpointer = checkpointer
+        self._history_store = history_store
+        logger.info(
+            "[fred-runtime] SQL checkpointer and history store ready (dialect=%s)",
+            engine.dialect.name,
+        )
 
     def start_metrics_exporter(self) -> None:
         """Start the Prometheus scrape endpoint when configured."""

{fred_runtime-3.4.0 → fred_runtime-3.4.1}/fred_runtime/app/openai_compat_router.py

@@ -188,7 +188,10 @@ def create_openai_compat_router(
 
         # F-A: gate this surface with the same pod-side OpenFGA check as
         # /agents/execute. When ReBAC is active, the caller must present a team
-        # (X-Fred-Team-Id) and hold CAN_READ on it; identity is the validated JWT.
+        # (X-Fred-Team-Id) and hold CAN_USE_TEAM_AGENTS on it (team_member-only,
+        # unlike CAN_READ which also admits the public marketplace-discovery
+        # relation — this is real execution, not discovery); identity is the
+        # validated JWT.
         rebac = get_runtime_context().config.rebac_engine
         if authenticated_user is not None and rebac is not None and rebac.enabled:
             if not team_id:
@@ -198,7 +201,7 @@ def create_openai_compat_router(
                 )
             try:
                 await rebac.check_user_team_permission_or_raise(
-                    authenticated_user, TeamPermission.CAN_READ, team_id
+                    authenticated_user, TeamPermission.CAN_USE_TEAM_AGENTS, team_id
                 )
             except AuthorizationError as exc:
                 raise HTTPException(

{fred_runtime-3.4.0 → fred_runtime-3.4.1}/fred_runtime/capabilities/assembly.py

@@ -373,7 +373,9 @@ def build_capability_agent_block(
         # combining capability's tools() output from the ReAct binding
         # (block.tools/HITL still saw it, but create_agent() never did).
         if capability_tools:
-            middleware.append(ToolCarrierMiddleware(capability_tools))
+            middleware.append(
+                ToolCarrierMiddleware(capability_tools, capability_id=cap_id)
+            )
         if type(capability).middleware is not AgentCapability.middleware:
             # Overridden: call it too, for the hook tools() cannot express.
             # No double-computation risk — the default middleware() (which