fred-runtime 2.0.0__tar.gz → 2.0.2__tar.gz

{fred_runtime-2.0.0 → fred_runtime-2.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fred-runtime
-Version: 2.0.0
+Version: 2.0.2
 Summary: Runtime adapters and infrastructure wiring for Fred v2 agents.
 Author-email: Thales <noreply@thalesgroup.com>
 License: Apache-2.0

{fred_runtime-2.0.0 → fred_runtime-2.0.2}/fred_runtime/app/agent_app.py

@@ -62,6 +62,7 @@ from fred_core.logs.log_setup import log_setup
 from fred_core.logs.memory_log_store import RamLogStore
 from fred_core.security.oidc import get_keycloak_client_id, get_keycloak_url
 from fred_core.security.structure import KeycloakUser
+from fred_sdk.contracts.eval import EvalStep, EvalTrace
 from fred_sdk.contracts.context import (
     AgentInvocationRequest,
     AgentInvocationResult,
@@ -558,11 +559,13 @@ class LocalRegistryAgentInvoker(AgentInvokerPort):
                 is_error=True,
             )
 
+        context_dict = request.context.model_dump(mode="json")
+        context_dict.setdefault("execution_action", ExecutionGrantAction.EXECUTE.value)
         execute_request = _AgentExecuteRequest.model_construct(
             agent_id=request.agent_id,
             agent_instance_id=None,
             message=request.message,
-            context=request.context.model_dump(mode="json"),
+            context=context_dict,
             resume_payload=None,
         )
 
@@ -838,17 +841,19 @@ def _apply_runtime_tuning(
     - `definition = _apply_runtime_tuning(template_definition, resolution.tuning)`
     """
 
-    return definition.model_copy(
-        update={
-            "role": tuning.role,
-            "description": tuning.description,
-            "tags": tuple(tuning.tags),
-            "fields": tuple(field.model_copy(deep=True) for field in tuning.fields),
-            "default_mcp_servers": tuple(
-                server.model_copy(deep=True) for server in tuning.mcp_servers
-            ),
-        }
-    )
+    update: dict[str, object] = {
+        "role": tuning.role,
+        "description": tuning.description,
+        "tags": tuple(tuning.tags),
+        "fields": tuple(field.model_copy(deep=True) for field in tuning.fields),
+        "default_mcp_servers": tuple(
+            server.model_copy(deep=True) for server in tuning.mcp_servers
+        ),
+    }
+    system_prompt = tuning.values.get("prompts.system")
+    if isinstance(system_prompt, str) and system_prompt.strip():
+        update["system_prompt_template"] = system_prompt
+    return definition.model_copy(update=update)
 
 
 def _available_mcp_servers_for_definition(
@@ -1377,6 +1382,120 @@ def _sse(payload: str) -> str:
     return f"data: {payload}\n\n"
 
 
+@dataclass(frozen=True)
+class _TurnOutcome:
+    model_name: str | None
+    finish_reason: str
+    token_usage: dict[str, Any] | None
+    input_tokens: int | None
+    output_tokens: int | None
+    tool_count: int
+    is_error: bool
+    total_ms: int
+    final_content: str | None
+
+
+def _parse_turn_outcome(
+    payloads: list[dict[str, Any]],
+    turn_start: float,
+) -> _TurnOutcome:
+    total_ms = int((time.monotonic() - turn_start) * 1000)
+    tool_count = sum(1 for p in payloads if p.get("kind") == "tool_call")
+    final = next((p for p in reversed(payloads) if p.get("kind") == "final"), None)
+    is_error = any(p.get("kind") == "execution_error" for p in payloads)
+    token_usage: dict[str, Any] | None = final.get("token_usage") if final else None
+    return _TurnOutcome(
+        model_name=final.get("model_name") if final else None,
+        finish_reason="error"
+        if is_error
+        else ((final.get("finish_reason") or "") if final else ""),
+        token_usage=token_usage,
+        input_tokens=token_usage.get("input_tokens") if token_usage else None,
+        output_tokens=token_usage.get("output_tokens") if token_usage else None,
+        tool_count=tool_count,
+        is_error=is_error,
+        total_ms=total_ms,
+        final_content=(final.get("content") or None) if final else None,
+    )
+
+
+def _build_eval_trace(
+    payloads: list[dict[str, Any]],
+    input_text: str,
+    agent_id: str,
+    session_id: str,
+    turn_start: float,
+) -> EvalTrace:
+    outcome = _parse_turn_outcome(payloads, turn_start)
+    steps: list[EvalStep] = []
+    retrieval_context: list[str] = []
+    tools_called: list[str] = []
+    error: str | None = None
+
+    for p in payloads:
+        kind = p.get("kind")
+        if kind == "tool_call":
+            steps.append(
+                EvalStep(
+                    kind="tool_call",
+                    tool_name=p.get("tool_name"),
+                    call_id=p.get("call_id"),
+                    arguments=p.get("arguments") or {},
+                )
+            )
+            if p.get("tool_name"):
+                tools_called.append(p["tool_name"])
+        elif kind == "tool_result":
+            content = p.get("content", "")
+            is_err = p.get("is_error", False)
+            steps.append(
+                EvalStep(
+                    kind="tool_result",
+                    tool_name=p.get("tool_name"),
+                    call_id=p.get("call_id"),
+                    content=content,
+                    is_error=is_err,
+                )
+            )
+            if not is_err:
+                sources = p.get("sources") or []
+                if sources:
+                    retrieval_context.extend(
+                        s["content"] for s in sources if s.get("content")
+                    )
+                elif content:
+                    retrieval_context.append(content)
+        elif kind == "final":
+            steps.append(EvalStep(kind="final", content=p.get("content")))
+        elif kind == "node_error":
+            steps.append(
+                EvalStep(
+                    kind="node_error",
+                    node_id=p.get("node_id"),
+                    error_message=p.get("error_message"),
+                )
+            )
+        elif kind == "awaiting_human":
+            steps.append(EvalStep(kind="awaiting_human"))
+        elif kind == "execution_error":
+            error = p.get("message")
+
+    return EvalTrace(
+        session_id=session_id,
+        agent_id=agent_id,
+        input=input_text,
+        output=outcome.final_content,
+        error=error,
+        latency_ms=outcome.total_ms,
+        model_name=outcome.model_name,
+        token_usage=outcome.token_usage,
+        finish_reason=outcome.finish_reason or None,
+        steps=tuple(steps),
+        retrieval_context=tuple(retrieval_context),
+        tools_called=tuple(tools_called),
+    )
+
+
 def _emit_turn_completed(
     container: PodApplicationContext,
     *,
@@ -1408,21 +1527,7 @@ def _emit_turn_completed(
     """
     try:
         kpi = get_runtime_context().get_kpi_writer()
-        total_ms = int((time.monotonic() - turn_start) * 1000)
-        tool_count = sum(1 for p in payloads if p.get("kind") == "tool_call")
-        final = next((p for p in reversed(payloads) if p.get("kind") == "final"), None)
-        is_error = any(p.get("kind") == "execution_error" for p in payloads)
-        model_name: str | None = final.get("model_name") if final else None
-        finish_reason: str = (
-            "error" if is_error else (final.get("finish_reason") or "") if final else ""
-        )
-        token_usage: dict[str, Any] | None = final.get("token_usage") if final else None
-        input_tokens: int | None = (
-            token_usage.get("input_tokens") if token_usage else None
-        )
-        output_tokens: int | None = (
-            token_usage.get("output_tokens") if token_usage else None
-        )
+        outcome = _parse_turn_outcome(payloads, turn_start)
         runtime_id = get_runtime_context().config.service_name
 
         # Prometheus-safe dims: low-cardinality only.
@@ -1433,25 +1538,25 @@ def _emit_turn_completed(
             "team_id": team_id,
             "template_agent_id": template_agent_id,
             "runtime_id": runtime_id,
-            "model_name": model_name,
-            "finish_reason": finish_reason,
+            "model_name": outcome.model_name,
+            "finish_reason": outcome.finish_reason,
         }
 
         kpi.emit(
             name="agent.turn_completed",
             type="timer",
-            value=total_ms,
+            value=outcome.total_ms,
             unit="ms",
             dims=prom_dims,
             quantities={
-                "tool_count": tool_count,
-                "input_tokens": input_tokens,
-                "output_tokens": output_tokens,
+                "tool_count": outcome.tool_count,
+                "input_tokens": outcome.input_tokens,
+                "output_tokens": outcome.output_tokens,
             },
             actor=KPIActor(type="system"),
         )
 
-        if is_error:
+        if outcome.is_error:
             kpi.emit(
                 name="agent.turn_error_total",
                 type="counter",
@@ -1468,12 +1573,12 @@ def _emit_turn_completed(
                 "session_id": session_id,
                 "exchange_id": exchange_id,
                 "user_id": user_id,
-                "total_ms": total_ms,
-                "is_error": is_error,
+                "total_ms": outcome.total_ms,
+                "is_error": outcome.is_error,
                 **prom_dims,
-                "tool_count": tool_count,
-                "input_tokens": input_tokens,
-                "output_tokens": output_tokens,
+                "tool_count": outcome.tool_count,
+                "input_tokens": outcome.input_tokens,
+                "output_tokens": outcome.output_tokens,
             },
         )
         with container._kpi_turns_lock:
@@ -2328,6 +2433,113 @@ def _build_agent_router(
                 )
         return _terminal_execute_payload(payloads)
 
+    @router.post(
+        "/evaluate",
+        response_model=EvalTrace,
+    )
+    async def evaluate(
+        request: RuntimeExecuteRequest,
+        http_request: Request,
+        authenticated_user: KeycloakUser | None = Depends(_authenticated_user),
+        container: PodApplicationContext = Depends(get_pod_container),
+    ) -> EvalTrace:
+        """
+        Execute one agent turn and return a complete EvalTrace as JSON.
+
+        POST <configured base_url>/agents/evaluate
+        Authorization: Bearer <user JWT>
+        Body: RuntimeExecuteRequest
+        Response: EvalTrace — synchronous, no SSE, no Langfuse dependency
+
+        Intended for evaluation harnesses (DeepEval, Promptfoo) that need
+        input, output, retrieval_context, tools_called, and steps in one response.
+        """
+        auth = http_request.headers.get("Authorization", "")
+        access_token = auth.removeprefix("Bearer ").strip() or None
+
+        expected_action = _expected_execution_action(request)
+
+        try:
+            validate_execution_grant(request, expected_action=expected_action)
+        except ExecutionGrantViolation as exc:
+            _emit_audit_event(
+                container,
+                "warning",
+                "grant_validation_failed",
+                agent_instance_id=request.agent_instance_id,
+                user_id=request.effective_user_id(),
+                action=expected_action.value,
+                reason=str(exc),
+            )
+            raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
+        if request.execution_grant is not None:
+            _emit_audit_event(
+                container,
+                "info",
+                "grant_validated",
+                agent_instance_id=request.agent_instance_id,
+                user_id=request.effective_user_id(),
+                action=expected_action.value,
+            )
+        _validate_grant_user_correlation(request, authenticated_user, container)
+        await _validate_session_checkpoint_access(request)
+
+        exchange_id = str(uuid4())
+        turn_start = time.monotonic()
+        internal_req = _to_internal_request(request)
+        target = await _resolve_agent_instance(
+            request=internal_req,
+            registry=registry,
+            access_token=access_token,
+            control_plane_url=get_runtime_context().config.control_plane_url,
+        )
+        payloads = [
+            payload
+            async for payload in _iterate_runtime_event_payloads(
+                target.definition,
+                internal_req,
+                access_token=access_token,
+                team_id=target.team_id,
+                registry=registry,
+                exchange_id=exchange_id,
+            )
+        ]
+        session_id: str | None = request.effective_session_id()
+        eval_session_id = session_id or str(uuid4())
+        user_id_str = request.effective_user_id() or "unknown"
+        _emit_turn_completed(
+            container,
+            session_id=session_id,
+            exchange_id=exchange_id,
+            user_id=user_id_str,
+            team_id=target.team_id,
+            agent_instance_id=request.agent_instance_id,
+            template_agent_id=target.definition.agent_id,
+            payloads=payloads,
+            turn_start=turn_start,
+        )
+        if session_id:
+            history_store = get_runtime_context().config.history_store
+            if history_store is not None:
+                await _write_turn_history(
+                    session_id=session_id,
+                    user_id=user_id_str,
+                    request_message=request.input,
+                    payloads=payloads,
+                    history_store=history_store,
+                    team_id=target.team_id,
+                    agent_instance_id=request.agent_instance_id,
+                    exchange_id=exchange_id,
+                    resume_payload=request.resume_payload,
+                )
+        return _build_eval_trace(
+            payloads=payloads,
+            input_text=request.input or "",
+            agent_id=target.definition.agent_id,
+            session_id=eval_session_id,
+            turn_start=turn_start,
+        )
+
     @router.post(
         "/execute/stream",
     )

{fred_runtime-2.0.0 → fred_runtime-2.0.2}/fred_runtime/cli/__init__.py

@@ -2,8 +2,10 @@ from .completion import completion_candidates
 from .entrypoint import build_parser, main
 from .history_display import (
     build_hitl_resume_payload,
+    print_eval_trace,
     print_history,
     print_runtime_event,
+    run_eval_turn,
     run_single_turn,
 )
 from .kpi_display import (
@@ -19,6 +21,8 @@ from .kpi_display import (
 from .pod_client import DEFAULT_AGENT_POD_BASE_URL, AgentPodClient
 from .repl import run_interactive_chat
 from .repl_helpers import (
+    ExecutionMode,
+    execution_mode_color,
     execution_mode_label,
     fmt_bytes,
     parse_mode_command,
@@ -33,6 +37,7 @@ from .url_helpers import (
 __all__ = [
     "AgentPodClient",
     "DEFAULT_AGENT_POD_BASE_URL",
+    "ExecutionMode",
     "HistogramSeriesSummary",
     "PrometheusSample",
     "build_hitl_resume_payload",
@@ -40,7 +45,9 @@ __all__ = [
     "completion_candidates",
     "default_agent_metrics_url",
     "default_agent_pod_base_url",
+    "execution_mode_color",
     "execution_mode_label",
+    "print_eval_trace",
     "filter_prometheus_samples",
     "fmt_bytes",
     "format_metric_value",
@@ -53,6 +60,7 @@ __all__ = [
     "print_history",
     "print_runtime_event",
     "render_kpi_report",
+    "run_eval_turn",
     "run_interactive_chat",
     "run_single_turn",
     "summarize_prometheus_histograms",

{fred_runtime-2.0.0 → fred_runtime-2.0.2}/fred_runtime/cli/completion.py

@@ -49,7 +49,7 @@ def completion_candidates(
         return [sid for sid in session_ids if sid.startswith(prefix)]
     if stripped.startswith("/mode "):
         prefix = stripped.removeprefix("/mode ").strip()
-        return [mode for mode in ("final", "stream") if mode.startswith(prefix)]
+        return [mode for mode in ("eval", "final", "stream") if mode.startswith(prefix)]
     if stripped.startswith("/"):
         return complete_slash_commands(stripped, commands=_COMMANDS)
     return []

{fred_runtime-2.0.0 → fred_runtime-2.0.2}/fred_runtime/cli/history_display.py

@@ -408,3 +408,152 @@ def build_hitl_resume_payload(
         if 0 <= idx < len(choices):
             selected_choice_id = str(choices[idx].get("id", raw_response))
     return {"choice_id": selected_choice_id}
+
+
+def print_eval_trace(trace: dict[str, Any], *, color_enabled: bool) -> None:
+    """Render one EvalTrace dict (from POST /agents/evaluate) to the terminal."""
+    sep = colorize("  " + "─" * 62, color=ANSI_DIM, enabled=color_enabled)
+    print(colorize("  EvalTrace", color=ANSI_CYAN, enabled=color_enabled, bold=True))
+    print(sep)
+
+    def _field(label: str, value: str, color: str = ANSI_DIM) -> None:
+        print(
+            colorize(f"  {label:<18}", color=ANSI_DIM, enabled=color_enabled)
+            + colorize(value, color=color, enabled=color_enabled)
+        )
+
+    _field("agent", trace.get("agent_id") or "-", ANSI_CYAN)
+    _field("session", trace.get("session_id") or "-")
+    _field("latency", f"{trace.get('latency_ms', 0)} ms")
+    _field("model", trace.get("model_name") or "-")
+    _field("finish", trace.get("finish_reason") or "-")
+
+    tu = trace.get("token_usage") or {}
+    if tu:
+        _field(
+            "tokens",
+            f"{tu.get('input_tokens', 0)}↑ in  {tu.get('output_tokens', 0)}↓ out",
+        )
+
+    tools_called: list[str] = list(trace.get("tools_called") or [])
+    if tools_called:
+        _field("tools_called", "  ".join(tools_called), ANSI_YELLOW)
+
+    retrieval_ctx: list[str] = list(trace.get("retrieval_context") or [])
+    _field("retrieval_ctx", str(len(retrieval_ctx)) + " chunk(s)")
+
+    steps: list[dict[str, Any]] = list(trace.get("steps") or [])
+    _field("steps", str(len(steps)))
+
+    err = trace.get("error")
+    _field("error", err or "none", ANSI_RED if err else ANSI_DIM)
+
+    if steps:
+        print()
+        print(colorize("  Steps:", color=ANSI_DIM, enabled=color_enabled, bold=True))
+        for i, step in enumerate(steps, 1):
+            kind = step.get("kind", "?")
+            name = step.get("tool_name") or ""
+            if kind == "tool_call":
+                raw_args = step.get("arguments")
+                args_str = (
+                    json.dumps(raw_args, ensure_ascii=False)
+                    if raw_args is not None
+                    else ""
+                )
+                args_str = (args_str[:80] + "…") if len(args_str) > 80 else args_str
+                print(
+                    colorize(f"  {i:>3}  ", color=ANSI_DIM, enabled=color_enabled)
+                    + colorize(
+                        "[tool_call]   ", color=ANSI_YELLOW, enabled=color_enabled
+                    )
+                    + colorize(
+                        name, color=ANSI_YELLOW, enabled=color_enabled, bold=True
+                    )
+                )
+                if args_str:
+                    print(
+                        colorize(
+                            f"       {args_str}", color=ANSI_DIM, enabled=color_enabled
+                        )
+                    )
+            elif kind == "tool_result":
+                is_err = step.get("is_error", False)
+                rc = ANSI_RED if is_err else ANSI_GREEN
+                content = str(step.get("content") or "")
+                content = (content[:80] + "…") if len(content) > 80 else content
+                print(
+                    colorize(f"  {i:>3}  ", color=ANSI_DIM, enabled=color_enabled)
+                    + colorize("[tool_result] ", color=rc, enabled=color_enabled)
+                    + colorize(name, color=rc, enabled=color_enabled, bold=True)
+                    + colorize(
+                        "  (error)" if is_err else "  (ok)",
+                        color=rc,
+                        enabled=color_enabled,
+                    )
+                )
+                if content:
+                    print(
+                        colorize(
+                            f"       {content}", color=ANSI_DIM, enabled=color_enabled
+                        )
+                    )
+            elif kind == "node_error":
+                msg = str(step.get("error_message") or "")
+                print(
+                    colorize(f"  {i:>3}  ", color=ANSI_DIM, enabled=color_enabled)
+                    + colorize("[node_error]  ", color=ANSI_RED, enabled=color_enabled)
+                    + colorize(
+                        step.get("node_id") or "",
+                        color=ANSI_RED,
+                        enabled=color_enabled,
+                        bold=True,
+                    )
+                )
+                if msg:
+                    print(
+                        colorize(f"       {msg}", color=ANSI_DIM, enabled=color_enabled)
+                    )
+            elif kind == "final":
+                print(
+                    colorize(f"  {i:>3}  ", color=ANSI_DIM, enabled=color_enabled)
+                    + colorize(
+                        "[final]", color=ANSI_GREEN, enabled=color_enabled, bold=True
+                    )
+                )
+            else:
+                print(
+                    colorize(f"  {i:>3}  ", color=ANSI_DIM, enabled=color_enabled)
+                    + colorize(f"[{kind}]", color=ANSI_DIM, enabled=color_enabled)
+                )
+
+    output = trace.get("output") or ""
+    if output:
+        print()
+        print(colorize("  Output:", color=ANSI_DIM, enabled=color_enabled, bold=True))
+        print(sep)
+        print(output)
+
+    print(sep)
+
+
+def run_eval_turn(
+    *,
+    client: AgentPodClient,
+    agent_id: str,
+    message: str,
+    session_id: str,
+    user_id: str,
+    team_id: str | None,
+    color_enabled: bool,
+) -> int:
+    """Call /agents/evaluate and pretty-print the EvalTrace."""
+    result = client.evaluate(
+        agent_id=agent_id,
+        message=message,
+        session_id=session_id,
+        user_id=user_id,
+        team_id=team_id,
+    )
+    print_eval_trace(result, color_enabled=color_enabled)
+    return 0 if result.get("error") is None else 1

{fred_runtime-2.0.0 → fred_runtime-2.0.2}/fred_runtime/cli/pod_client.py

@@ -96,6 +96,41 @@ class AgentPodClient:
             raise RuntimeError("Execute response must be a JSON object.")
         return result
 
+    def evaluate(
+        self,
+        *,
+        agent_id: str,
+        message: str,
+        session_id: str,
+        user_id: str,
+        team_id: str | None = None,
+        agent_instance_id: str | None = None,
+        checkpoint_id: str | None = None,
+    ) -> dict[str, Any]:
+        runtime_context: dict[str, Any] = {"user_id": user_id}
+        if team_id:
+            runtime_context["team_id"] = team_id
+        payload: dict[str, Any] = {
+            "agent_id": agent_id,
+            "input": message,
+            "session_id": session_id,
+            "runtime_context": runtime_context,
+        }
+        if agent_instance_id is not None:
+            payload["agent_instance_id"] = agent_instance_id
+        if checkpoint_id is not None:
+            payload["checkpoint_id"] = checkpoint_id
+        response = self.http_client.post(
+            f"{self.base_url}/agents/evaluate",
+            json=payload,
+            headers=self._auth_headers(),
+        )
+        response.raise_for_status()
+        result = response.json()
+        if not isinstance(result, dict):
+            raise RuntimeError("Evaluate response must be a JSON object.")
+        return result
+
     def stream_events(
         self,
         *,