synapse-orch-ai 1.6.1 → 1.6.3

package/backend/core/builder_tools.py

@@ -637,9 +637,10 @@ BUILDER_TOOL_SCHEMAS = [
                     "include_full_history": {
                         "type": "boolean",
                         "description": (
-                            "For agent/llm/tool steps: on re-invocation (evaluator loop-back or retry), "
-                            "include every prior turn's inputs, tools, and output instead of only the last attempt. "
-                            "Useful for feedback loops; increases prompt length."
+                            "For agent/llm/tool steps: controls revision-history rendering on re-runs. "
+                            "Default (unset) auto-enables full history whenever this step runs more than once. "
+                            "Set false to force last-attempt only (smaller prompt); set true is equivalent to default. "
+                            "Only relevant if you want to explicitly opt OUT of full history."
                         ),
                     },
                 },
@@ -698,7 +699,7 @@ BUILDER_TOOL_SCHEMAS = [
                                 "max_turns": {"type": "integer"},
                                 "timeout_seconds": {"type": "integer"},
                                 "model": {"type": "string"},
-                                "include_full_history": {"type": "boolean", "description": "Include full revision history on re-invocation (agent/llm/tool steps)"},
+                                "include_full_history": {"type": "boolean", "description": "Opt OUT of full revision history on re-runs (default: auto-on for any re-run). Set false to keep prompts small."},
                             },
                             "required": ["step_id", "name", "type"],
                         },

package/backend/core/mcp_client.py

@@ -441,6 +441,13 @@ class MCPClientManager:
             if session:
                 self._set_status(name, "connected")
                 await self._auto_register(name)   # ← register on startup
+            else:
+                # Remote servers without a bearer token use OAuth — failure likely means re-auth needed.
+                # Stdio and bearer-token servers just go disconnected.
+                if server_type == "remote" and not config.get("token"):
+                    self._set_status(name, "reauth_needed")
+                else:
+                    self._set_status(name, "disconnected")
         return self.sessions
 
     # ── add_server ─────────────────────────────────────────────────────────────

package/backend/core/models_orchestration.py

@@ -100,7 +100,9 @@ class StepConfig(BaseModel):
 
     # On re-invocation (evaluator feedback or loop), include every prior turn's
     # inputs/tools/output in the prompt instead of only the last attempt.
-    include_full_history: bool = False
+    # Tri-state: True = always include, False = always last-attempt only,
+    # None (default) = auto (full history on any re-run).
+    include_full_history: bool | None = None
 
     # Graph routing
     next_step_id: str | None = None  # Linear next step / loop "done" path
@@ -145,6 +147,9 @@ class OrchestrationRun(BaseModel):
     waiting_for_human: bool = False
     human_prompt: str | None = None
     human_fields: list[dict[str, str]] = []
+    # Nested-orchestration human-in-the-loop tracking
+    nested_run_id: str | None = None   # sub-run paused waiting for human input
+    nested_orch_id: str | None = None  # sub-orchestration definition ID
 
     # Cost tracking
     total_tokens_used: int = 0

package/backend/core/native_builder/__init__.py

@@ -24,7 +24,7 @@ __all__ = [
 # their zero-defaults (`route_map: {}`, `parallel_branches: []`, etc.) so the
 # engine deserialises cleanly.
 STEP_TYPE_CHEATSHEET = """\
-- **agent**: runs a configured sub-agent with a prompt + its tool set. Required: `agent_id`, `prompt_template`. Optional: `include_full_history` (bool) — on re-invocation show every prior turn's inputs/tools/output instead of only the last attempt; useful for feedback/retry loops. Use for any step that needs multi-turn reasoning or tool use.
+- **agent**: runs a configured sub-agent with a prompt + its tool set. Required: `agent_id`, `prompt_template`. Optional: `include_full_history` (bool) — default behaviour auto-shows full revision history on any re-run; set this to `false` only if you need to keep the prompt small. Use for any step that needs multi-turn reasoning or tool use.
 - **llm**: single one-shot LLM call, no tools. Required: `prompt_template` (optional `model`). Use for lightweight summarisation, rewriting, or deterministic prose generation.
 - **tool**: forces a single tool call with no LLM reasoning. Required: `forced_tool` (+ `agent_id` for tool-resolution). Use when the arguments are already in state and just need forwarding.
 - **evaluator**: pure routing node. Required: `route_map_json` (JSON-encoded `{label: target_step_id}`), `route_descriptions_json` (JSON), `evaluator_prompt`. Output_key stores the bare route label. Use to fork on a classifier decision.

package/backend/core/orchestration/context.py

@@ -472,14 +472,22 @@ def _origin_label(origin: dict | None, execution: int) -> str:
     return otype or "re-invocation"
 
 
-def _render_full_history(memory: list[dict]) -> str:
-    """Render every recorded turn as inputs → tools → output."""
+def _render_full_history(memory: list[dict], skip_last_output: bool = False) -> str:
+    """Render every recorded turn as inputs -> tools -> output.
+
+    `skip_last_output`: when the caller is also rendering the most recent
+    turn's output as a standalone YOUR PREVIOUS OUTPUT section, set this to
+    True so the history doesn't duplicate that body. The last turn still
+    shows its tools/inputs/label; only the Output: block is replaced with
+    a pointer back to the standalone section.
+    """
     lines: list[str] = ["## REVISION HISTORY (all prior turns)"]
-    for entry in memory:
+    last_idx = len(memory) - 1
+    for idx, entry in enumerate(memory):
         execution = entry.get("execution", 0)
         origin = entry.get("origin") or {}
         label = _origin_label(origin, execution)
-        lines.append(f"\n### Turn {execution} — {label}")
+        lines.append(f"\n### Turn {execution} - {label}")
 
         # Inputs at the time of this turn
         inputs = entry.get("inputs") or {}
@@ -509,10 +517,13 @@ def _render_full_history(memory: list[dict]) -> str:
         # Final output produced on that turn
         final_out = str(trace.get("final_output") or "")
         if final_out:
-            if len(final_out) > 800:
-                from .summarizer import smart_truncate
-                final_out = smart_truncate(final_out, 800)
-            lines.append(f"Output:\n{final_out}")
+            if skip_last_output and idx == last_idx:
+                lines.append("Output: (see YOUR PREVIOUS OUTPUT section above)")
+            else:
+                if len(final_out) > 800:
+                    from .summarizer import smart_truncate
+                    final_out = smart_truncate(final_out, 800)
+                lines.append(f"Output:\n{final_out}")
 
     return "\n".join(lines)
 
@@ -603,18 +614,21 @@ def build_origin_aware_context(
     run: "OrchestrationRun",
     engine: "OrchestrationEngine",
     transition: TransitionContext,
-) -> tuple[str, str]:
+) -> tuple[str, str, str]:
     """
-    Build (prompt, system_prompt_extra) for an agent/tool/llm step using
-    the structured, origin-aware format.
-
-    Returns two strings:
-      prompt              — the full user-turn message to send to the LLM
-      system_prompt_extra — short orchestration-awareness block to append to
-                            the agent's system prompt (can be empty string)
+    Build (prompt, system_prompt_extra, system_prompt_prefix) for an
+    agent/tool/llm step using the structured, origin-aware format.
+
+    Returns three strings:
+      prompt               full user-turn message to send to the LLM
+      system_prompt_extra  short orchestration-awareness block APPENDED to
+                           the agent's system prompt (can be empty)
+      system_prompt_prefix iteration banner PREPENDED to the system prompt
+                           on re-runs (execution_number > 1); empty otherwise
     """
     import re
     sections: list[str] = []
+    is_rerun = transition.execution_number > 1
 
     # ------------------------------------------------------------------
     # Section: ROLE
@@ -659,36 +673,99 @@ def build_origin_aware_context(
     sections.append("\n".join(role_lines))
 
     # ------------------------------------------------------------------
-    # Section: EVALUATOR FEEDBACK (only on evaluator re-invocation)
+    # Section: MAIN GOAL (always present)
+    # The user's original request is the root of every step in this
+    # workflow. Keep it at the top of every prompt so re-runs, branching,
+    # and deep step chains do not lose sight of what we are ultimately
+    # solving for. Sourced from run.shared_state["user_input"] which
+    # the engine seeds on the first step.
     # ------------------------------------------------------------------
-    if transition.origin_type == "evaluator" and (
-        transition.routing_decision or transition.routing_reasoning
-    ):
-        feedback_lines = ["## EVALUATOR FEEDBACK"]
-        if transition.routing_decision:
-            feedback_lines.append(f"Decision: \"{transition.routing_decision}\"")
-        if transition.routing_reasoning:
-            feedback_lines.append(f"Reason: {transition.routing_reasoning}")
-
-        # Previous output summary
-        if step.output_key and step.output_key in run.shared_state:
-            prev_out = str(run.shared_state[step.output_key])
-            if len(prev_out) > 2000:
-                from .summarizer import smart_truncate
-                prev_out = smart_truncate(prev_out, 2000)
-            feedback_lines.append(f"\n### Your previous output\n{prev_out}")
+    main_goal = run.shared_state.get("user_input")
+    if main_goal:
+        goal_text = str(main_goal).strip()
+        if len(goal_text) > 2000:
+            from .summarizer import smart_truncate
+            goal_text = smart_truncate(goal_text, 2000)
+        sections.append(
+            "## MAIN GOAL (original user request)\n"
+            "Everything this workflow does exists to solve this. "
+            "Keep it in mind when deciding what to produce:\n\n"
+            f"{goal_text}"
+        )
 
-        sections.append("\n".join(feedback_lines))
+    # ------------------------------------------------------------------
+    # Section: WHY YOU ARE RUNNING AGAIN (any re-run, any origin)
+    # ------------------------------------------------------------------
+    if is_rerun:
+        why_lines = [
+            "## WHY YOU ARE RUNNING AGAIN",
+            f"This is execution #{transition.execution_number} of this step.",
+        ]
+        if transition.origin_type == "evaluator":
+            if transition.routing_decision:
+                why_lines.append(f"Evaluator decision: \"{transition.routing_decision}\"")
+            if transition.routing_reasoning:
+                why_lines.append(f"Evaluator reasoning: {transition.routing_reasoning}")
+            why_lines.append(
+                "An evaluator reviewed your previous output and routed you back. "
+                "See HOW TO PROCEED below."
+            )
+        elif transition.origin_type == "loop":
+            iter_str = (
+                f"iteration {transition.loop_iteration} of {transition.loop_total}"
+                if transition.loop_iteration and transition.loop_total
+                else f"iteration #{transition.execution_number}"
+            )
+            why_lines.append(
+                f"You are inside a loop ({iter_str}). See HOW TO PROCEED below."
+            )
+        else:
+            why_lines.append(
+                "The workflow has routed control back to this step. See HOW TO PROCEED below."
+            )
+        sections.append("\n".join(why_lines))
+
+    # ------------------------------------------------------------------
+    # Section: YOUR PREVIOUS OUTPUT (any re-run)
+    # ------------------------------------------------------------------
+    if is_rerun and step.output_key and step.output_key in run.shared_state:
+        prev_out = str(run.shared_state[step.output_key])
+        if len(prev_out) > 2000:
+            from .summarizer import smart_truncate
+            prev_out = smart_truncate(prev_out, 2000)
+        sections.append(
+            "## YOUR PREVIOUS OUTPUT\n"
+            "This is what you produced last turn. Read it before responding:\n\n"
+            f"{prev_out}"
+        )
 
     # ------------------------------------------------------------------
     # Section: YOUR PREVIOUS WORK / REVISION HISTORY (on any re-invocation)
     # ------------------------------------------------------------------
-    if transition.execution_number > 1:
+    if is_rerun:
         memory = get_execution_memory(run, step.id)
         if memory:
-            if step.include_full_history:
-                sections.append(_render_full_history(memory))
+            use_full = (
+                step.include_full_history
+                if step.include_full_history is not None
+                else True
+            )
+            # Detect whether the standalone YOUR PREVIOUS OUTPUT section was
+            # rendered above (same condition used there). If so, avoid
+            # duplicating the most recent turn's body in the history.
+            previous_output_rendered = bool(
+                step.output_key and step.output_key in run.shared_state
+            )
+            if use_full:
+                # With only one prior turn, the history would just restate
+                # YOUR PREVIOUS OUTPUT under a turn label. Skip it then.
+                if len(memory) > 1:
+                    sections.append(_render_full_history(
+                        memory, skip_last_output=previous_output_rendered
+                    ))
             else:
+                # Last-attempt mode shows only tools/inputs (no output body),
+                # so it never duplicates YOUR PREVIOUS OUTPUT.
                 sections.append(_render_last_attempt(memory[-1]))
 
     # ------------------------------------------------------------------
@@ -722,11 +799,10 @@ def build_origin_aware_context(
     # ------------------------------------------------------------------
     context_parts = []
 
-    # Always include user_input unless explicitly in input_keys
-    if "user_input" in run.shared_state and "user_input" not in (step.input_keys or []):
-        context_parts.append(
-            f"### user_input\nSource: initial input\n{run.shared_state['user_input']}"
-        )
+    # NOTE: user_input is rendered at the top under "## MAIN GOAL" so it is
+    # not duplicated here. If a step explicitly lists user_input in its
+    # input_keys, the explicit-input loop below will skip it for the same
+    # reason.
 
     # Human response keys (always inject unless already listed)
     human_keys = {"human_response"}
@@ -745,9 +821,9 @@ def build_origin_aware_context(
                 val = smart_truncate(val, 3000)
             context_parts.append(f"### {hkey}\nSource: human response\n{val}")
 
-    # Explicitly declared input_keys
+    # Explicitly declared input_keys (skip user_input — shown under MAIN GOAL)
     for key in (step.input_keys or []):
-        if key not in run.shared_state:
+        if key not in run.shared_state or key == "user_input":
             continue
         val = run.shared_state[key]
         label = key
@@ -755,14 +831,48 @@ def build_origin_aware_context(
             (s for s in engine.step_map.values() if s.output_key == key), None
         )
         if producer and producer.agent_id and producer.agent_id in engine.agent_names:
-            label = f"{engine.agent_names[producer.agent_id]} \u2192 {key}"
+            label = f"{engine.agent_names[producer.agent_id]} → {key}"
         context_parts.append(_format_context_value(key, val, label))
 
     if context_parts:
         sections.append("## CONTEXT FROM PREVIOUS STEPS\n" + "\n\n".join(context_parts))
 
     # ------------------------------------------------------------------
-    # Section: TASK
+    # Section: HOW TO PROCEED (any re-run)
+    # Loose framing: agent decides between refine / redo / push back. The
+    # only firm requirement is that the new output explain the change (or
+    # the deliberate non-change) relative to the previous attempt.
+    # ------------------------------------------------------------------
+    if is_rerun:
+        if transition.origin_type == "loop":
+            proceed_block = (
+                "## HOW TO PROCEED\n"
+                "This is another iteration of a loop. Your previous iteration's output and "
+                "any sibling iterations are shown above.\n\n"
+                "Use your judgement: produce the next item, refine, or take a different angle. "
+                "Just don't silently re-emit what you produced before.\n\n"
+                "At the top of your output, include a brief note (1-3 lines) explaining what "
+                "this iteration adds or changes relative to the previous one."
+            )
+        else:
+            proceed_block = (
+                "## HOW TO PROCEED\n"
+                "You are running this step again. Your previous output and the reason for re-running "
+                "are shown above. Read them before responding.\n\n"
+                "You decide how to respond — any of these are valid:\n"
+                "  - Refine the previous output (small targeted edits).\n"
+                "  - Redo it from scratch if it was fundamentally off.\n"
+                "  - Push back if you believe the feedback is mistaken: explain why and either "
+                "defend the previous output or propose a different correction.\n\n"
+                "Whichever path you take, include a brief reasoning note at the top of your output "
+                "(1-3 lines) that explains:\n"
+                "  - What changed from your previous output (or why you kept it), and\n"
+                "  - Why you believe this is the right response to the feedback."
+            )
+        sections.append(proceed_block)
+
+    # ------------------------------------------------------------------
+    # Section: TASK (first run) / ORIGINAL TASK reference (re-run)
     # ------------------------------------------------------------------
     prompt_template = step.prompt_template or run.shared_state.get("user_input", "")
 
@@ -773,12 +883,17 @@ def build_origin_aware_context(
 
     task_text = re.sub(r"\{state\.(\w+)\}", replace_ref, prompt_template)
 
-    task_header = "## TASK (REVISION)" if transition.origin_type == "evaluator" else "## TASK"
-    task_suffix = ""
-    if transition.origin_type == "evaluator" and transition.routing_reasoning:
-        task_suffix = "\n\nAddress the evaluator's feedback above in your revised output."
-    elif transition.origin_type == "human_response":
-        task_suffix = "\n\nIncorporate the human's input above."
+    if is_rerun:
+        task_header = "## ORIGINAL TASK (reference only)"
+        task_suffix = (
+            "\n\nThis is the original task statement, included for reference. "
+            "Use it together with YOUR PREVIOUS OUTPUT and HOW TO PROCEED above to decide your response."
+        )
+    else:
+        task_header = "## TASK"
+        task_suffix = ""
+        if transition.origin_type == "human_response":
+            task_suffix = "\n\nIncorporate the human's input above."
 
     sections.append(f"{task_header}\n{task_text}{task_suffix}")
 
@@ -799,7 +914,7 @@ def build_origin_aware_context(
         if sid:
             exec_counts[sid] = exec_counts.get(sid, 0) + 1
     # Use execution_number for the active step (counts this in-flight run).
-    if transition.execution_number > 1:
+    if is_rerun:
         exec_counts[step.id] = transition.execution_number
 
     graph_md = build_workflow_graph_markdown(engine.orch, step.id, exec_counts)
@@ -816,4 +931,26 @@ def build_origin_aware_context(
         )
     system_prompt_extra = "\n".join(sys_lines)
 
-    return prompt, system_prompt_extra
+    # ------------------------------------------------------------------
+    # System prompt PREFIX: iteration banner (re-runs only).
+    # Prepended to the system prompt so a long agent role description
+    # cannot drown out the iteration signal.
+    # ------------------------------------------------------------------
+    system_prompt_prefix = ""
+    if is_rerun:
+        why_short = ""
+        if transition.origin_type == "evaluator" and transition.routing_decision:
+            why_short = f" (evaluator routed back: \"{transition.routing_decision}\")"
+        elif transition.origin_type == "loop" and transition.loop_iteration:
+            why_short = f" (loop iteration {transition.loop_iteration})"
+        system_prompt_prefix = (
+            "ITERATION CONTEXT - READ BEFORE RESPONDING\n"
+            f"You are on execution #{transition.execution_number} of step "
+            f"\"{step_name}\"{why_short}. This is NOT your first attempt.\n"
+            "In the user message below, read YOUR PREVIOUS OUTPUT, WHY YOU ARE RUNNING AGAIN, "
+            "and HOW TO PROCEED. Then decide how to respond — refine, redo, or push back on the "
+            "feedback if you think it is mistaken. Whichever you choose, start your output with a "
+            "brief reasoning note explaining what changed (or why you kept the previous output)."
+        )
+
+    return prompt, system_prompt_extra, system_prompt_prefix

package/backend/core/orchestration/engine.py

@@ -175,6 +175,14 @@ class OrchestrationEngine:
                                 run.waiting_for_human = True
                                 run.status = "paused"
                                 run.current_step_id = step.id
+                                # Track the sub-run that needs human input when the
+                                # request originated inside a nested orchestration.
+                                if event.get("nested_run_id"):
+                                    run.nested_run_id = event["nested_run_id"]
+                                    run.nested_orch_id = event.get("nested_orch_id")
+                                else:
+                                    run.nested_run_id = None
+                                    run.nested_orch_id = None
                                 state.checkpoint()
                                 if logger:
                                     logger.step_end(step.id, "paused")
@@ -370,6 +378,14 @@ class OrchestrationEngine:
             session_id=run.session_id,
         )
 
+        # If the parent was paused because a NESTED orchestration hit a human step,
+        # resume the sub-run and let it complete before continuing the parent.
+        if run.nested_run_id:
+            async for event in cls._resume_nested_orch(run, engine, human_response, server_module):
+                yield event
+            return
+
+        # Normal path: human step is directly in this orchestration.
         # Move to next step after the HUMAN step
         current_step = engine.step_map.get(run.current_step_id)
 
@@ -392,6 +408,120 @@ class OrchestrationEngine:
         async for event in engine._execute_loop(run, state):
             yield event
 
+    @classmethod
+    async def _resume_nested_orch(
+        cls,
+        run: OrchestrationRun,
+        engine: "OrchestrationEngine",
+        human_response: dict,
+        server_module,
+    ) -> AsyncGenerator[dict, None]:
+        """Resume a parent run whose nested sub-orchestration was paused at a human step.
+
+        Resumes the sub-run, forwards its events tagged with the parent step's
+        context, then writes the sub-orch result to parent shared_state and
+        continues the parent execution loop.
+        """
+        parent_step = engine.step_map.get(run.current_step_id)
+        nested_run_id = run.nested_run_id
+        nested_orch_id = run.nested_orch_id
+        _NESTED_FILTER_TYPES = {"orchestration_start", "orchestration_complete", "orchestration_end"}
+
+        final_response: str | None = None
+        sub_events: list[dict] = []
+
+        async for sub_event in cls.resume(nested_run_id, human_response, server_module):
+            sub_events.append(sub_event)
+
+            # Sub-orch hit another human step (multi-turn human interaction inside nested orch)
+            if sub_event.get("type") == "human_input_required":
+                run.nested_run_id = sub_event.get("nested_run_id") or nested_run_id
+                run.nested_orch_id = sub_event.get("nested_orch_id") or nested_orch_id
+                state = SharedState(run)
+                state.checkpoint()
+                yield {
+                    **sub_event,
+                    "run_id": run.run_id,
+                    "orch_step_id": run.current_step_id,
+                    "step_name": parent_step.name if parent_step else "",
+                    "nested_run_id": nested_run_id,
+                    "nested_orch_id": nested_orch_id,
+                }
+                return
+
+            if sub_event.get("type") == "final" and sub_event.get("intent") == "orchestration":
+                final_response = sub_event.get("response", "")
+
+            # Skip sub-orch lifecycle meta-events (same filter as initial execution)
+            if sub_event.get("type") in _NESTED_FILTER_TYPES:
+                continue
+
+            yield {
+                **sub_event,
+                "run_id": run.run_id,
+                "orch_step_id": run.current_step_id,
+                "step_name": parent_step.name if parent_step else "",
+                "nested_run_id": nested_run_id,
+                "nested_orch_id": nested_orch_id,
+            }
+
+        # Fallback: extract result from orchestration_complete if no "final" event was emitted
+        if final_response is None:
+            from core.routes.orchestrations import load_orchestrations
+            from core.models_orchestration import Orchestration
+            orchs = load_orchestrations()
+            sub_orch_data = next((o for o in orchs if o["id"] == nested_orch_id), None)
+            if sub_orch_data:
+                sub_orch = Orchestration.model_validate(sub_orch_data)
+                for ev in reversed(sub_events):
+                    if ev.get("type") == "orchestration_complete":
+                        state_data = ev.get("final_state") or {}
+                        for sub_step in reversed(sub_orch.steps):
+                            if sub_step.output_key and sub_step.output_key in state_data:
+                                final_response = str(state_data[sub_step.output_key])
+                                break
+                        break
+
+        if final_response is None:
+            run.status = "failed"
+            state = SharedState(run)
+            state.checkpoint()
+            yield {
+                "type": "orchestration_error",
+                "error": "Nested orchestration failed to produce a result after human input",
+            }
+            return
+
+        # Write sub-orch result into parent shared_state
+        if parent_step and parent_step.output_key:
+            run.shared_state[parent_step.output_key] = final_response
+
+        # Record agent step completion in history
+        import time as _time
+        run.step_history.append({
+            "step_id": run.current_step_id,
+            "step_name": parent_step.name if parent_step else "",
+            "step_type": "agent",
+            "status": "completed",
+            "ended_at": _time.strftime("%Y-%m-%dT%H:%M:%SZ", _time.gmtime()),
+        })
+
+        # Clear nested context and resume parent from next step after agent step
+        run.nested_run_id = None
+        run.nested_orch_id = None
+        run.waiting_for_human = False
+        run.status = "running"
+
+        if parent_step:
+            next_id, extra_event = engine._resolve_next(parent_step, run)
+            if extra_event:
+                yield extra_event
+            run.current_step_id = next_id
+
+        state = SharedState(run)
+        async for event in engine._execute_loop(run, state):
+            yield event
+
     def _init_state(self, user_input: str) -> dict:
         """Initialize shared state from schema defaults + user input."""
         state = {}