synapse-orch-ai 1.8.6 → 1.9.1

package/README.md

@@ -64,6 +64,8 @@ docker run -d \
 
 Then open `http://localhost:3000`. See the [Docker guide](https://docs.synapseorch.com/getting-started/installation#docker) in the docs for custom ports and environment variable configuration.
 
+> **Security:** publish only the frontend port `3000` — the backend (`8765`) serves an internal API and should not be exposed on `0.0.0.0` (docker-compose binds it to `127.0.0.1`). The container auto-generates a shared internal token so that surface is authenticated by default. For any network-reachable deployment, enable **login** (Settings → Security) and/or set `allow_stdio_mcp=false`, since registering a stdio MCP server launches local commands.
+
 ### Upgrading
 
 | Install method | Upgrade command |
@@ -73,6 +75,8 @@ Then open `http://localhost:3000`. See the [Docker guide](https://docs.synapseor
 | npm | `npm update -g synapse-orch-ai` |
 | Docker | `docker pull synapseorchai/synapse-ai:latest` |
 
+> **Upgrade note (security hardening):** the backend now auto-generates a shared internal token and, under docker-compose, binds its port to `127.0.0.1` by default. The UI and its API are unaffected (the frontend proxies the backend internally). If you previously called the **backend directly** on `:8765` (e.g. external API clients), either point them at the frontend instead — `http://host:3000/api/v1|v2/...` — or set `SYNAPSE_BACKEND_BIND=0.0.0.0` in `.env` to keep publishing `:8765`.
+
 ---
 
 ## Scale Mode

package/backend/core/config.py

@@ -108,6 +108,16 @@ def load_settings():
         "messaging_enabled": True,
         "embed_code": False,
         "bash_allowed_dirs": [],
+        # ── MCP stdio server registration (security) ──
+        # stdio MCP servers launch local OS processes (npx/uvx/python/…). This is
+        # a powerful, admin-only capability. Allowed by default for local
+        # single-user use, but force-disabled in scale mode (multi-tenant /
+        # network-exposed) — see stdio_mcp_allowed(). See GHSA-3j67-x3j8-r32x.
+        "allow_stdio_mcp": True,
+        # Optional hardening: when non-empty, only these command basenames may be
+        # registered as stdio servers. Empty = allow any command in PATH.
+        # Recommended for exposed UIs: ["npx","uvx","uv","node","python","python3","docker"]
+        "mcp_command_allowlist": [],
         "login_enabled": False,
         "login_username": "",
         "login_password_hash": "",

package/backend/core/internal_auth.py

@@ -10,9 +10,23 @@ that try to hit /api/settings, /api/agents, etc. directly will get 403.
 Rules:
 - /api/v1/*, /api/v2/*, ... → SKIP (external versioned API; uses API key auth instead)
 - /docs, /openapi.json, /redoc  → SKIP (FastAPI docs)
-- /chat*, /auth/*    → SKIP (direct backend routes, not under /api/)
-- /api/*             → REQUIRE X-Synapse-Internal header
-- If SYNAPSE_INTERNAL_TOKEN is not set → permissive (backward compatible)
+- /auth/*            → SKIP (OAuth redirects; browser-navigated, cannot carry a header)
+- /api/*, /chat*     → REQUIRE X-Synapse-Internal header
+- If SYNAPSE_INTERNAL_TOKEN is not set → LOOPBACK-ONLY: allow requests whose
+  direct peer is 127.0.0.1/::1 (local dev, same-container proxy) and 403 any
+  remote caller. This closes the unauthenticated-RCE hole on network-exposed
+  token-less deployments (GHSA-3j67-x3j8-r32x). Docker images auto-generate a
+  token (docker/entrypoint.sh) so they never rely on this fallback.
+
+Why /chat is gated even though it is not under /api/:
+  POST /chat and /chat/stream (core/routes/chat.py) run the full ReAct loop with
+  the complete tool surface — bash, execute_python, SQL, every MCP server. They
+  are mounted at the root, so the "not /api/ → skip" rule let any caller who
+  could reach the port drive them with no credentials at all.
+  The public, externally-authenticated chat API is /api/v1/chat (require_api_key)
+  — a different route. Root /chat is reached only by the Next.js route handler
+  (frontend/src/app/api/chat/**), which already sends X-Synapse-Internal via
+  backendHeaders(), so gating it is transparent to the UI.
 """
 import os
 import re
@@ -21,6 +35,27 @@ from fastapi import Request
 from fastapi.responses import JSONResponse
 from starlette.middleware.base import BaseHTTPMiddleware
 
+# Direct-peer addresses treated as local. We deliberately check request.client
+# (the immediate TCP peer), NOT X-Forwarded-For, which is attacker-spoofable.
+_LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
+
+# Root-mounted routes that are internal despite not living under /api/.
+# Matched as exact path or path-prefixed segment so /chat and /chat/stream are
+# both covered while an unrelated future route like /chatbot-docs is not.
+_INTERNAL_ROOT_PREFIXES = ("/chat",)
+
+
+def _is_loopback(request: Request) -> bool:
+    client = request.client
+    return bool(client) and client.host in _LOOPBACK_HOSTS
+
+
+def _is_internal_root_route(path: str) -> bool:
+    return any(
+        path == prefix or path.startswith(prefix + "/")
+        for prefix in _INTERNAL_ROOT_PREFIXES
+    )
+
 
 class InternalTokenMiddleware(BaseHTTPMiddleware):
     """Block direct access to internal /api/* routes without the internal token."""
@@ -32,29 +67,48 @@ class InternalTokenMiddleware(BaseHTTPMiddleware):
     async def dispatch(self, request: Request, call_next):
         path = request.url.path
 
-        # If no token configured, be permissive (local dev / backward compat)
-        if not self.token:
-            return await call_next(request)
+        # ── Skips run FIRST, independent of the internal token ──────────────────
+        # These routes are either externally authenticated (versioned API → API
+        # key) or intentionally public, so they must stay reachable even when the
+        # token is unset/loopback-gated. This lets external clients hit the open
+        # /api/v1|v2 API through the frontend proxy (or a directly-published
+        # backend port) without the internal frontend token.
 
-        # Skip: external versioned API routes (v1, v2, ...) — they use API key
-        # auth (require_api_key), not the internal frontend token. Match any
-        # /api/v<N> prefix so future versions are exempt automatically.
+        # External versioned API (v1, v2, ...) — uses require_api_key, not the
+        # internal token. Match any /api/v<N> prefix so future versions are exempt.
         if re.match(r"^/api/v\d+(/|$)", path):
             return await call_next(request)
 
-        # Skip: MCP OAuth callback — called by external OAuth providers, not frontend
+        # MCP OAuth callback — called by external OAuth providers, not the frontend.
         if path == "/api/mcp/oauth/callback":
             return await call_next(request)
 
-        # Skip: FastAPI docs
+        # FastAPI docs.
         if path in ("/docs", "/openapi.json", "/redoc"):
             return await call_next(request)
 
-        # Skip: non-API routes (chat, auth, health, websocket, etc.)
-        if not path.startswith("/api/"):
+        # Non-API routes (auth redirects, health, websocket, etc.) — EXCEPT the
+        # root-mounted internal routes, which are gated below. /chat drives the
+        # full agent tool surface and must never be reachable unauthenticated.
+        if not path.startswith("/api/") and not _is_internal_root_route(path):
             return await call_next(request)
 
-        # This is an /api/* route — require internal token
+        # ── Internal surface — /api/* plus the root-mounted internal routes.
+        # The sensitive routes (settings, agents, mcp/servers, chat, …) that rely
+        # on the internal token for protection ──────────────────────────────────
+        if not self.token:
+            # No token configured → permissive ONLY for loopback callers. A remote
+            # client hitting a token-less backend directly is rejected, so the
+            # internal surface is never exposed unauthenticated over the network
+            # even in a misconfigured/bare-metal deployment (GHSA-3j67-x3j8-r32x).
+            if _is_loopback(request):
+                return await call_next(request)
+            return JSONResponse(
+                status_code=403,
+                content={"detail": "Forbidden: internal token not configured"},
+            )
+
+        # Token configured → require the matching header.
         provided = request.headers.get("X-Synapse-Internal", "")
         if provided != self.token:
             return JSONResponse(

package/backend/core/mcp_client.py

@@ -39,13 +39,44 @@ from mcp.client.stdio import stdio_client
 from mcp.client.streamable_http import streamable_http_client
 from mcp.shared.auth import OAuthClientInformationFull, OAuthClientMetadata, OAuthToken
 
-from core.config import DATA_DIR, MCP_SESSION_READ_TIMEOUT
+from core.config import DATA_DIR, MCP_SESSION_READ_TIMEOUT, load_settings
 import core.mcp_oauth_state as oauth_state
 
 # ── Constants ──────────────────────────────────────────────────────────────────
 
 _SESSION_READ_TIMEOUT = timedelta(seconds=MCP_SESSION_READ_TIMEOUT)
 
+
+# ── stdio MCP registration policy (security) ────────────────────────────────────
+# stdio MCP servers launch arbitrary local OS processes (npx/uvx/python -c/…),
+# which is the whole point of the transport but also an RCE-shaped capability.
+# These helpers gate every path that can spawn one. See GHSA-3j67-x3j8-r32x.
+
+def stdio_mcp_allowed() -> bool:
+    """Whether stdio MCP servers may be registered/spawned on this deployment.
+
+    Disabled in scale mode (multi-tenant / network-exposed) and toggleable via
+    the ``allow_stdio_mcp`` setting. Fails open only if settings are unreadable,
+    to preserve local behaviour when config is unavailable.
+    """
+    try:
+        s = load_settings()
+    except Exception:
+        return True
+    if s.get("scale_mode_enabled"):
+        return False
+    return bool(s.get("allow_stdio_mcp", True))
+
+
+def check_stdio_command_allowed(command: str) -> None:
+    """Enforce the optional ``mcp_command_allowlist`` (empty list = allow any)."""
+    try:
+        allowlist = load_settings().get("mcp_command_allowlist") or []
+    except Exception:
+        allowlist = []
+    if allowlist and os.path.basename(command) not in allowlist:
+        raise ValueError(f"Command '{command}' is not in the allowed MCP command list.")
+
 MCP_SERVERS_FILE = os.path.join(DATA_DIR, "mcp_servers.json")
 MCP_TOKENS_DIR   = os.path.join(DATA_DIR, "mcp_tokens")
 
@@ -231,6 +262,24 @@ class MCPClientManager:
             print(f"Skipping '{name}': no command")
             return None
 
+        # Gate at the sink, not only at the callers. Spawning a stdio server is
+        # arbitrary local command execution, so the check belongs on the code
+        # path that actually spawns — otherwise any new (or overlooked) call site
+        # silently bypasses it. POST /api/import did exactly that: it wrote a
+        # bundle's MCP configs to disk and called this method directly, skipping
+        # the add_server/reconnect_server/connect_all gates entirely.
+        if not stdio_mcp_allowed():
+            print(f"[MCP] Refusing to spawn stdio server '{name}': "
+                  f"stdio MCP servers are disabled on this deployment.")
+            self._set_status(name, "disconnected")
+            return None
+        try:
+            check_stdio_command_allowed(command)
+        except ValueError as e:
+            print(f"[MCP] Refusing to spawn stdio server '{name}': {e}")
+            self._set_status(name, "disconnected")
+            return None
+
         env = os.environ.copy()
         env.update(env_vars)
 
@@ -432,6 +481,13 @@ class MCPClientManager:
 
             server_type = config.get("server_type", "stdio")
             if server_type == "stdio":
+                # Do not re-spawn persisted stdio servers when the capability is
+                # disabled — otherwise a saved (possibly malicious) config would
+                # execute on every boot. See GHSA-3j67-x3j8-r32x.
+                if not stdio_mcp_allowed():
+                    print(f"[MCP] Skipping stdio server '{name}': stdio MCP disabled on this deployment.")
+                    self._set_status(name, "disabled")
+                    continue
                 session = await self.connect_stdio_server(config)
             elif config.get("token"):
                 session = await self.connect_remote_server(config)
@@ -472,8 +528,11 @@ class MCPClientManager:
         new_config: Dict[str, Any] = {"name": name, "label": label or name, "server_type": server_type, "status": "disconnected"}
 
         if server_type == "stdio":
+            if not stdio_mcp_allowed():
+                raise PermissionError("stdio MCP server registration is disabled on this deployment.")
             if not command:
                 raise ValueError("Command is required for stdio servers.")
+            check_stdio_command_allowed(command)
             if shutil.which(command) is None:
                 hints = {"uvx": "Install uv: curl -LsSf https://astral.sh/uv/install.sh | sh",
                          "npx": "Install Node.js/npm."}
@@ -527,6 +586,8 @@ class MCPClientManager:
 
         server_type = config.get("server_type", "stdio")
         if server_type == "stdio":
+            if not stdio_mcp_allowed():
+                raise PermissionError("stdio MCP servers are disabled on this deployment.")
             session = await self.connect_stdio_server(config)
         elif config.get("token"):
             session = await self.connect_remote_server(config)

package/backend/core/models.py

@@ -131,6 +131,10 @@ class Settings(BaseModel):
     login_username: str = ""
     login_password_hash: str = ""
 
+    # Run notifications: messaging channel id to mirror run notifications to
+    # (human-input pauses, completions, failures). Empty = in-app only.
+    run_notification_channel_id: str = ""
+
     # Scale / distributed execution settings
     redis_url: str = ""
     scale_postgres_url: str = ""

package/backend/core/notifications.py

@@ -0,0 +1,251 @@
+"""User-facing run notifications.
+
+A process-wide hub that turns orchestration lifecycle events into
+notifications the UI (and optionally a messaging channel) can consume:
+
+- `human_input_required`  → "… needs your input"
+- `orchestration_complete`→ run completed / run failed
+- `orchestration_error`   → run failed
+
+Fed by the runner pump via `observe_run_event` (registered on
+`core.orchestration.runner.event_observers` at server startup), so every
+entry point — UI, API, scheduler — produces notifications with no extra
+wiring. Standalone-safe: a bounded in-memory ring with best-effort JSON
+persistence, no external dependencies.
+
+Delivery surfaces:
+- `GET /api/notifications` + `/api/notifications/stream` (routes/notifications.py)
+  for the in-app provider (toasts, badges, browser notifications).
+- Optional messaging dispatch: when settings.json has
+  `run_notification_channel_id` set to a configured messaging channel id,
+  each notification is also sent there (`notify_chat_id` or the channel's
+  last active chat).
+"""
+import asyncio
+import json
+import time
+from pathlib import Path
+
+NOTIFICATIONS_FILE = Path(__file__).parent.parent / "logs" / "notifications.json"
+CAPACITY = 200
+
+
+class NotificationHub:
+    def __init__(self, capacity: int = CAPACITY, persist_path: Path = NOTIFICATIONS_FILE):
+        self._capacity = capacity
+        self._path = persist_path
+        self._items: list[dict] = self._load()
+        self._next_id = (self._items[-1]["id"] + 1) if self._items else 1
+        self._subs: set[asyncio.Queue] = set()
+        # run_id → orchestration display name (cached from orchestration_start)
+        self._orch_names: dict[str, str] = {}
+        # Runs whose terminal notification already fired — engine failure paths
+        # can emit both orchestration_error and orchestration_complete(failed).
+        self._terminal_notified: set[str] = set()
+        self._server_module = None
+
+    # ── persistence ──────────────────────────────────────────────────────
+    def _load(self) -> list[dict]:
+        try:
+            data = json.loads(self._path.read_text(encoding="utf-8"))
+            return data if isinstance(data, list) else []
+        except Exception:
+            return []
+
+    def _persist(self):
+        try:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            self._path.write_text(json.dumps(self._items), encoding="utf-8")
+        except Exception:
+            pass  # best-effort — the ring keeps working in memory
+
+    # ── core API ─────────────────────────────────────────────────────────
+    def publish(self, kind: str, title: str, body: str = "",
+                run_id: str | None = None, data: dict | None = None) -> dict:
+        item = {
+            "id": self._next_id,
+            "ts": time.time(),
+            "kind": kind,          # human_input | run_completed | run_failed
+            "title": title[:200],
+            "body": (body or "")[:500],
+            "run_id": run_id,
+            "data": data or {},
+        }
+        self._next_id += 1
+        self._items.append(item)
+        if len(self._items) > self._capacity:
+            self._items = self._items[-self._capacity:]
+        self._persist()
+        for queue in list(self._subs):
+            queue.put_nowait(item)
+        self._dispatch_messaging(item)
+        return item
+
+    def list(self, after_id: int = 0) -> list[dict]:
+        return [i for i in self._items if i["id"] > after_id]
+
+    def last_id(self) -> int:
+        return self._next_id - 1
+
+    def subscribe(self) -> asyncio.Queue:
+        queue: asyncio.Queue = asyncio.Queue()
+        self._subs.add(queue)
+        return queue
+
+    def unsubscribe(self, queue: asyncio.Queue):
+        self._subs.discard(queue)
+
+    def configure(self, server_module):
+        """Give the hub access to the messaging manager (server startup)."""
+        self._server_module = server_module
+
+    # ── engine event observer (runner pump) ──────────────────────────────
+    def observe_run_event(self, run_id: str, event: dict):
+        etype = event.get("type", "")
+        if etype == "orchestration_start":
+            self._orch_names[run_id] = (
+                event.get("orchestration_name") or event.get("orchestration_id") or run_id
+            )
+            self._terminal_notified.discard(run_id)
+        elif etype == "step_start":
+            # Any forward progress re-arms terminal notification (a resumed
+            # run emits no orchestration_start) and means any pending
+            # human-input request for this run was answered.
+            self._terminal_notified.discard(run_id)
+            self._resolve_human_input(run_id)
+        elif etype == "human_input_required":
+            name = self._orch_name(run_id)
+            self.publish(
+                "human_input", f"{name} needs your input",
+                body=str(event.get("prompt") or ""),
+                run_id=run_id,
+                data={"step_id": event.get("orch_step_id")},
+            )
+        elif etype == "orchestration_complete":
+            self._resolve_human_input(run_id)
+            if run_id in self._terminal_notified:
+                return
+            self._terminal_notified.add(run_id)
+            name = self._orch_name(run_id)
+            status = event.get("status", "completed")
+            if status == "completed":
+                self.publish("run_completed", f"{name} completed", run_id=run_id)
+            else:
+                self.publish("run_failed", f"{name} {status}", run_id=run_id)
+        elif etype == "orchestration_error":
+            self._resolve_human_input(run_id)
+            if run_id in self._terminal_notified:
+                return
+            self._terminal_notified.add(run_id)
+            name = self._orch_name(run_id)
+            self.publish("run_failed", f"{name} failed",
+                         body=str(event.get("error") or ""), run_id=run_id)
+        elif etype == "run_stream_end" and not event.get("paused"):
+            self._orch_names.pop(run_id, None)
+
+    def _resolve_human_input(self, run_id: str):
+        """Mark this run's human-input notifications answered. The updated
+        items (same id, resolved=true) are re-pushed to live subscribers;
+        clients upsert by id so the bell stops showing them as actionable."""
+        updated = [
+            item for item in self._items
+            if item["run_id"] == run_id and item["kind"] == "human_input"
+            and not item.get("resolved")
+        ]
+        if not updated:
+            return
+        for item in updated:
+            item["resolved"] = True
+        self._persist()
+        for queue in list(self._subs):
+            for item in updated:
+                queue.put_nowait(item)
+
+    def _orch_name(self, run_id: str) -> str:
+        """Cached name, with a checkpoint fallback for resumed runs (which
+        emit no orchestration_start)."""
+        name = self._orch_names.get(run_id)
+        if name:
+            return name
+        try:
+            from core.orchestration.state import SharedState
+            from core.routes.orchestrations import load_orchestrations
+            orch_id = SharedState.restore(run_id).run.orchestration_id
+            orch = next((o for o in load_orchestrations() if o["id"] == orch_id), None)
+            name = (orch or {}).get("name") or orch_id
+        except Exception:
+            name = "Orchestration"
+        self._orch_names[run_id] = name
+        return name
+
+    # ── startup: reconstruct missed human-input notifications ────────────
+    def reconstruct_missed(self, limit: int = 100):
+        """Re-publish 'needs your input' for runs still paused on a human step
+        (the server may have restarted since the pause fired)."""
+        try:
+            from core.orchestration.state import SharedState
+            # Only an *unresolved* item counts as already-notified: a run that
+            # paused again at a later step deserves a fresh notification.
+            notified_runs = {
+                i["run_id"] for i in self._items
+                if i["kind"] == "human_input" and not i.get("resolved")
+            }
+            paused_waiting: set[str] = set()
+            for summary in SharedState.list_runs(limit=limit):
+                if summary.get("status") != "paused":
+                    continue
+                run_id = summary.get("run_id") or ""
+                try:
+                    run = SharedState.restore(run_id).run
+                except Exception:
+                    continue
+                if not run.waiting_for_human:
+                    continue
+                paused_waiting.add(run_id)
+                if run_id not in notified_runs:
+                    self.publish(
+                        "human_input",
+                        f"{self._orch_name(run_id)} needs your input",
+                        body=str(run.human_prompt or ""),
+                        run_id=run_id,
+                    )
+            # Inverse sweep: unresolved items whose run is no longer waiting
+            # (answered/failed/swept while we were down) get resolved.
+            for run_id in notified_runs - paused_waiting:
+                self._resolve_human_input(run_id)
+        except Exception as e:
+            print(f"Warning: notification reconstruction failed: {e}", flush=True)
+
+    # ── optional messaging dispatch ──────────────────────────────────────
+    def _dispatch_messaging(self, item: dict):
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            return
+        loop.create_task(self._send_messaging(item))
+
+    async def _send_messaging(self, item: dict):
+        try:
+            from core.config import load_settings
+            channel_id = (load_settings() or {}).get("run_notification_channel_id") or ""
+            if not channel_id:
+                return
+            manager = getattr(self._server_module, "messaging_manager", None)
+            adapter = manager._adapters.get(channel_id) if manager else None
+            if adapter is None:
+                return
+            from core.messaging import store as channel_store
+            channel = channel_store.get_channel(channel_id) or {}
+            chat_id = channel.get("notify_chat_id") or getattr(adapter, "_last_chat_id", None)
+            if not chat_id:
+                return
+            icon = {"human_input": "⏸", "run_completed": "✅", "run_failed": "❌"}.get(item["kind"], "🔔")
+            text = f"{icon} {item['title']}"
+            if item.get("body"):
+                text += f"\n{item['body']}"
+            await adapter.send_message(chat_id, text)
+        except Exception as e:
+            print(f"Warning: messaging notification failed: {e}", flush=True)
+
+
+hub = NotificationHub()

package/backend/core/orchestration/engine.py

@@ -78,6 +78,14 @@ class OrchestrationEngine:
         )
         state = SharedState(run)
 
+        # Initial checkpoint: make the run visible to /runs listings and
+        # status polling from second zero. Without it the run file only
+        # appears at the first step boundary, so a run inside a long first
+        # step is invisible to the active-runs UI (and a crash there leaves
+        # no file to resume from). The startup zombie sweep handles these
+        # like any other interrupted "running" checkpoint.
+        state.checkpoint()
+
         self.logger = OrchestrationLogger(
             run_id=run_id,
             orchestration_id=self.orch.id,
@@ -356,6 +364,13 @@ class OrchestrationEngine:
         For failed runs: the failed step's history entry is removed so it re-executes.
         For cancelled runs: resumes from current_step_id (which was never started).
         All state accumulated by prior steps is preserved.
+
+        Mid-node continuity: agent/loop/parallel executors checkpoint private
+        progress keys in shared_state (_step_progress_*, _loop_progress_*,
+        _parallel_progress_*) as work completes, so the re-executed node
+        continues from its last turn/iteration/branch instead of starting
+        from scratch. The seeding happens inside the executors — this method
+        just replays the node.
         """
         from .state import SharedState as SS, _cancelled_run_ids
 
@@ -395,6 +410,11 @@ class OrchestrationEngine:
         run.status = "running"
 
         state = SharedState(run)
+        # Persist "running" before the first step so the run reappears in the
+        # active-runs listing immediately. Without this the checkpoint still
+        # reads failed/cancelled until the first step boundary, and a resumed
+        # run looks dead for as long as its first step takes.
+        state.checkpoint()
         async for event in engine._execute_loop(run, state):
             yield event
 
@@ -466,6 +486,9 @@ class OrchestrationEngine:
 
         print(f"[engine.resume] 🚀 entering _execute_loop with current_step_id={run.current_step_id!r} status={run.status}", flush=True)
         state = SharedState(run)
+        # Persist "running" + the cleared human flag before the first step, so
+        # the run stops showing as "needs input" the moment it resumes.
+        state.checkpoint()
         async for event in engine._execute_loop(run, state):
             yield event
         print(f"[engine.resume] 🏁 _execute_loop finished, run.status={run.status} run.current_step_id={run.current_step_id!r}", flush=True)
@@ -581,6 +604,8 @@ class OrchestrationEngine:
             run.current_step_id = next_id
 
         state = SharedState(run)
+        # Same as resume(): make the un-paused state visible before step one.
+        state.checkpoint()
         async for event in engine._execute_loop(run, state):
             yield event