synapse-orch-ai 1.8.5 → 1.9.0

package/backend/core/models.py

@@ -131,6 +131,10 @@ class Settings(BaseModel):
     login_username: str = ""
     login_password_hash: str = ""
 
+    # Run notifications: messaging channel id to mirror run notifications to
+    # (human-input pauses, completions, failures). Empty = in-app only.
+    run_notification_channel_id: str = ""
+
     # Scale / distributed execution settings
     redis_url: str = ""
     scale_postgres_url: str = ""

package/backend/core/notifications.py

@@ -0,0 +1,251 @@
+"""User-facing run notifications.
+
+A process-wide hub that turns orchestration lifecycle events into
+notifications the UI (and optionally a messaging channel) can consume:
+
+- `human_input_required`  → "… needs your input"
+- `orchestration_complete`→ run completed / run failed
+- `orchestration_error`   → run failed
+
+Fed by the runner pump via `observe_run_event` (registered on
+`core.orchestration.runner.event_observers` at server startup), so every
+entry point — UI, API, scheduler — produces notifications with no extra
+wiring. Standalone-safe: a bounded in-memory ring with best-effort JSON
+persistence, no external dependencies.
+
+Delivery surfaces:
+- `GET /api/notifications` + `/api/notifications/stream` (routes/notifications.py)
+  for the in-app provider (toasts, badges, browser notifications).
+- Optional messaging dispatch: when settings.json has
+  `run_notification_channel_id` set to a configured messaging channel id,
+  each notification is also sent there (`notify_chat_id` or the channel's
+  last active chat).
+"""
+import asyncio
+import json
+import time
+from pathlib import Path
+
+NOTIFICATIONS_FILE = Path(__file__).parent.parent / "logs" / "notifications.json"
+CAPACITY = 200
+
+
+class NotificationHub:
+    def __init__(self, capacity: int = CAPACITY, persist_path: Path = NOTIFICATIONS_FILE):
+        self._capacity = capacity
+        self._path = persist_path
+        self._items: list[dict] = self._load()
+        self._next_id = (self._items[-1]["id"] + 1) if self._items else 1
+        self._subs: set[asyncio.Queue] = set()
+        # run_id → orchestration display name (cached from orchestration_start)
+        self._orch_names: dict[str, str] = {}
+        # Runs whose terminal notification already fired — engine failure paths
+        # can emit both orchestration_error and orchestration_complete(failed).
+        self._terminal_notified: set[str] = set()
+        self._server_module = None
+
+    # ── persistence ──────────────────────────────────────────────────────
+    def _load(self) -> list[dict]:
+        try:
+            data = json.loads(self._path.read_text(encoding="utf-8"))
+            return data if isinstance(data, list) else []
+        except Exception:
+            return []
+
+    def _persist(self):
+        try:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            self._path.write_text(json.dumps(self._items), encoding="utf-8")
+        except Exception:
+            pass  # best-effort — the ring keeps working in memory
+
+    # ── core API ─────────────────────────────────────────────────────────
+    def publish(self, kind: str, title: str, body: str = "",
+                run_id: str | None = None, data: dict | None = None) -> dict:
+        item = {
+            "id": self._next_id,
+            "ts": time.time(),
+            "kind": kind,          # human_input | run_completed | run_failed
+            "title": title[:200],
+            "body": (body or "")[:500],
+            "run_id": run_id,
+            "data": data or {},
+        }
+        self._next_id += 1
+        self._items.append(item)
+        if len(self._items) > self._capacity:
+            self._items = self._items[-self._capacity:]
+        self._persist()
+        for queue in list(self._subs):
+            queue.put_nowait(item)
+        self._dispatch_messaging(item)
+        return item
+
+    def list(self, after_id: int = 0) -> list[dict]:
+        return [i for i in self._items if i["id"] > after_id]
+
+    def last_id(self) -> int:
+        return self._next_id - 1
+
+    def subscribe(self) -> asyncio.Queue:
+        queue: asyncio.Queue = asyncio.Queue()
+        self._subs.add(queue)
+        return queue
+
+    def unsubscribe(self, queue: asyncio.Queue):
+        self._subs.discard(queue)
+
+    def configure(self, server_module):
+        """Give the hub access to the messaging manager (server startup)."""
+        self._server_module = server_module
+
+    # ── engine event observer (runner pump) ──────────────────────────────
+    def observe_run_event(self, run_id: str, event: dict):
+        etype = event.get("type", "")
+        if etype == "orchestration_start":
+            self._orch_names[run_id] = (
+                event.get("orchestration_name") or event.get("orchestration_id") or run_id
+            )
+            self._terminal_notified.discard(run_id)
+        elif etype == "step_start":
+            # Any forward progress re-arms terminal notification (a resumed
+            # run emits no orchestration_start) and means any pending
+            # human-input request for this run was answered.
+            self._terminal_notified.discard(run_id)
+            self._resolve_human_input(run_id)
+        elif etype == "human_input_required":
+            name = self._orch_name(run_id)
+            self.publish(
+                "human_input", f"{name} needs your input",
+                body=str(event.get("prompt") or ""),
+                run_id=run_id,
+                data={"step_id": event.get("orch_step_id")},
+            )
+        elif etype == "orchestration_complete":
+            self._resolve_human_input(run_id)
+            if run_id in self._terminal_notified:
+                return
+            self._terminal_notified.add(run_id)
+            name = self._orch_name(run_id)
+            status = event.get("status", "completed")
+            if status == "completed":
+                self.publish("run_completed", f"{name} completed", run_id=run_id)
+            else:
+                self.publish("run_failed", f"{name} {status}", run_id=run_id)
+        elif etype == "orchestration_error":
+            self._resolve_human_input(run_id)
+            if run_id in self._terminal_notified:
+                return
+            self._terminal_notified.add(run_id)
+            name = self._orch_name(run_id)
+            self.publish("run_failed", f"{name} failed",
+                         body=str(event.get("error") or ""), run_id=run_id)
+        elif etype == "run_stream_end" and not event.get("paused"):
+            self._orch_names.pop(run_id, None)
+
+    def _resolve_human_input(self, run_id: str):
+        """Mark this run's human-input notifications answered. The updated
+        items (same id, resolved=true) are re-pushed to live subscribers;
+        clients upsert by id so the bell stops showing them as actionable."""
+        updated = [
+            item for item in self._items
+            if item["run_id"] == run_id and item["kind"] == "human_input"
+            and not item.get("resolved")
+        ]
+        if not updated:
+            return
+        for item in updated:
+            item["resolved"] = True
+        self._persist()
+        for queue in list(self._subs):
+            for item in updated:
+                queue.put_nowait(item)
+
+    def _orch_name(self, run_id: str) -> str:
+        """Cached name, with a checkpoint fallback for resumed runs (which
+        emit no orchestration_start)."""
+        name = self._orch_names.get(run_id)
+        if name:
+            return name
+        try:
+            from core.orchestration.state import SharedState
+            from core.routes.orchestrations import load_orchestrations
+            orch_id = SharedState.restore(run_id).run.orchestration_id
+            orch = next((o for o in load_orchestrations() if o["id"] == orch_id), None)
+            name = (orch or {}).get("name") or orch_id
+        except Exception:
+            name = "Orchestration"
+        self._orch_names[run_id] = name
+        return name
+
+    # ── startup: reconstruct missed human-input notifications ────────────
+    def reconstruct_missed(self, limit: int = 100):
+        """Re-publish 'needs your input' for runs still paused on a human step
+        (the server may have restarted since the pause fired)."""
+        try:
+            from core.orchestration.state import SharedState
+            # Only an *unresolved* item counts as already-notified: a run that
+            # paused again at a later step deserves a fresh notification.
+            notified_runs = {
+                i["run_id"] for i in self._items
+                if i["kind"] == "human_input" and not i.get("resolved")
+            }
+            paused_waiting: set[str] = set()
+            for summary in SharedState.list_runs(limit=limit):
+                if summary.get("status") != "paused":
+                    continue
+                run_id = summary.get("run_id") or ""
+                try:
+                    run = SharedState.restore(run_id).run
+                except Exception:
+                    continue
+                if not run.waiting_for_human:
+                    continue
+                paused_waiting.add(run_id)
+                if run_id not in notified_runs:
+                    self.publish(
+                        "human_input",
+                        f"{self._orch_name(run_id)} needs your input",
+                        body=str(run.human_prompt or ""),
+                        run_id=run_id,
+                    )
+            # Inverse sweep: unresolved items whose run is no longer waiting
+            # (answered/failed/swept while we were down) get resolved.
+            for run_id in notified_runs - paused_waiting:
+                self._resolve_human_input(run_id)
+        except Exception as e:
+            print(f"Warning: notification reconstruction failed: {e}", flush=True)
+
+    # ── optional messaging dispatch ──────────────────────────────────────
+    def _dispatch_messaging(self, item: dict):
+        try:
+            loop = asyncio.get_running_loop()
+        except RuntimeError:
+            return
+        loop.create_task(self._send_messaging(item))
+
+    async def _send_messaging(self, item: dict):
+        try:
+            from core.config import load_settings
+            channel_id = (load_settings() or {}).get("run_notification_channel_id") or ""
+            if not channel_id:
+                return
+            manager = getattr(self._server_module, "messaging_manager", None)
+            adapter = manager._adapters.get(channel_id) if manager else None
+            if adapter is None:
+                return
+            from core.messaging import store as channel_store
+            channel = channel_store.get_channel(channel_id) or {}
+            chat_id = channel.get("notify_chat_id") or getattr(adapter, "_last_chat_id", None)
+            if not chat_id:
+                return
+            icon = {"human_input": "⏸", "run_completed": "✅", "run_failed": "❌"}.get(item["kind"], "🔔")
+            text = f"{icon} {item['title']}"
+            if item.get("body"):
+                text += f"\n{item['body']}"
+            await adapter.send_message(chat_id, text)
+        except Exception as e:
+            print(f"Warning: messaging notification failed: {e}", flush=True)
+
+
+hub = NotificationHub()

package/backend/core/orchestration/engine.py

@@ -78,6 +78,14 @@ class OrchestrationEngine:
         )
         state = SharedState(run)
 
+        # Initial checkpoint: make the run visible to /runs listings and
+        # status polling from second zero. Without it the run file only
+        # appears at the first step boundary, so a run inside a long first
+        # step is invisible to the active-runs UI (and a crash there leaves
+        # no file to resume from). The startup zombie sweep handles these
+        # like any other interrupted "running" checkpoint.
+        state.checkpoint()
+
         self.logger = OrchestrationLogger(
             run_id=run_id,
             orchestration_id=self.orch.id,
@@ -356,6 +364,13 @@ class OrchestrationEngine:
         For failed runs: the failed step's history entry is removed so it re-executes.
         For cancelled runs: resumes from current_step_id (which was never started).
         All state accumulated by prior steps is preserved.
+
+        Mid-node continuity: agent/loop/parallel executors checkpoint private
+        progress keys in shared_state (_step_progress_*, _loop_progress_*,
+        _parallel_progress_*) as work completes, so the re-executed node
+        continues from its last turn/iteration/branch instead of starting
+        from scratch. The seeding happens inside the executors — this method
+        just replays the node.
         """
         from .state import SharedState as SS, _cancelled_run_ids
 
@@ -395,6 +410,11 @@ class OrchestrationEngine:
         run.status = "running"
 
         state = SharedState(run)
+        # Persist "running" before the first step so the run reappears in the
+        # active-runs listing immediately. Without this the checkpoint still
+        # reads failed/cancelled until the first step boundary, and a resumed
+        # run looks dead for as long as its first step takes.
+        state.checkpoint()
         async for event in engine._execute_loop(run, state):
             yield event
 
@@ -466,6 +486,9 @@ class OrchestrationEngine:
 
         print(f"[engine.resume] 🚀 entering _execute_loop with current_step_id={run.current_step_id!r} status={run.status}", flush=True)
         state = SharedState(run)
+        # Persist "running" + the cleared human flag before the first step, so
+        # the run stops showing as "needs input" the moment it resumes.
+        state.checkpoint()
         async for event in engine._execute_loop(run, state):
             yield event
         print(f"[engine.resume] 🏁 _execute_loop finished, run.status={run.status} run.current_step_id={run.current_step_id!r}", flush=True)
@@ -581,6 +604,8 @@ class OrchestrationEngine:
             run.current_step_id = next_id
 
         state = SharedState(run)
+        # Same as resume(): make the un-paused state visible before step one.
+        state.checkpoint()
         async for event in engine._execute_loop(run, state):
             yield event
 

package/backend/core/orchestration/journal.py

@@ -0,0 +1,263 @@
+"""Append-only per-run event journal + in-process broker for live tailing.
+
+Every event the engine emits for a run is journaled as one JSONL line in
+`backend/logs/orchestration_events/{run_id}.jsonl`:
+
+    {"id": <monotonic seq>, "ts": <epoch seconds>, "event": {...}}
+
+The id is embedded in the line (not derived from the byte offset) so a torn
+final line — a crash mid-write — costs at most that one event: `read()` skips
+unparseable lines, and the next pump recovers the sequence from the last valid
+line and continues it. Run → resume → human-input pumps for the same run share
+one journal instance (see `get_journal`), so ids stay monotonic across the
+whole lifetime of a run.
+
+This is the standalone-mode counterpart of the Redis Stream published by scale
+workers (`core.scale.pubsub.RunEventPublisher`): the read endpoints in
+`routes/orchestrations.py` serve whichever of the two exists for a run.
+
+Builder flows (`builder_*` run ids) are intentionally NOT journaled: they are
+ephemeral UI conversations driven by `routes/builder.py`, which bypasses the
+runner pump on purpose.
+
+Writes are synchronous: a flushed append of a bounded line is microseconds,
+which is noise next to the LLM/tool latency between events, and it keeps this
+module free of extra dependencies. The pump wraps every append in try/except —
+a journal failure must never kill a run.
+"""
+import json
+import time
+from pathlib import Path
+
+import asyncio
+
+EVENTS_DIR = Path(__file__).parent.parent.parent / "logs" / "orchestration_events"
+
+# A single serialized event line larger than this gets its heavy fields
+# truncated (the UI fetches full run state from the checkpoint endpoint).
+MAX_EVENT_BYTES = 32 * 1024
+
+# Keys that carry potentially unbounded payloads, shrunk deepest-first when a
+# line exceeds MAX_EVENT_BYTES.
+_HEAVY_KEYS = ("final_state", "shared_state", "data", "preview", "response",
+               "reasoning", "thought", "output", "result", "agent_context")
+
+# After this many events a run journals only structural events — enough to
+# reconstruct the run's shape without letting a chatty agent grow the file
+# unboundedly.
+MAX_EVENTS_PER_RUN = 20_000
+
+STRUCTURAL_EVENT_TYPES = frozenset({
+    "orchestration_start", "step_start", "step_complete", "step_error",
+    "loop_iteration", "loop_complete", "loop_limit_reached",
+    "parallel_start", "parallel_complete", "branch_start", "merge_complete",
+    "routing_decision", "if_decision", "switch_decision",
+    "human_input_required", "orchestration_complete", "orchestration_error",
+    "orchestration_end", "final", "run_stream_end", "journal_truncated",
+})
+
+
+def _ensure_events_dir():
+    EVENTS_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def _truncate_strings(value, limit: int):
+    """Recursively cap every string in `value` to `limit` chars."""
+    if isinstance(value, str):
+        return value[:limit] + "…[truncated]" if len(value) > limit else value
+    if isinstance(value, dict):
+        return {k: _truncate_strings(v, limit) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_truncate_strings(v, limit) for v in value]
+    return value
+
+
+def _shrink(event: dict) -> dict:
+    """Bound an event to roughly MAX_EVENT_BYTES by truncating heavy fields.
+
+    The full payload still reaches the live SSE consumer untouched — only the
+    journaled copy is shrunk, and it is marked `"truncated": true` so replay
+    UIs can say so.
+    """
+    try:
+        if len(json.dumps(event, default=str)) <= MAX_EVENT_BYTES:
+            return event
+    except Exception:
+        return {"type": event.get("type", "unknown"), "truncated": True,
+                "error": "unserializable event"}
+    shrunk = dict(event)
+    for key in _HEAVY_KEYS:
+        if key in shrunk:
+            shrunk[key] = _truncate_strings(shrunk[key], 2_000)
+    shrunk["truncated"] = True
+    # Last resort: whole-event string cap if still oversized (e.g. many
+    # unlisted large fields).
+    if len(json.dumps(shrunk, default=str)) > MAX_EVENT_BYTES:
+        shrunk = {"type": event.get("type", "unknown"), "truncated": True}
+    return shrunk
+
+
+class FileRunJournal:
+    """JSONL journal for one run. Instances are shared via `get_journal`."""
+
+    def __init__(self, run_id: str):
+        self.run_id = run_id
+        self._path = EVENTS_DIR / f"{run_id}.jsonl"
+        self._fh = None
+        self._seq = self._recover_seq()
+        self._capped = self._seq >= MAX_EVENTS_PER_RUN
+        self._cap_marked = self._capped
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    def _recover_seq(self) -> int:
+        """Continue the id sequence from the last valid line of the file."""
+        if not self._path.exists():
+            return 0
+        try:
+            with open(self._path, "rb") as f:
+                f.seek(0, 2)
+                size = f.tell()
+                f.seek(max(0, size - 64 * 1024))
+                tail = f.read().decode("utf-8", errors="replace")
+        except OSError:
+            return 0
+        for line in reversed(tail.splitlines()):
+            try:
+                entry = json.loads(line)
+                return int(entry["id"])
+            except Exception:
+                continue  # torn/partial line — keep scanning backwards
+        return 0
+
+    def append(self, event: dict) -> int:
+        """Append one event; returns its journal id (0 = skipped by the cap)."""
+        etype = event.get("type", "")
+        if self._capped and etype not in STRUCTURAL_EVENT_TYPES:
+            return 0
+        _ensure_events_dir()
+        if self._fh is None:
+            self._fh = open(self._path, "a", encoding="utf-8")
+        if not self._capped and self._seq >= MAX_EVENTS_PER_RUN:
+            self._capped = True
+            if not self._cap_marked:
+                self._cap_marked = True
+                self._seq += 1
+                self._fh.write(json.dumps({
+                    "id": self._seq, "ts": time.time(),
+                    "event": {"type": "journal_truncated",
+                              "message": "event cap reached; only structural events are journaled from here"},
+                }) + "\n")
+            if etype not in STRUCTURAL_EVENT_TYPES:
+                self._fh.flush()
+                return 0
+        self._seq += 1
+        line = json.dumps(
+            {"id": self._seq, "ts": time.time(), "event": _shrink(event)},
+            default=str,
+        )
+        self._fh.write(line + "\n")
+        self._fh.flush()
+        return self._seq
+
+    def read(self, after_id: int = 0, limit: int = 5000) -> list[dict]:
+        """Return entries with id > after_id, oldest first, skipping torn lines."""
+        if self._fh is not None:
+            self._fh.flush()
+        if not self._path.exists():
+            return []
+        entries = []
+        try:
+            with open(self._path, "r", encoding="utf-8", errors="replace") as f:
+                for line in f:
+                    try:
+                        entry = json.loads(line)
+                    except Exception:
+                        continue  # torn line (crash mid-write)
+                    if entry.get("id", 0) > after_id:
+                        entries.append(entry)
+                        if len(entries) >= limit:
+                            break
+        except OSError:
+            return []
+        return entries
+
+    def last_id(self) -> int:
+        return self._seq
+
+    def close(self):
+        if self._fh is not None:
+            try:
+                self._fh.close()
+            except OSError:
+                pass
+            self._fh = None
+
+    @classmethod
+    def exists(cls, run_id: str) -> bool:
+        return (EVENTS_DIR / f"{run_id}.jsonl").exists()
+
+    @classmethod
+    def delete(cls, run_id: str) -> bool:
+        close_journal(run_id)
+        path = EVENTS_DIR / f"{run_id}.jsonl"
+        try:
+            path.unlink()
+            return True
+        except FileNotFoundError:
+            return False
+
+
+# ── Per-run journal registry ─────────────────────────────────────────
+# Run/resume/human-input pumps for one run share an instance so the id
+# sequence and file handle are continuous. Evicted when the pump ends.
+
+_journals: dict[str, FileRunJournal] = {}
+
+
+def get_journal(run_id: str) -> FileRunJournal:
+    journal = _journals.get(run_id)
+    if journal is None:
+        journal = FileRunJournal(run_id)
+        _journals[run_id] = journal
+    return journal
+
+
+def close_journal(run_id: str):
+    journal = _journals.pop(run_id, None)
+    if journal is not None:
+        journal.close()
+
+
+class RunEventBroker:
+    """In-process pub/sub keyed by run_id, for live-tailing SSE endpoints.
+
+    Subscribers get `{"id": n, "event": {...}}` entries. Queues are unbounded:
+    entries are small (journal-shrunk) and subscribers are SSE writers that
+    drain continuously; dropping events would break the replay→live dedupe
+    contract (monotonic ids with no gaps after the subscription point).
+    """
+
+    def __init__(self):
+        self._subs: dict[str, set[asyncio.Queue]] = {}
+
+    def publish(self, run_id: str, entry: dict):
+        for queue in self._subs.get(run_id, ()):
+            queue.put_nowait(entry)
+
+    def subscribe(self, run_id: str) -> asyncio.Queue:
+        queue: asyncio.Queue = asyncio.Queue()
+        self._subs.setdefault(run_id, set()).add(queue)
+        return queue
+
+    def unsubscribe(self, run_id: str, queue: asyncio.Queue):
+        subs = self._subs.get(run_id)
+        if subs is not None:
+            subs.discard(queue)
+            if not subs:
+                self._subs.pop(run_id, None)
+
+
+broker = RunEventBroker()

package/backend/core/orchestration/runner.py

@@ -8,11 +8,17 @@ in a fresh task raises "Attempted to exit cancel scope in a different task"
 (issue #356).
 
 Every orchestration entry point goes through here, so the launch path can no
-longer change how the engine is driven.
+longer change how the engine is driven. That makes the pump the single choke
+point for run observability: every event is appended to the run's journal
+(`journal.FileRunJournal`) and fanned out to live tails (`journal.broker`)
+before it reaches the SSE queue. A journal/broker/observer failure must never
+kill a run -- all three are best-effort.
 """
 import asyncio
 from typing import AsyncGenerator
 
+from core.orchestration.journal import broker, close_journal, get_journal
+
 # Live in-process runs, keyed by run_id. The cancel endpoint cancels the task
 # to interrupt an in-progress await; the run-status endpoint consults it to
 # report a run that is still inside its first step (no checkpoint yet).
@@ -21,6 +27,29 @@ _active_tasks: dict[str, asyncio.Task] = {}
 # Queue marker meaning "the engine generator is exhausted".
 SENTINEL = object()
 
+# Observers called as fn(run_id, event) for every pumped event (best-effort).
+# core.notifications registers here to derive user-facing notifications from
+# lifecycle events without the pump knowing about notification semantics.
+event_observers: list = []
+
+
+def _observe(run_id: str, event: dict):
+    """Journal + broadcast + notify for one event. Never raises."""
+    entry_id = 0
+    try:
+        entry_id = get_journal(run_id).append(event)
+    except Exception as e:
+        print(f"WARNING: journal append failed for '{run_id}': {e}", flush=True)
+    try:
+        broker.publish(run_id, {"id": entry_id, "event": event})
+    except Exception:
+        pass
+    for observer in event_observers:
+        try:
+            observer(run_id, event)
+        except Exception:
+            pass
+
 
 def spawn_engine_run(agen, run_id: str) -> tuple[asyncio.Task, asyncio.Queue]:
     """Advance `agen` in one dedicated task, returning (task, queue).
@@ -32,19 +61,39 @@ def spawn_engine_run(agen, run_id: str) -> tuple[asyncio.Task, asyncio.Queue]:
     queue: asyncio.Queue = asyncio.Queue()
 
     async def _pump():
+        last_type = None
         try:
             async for event in agen:
+                _observe(run_id, event)
+                last_type = event.get("type")
                 await queue.put(event)
                 # Yield so the SSE consumer can dequeue and flush this event
                 # to the HTTP response before the next one is enqueued.
                 await asyncio.sleep(0)
         except asyncio.CancelledError:
-            await queue.put({"type": "orchestration_error", "error": "Cancelled"})
+            event = {"type": "orchestration_error", "error": "Cancelled"}
+            _observe(run_id, event)
+            last_type = event["type"]
+            await queue.put(event)
         except FileNotFoundError:
-            await queue.put({"type": "orchestration_error", "error": "Run not found"})
+            event = {"type": "orchestration_error", "error": "Run not found"}
+            _observe(run_id, event)
+            last_type = event["type"]
+            await queue.put(event)
         except Exception as e:
-            await queue.put({"type": "orchestration_error", "error": str(e)})
+            event = {"type": "orchestration_error", "error": str(e)}
+            _observe(run_id, event)
+            last_type = event["type"]
+            await queue.put(event)
         finally:
+            # Journal/broker-only terminator (never sent on the POST SSE):
+            # tells tailing GET clients whether to stay open for a resume pump
+            # (paused on human input) or close (terminal).
+            _observe(run_id, {
+                "type": "run_stream_end",
+                "paused": last_type == "human_input_required",
+            })
+            close_journal(run_id)
             _active_tasks.pop(run_id, None)
             print(f"DEBUG SSE QUEUE: sentinel sent for '{run_id}', stream closing", flush=True)
             await queue.put(SENTINEL)