synapse-orch-ai 1.8.4 → 1.8.6

package/backend/core/orchestration/engine.py

@@ -182,12 +182,18 @@ class OrchestrationEngine:
                 # This properly interrupts a stuck session.call_tool() inside
                 # the executor when the step deadline is reached.
                 #
-                # IMPORTANT: never yield human_input_required inside the
-                # fail_after scope. Python's GC finalizer runs aclose() on
-                # abandoned async generators in a NEW asyncio Task; if a
-                # cancel scope is active at the suspension point, anyio raises
-                # "cancel scope in a different task". We break out of the scope
-                # cleanly (in the current task) and yield the event below.
+                # IMPORTANT: this scope stays open across the yields below, so
+                # this generator must be advanced — and closed — from a single
+                # task, or anyio raises "cancel scope in a different task".
+                # core.orchestration.runner is what guarantees that; every
+                # entry point runs us through it (issue #356). Do not drive
+                # the engine directly from a task-per-pull consumer.
+                #
+                # human_input_required is still yielded outside the scope: the
+                # generator is abandoned at that point, and Python's GC
+                # finalizer runs aclose() on abandoned async generators in a
+                # NEW asyncio Task. We break out of the scope cleanly (in the
+                # current task) and yield the event below.
                 human_input_event: dict | None = None
                 try:
                     with anyio.fail_after(step_timeout):

package/backend/core/orchestration/runner.py

@@ -0,0 +1,70 @@
+"""Single-task runner for orchestration engine streams.
+
+The engine holds anyio cancel scopes across `yield`s (the per-step timeout at
+`engine._execute_loop`). anyio binds a cancel scope to the task that entered
+it, so the engine's generator must be advanced -- and closed -- from ONE task,
+whatever the HTTP layer does with the events. A consumer that pulls each event
+in a fresh task raises "Attempted to exit cancel scope in a different task"
+(issue #356).
+
+Every orchestration entry point goes through here, so the launch path can no
+longer change how the engine is driven.
+"""
+import asyncio
+from typing import AsyncGenerator
+
+# Live in-process runs, keyed by run_id. The cancel endpoint cancels the task
+# to interrupt an in-progress await; the run-status endpoint consults it to
+# report a run that is still inside its first step (no checkpoint yet).
+_active_tasks: dict[str, asyncio.Task] = {}
+
+# Queue marker meaning "the engine generator is exhausted".
+SENTINEL = object()
+
+
+def spawn_engine_run(agen, run_id: str) -> tuple[asyncio.Task, asyncio.Queue]:
+    """Advance `agen` in one dedicated task, returning (task, queue).
+
+    The task is detached: it runs to completion even if nobody drains the
+    queue, so a client disconnect never kills a run. Only the cancel endpoint
+    stops it.
+    """
+    queue: asyncio.Queue = asyncio.Queue()
+
+    async def _pump():
+        try:
+            async for event in agen:
+                await queue.put(event)
+                # Yield so the SSE consumer can dequeue and flush this event
+                # to the HTTP response before the next one is enqueued.
+                await asyncio.sleep(0)
+        except asyncio.CancelledError:
+            await queue.put({"type": "orchestration_error", "error": "Cancelled"})
+        except FileNotFoundError:
+            await queue.put({"type": "orchestration_error", "error": "Run not found"})
+        except Exception as e:
+            await queue.put({"type": "orchestration_error", "error": str(e)})
+        finally:
+            _active_tasks.pop(run_id, None)
+            print(f"DEBUG SSE QUEUE: sentinel sent for '{run_id}', stream closing", flush=True)
+            await queue.put(SENTINEL)
+
+    task = asyncio.create_task(_pump())
+    _active_tasks[run_id] = task
+    return task, queue
+
+
+async def stream_engine_events(agen, run_id: str) -> AsyncGenerator[dict, None]:
+    """Yield the engine's event dicts while a dedicated task advances it.
+
+    Deliberately does NOT cancel that task on close: the run is detached, so
+    abandoning this generator (client disconnect, GC finalization) must leave
+    it running -- the run still checkpoints and stays cancellable and
+    resumable.
+    """
+    _task, queue = spawn_engine_run(agen, run_id)
+    while True:
+        event = await queue.get()
+        if event is SENTINEL:
+            return
+        yield event

package/backend/core/react_engine.py

@@ -4,6 +4,7 @@ Yields structured event dicts that callers can handle differently
 (collect for sync response, or stream as SSE).
 """
 import asyncio
+import contextlib
 import json
 import sys
 import time
@@ -47,50 +48,74 @@ MAX_TURNS = 30
 # Heartbeats are yielded as the raw SSE string, so endpoints can pass them
 # straight through; real events remain dicts (`isinstance(item, str)`
 # distinguishes them).
+#
+# The source is advanced by a single dedicated pump task rather than a fresh
+# task per pull. anyio binds a cancel scope to the task that entered it, so a
+# source holding a scope across a yield (the orchestration engine's per-step
+# timeout) must be advanced — and closed — from one task. See issue #356.
 # ──────────────────────────────────────────────────────────────────────────────
 SSE_HEARTBEAT = ": ping\n\n"
 
 
+class _StreamEnd:
+    """Sentinel handing the source's terminal state back to the consumer."""
+    __slots__ = ("error",)
+
+    def __init__(self, error):
+        self.error = error
+
+
 async def iter_with_heartbeat(agen, interval: float = 10.0):
     """Yield items from `agen`, injecting `SSE_HEARTBEAT` whenever no item
     arrives within `interval` seconds.
 
     The underlying generator is never cancelled on a heartbeat timeout — only
     when the consumer stops early (e.g. a client disconnect closes this
-    generator), in which case the pending pull is cancelled and the source is
-    closed in the finally block.
+    generator), in which case the pump is cancelled and closes the source.
     """
-    ait = agen.__aiter__()
-    pending = None
+    queue: asyncio.Queue = asyncio.Queue(maxsize=1)  # maxsize=1 keeps backpressure
+
+    async def _pump():
+        # Advances `agen` in THIS single task, so a cancel scope the source
+        # holds across a yield is never exited from another task — and
+        # aclose() below runs in that scope's host task too.
+        try:
+            async for item in agen:
+                await queue.put(item)
+            await queue.put(_StreamEnd(None))
+        except asyncio.CancelledError:
+            raise
+        except BaseException as exc:
+            with contextlib.suppress(BaseException):
+                await queue.put(_StreamEnd(exc))
+        finally:
+            aclose = getattr(agen, "aclose", None)
+            if aclose is not None:
+                with contextlib.suppress(BaseException):
+                    await aclose()
+
+    pump = asyncio.ensure_future(_pump())
     try:
         while True:
-            if pending is None:
-                pending = asyncio.ensure_future(ait.__anext__())
-            done, _ = await asyncio.wait({pending}, timeout=interval)
-            if not done:
+            try:
+                item = await asyncio.wait_for(queue.get(), timeout=interval)
+            except asyncio.TimeoutError:
+                if pump.done():
+                    # Source ended without a sentinel — it was cancelled.
+                    return
                 # No event within the interval — keep the stream warm.
                 yield SSE_HEARTBEAT
                 continue
-            task = pending
-            pending = None
-            try:
-                item = task.result()
-            except StopAsyncIteration:
+            if isinstance(item, _StreamEnd):
+                if item.error is not None:
+                    raise item.error
                 return
             yield item
     finally:
-        if pending is not None:
-            pending.cancel()
-            try:
-                await pending
-            except BaseException:
-                pass
-        aclose = getattr(agen, "aclose", None)
-        if aclose is not None:
-            try:
-                await aclose()
-            except BaseException:
-                pass
+        if not pump.done():
+            pump.cancel()
+        with contextlib.suppress(BaseException):
+            await pump
 
 
 async def drain_queue_with_heartbeat(queue, sentinel, interval: float = 10.0):
@@ -1591,6 +1616,7 @@ async def run_react_loop(request, server_module):
                 from core.routes.orchestrations import load_orchestrations
                 from core.models_orchestration import Orchestration
                 from core.orchestration.engine import OrchestrationEngine
+                from core.orchestration.runner import stream_engine_events
 
                 orchs = load_orchestrations()
                 orch_data = next((o for o in orchs if o["id"] == orch_id), None)
@@ -1598,7 +1624,14 @@ async def run_react_loop(request, server_module):
                     orch = Orchestration.model_validate(orch_data)
                     engine = OrchestrationEngine(orch, server_module)
                     run_id = f"run_{orch_id}_{int(time.time() * 1000)}"
-                    async for event in engine.run(user_message, run_id, session_id=session_id):
+                    # Route through the shared runner, exactly as the editor's
+                    # run endpoint does: the engine holds anyio cancel scopes
+                    # across yields, so it must be advanced from a single task
+                    # no matter how the SSE layer consumes us (issue #356).
+                    # It also registers the run for cancel/reconnect.
+                    async for event in stream_engine_events(
+                        engine.run(user_message, run_id, session_id=session_id), run_id
+                    ):
                         yield event
                     return
                 else:

package/backend/core/routes/api_v1.py

@@ -113,8 +113,9 @@ async def v1_chat(body: V1ChatRequest, key_record: dict = Depends(require_api_ke
             etype = event.get("type", "")
             if etype == "final":
                 final_event = event
-            elif etype == "error":
-                error_msg = event.get("message", "Unknown error")
+            elif etype in ("error", "orchestration_error"):
+                # orchestration_error is what an orchestrator agent fails with.
+                error_msg = event.get("message") or event.get("error") or "Unknown error"
     except Exception as exc:
         log.exception("[v1/chat] Unhandled error for session=%s", chat_request.session_id)
         raise HTTPException(status_code=500, detail="An internal error occurred. Check server logs for details.")
@@ -204,8 +205,10 @@ async def v1_chat_stream(body: V1ChatRequest, key_record: dict = Depends(require
                     })
                     yield _format_sse_event({"type": "done"})
 
-                elif etype == "error":
-                    log.error("[v1/chat/stream] Agent error session=%s: %s", chat_request.session_id, event.get("message"))
+                elif etype in ("error", "orchestration_error"):
+                    # orchestration_error is what an orchestrator agent fails with.
+                    log.error("[v1/chat/stream] Agent error session=%s: %s", chat_request.session_id,
+                              event.get("message") or event.get("error"))
                     yield _format_sse_event({"type": "error", "message": "The agent encountered an error processing your request."})
 
                 await asyncio.sleep(0)
@@ -314,8 +317,11 @@ async def v1_orchestration_run_stream(
 
     async def event_stream():
         from core.react_engine import iter_with_heartbeat
+        from core.orchestration.runner import stream_engine_events
         try:
-            async for event in iter_with_heartbeat(engine.run(body.message, run_id)):
+            async for event in iter_with_heartbeat(
+                stream_engine_events(engine.run(body.message, run_id), run_id)
+            ):
                 # Heartbeat comments are yielded as raw SSE strings — pass through.
                 if isinstance(event, str):
                     yield event
@@ -436,8 +442,13 @@ async def v1_orchestration_resume_stream(
 
     async def event_stream():
         from core.react_engine import iter_with_heartbeat
+        from core.orchestration.runner import stream_engine_events
         try:
-            async for event in iter_with_heartbeat(OrchestrationEngine.resume(run_id, human_response, _server)):
+            async for event in iter_with_heartbeat(
+                stream_engine_events(
+                    OrchestrationEngine.resume(run_id, human_response, _server), run_id
+                )
+            ):
                 # Heartbeat comments are yielded as raw SSE strings — pass through.
                 if isinstance(event, str):
                     yield event

package/backend/core/routes/chat.py

@@ -39,6 +39,9 @@ async def chat(request: ChatRequest):
             final_event = event
         elif event["type"] == "error":
             return ChatResponse(response=event["message"], intent="chat", data=None, tool_name=None)
+        elif event["type"] == "orchestration_error":
+            # An orchestrator agent failed — same outcome, different event shape.
+            return ChatResponse(response=event["error"], intent="chat", data=None, tool_name=None)
 
     if not final_event:
         return ChatResponse(response="I completed the requested actions.", intent="chat", data=None, tool_name=None)

package/backend/core/routes/orchestrations.py

@@ -1,7 +1,6 @@
 """
 Orchestration management endpoints: CRUD, run, human-input, cancel.
 """
-import asyncio
 import os
 import json
 import time
@@ -13,14 +12,14 @@ from core.models_orchestration import Orchestration
 from core.config import DATA_DIR
 from core.json_store import JsonStore
 from core.react_engine import drain_queue_with_heartbeat
+# Registry of active run tasks (for mid-step cancellation), owned by the runner
+# so non-route callers can register runs too.
+from core.orchestration.runner import SENTINEL, _active_tasks, spawn_engine_run
 
 router = APIRouter()
 
 _orch_store = JsonStore(os.path.join(DATA_DIR, "orchestrations.json"), cache_ttl=2.0)
 
-# In-memory registry of active run tasks (for mid-step cancellation)
-_active_tasks: dict[str, asyncio.Task] = {}
-
 
 def load_orchestrations() -> list[dict]:
     return _orch_store.load()
@@ -153,36 +152,14 @@ async def run_orchestration(orch_id: str, request: Request):
     from core.orchestration.engine import OrchestrationEngine
     engine = OrchestrationEngine(orch, server_module)
 
-    queue: asyncio.Queue = asyncio.Queue()
-    _SENTINEL = object()
-
-    async def _run_engine():
-        try:
-            async for event in engine.run(user_input, run_id):
-                etype = event.get("type", "")
-                if etype not in ("chunk", "thinking", "token_usage"):
-                    print(f"DEBUG SSE QUEUE: → {etype} step={event.get('orch_step_id', '')}", flush=True)
-                await queue.put(event)
-                # Yield so event_stream() can dequeue and flush this event
-                # to the HTTP response before the next event is enqueued.
-                await asyncio.sleep(0)
-        except asyncio.CancelledError:
-            await queue.put({"type": "orchestration_error", "error": "Cancelled"})
-        except Exception as e:
-            await queue.put({"type": "orchestration_error", "error": str(e)})
-        finally:
-            _active_tasks.pop(run_id, None)
-            print(f"DEBUG SSE QUEUE: sentinel sent, stream closing", flush=True)
-            await queue.put(_SENTINEL)
-
-    # Engine runs independently of SSE consumer; store task for cancellation
-    task = asyncio.create_task(_run_engine())
-    _active_tasks[run_id] = task
+    # Engine runs in its own task, independently of the SSE consumer, and is
+    # registered for cancellation.
+    _task, queue = spawn_engine_run(engine.run(user_input, run_id), run_id)
 
     async def event_stream():
         # Heartbeats keep the stream warm during long LLM/compaction steps so a
         # busy-but-alive run is not mistaken for a dropped connection.
-        async for chunk in drain_queue_with_heartbeat(queue, _SENTINEL):
+        async for chunk in drain_queue_with_heartbeat(queue, SENTINEL):
             yield chunk
         yield "data: {\"type\": \"done\"}\n\n"
 
@@ -222,29 +199,14 @@ async def resume_failed_run(run_id: str, request: Request):
 
     from core.orchestration.engine import OrchestrationEngine
 
-    queue: asyncio.Queue = asyncio.Queue()
-    _SENTINEL = object()
-
-    async def _run_engine():
-        try:
-            async for event in OrchestrationEngine.resume_failed(run_id, server_module):
-                await queue.put(event)
-                await asyncio.sleep(0)
-        except FileNotFoundError:
-            await queue.put({"type": "orchestration_error", "error": "Run not found"})
-        except Exception as e:
-            await queue.put({"type": "orchestration_error", "error": str(e)})
-        finally:
-            _active_tasks.pop(run_id, None)
-            await queue.put(_SENTINEL)
-
-    task = asyncio.create_task(_run_engine())
-    _active_tasks[run_id] = task
+    _task, queue = spawn_engine_run(
+        OrchestrationEngine.resume_failed(run_id, server_module), run_id
+    )
 
     async def event_stream():
         # Heartbeats keep the stream warm during long LLM/compaction steps so a
         # busy-but-alive run is not mistaken for a dropped connection.
-        async for chunk in drain_queue_with_heartbeat(queue, _SENTINEL):
+        async for chunk in drain_queue_with_heartbeat(queue, SENTINEL):
             yield chunk
         yield "data: {\"type\": \"done\"}\n\n"
 
@@ -341,27 +303,14 @@ async def submit_human_input(run_id: str, request: Request):
     # --- V1 in-process path (unchanged) ---
     from core.orchestration.engine import OrchestrationEngine
 
-    queue: asyncio.Queue = asyncio.Queue()
-    _SENTINEL = object()
-
-    async def _run_engine():
-        try:
-            async for event in OrchestrationEngine.resume(run_id, human_response, server_module):
-                await queue.put(event)
-                await asyncio.sleep(0)
-        except FileNotFoundError:
-            await queue.put({"type": "orchestration_error", "error": "Run not found"})
-        except Exception as e:
-            await queue.put({"type": "orchestration_error", "error": str(e)})
-        finally:
-            await queue.put(_SENTINEL)
-
-    asyncio.create_task(_run_engine())
+    _task, queue = spawn_engine_run(
+        OrchestrationEngine.resume(run_id, human_response, server_module), run_id
+    )
 
     async def event_stream():
         # Heartbeats keep the stream warm during long LLM/compaction steps so a
         # busy-but-alive run is not mistaken for a dropped connection.
-        async for chunk in drain_queue_with_heartbeat(queue, _SENTINEL):
+        async for chunk in drain_queue_with_heartbeat(queue, SENTINEL):
             yield chunk
         yield "data: {\"type\": \"done\"}\n\n"
 

package/backend/requirements-coding.txt

@@ -1,4 +1,6 @@
-psycopg[binary]
-psycopg_pool
+# Coding-agent extras (code indexing). Installed alongside requirements.txt.
+# Upper bounds are mandatory here too — see the header of requirements.txt.
+psycopg[binary]>=3.0,<4
+psycopg_pool>=3.0,<4
 cocoindex>=0.3.30,<1.0
-numpy
+numpy>=1.26,<3

package/backend/requirements-messaging.lock

@@ -0,0 +1,167 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --universal --python-version 3.11 requirements-messaging.txt --constraint requirements.lock -o requirements-messaging.lock
+aiohappyeyeballs==2.7.1
+    # via
+    #   -c requirements.lock
+    #   aiohttp
+aiohttp==3.14.3
+    # via
+    #   -c requirements.lock
+    #   aiohttp-retry
+    #   botbuilder-integration-aiohttp
+    #   discord-py
+    #   twilio
+aiohttp-retry==2.9.1
+    # via twilio
+aiosignal==1.4.0
+    # via
+    #   -c requirements.lock
+    #   aiohttp
+anyio==4.14.2
+    # via
+    #   -c requirements.lock
+    #   httpx
+attrs==26.1.0
+    # via
+    #   -c requirements.lock
+    #   aiohttp
+audioop-lts==0.2.2 ; python_full_version >= '3.13'
+    # via discord-py
+azure-core==1.41.0
+    # via msrest
+botbuilder-core==4.17.1
+    # via botbuilder-integration-aiohttp
+botbuilder-integration-aiohttp==4.17.1
+    # via -r requirements-messaging.txt
+botbuilder-schema==4.17.1
+    # via
+    #   botbuilder-core
+    #   botbuilder-integration-aiohttp
+    #   botframework-connector
+    #   botframework-streaming
+botframework-connector==4.17.1
+    # via
+    #   -r requirements-messaging.txt
+    #   botbuilder-core
+    #   botbuilder-integration-aiohttp
+    #   botframework-streaming
+botframework-streaming==4.17.1
+    # via botbuilder-core
+certifi==2026.7.22
+    # via
+    #   -c requirements.lock
+    #   httpcore
+    #   httpx
+    #   msrest
+    #   requests
+cffi==2.1.1 ; platform_python_implementation != 'PyPy'
+    # via
+    #   -c requirements.lock
+    #   cryptography
+charset-normalizer==3.4.9
+    # via
+    #   -c requirements.lock
+    #   requests
+cryptography==50.0.0
+    # via
+    #   -c requirements.lock
+    #   msal
+    #   pyjwt
+discord-py==2.7.1
+    # via -r requirements-messaging.txt
+frozenlist==1.8.0
+    # via
+    #   -c requirements.lock
+    #   aiohttp
+    #   aiosignal
+h11==0.16.0
+    # via
+    #   -c requirements.lock
+    #   httpcore
+httpcore==1.0.9
+    # via
+    #   -c requirements.lock
+    #   httpx
+    #   python-telegram-bot
+httpx==0.28.1
+    # via
+    #   -c requirements.lock
+    #   python-telegram-bot
+idna==3.18
+    # via
+    #   -c requirements.lock
+    #   anyio
+    #   httpx
+    #   requests
+    #   yarl
+isodate==0.7.2
+    # via msrest
+jsonpickle==3.4.2
+    # via botbuilder-core
+msal==1.37.0
+    # via botframework-connector
+msrest==0.7.1
+    # via
+    #   botbuilder-schema
+    #   botframework-connector
+multidict==6.7.1
+    # via
+    #   -c requirements.lock
+    #   aiohttp
+    #   yarl
+oauthlib==3.3.1
+    # via
+    #   -c requirements.lock
+    #   requests-oauthlib
+propcache==0.5.2
+    # via
+    #   -c requirements.lock
+    #   aiohttp
+    #   yarl
+pycparser==3.0 ; implementation_name != 'PyPy' and platform_python_implementation != 'PyPy'
+    # via
+    #   -c requirements.lock
+    #   cffi
+pyjwt==2.13.0
+    # via
+    #   -c requirements.lock
+    #   botframework-connector
+    #   msal
+    #   twilio
+python-telegram-bot==22.8
+    # via -r requirements-messaging.txt
+requests==2.34.2
+    # via
+    #   -c requirements.lock
+    #   azure-core
+    #   msal
+    #   msrest
+    #   requests-oauthlib
+    #   twilio
+requests-oauthlib==2.0.0
+    # via
+    #   -c requirements.lock
+    #   msrest
+slack-bolt==1.30.0
+    # via -r requirements-messaging.txt
+slack-sdk==3.43.0
+    # via slack-bolt
+twilio==9.10.9
+    # via -r requirements-messaging.txt
+typing-extensions==4.16.0
+    # via
+    #   -c requirements.lock
+    #   aiohttp
+    #   aiosignal
+    #   anyio
+    #   azure-core
+urllib3==2.7.0
+    # via
+    #   -c requirements.lock
+    #   botbuilder-schema
+    #   requests
+yarl==1.24.5
+    # via
+    #   -c requirements.lock
+    #   aiohttp
+    #   botbuilder-integration-aiohttp

package/backend/requirements-messaging.txt

@@ -1,6 +1,8 @@
-python-telegram-bot>=20.0
-discord.py>=2.3
-slack-bolt>=1.18
-botframework-connector>=4.15
-botbuilder-integration-aiohttp>=4.15
-twilio>=8.0
+# Messaging-channel extras (Telegram / Discord / Slack / Teams / Twilio).
+# Upper bounds are mandatory here too — see the header of requirements.txt.
+python-telegram-bot>=20.0,<23
+discord.py>=2.3,<3
+slack-bolt>=1.18,<2
+botframework-connector>=4.15,<5
+botbuilder-integration-aiohttp>=4.15,<5
+twilio>=8.0,<10