synapse-orch-ai 1.6.6 → 1.7.0

package/backend/core/agent_logger.py

@@ -96,6 +96,37 @@ class AgentLogger:
 {'='*80}
 """)
 
+    def close(self) -> None:
+        """Drain the write queue, then upload the completed log to S3 (scale mode)."""
+        self._q.put(None)  # poison pill — stops the drain thread
+        self._thread.join(timeout=10)
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3 and self.path.exists():
+                head = self.path.read_text(encoding="utf-8", errors="replace")[:1000]
+
+                def _extract(label: str) -> str:
+                    for line in head.split("\n"):
+                        if label in line:
+                            return line.split(":", 1)[1].strip()
+                    return ""
+
+                s3.upload_text(
+                    f"logs/agent/{self.path.name}",
+                    self.path.read_text(encoding="utf-8"),
+                    metadata={
+                        "agent_name": _extract("Agent Name      :"),
+                        "agent_id": _extract("Agent ID        :"),
+                        "source": _extract("Source          :"),
+                        "session_id": _extract("Session ID      :"),
+                        "started_at": _extract("Started at      :"),
+                        "user_input": _extract("User Input      :")[:200],
+                    },
+                )
+        except Exception:
+            pass  # S3 upload failure must never surface to callers
+
     # ── Event logging ──────────────────────────────────────────────
 
     def log_event(self, event: dict):
@@ -212,44 +243,101 @@ class AgentLogger:
     @staticmethod
     def get_log(run_id: str) -> str | None:
         path = LOGS_DIR / f"{run_id}.log"
-        if not path.exists():
-            return None
-        return path.read_text(encoding="utf-8")
+        if path.exists():
+            return path.read_text(encoding="utf-8")
+        # Fall back to S3 for logs written by other workers
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                return s3.download_text(f"logs/agent/{run_id}.log")
+        except Exception:
+            pass
+        return None
 
     @staticmethod
     def list_logs(limit: int = 100, offset: int = 0) -> list[dict]:
         _ensure_logs_dir()
-        logs = []
+
+        def _parse_head(head: str, run_id: str, size_kb: float) -> dict:
+            def _extract(label: str) -> str:
+                for line in head.split("\n"):
+                    if label in line:
+                        return line.split(":", 1)[1].strip()
+                return ""
+            return {
+                "run_id": run_id,
+                "agent_name": _extract("Agent Name      :"),
+                "agent_id": _extract("Agent ID        :"),
+                "source": _extract("Source          :"),
+                "session_id": _extract("Session ID      :"),
+                "started_at": _extract("Started at      :"),
+                "user_input": _extract("User Input      :")[:200],
+                "file_size_kb": size_kb,
+            }
+
+        # Collect local entries
+        local_ids: set[str] = set()
+        logs: list[dict] = []
         files = sorted(LOGS_DIR.glob("*.log"), key=lambda p: p.stat().st_mtime, reverse=True)
-        for f in files[offset : offset + limit]:
+        for f in files:
             run_id = f.stem
+            local_ids.add(run_id)
             try:
                 head = f.read_text(encoding="utf-8", errors="replace")[:1000]
-
-                def _extract(label: str) -> str:
-                    for line in head.split("\n"):
-                        if label in line:
-                            return line.split(":", 1)[1].strip()
-                    return ""
-
-                logs.append({
-                    "run_id": run_id,
-                    "agent_name": _extract("Agent Name      :"),
-                    "agent_id": _extract("Agent ID        :"),
-                    "source": _extract("Source          :"),
-                    "session_id": _extract("Session ID      :"),
-                    "started_at": _extract("Started at      :"),
-                    "user_input": _extract("User Input      :")[:200],
-                    "file_size_kb": round(f.stat().st_size / 1024, 1),
-                })
+                logs.append(_parse_head(head, run_id, round(f.stat().st_size / 1024, 1)))
             except Exception:
-                logs.append({"run_id": run_id})
-        return logs
+                logs.append({"run_id": run_id, "file_size_kb": 0})
+
+        # Merge S3 entries that aren't already local
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                from concurrent.futures import ThreadPoolExecutor
+                s3_keys = s3.list_keys("logs/agent/")
+                missing_keys = [k for k in s3_keys if k.endswith(".log") and Path(k).stem not in local_ids]
+
+                def _fetch_meta(rel_key: str) -> dict:
+                    run_id = Path(rel_key).stem
+                    try:
+                        meta = s3.get_metadata(rel_key) or {}
+                        return {
+                            "run_id": run_id,
+                            "agent_name": meta.get("agent_name", ""),
+                            "agent_id": meta.get("agent_id", ""),
+                            "source": meta.get("source", ""),
+                            "session_id": meta.get("session_id", ""),
+                            "started_at": meta.get("started_at", ""),
+                            "user_input": meta.get("user_input", "")[:200],
+                            "file_size_kb": 0,
+                        }
+                    except Exception:
+                        return {"run_id": run_id}
+
+                with ThreadPoolExecutor(max_workers=10) as pool:
+                    s3_entries = list(pool.map(_fetch_meta, missing_keys))
+                logs.extend(s3_entries)
+        except Exception:
+            pass
+
+        # Sort by started_at descending (local files already sorted; blend with S3)
+        logs.sort(key=lambda x: x.get("started_at", ""), reverse=True)
+        return logs[offset: offset + limit]
 
     @staticmethod
     def delete_log(run_id: str) -> bool:
         path = LOGS_DIR / f"{run_id}.log"
+        deleted = False
         if path.exists():
             path.unlink()
-            return True
-        return False
+            deleted = True
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                s3.delete(f"logs/agent/{run_id}.log")
+                deleted = True
+        except Exception:
+            pass
+        return deleted

package/backend/core/config.py

@@ -77,19 +77,45 @@ def load_settings():
         "login_enabled": False,
         "login_username": "",
         "login_password_hash": "",
+        # Scale / distributed execution
+        "redis_url": "",
+        "scale_postgres_url": "",
+        "scale_mode_enabled": False,
+        "scale_auto_sync": False,
+        "worker_concurrency": 10,
+        "otlp_endpoint": "",
+        "metrics_token": "",
+        "max_global_queue_depth": 1_000_000,
+        "rate_limit_per_tenant_rps": 1000,
+        "pgbouncer_mode": False,
+        "num_queue_shards": 1,
     }
     
     if not os.path.exists(SETTINGS_FILE):
-        return default_settings
-    
-    try:
-        with open(SETTINGS_FILE, 'r') as f:
-            data = json.load(f)
-            # Merge defaults
-            return {**default_settings, **data}
-    except Exception as e:
-        print(f"DEBUG: Error loading settings: {e}")
-        return default_settings
+        file_settings = {}
+    else:
+        try:
+            with open(SETTINGS_FILE, 'r') as f:
+                file_settings = json.load(f)
+        except Exception as e:
+            print(f"DEBUG: Error loading settings: {e}")
+            file_settings = {}
+
+    settings = {**default_settings, **file_settings}
+
+    # In scale worker mode, inject_llm_env() populates SYNAPSE_SETTING_* env vars
+    # from Postgres. Overlay them here so all callers of load_settings() see the
+    # Postgres-sourced values without needing access to the local settings.json.
+    _prefix = "SYNAPSE_SETTING_"
+    for _env_key, _env_val in os.environ.items():
+        if _env_key.startswith(_prefix):
+            _setting_key = _env_key[len(_prefix):].lower()
+            try:
+                settings[_setting_key] = json.loads(_env_val)
+            except Exception:
+                settings[_setting_key] = _env_val
+
+    return settings
 
 
 def get_or_create_jwt_secret() -> str:

package/backend/core/models.py

@@ -124,6 +124,19 @@ class Settings(BaseModel):
     login_username: str = ""
     login_password_hash: str = ""
 
+    # Scale / distributed execution settings
+    redis_url: str = ""
+    scale_postgres_url: str = ""
+    scale_mode_enabled: bool = False
+    scale_auto_sync: bool = False
+    worker_concurrency: int = 10
+    otlp_endpoint: str = ""
+    metrics_token: str = ""
+    max_global_queue_depth: int = 1_000_000
+    rate_limit_per_tenant_rps: int = 1000
+    pgbouncer_mode: bool = False
+    num_queue_shards: int = 1
+
 
 class PersonalAddress(BaseModel):
     address1: str = ""

package/backend/core/orchestration/engine.py

@@ -4,7 +4,7 @@ checkpointing, loop guards, and yielding SSE events.
 """
 import asyncio
 import time
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Awaitable, Callable
 
 import anyio
 
@@ -20,10 +20,20 @@ MAX_NESTED_DEPTH = 3
 class OrchestrationEngine:
     """Runs an orchestration by walking through its step graph."""
 
-    def __init__(self, orchestration: Orchestration, server_module, depth: int = 0):
+    def __init__(
+        self,
+        orchestration: Orchestration,
+        server_module,
+        depth: int = 0,
+        cancel_hook: Callable[[], Awaitable[bool]] | None = None,
+    ):
         self.orch = orchestration
         self.server_module = server_module
         self.depth = depth  # 0 = top-level run; >0 = nested sub-orchestration
+        # Optional async callable returning True when the run should be cancelled.
+        # Used by distributed workers to check Redis instead of the in-memory set.
+        # V1 never sets this — cancel_hook=None uses the existing in-memory check.
+        self.cancel_hook = cancel_hook
         self.step_map: dict[str, StepConfig] = {s.id: s for s in orchestration.steps}
         self.executors = STEP_EXECUTORS
         self.agent_names: dict[str, str] = self._load_agent_names()
@@ -98,10 +108,20 @@ class OrchestrationEngine:
             # before the next step starts sending new requests.
             await anyio.sleep(0)
 
-            # Check if this run was cancelled via the cancel endpoint
+            # Check if this run was cancelled via the cancel endpoint.
+            # cancel_hook (set by distributed workers) checks Redis; the default
+            # path checks the in-memory _cancelled_run_ids set (V1 / standalone).
             from .state import _cancelled_run_ids
-            if run.run_id in _cancelled_run_ids:
+            _is_cancelled = False
+            if self.cancel_hook:
+                try:
+                    _is_cancelled = await self.cancel_hook()
+                except Exception:
+                    pass
+            if not _is_cancelled and run.run_id in _cancelled_run_ids:
                 _cancelled_run_ids.discard(run.run_id)
+                _is_cancelled = True
+            if _is_cancelled:
                 run.status = "cancelled"
                 print(f"DEBUG ENGINE: 🛑 run '{run.run_id}' cancelled by request", flush=True)
                 break
@@ -195,6 +215,7 @@ class OrchestrationEngine:
                                 if logger:
                                     logger.step_end(step.id, "paused")
                                     logger.run_end("paused")
+                                    logger.close()
                                 # Break exits the async-for and then the
                                 # fail_after scope cleanly in this task.
                                 # The event is yielded below, outside the scope,
@@ -295,6 +316,7 @@ class OrchestrationEngine:
 
         if logger:
             logger.run_end(run.status)
+            logger.close()
 
         final_output = self._build_final_response(run)
 
@@ -376,19 +398,24 @@ class OrchestrationEngine:
         """Resume a paused orchestration after human input."""
         from .state import SharedState as SS
 
+        print(f"[engine.resume] ▶ restoring state from JSON checkpoint for run_id={run_id}", flush=True)
         restored = SS.restore(run_id)
         run = restored.run
+        print(f"[engine.resume] 📋 restored run: orch_id={run.orchestration_id} status={run.status} current_step_id={run.current_step_id} waiting_for_human={run.waiting_for_human} step_history_len={len(run.step_history)}", flush=True)
 
         # Load the orchestration definition
         from core.routes.orchestrations import load_orchestrations
         orchestrations = load_orchestrations()
+        print(f"[engine.resume] 📦 loaded {len(orchestrations)} orchestrations from disk, looking for id={run.orchestration_id}", flush=True)
         orch_data = next((o for o in orchestrations if o["id"] == run.orchestration_id), None)
         if not orch_data:
+            print(f"[engine.resume] ❌ orchestration '{run.orchestration_id}' NOT FOUND on disk — available ids: {[o.get('id') for o in orchestrations]}", flush=True)
             yield {"type": "orchestration_error", "error": f"Orchestration '{run.orchestration_id}' not found"}
             return
 
         orch = Orchestration.model_validate(orch_data)
         engine = cls(orch, server_module)
+        print(f"[engine.resume] 🗺  step_map keys: {list(engine.step_map.keys())}", flush=True)
 
         # Create logger — appends to existing log file if present
         engine.logger = OrchestrationLogger(
@@ -402,6 +429,7 @@ class OrchestrationEngine:
         # If the parent was paused because a NESTED orchestration hit a human step,
         # resume the sub-run and let it complete before continuing the parent.
         if run.nested_run_id:
+            print(f"[engine.resume] 🔗 nested run detected: nested_run_id={run.nested_run_id}, delegating to _resume_nested_orch", flush=True)
             async for event in cls._resume_nested_orch(run, engine, human_response, server_module):
                 yield event
             return
@@ -409,6 +437,7 @@ class OrchestrationEngine:
         # Normal path: human step is directly in this orchestration.
         # Move to next step after the HUMAN step
         current_step = engine.step_map.get(run.current_step_id)
+        print(f"[engine.resume] 🔍 current_step lookup: current_step_id={run.current_step_id!r} → found={current_step is not None} type={current_step.type.value if current_step else 'N/A'}", flush=True)
 
         # Write human response to shared state under the step's configured output_key.
         # Falling back to "human_response" preserves backward-compat for steps with no output_key.
@@ -424,10 +453,15 @@ class OrchestrationEngine:
         if current_step:
             next_id, _ = engine._resolve_next(current_step, run)
             run.current_step_id = next_id
+            print(f"[engine.resume] ➡  _resolve_next({run.current_step_id!r} was HUMAN) → next_step_id={next_id!r}", flush=True)
+        else:
+            print(f"[engine.resume] ⚠️  current_step not found in step_map — current_step_id stays as {run.current_step_id!r}, loop will likely fail immediately", flush=True)
 
+        print(f"[engine.resume] 🚀 entering _execute_loop with current_step_id={run.current_step_id!r} status={run.status}", flush=True)
         state = SharedState(run)
         async for event in engine._execute_loop(run, state):
             yield event
+        print(f"[engine.resume] 🏁 _execute_loop finished, run.status={run.status} run.current_step_id={run.current_step_id!r}", flush=True)
 
     @classmethod
     async def _resume_nested_orch(

package/backend/core/orchestration/logger.py

@@ -67,6 +67,34 @@ class OrchestrationLogger:
 {'='*80}
 """)
 
+    def close(self) -> None:
+        """Upload the completed log to S3 (scale mode). No-op in standalone mode."""
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3 and self.path.exists():
+                head = self.path.read_text(encoding="utf-8", errors="replace")[:1000]
+
+                def _extract(label: str) -> str:
+                    for line in head.split("\n"):
+                        if label in line:
+                            return line.split(":", 1)[1].strip()
+                    return ""
+
+                s3.upload_text(
+                    f"logs/orchestration/{self.path.name}",
+                    self.path.read_text(encoding="utf-8"),
+                    metadata={
+                        "orchestration_name": _extract("Orchestration   :"),
+                        "orchestration_id": _extract("Orchestration ID:"),
+                        "session_id": _extract("Session ID      :"),
+                        "started_at": _extract("Started at      :"),
+                        "user_input": _extract("User Input      :")[:200],
+                    },
+                )
+        except Exception:
+            pass
+
     # ── Step lifecycle ─────────────────────────────────────────────
 
     def step_start(self, step_id: str, step_name: str, step_type: str,
@@ -295,43 +323,94 @@ class OrchestrationLogger:
     @staticmethod
     def get_log(run_id: str) -> str | None:
         path = LOGS_DIR / f"{run_id}.log"
-        if not path.exists():
-            return None
-        return path.read_text(encoding="utf-8")
+        if path.exists():
+            return path.read_text(encoding="utf-8")
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                return s3.download_text(f"logs/orchestration/{run_id}.log")
+        except Exception:
+            pass
+        return None
 
     @staticmethod
     def list_logs(limit: int = 100, offset: int = 0) -> list[dict]:
         _ensure_logs_dir()
-        logs = []
+
+        def _parse_head(head: str, run_id: str, size_kb: float) -> dict:
+            def _extract(label: str) -> str:
+                for line in head.split("\n"):
+                    if label in line:
+                        return line.split(":", 1)[1].strip()
+                return ""
+            return {
+                "run_id": run_id,
+                "orchestration_name": _extract("Orchestration   :"),
+                "orchestration_id": _extract("Orchestration ID:"),
+                "session_id": _extract("Session ID      :"),
+                "started_at": _extract("Started at      :"),
+                "user_input": _extract("User Input      :")[:200],
+                "file_size_kb": size_kb,
+            }
+
+        local_ids: set[str] = set()
+        logs: list[dict] = []
         files = sorted(LOGS_DIR.glob("*.log"), key=lambda p: p.stat().st_mtime, reverse=True)
-        for f in files[offset : offset + limit]:
+        for f in files:
             run_id = f.stem
-            # Parse header lines for summary
+            local_ids.add(run_id)
             try:
                 head = f.read_text(encoding="utf-8", errors="replace")[:1000]
-                def _extract(label: str) -> str:
-                    for line in head.split("\n"):
-                        if label in line:
-                            return line.split(":", 1)[1].strip()
-                    return ""
-
-                logs.append({
-                    "run_id": run_id,
-                    "orchestration_name": _extract("Orchestration   :"),
-                    "orchestration_id": _extract("Orchestration ID:"),
-                    "session_id": _extract("Session ID      :"),
-                    "started_at": _extract("Started at      :"),
-                    "user_input": _extract("User Input      :")[:200],
-                    "file_size_kb": round(f.stat().st_size / 1024, 1),
-                })
+                logs.append(_parse_head(head, run_id, round(f.stat().st_size / 1024, 1)))
             except Exception:
-                logs.append({"run_id": run_id})
-        return logs
+                logs.append({"run_id": run_id, "file_size_kb": 0})
+
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                from concurrent.futures import ThreadPoolExecutor
+                s3_keys = s3.list_keys("logs/orchestration/")
+                missing_keys = [k for k in s3_keys if k.endswith(".log") and Path(k).stem not in local_ids]
+
+                def _fetch_meta(rel_key: str) -> dict:
+                    run_id = Path(rel_key).stem
+                    try:
+                        meta = s3.get_metadata(rel_key) or {}
+                        return {
+                            "run_id": run_id,
+                            "orchestration_name": meta.get("orchestration_name", ""),
+                            "orchestration_id": meta.get("orchestration_id", ""),
+                            "session_id": meta.get("session_id", ""),
+                            "started_at": meta.get("started_at", ""),
+                            "user_input": meta.get("user_input", "")[:200],
+                            "file_size_kb": 0,
+                        }
+                    except Exception:
+                        return {"run_id": run_id}
+
+                with ThreadPoolExecutor(max_workers=10) as pool:
+                    logs.extend(pool.map(_fetch_meta, missing_keys))
+        except Exception:
+            pass
+
+        logs.sort(key=lambda x: x.get("started_at", ""), reverse=True)
+        return logs[offset: offset + limit]
 
     @staticmethod
     def delete_log(run_id: str) -> bool:
         path = LOGS_DIR / f"{run_id}.log"
+        deleted = False
         if path.exists():
             path.unlink()
-            return True
-        return False
+            deleted = True
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                s3.delete(f"logs/orchestration/{run_id}.log")
+                deleted = True
+        except Exception:
+            pass
+        return deleted

package/backend/core/orchestration/steps.py

@@ -45,7 +45,7 @@ class AgentStepExecutor:
     ) -> AsyncGenerator[dict, None]:
         from core.react_engine import run_agent_step
         from core.agent_logger import AgentLogger
-        from core.routes.agents import load_user_agents
+        from core.scale.context import resolve_agent
         print(f"DEBUG AGENT EXEC: agent_id={step.agent_id} step={step.id}", flush=True)
 
         from .context import build_origin_aware_context, snapshot_inputs
@@ -64,9 +64,7 @@ class AgentStepExecutor:
                "system_prompt_prefix": system_prompt_prefix}
 
         # ── Orchestrator-as-agent: delegate to a nested OrchestrationEngine ──
-        target_agent = next(
-            (a for a in load_user_agents() if a.get("id") == step.agent_id), None
-        )
+        target_agent = await resolve_agent(step.agent_id) if step.agent_id else None
         if target_agent and target_agent.get("type") == "orchestrator":
             async for ev in self._execute_nested_orchestration(
                 step, run, engine, transition, target_agent, prompt, inputs_snapshot
@@ -115,6 +113,7 @@ class AgentStepExecutor:
             raise
         finally:
             agent_log.run_end(_log_status)
+            agent_log.close()
 
         if final_response is None:
             raise RuntimeError(f"Agent step '{step.name}' ended without producing a final response")
@@ -277,7 +276,7 @@ class ToolStepExecutor:
         from core.config import load_settings
         from core.react_engine import parse_tool_call
         from core.tools import aggregate_all_tools
-        from core.routes.agents import load_user_agents
+        from core.scale.context import resolve_agent, resolve_custom_tools
 
         if not step.forced_tool:
             yield {"type": "step_warning", "orch_step_id": step.id,
@@ -302,9 +301,8 @@ class ToolStepExecutor:
         mode = detect_mode_from_model(model)
 
         # Load only the forced tool's schema — aggregate all tools then filter to one
-        agents = load_user_agents()
-        active_agent = next((a for a in agents if a.get("id") == step.agent_id), agents[0] if agents else {})
-        custom_tools = self._load_custom_tools()
+        active_agent = (await resolve_agent(step.agent_id) if step.agent_id else None) or {}
+        custom_tools = await resolve_custom_tools()
         all_tools, _, _, _ = await aggregate_all_tools(
             engine.server_module.agent_sessions, active_agent, custom_tools
         )
@@ -609,10 +607,10 @@ class ToolStepExecutor:
         except Exception:
             return resp.text or json.dumps({"error": f"Empty response (Status: {resp.status_code})"})
 
-    def _load_custom_tools(self) -> list:
+    async def _load_custom_tools(self) -> list:
         try:
-            from core.routes.tools import load_custom_tools
-            return load_custom_tools()
+            from core.scale.context import resolve_custom_tools
+            return await resolve_custom_tools()
         except Exception:
             return []
 

package/backend/core/react_engine.py

@@ -529,8 +529,13 @@ async def run_agent_step(
     if max_turns is None:
         max_turns = MAX_TURNS
 
-    # Resolve agent
-    active_agent = agent_override if agent_override is not None else _resolve_agent_by_id(agent_id)
+    # Resolve agent — Postgres first (worker mode), JSON fallback
+    if agent_override is not None:
+        active_agent = agent_override
+    else:
+        from core.scale.context import resolve_agent as _ctx_resolve_agent
+        resolved = await _ctx_resolve_agent(agent_id) if agent_id else None
+        active_agent = resolved if resolved is not None else _resolve_agent_by_id(agent_id)
     agent_id_for_session = active_agent.get("id", agent_id or "unknown")
 
     # Build system prompt with repo and DB context injection
@@ -564,7 +569,8 @@ async def run_agent_step(
         tools_json = json.dumps(tools_override)
         print(f"DEBUG RUN_AGENT: start agent_id={agent_id_for_session}, tools_override={len(tools_override)} tools", flush=True)
     else:
-        custom_tools = load_custom_tools()
+        from core.scale.context import resolve_custom_tools as _ctx_resolve_tools
+        custom_tools = await _ctx_resolve_tools()
         print(f"DEBUG RUN_AGENT: start agent_id={agent_id_for_session}, sessions={list(server_module.agent_sessions.keys())}", flush=True)
         all_tools, tool_schema_map, ollama_tools, tools_json = await aggregate_all_tools(
             server_module.agent_sessions, active_agent, custom_tools
@@ -1073,7 +1079,8 @@ async def run_agent_step(
                 # ===== CUSTOM TOOLS (Webhook + Python) =====
 
                 if tool_name not in server_module.tool_router:
-                    custom_tools_list = load_custom_tools()
+                    from core.scale.context import resolve_custom_tools as _ctx_resolve_tools2
+                    custom_tools_list = await _ctx_resolve_tools2()
                     target_tool = next((t for t in custom_tools_list if t["name"] == tool_name), None)
 
                     if target_tool:
@@ -1479,8 +1486,10 @@ async def run_react_loop(request, server_module):
 
     yield {"type": "status", "message": "Processing your request..."}
 
-    # Resolve active agent
-    active_agent = _resolve_agent_by_id(request.agent_id)
+    # Resolve active agent — Postgres first (worker mode), JSON fallback
+    from core.scale.context import resolve_agent as _ctx_resolve_agent
+    _resolved = await _ctx_resolve_agent(request.agent_id) if request.agent_id else None
+    active_agent = _resolved if _resolved is not None else _resolve_agent_by_id(request.agent_id)
 
     # --- Orchestrator delegation ---
     if active_agent.get("type") == "orchestrator":
@@ -1537,3 +1546,4 @@ async def run_react_loop(request, server_module):
         raise
     finally:
         _agent_log.run_end(_log_status)
+        _agent_log.close()