synapse-orch-ai 1.6.6 → 1.7.1

package/README.md

@@ -75,12 +75,38 @@ Then open `http://localhost:3000`. See the [Docker guide](https://docs.synapseor
 
 ---
 
+## Scale Mode
+
+Run unlimited agents and orchestrations concurrently. When you need to go beyond a single process, the distributed scale layer handles the load:
+
+- **Redis Cluster** — job queue, SSE event streams, pub/sub cancellation signals, auto-failover
+- **ARQ worker fleet** — 1 to 100+ independent workers, each running up to 20 concurrent orchestrations; autoscale with KEDA on queue depth
+- **PgBouncer** — multiplexes hundreds of worker connections into a small, stable Postgres pool
+- **S3 artifact storage** — stream large file outputs directly to AWS S3, Cloudflare R2, or MinIO
+- **Multi-tenant quotas** — per-team or per-customer concurrent run limits with HTTP 429 enforcement
+- **Per-step checkpoint recovery** — worker crashes don't lose jobs; the next worker resumes from the last completed step
+
+Three Docker images — pull only what you need:
+
+```bash
+docker pull synapseorchai/synapse-ai:latest             # full app (standalone mode)
+docker pull synapseorchai/synapse-ai-api-server:latest  # stateless API server (scale mode)
+docker pull synapseorchai/synapse-ai-worker:latest      # worker process (scale mode, run as many as needed)
+```
+
+The `docker-compose.yml` in the repo spins up the full stack. Production K8s manifests are in `infra/k8s/`.
+
+📖 [**Scale Mode docs →**](https://docs.synapseorch.com/scale/overview)
+
+---
+
 ## What Makes Synapse Different
 
 - **Multi-Model Orchestrations** — Run a different LLM at every step. Use a fast model for routing, a powerful one for analysis. You control where the compute goes.
 - **Deterministic DAG Execution** — Orchestrations follow the exact path you designed. No hallucinated detours.
 - **Turn Anything Into a Tool** — Python scripts, REST APIs, webhooks, MCP servers, or entire orchestrations — all become agent-callable tools.
 - **Human-in-the-Loop** — Pause workflows for human review. Resumable across restarts. Connect via UI, Slack, Telegram, or any messaging channel.
+- **Scales to Millions of Requests** — The distributed scale layer separates API servers, Redis job queues, and independent worker processes so you can run any number of agents or orchestrations concurrently. Start on one machine, grow to a Kubernetes cluster — the V2 API never changes.
 - **Local-First, No Lock-In** — Full local operation with Ollama. Mix local and cloud models freely. Your data stays yours.
 - **Built-In Scheduling & Messaging** — Cron-based automation with results pushed to Slack, Discord, Telegram, Teams, or WhatsApp.
 - **14+ LLM Providers** — Cloud, local, and CLI providers including Ollama, OpenAI, Anthropic, Gemini, xAI, DeepSeek, AWS Bedrock, and more.
@@ -122,7 +148,8 @@ https://github.com/user-attachments/assets/282cc99d-cdea-4ad0-b648-f22112c6e295
 | **AI Builder** | A meta-agent that designs and materializes orchestrations from natural language. [Docs →](https://docs.synapseorch.com/orchestrations/ai-builder) |
 | **Schedules** | Cron/interval automation with messaging notifications. [Docs →](https://docs.synapseorch.com/integrations/scheduling) |
 | **Messaging** | Slack, Discord, Telegram, Teams, WhatsApp — with multi-agent mode. [Docs →](https://docs.synapseorch.com/integrations/messaging) |
-| **REST API** | Trigger agents and orchestrations programmatically from any application. [Docs →](https://docs.synapseorch.com/api/overview) |
+| **Scale Mode** | Distributed execution layer: Redis job queue, independent worker fleet, per-step Postgres checkpoints, S3 artifact storage, and multi-tenant quotas. [Docs →](https://docs.synapseorch.com/scale/overview) |
+| **V2 API** | Stable versioned REST API for building products on top of Synapse — enqueue, stream, cancel, webhooks. [Docs →](https://docs.synapseorch.com/api/overview) |
 | **Vault** | Persistent file storage shared across agents and sessions. [Docs →](https://docs.synapseorch.com/vault) |
 
 ---

package/backend/core/agent_logger.py

@@ -96,6 +96,37 @@ class AgentLogger:
 {'='*80}
 """)
 
+    def close(self) -> None:
+        """Drain the write queue, then upload the completed log to S3 (scale mode)."""
+        self._q.put(None)  # poison pill — stops the drain thread
+        self._thread.join(timeout=10)
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3 and self.path.exists():
+                head = self.path.read_text(encoding="utf-8", errors="replace")[:1000]
+
+                def _extract(label: str) -> str:
+                    for line in head.split("\n"):
+                        if label in line:
+                            return line.split(":", 1)[1].strip()
+                    return ""
+
+                s3.upload_text(
+                    f"logs/agent/{self.path.name}",
+                    self.path.read_text(encoding="utf-8"),
+                    metadata={
+                        "agent_name": _extract("Agent Name      :"),
+                        "agent_id": _extract("Agent ID        :"),
+                        "source": _extract("Source          :"),
+                        "session_id": _extract("Session ID      :"),
+                        "started_at": _extract("Started at      :"),
+                        "user_input": _extract("User Input      :")[:200],
+                    },
+                )
+        except Exception:
+            pass  # S3 upload failure must never surface to callers
+
     # ── Event logging ──────────────────────────────────────────────
 
     def log_event(self, event: dict):
@@ -212,44 +243,101 @@ class AgentLogger:
     @staticmethod
     def get_log(run_id: str) -> str | None:
         path = LOGS_DIR / f"{run_id}.log"
-        if not path.exists():
-            return None
-        return path.read_text(encoding="utf-8")
+        if path.exists():
+            return path.read_text(encoding="utf-8")
+        # Fall back to S3 for logs written by other workers
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                return s3.download_text(f"logs/agent/{run_id}.log")
+        except Exception:
+            pass
+        return None
 
     @staticmethod
     def list_logs(limit: int = 100, offset: int = 0) -> list[dict]:
         _ensure_logs_dir()
-        logs = []
+
+        def _parse_head(head: str, run_id: str, size_kb: float) -> dict:
+            def _extract(label: str) -> str:
+                for line in head.split("\n"):
+                    if label in line:
+                        return line.split(":", 1)[1].strip()
+                return ""
+            return {
+                "run_id": run_id,
+                "agent_name": _extract("Agent Name      :"),
+                "agent_id": _extract("Agent ID        :"),
+                "source": _extract("Source          :"),
+                "session_id": _extract("Session ID      :"),
+                "started_at": _extract("Started at      :"),
+                "user_input": _extract("User Input      :")[:200],
+                "file_size_kb": size_kb,
+            }
+
+        # Collect local entries
+        local_ids: set[str] = set()
+        logs: list[dict] = []
         files = sorted(LOGS_DIR.glob("*.log"), key=lambda p: p.stat().st_mtime, reverse=True)
-        for f in files[offset : offset + limit]:
+        for f in files:
             run_id = f.stem
+            local_ids.add(run_id)
             try:
                 head = f.read_text(encoding="utf-8", errors="replace")[:1000]
-
-                def _extract(label: str) -> str:
-                    for line in head.split("\n"):
-                        if label in line:
-                            return line.split(":", 1)[1].strip()
-                    return ""
-
-                logs.append({
-                    "run_id": run_id,
-                    "agent_name": _extract("Agent Name      :"),
-                    "agent_id": _extract("Agent ID        :"),
-                    "source": _extract("Source          :"),
-                    "session_id": _extract("Session ID      :"),
-                    "started_at": _extract("Started at      :"),
-                    "user_input": _extract("User Input      :")[:200],
-                    "file_size_kb": round(f.stat().st_size / 1024, 1),
-                })
+                logs.append(_parse_head(head, run_id, round(f.stat().st_size / 1024, 1)))
             except Exception:
-                logs.append({"run_id": run_id})
-        return logs
+                logs.append({"run_id": run_id, "file_size_kb": 0})
+
+        # Merge S3 entries that aren't already local
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                from concurrent.futures import ThreadPoolExecutor
+                s3_keys = s3.list_keys("logs/agent/")
+                missing_keys = [k for k in s3_keys if k.endswith(".log") and Path(k).stem not in local_ids]
+
+                def _fetch_meta(rel_key: str) -> dict:
+                    run_id = Path(rel_key).stem
+                    try:
+                        meta = s3.get_metadata(rel_key) or {}
+                        return {
+                            "run_id": run_id,
+                            "agent_name": meta.get("agent_name", ""),
+                            "agent_id": meta.get("agent_id", ""),
+                            "source": meta.get("source", ""),
+                            "session_id": meta.get("session_id", ""),
+                            "started_at": meta.get("started_at", ""),
+                            "user_input": meta.get("user_input", "")[:200],
+                            "file_size_kb": 0,
+                        }
+                    except Exception:
+                        return {"run_id": run_id}
+
+                with ThreadPoolExecutor(max_workers=10) as pool:
+                    s3_entries = list(pool.map(_fetch_meta, missing_keys))
+                logs.extend(s3_entries)
+        except Exception:
+            pass
+
+        # Sort by started_at descending (local files already sorted; blend with S3)
+        logs.sort(key=lambda x: x.get("started_at", ""), reverse=True)
+        return logs[offset: offset + limit]
 
     @staticmethod
     def delete_log(run_id: str) -> bool:
         path = LOGS_DIR / f"{run_id}.log"
+        deleted = False
         if path.exists():
             path.unlink()
-            return True
-        return False
+            deleted = True
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                s3.delete(f"logs/agent/{run_id}.log")
+                deleted = True
+        except Exception:
+            pass
+        return deleted

package/backend/core/config.py

@@ -39,6 +39,10 @@ def load_settings():
         "huggingface_token": "",
         "huggingface_models": "",
         "huggingface_max_new_tokens": 1024,
+        "anthropic_cli_models": "",
+        "gemini_cli_models": "",
+        "codex_cli_models": "",
+        "github_copilot_cli_models": "",
         "bedrock_api_key": "",
         "bedrock_inference_profile": "",
         "embedding_model": "",
@@ -77,19 +81,45 @@ def load_settings():
         "login_enabled": False,
         "login_username": "",
         "login_password_hash": "",
+        # Scale / distributed execution
+        "redis_url": "",
+        "scale_postgres_url": "",
+        "scale_mode_enabled": False,
+        "scale_auto_sync": False,
+        "worker_concurrency": 10,
+        "otlp_endpoint": "",
+        "metrics_token": "",
+        "max_global_queue_depth": 1_000_000,
+        "rate_limit_per_tenant_rps": 1000,
+        "pgbouncer_mode": False,
+        "num_queue_shards": 1,
     }
     
     if not os.path.exists(SETTINGS_FILE):
-        return default_settings
-    
-    try:
-        with open(SETTINGS_FILE, 'r') as f:
-            data = json.load(f)
-            # Merge defaults
-            return {**default_settings, **data}
-    except Exception as e:
-        print(f"DEBUG: Error loading settings: {e}")
-        return default_settings
+        file_settings = {}
+    else:
+        try:
+            with open(SETTINGS_FILE, 'r') as f:
+                file_settings = json.load(f)
+        except Exception as e:
+            print(f"DEBUG: Error loading settings: {e}")
+            file_settings = {}
+
+    settings = {**default_settings, **file_settings}
+
+    # In scale worker mode, inject_llm_env() populates SYNAPSE_SETTING_* env vars
+    # from Postgres. Overlay them here so all callers of load_settings() see the
+    # Postgres-sourced values without needing access to the local settings.json.
+    _prefix = "SYNAPSE_SETTING_"
+    for _env_key, _env_val in os.environ.items():
+        if _env_key.startswith(_prefix):
+            _setting_key = _env_key[len(_prefix):].lower()
+            try:
+                settings[_setting_key] = json.loads(_env_val)
+            except Exception:
+                settings[_setting_key] = _env_val
+
+    return settings
 
 
 def get_or_create_jwt_secret() -> str:

package/backend/core/llm_providers.py

@@ -347,11 +347,16 @@ async def call_cli_provider(
     AUTH_PATTERNS = ["login", "authenticate", "auth", "expired", "sign in", "api key", "unauthorized"]
     RATE_LIMIT_PATTERNS = ["429", "rate limit", "ratelimit", "quota", "too many requests", "capacity", "resource_exhausted", "ratelimitexceeded"]
 
-    parts = cli_model.lower().split(".")
+    # Split into base ("cli.<provider>") + variant, preserving dots in the
+    # variant (e.g. "cli.codex.gpt-5.4" → variant "gpt-5.4"). The base is matched
+    # case-insensitively; the variant is kept verbatim since model names can be
+    # case-sensitive and are passed straight through to the CLI's -m/--model flag.
+    parts = cli_model.split(".", 2)
     if len(parts) >= 2:
-        base_cli = f"{parts[0]}.{parts[1]}"
+        base_cli = f"{parts[0]}.{parts[1]}".lower()
     else:
         base_cli = cli_model.lower()
+    variant = parts[2] if len(parts) > 2 else None
 
     base_cmd = _CLI_COMMANDS.get(base_cli)
     if not base_cmd:
@@ -365,8 +370,7 @@ async def call_cli_provider(
         cmd.extend(["--resume", cli_session_id])
 
     # ── Dynamic Model & Options Parsing ──
-    if len(parts) > 2:
-        variant = parts[2]
+    if variant:
         if base_cli == "cli.claude":
             # Extract base model by removing any thinking suffixes
             clean_variant = variant.replace('-thinking', '').replace('-max', '').replace('-high', '')
@@ -577,10 +581,14 @@ async def call_cli_provider(
             )
 
         if not clean:
+            # Report the TAIL of stderr, not the head. CLIs like codex echo a
+            # startup banner (version/workdir/model) to the head of stderr and
+            # write the actual failure (e.g. an invalid_request_error for an
+            # unsupported model) at the tail — so [:200] would hide the real cause.
             raise LLMError(
                 f"CLI provider '{cli_model}' returned empty output. "
                 f"Return code: {process.returncode}. "
-                f"stderr: {stderr_text[:200]}"
+                f"stderr: {stderr_text[-600:]}"
             )
 
         print(f"DEBUG: ✅ CLI '{cli_model}' response ({len(clean)} chars)", flush=True)

package/backend/core/models.py

@@ -90,6 +90,13 @@ class Settings(BaseModel):
     local_compatible_embed_models: str = ""  # Comma-separated embedding model names for local v1-compatible
     huggingface_token: str = ""  # Optional HF access token (required for gated models like Llama, Gemma)
     huggingface_models: str = ""  # Comma- or newline-separated HF model IDs (e.g. Qwen/Qwen2.5-7B-Instruct)
+    # Comma-separated custom model names for the local CLI session providers.
+    # Each entry becomes a selectable cli.<provider>.<name> model (e.g. "gpt-5.4"
+    # → cli.codex.gpt-5.4), passed to the CLI via its -m/--model flag at call time.
+    anthropic_cli_models: str = ""
+    gemini_cli_models: str = ""
+    codex_cli_models: str = ""
+    github_copilot_cli_models: str = ""
     huggingface_max_new_tokens: int = 1024  # Max tokens to generate per HF call
     # Transform step Python execution runtime: "docker" (sandboxed, default) or
     # "host" (unsandboxed subprocess on host — needed for torch/GPU workloads,
@@ -124,6 +131,19 @@ class Settings(BaseModel):
     login_username: str = ""
     login_password_hash: str = ""
 
+    # Scale / distributed execution settings
+    redis_url: str = ""
+    scale_postgres_url: str = ""
+    scale_mode_enabled: bool = False
+    scale_auto_sync: bool = False
+    worker_concurrency: int = 10
+    otlp_endpoint: str = ""
+    metrics_token: str = ""
+    max_global_queue_depth: int = 1_000_000
+    rate_limit_per_tenant_rps: int = 1000
+    pgbouncer_mode: bool = False
+    num_queue_shards: int = 1
+
 
 class PersonalAddress(BaseModel):
     address1: str = ""

package/backend/core/orchestration/engine.py

@@ -4,7 +4,7 @@ checkpointing, loop guards, and yielding SSE events.
 """
 import asyncio
 import time
-from typing import AsyncGenerator
+from typing import AsyncGenerator, Awaitable, Callable
 
 import anyio
 
@@ -20,10 +20,20 @@ MAX_NESTED_DEPTH = 3
 class OrchestrationEngine:
     """Runs an orchestration by walking through its step graph."""
 
-    def __init__(self, orchestration: Orchestration, server_module, depth: int = 0):
+    def __init__(
+        self,
+        orchestration: Orchestration,
+        server_module,
+        depth: int = 0,
+        cancel_hook: Callable[[], Awaitable[bool]] | None = None,
+    ):
         self.orch = orchestration
         self.server_module = server_module
         self.depth = depth  # 0 = top-level run; >0 = nested sub-orchestration
+        # Optional async callable returning True when the run should be cancelled.
+        # Used by distributed workers to check Redis instead of the in-memory set.
+        # V1 never sets this — cancel_hook=None uses the existing in-memory check.
+        self.cancel_hook = cancel_hook
         self.step_map: dict[str, StepConfig] = {s.id: s for s in orchestration.steps}
         self.executors = STEP_EXECUTORS
         self.agent_names: dict[str, str] = self._load_agent_names()
@@ -98,10 +108,20 @@ class OrchestrationEngine:
             # before the next step starts sending new requests.
             await anyio.sleep(0)
 
-            # Check if this run was cancelled via the cancel endpoint
+            # Check if this run was cancelled via the cancel endpoint.
+            # cancel_hook (set by distributed workers) checks Redis; the default
+            # path checks the in-memory _cancelled_run_ids set (V1 / standalone).
             from .state import _cancelled_run_ids
-            if run.run_id in _cancelled_run_ids:
+            _is_cancelled = False
+            if self.cancel_hook:
+                try:
+                    _is_cancelled = await self.cancel_hook()
+                except Exception:
+                    pass
+            if not _is_cancelled and run.run_id in _cancelled_run_ids:
                 _cancelled_run_ids.discard(run.run_id)
+                _is_cancelled = True
+            if _is_cancelled:
                 run.status = "cancelled"
                 print(f"DEBUG ENGINE: 🛑 run '{run.run_id}' cancelled by request", flush=True)
                 break
@@ -195,6 +215,7 @@ class OrchestrationEngine:
                                 if logger:
                                     logger.step_end(step.id, "paused")
                                     logger.run_end("paused")
+                                    logger.close()
                                 # Break exits the async-for and then the
                                 # fail_after scope cleanly in this task.
                                 # The event is yielded below, outside the scope,
@@ -295,6 +316,7 @@ class OrchestrationEngine:
 
         if logger:
             logger.run_end(run.status)
+            logger.close()
 
         final_output = self._build_final_response(run)
 
@@ -376,19 +398,24 @@ class OrchestrationEngine:
         """Resume a paused orchestration after human input."""
         from .state import SharedState as SS
 
+        print(f"[engine.resume] ▶ restoring state from JSON checkpoint for run_id={run_id}", flush=True)
         restored = SS.restore(run_id)
         run = restored.run
+        print(f"[engine.resume] 📋 restored run: orch_id={run.orchestration_id} status={run.status} current_step_id={run.current_step_id} waiting_for_human={run.waiting_for_human} step_history_len={len(run.step_history)}", flush=True)
 
         # Load the orchestration definition
         from core.routes.orchestrations import load_orchestrations
         orchestrations = load_orchestrations()
+        print(f"[engine.resume] 📦 loaded {len(orchestrations)} orchestrations from disk, looking for id={run.orchestration_id}", flush=True)
         orch_data = next((o for o in orchestrations if o["id"] == run.orchestration_id), None)
         if not orch_data:
+            print(f"[engine.resume] ❌ orchestration '{run.orchestration_id}' NOT FOUND on disk — available ids: {[o.get('id') for o in orchestrations]}", flush=True)
             yield {"type": "orchestration_error", "error": f"Orchestration '{run.orchestration_id}' not found"}
             return
 
         orch = Orchestration.model_validate(orch_data)
         engine = cls(orch, server_module)
+        print(f"[engine.resume] 🗺  step_map keys: {list(engine.step_map.keys())}", flush=True)
 
         # Create logger — appends to existing log file if present
         engine.logger = OrchestrationLogger(
@@ -402,6 +429,7 @@ class OrchestrationEngine:
         # If the parent was paused because a NESTED orchestration hit a human step,
         # resume the sub-run and let it complete before continuing the parent.
         if run.nested_run_id:
+            print(f"[engine.resume] 🔗 nested run detected: nested_run_id={run.nested_run_id}, delegating to _resume_nested_orch", flush=True)
             async for event in cls._resume_nested_orch(run, engine, human_response, server_module):
                 yield event
             return
@@ -409,6 +437,7 @@ class OrchestrationEngine:
         # Normal path: human step is directly in this orchestration.
         # Move to next step after the HUMAN step
         current_step = engine.step_map.get(run.current_step_id)
+        print(f"[engine.resume] 🔍 current_step lookup: current_step_id={run.current_step_id!r} → found={current_step is not None} type={current_step.type.value if current_step else 'N/A'}", flush=True)
 
         # Write human response to shared state under the step's configured output_key.
         # Falling back to "human_response" preserves backward-compat for steps with no output_key.
@@ -424,10 +453,15 @@ class OrchestrationEngine:
         if current_step:
             next_id, _ = engine._resolve_next(current_step, run)
             run.current_step_id = next_id
+            print(f"[engine.resume] ➡  _resolve_next({run.current_step_id!r} was HUMAN) → next_step_id={next_id!r}", flush=True)
+        else:
+            print(f"[engine.resume] ⚠️  current_step not found in step_map — current_step_id stays as {run.current_step_id!r}, loop will likely fail immediately", flush=True)
 
+        print(f"[engine.resume] 🚀 entering _execute_loop with current_step_id={run.current_step_id!r} status={run.status}", flush=True)
         state = SharedState(run)
         async for event in engine._execute_loop(run, state):
             yield event
+        print(f"[engine.resume] 🏁 _execute_loop finished, run.status={run.status} run.current_step_id={run.current_step_id!r}", flush=True)
 
     @classmethod
     async def _resume_nested_orch(

package/backend/core/orchestration/logger.py

@@ -67,6 +67,34 @@ class OrchestrationLogger:
 {'='*80}
 """)
 
+    def close(self) -> None:
+        """Upload the completed log to S3 (scale mode). No-op in standalone mode."""
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3 and self.path.exists():
+                head = self.path.read_text(encoding="utf-8", errors="replace")[:1000]
+
+                def _extract(label: str) -> str:
+                    for line in head.split("\n"):
+                        if label in line:
+                            return line.split(":", 1)[1].strip()
+                    return ""
+
+                s3.upload_text(
+                    f"logs/orchestration/{self.path.name}",
+                    self.path.read_text(encoding="utf-8"),
+                    metadata={
+                        "orchestration_name": _extract("Orchestration   :"),
+                        "orchestration_id": _extract("Orchestration ID:"),
+                        "session_id": _extract("Session ID      :"),
+                        "started_at": _extract("Started at      :"),
+                        "user_input": _extract("User Input      :")[:200],
+                    },
+                )
+        except Exception:
+            pass
+
     # ── Step lifecycle ─────────────────────────────────────────────
 
     def step_start(self, step_id: str, step_name: str, step_type: str,
@@ -295,43 +323,94 @@ class OrchestrationLogger:
     @staticmethod
     def get_log(run_id: str) -> str | None:
         path = LOGS_DIR / f"{run_id}.log"
-        if not path.exists():
-            return None
-        return path.read_text(encoding="utf-8")
+        if path.exists():
+            return path.read_text(encoding="utf-8")
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                return s3.download_text(f"logs/orchestration/{run_id}.log")
+        except Exception:
+            pass
+        return None
 
     @staticmethod
     def list_logs(limit: int = 100, offset: int = 0) -> list[dict]:
         _ensure_logs_dir()
-        logs = []
+
+        def _parse_head(head: str, run_id: str, size_kb: float) -> dict:
+            def _extract(label: str) -> str:
+                for line in head.split("\n"):
+                    if label in line:
+                        return line.split(":", 1)[1].strip()
+                return ""
+            return {
+                "run_id": run_id,
+                "orchestration_name": _extract("Orchestration   :"),
+                "orchestration_id": _extract("Orchestration ID:"),
+                "session_id": _extract("Session ID      :"),
+                "started_at": _extract("Started at      :"),
+                "user_input": _extract("User Input      :")[:200],
+                "file_size_kb": size_kb,
+            }
+
+        local_ids: set[str] = set()
+        logs: list[dict] = []
         files = sorted(LOGS_DIR.glob("*.log"), key=lambda p: p.stat().st_mtime, reverse=True)
-        for f in files[offset : offset + limit]:
+        for f in files:
             run_id = f.stem
-            # Parse header lines for summary
+            local_ids.add(run_id)
             try:
                 head = f.read_text(encoding="utf-8", errors="replace")[:1000]
-                def _extract(label: str) -> str:
-                    for line in head.split("\n"):
-                        if label in line:
-                            return line.split(":", 1)[1].strip()
-                    return ""
-
-                logs.append({
-                    "run_id": run_id,
-                    "orchestration_name": _extract("Orchestration   :"),
-                    "orchestration_id": _extract("Orchestration ID:"),
-                    "session_id": _extract("Session ID      :"),
-                    "started_at": _extract("Started at      :"),
-                    "user_input": _extract("User Input      :")[:200],
-                    "file_size_kb": round(f.stat().st_size / 1024, 1),
-                })
+                logs.append(_parse_head(head, run_id, round(f.stat().st_size / 1024, 1)))
             except Exception:
-                logs.append({"run_id": run_id})
-        return logs
+                logs.append({"run_id": run_id, "file_size_kb": 0})
+
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                from concurrent.futures import ThreadPoolExecutor
+                s3_keys = s3.list_keys("logs/orchestration/")
+                missing_keys = [k for k in s3_keys if k.endswith(".log") and Path(k).stem not in local_ids]
+
+                def _fetch_meta(rel_key: str) -> dict:
+                    run_id = Path(rel_key).stem
+                    try:
+                        meta = s3.get_metadata(rel_key) or {}
+                        return {
+                            "run_id": run_id,
+                            "orchestration_name": meta.get("orchestration_name", ""),
+                            "orchestration_id": meta.get("orchestration_id", ""),
+                            "session_id": meta.get("session_id", ""),
+                            "started_at": meta.get("started_at", ""),
+                            "user_input": meta.get("user_input", "")[:200],
+                            "file_size_kb": 0,
+                        }
+                    except Exception:
+                        return {"run_id": run_id}
+
+                with ThreadPoolExecutor(max_workers=10) as pool:
+                    logs.extend(pool.map(_fetch_meta, missing_keys))
+        except Exception:
+            pass
+
+        logs.sort(key=lambda x: x.get("started_at", ""), reverse=True)
+        return logs[offset: offset + limit]
 
     @staticmethod
     def delete_log(run_id: str) -> bool:
         path = LOGS_DIR / f"{run_id}.log"
+        deleted = False
         if path.exists():
             path.unlink()
-            return True
-        return False
+            deleted = True
+        try:
+            from core.s3_storage import get_s3
+            s3 = get_s3()
+            if s3:
+                s3.delete(f"logs/orchestration/{run_id}.log")
+                deleted = True
+        except Exception:
+            pass
+        return deleted