meshcode 2.10.13__tar.gz → 2.10.15__tar.gz

{meshcode-2.10.13 → meshcode-2.10.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meshcode
-Version: 2.10.13
+Version: 2.10.15
 Summary: Real-time communication between AI agents — Supabase-backed CLI
 Author-email: MeshCode <hello@meshcode.io>
 License: MIT

{meshcode-2.10.13 → meshcode-2.10.15}/meshcode/__init__.py

@@ -1,2 +1,2 @@
 """MeshCode — Real-time communication between AI agents."""
-__version__ = "2.10.13"
+__version__ = "2.10.15"

{meshcode-2.10.13 → meshcode-2.10.15}/meshcode/meshcode_mcp/backend.py

@@ -5,6 +5,8 @@ Zero deps beyond stdlib (urllib).
 """
 import json
 import os
+import time as _time
+import threading as _threading
 from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, List, Optional
@@ -12,6 +14,54 @@ from urllib.error import HTTPError, URLError
 from urllib.parse import quote
 from urllib.request import Request, urlopen
 
+
+# ── Circuit Breaker ──────────────────────────────────────────────
+# Protects against cascading failures when Supabase is down.
+# States: CLOSED (normal) → OPEN (reject fast) → HALF_OPEN (probe)
+class _CircuitBreaker:
+    CLOSED = "closed"
+    OPEN = "open"
+    HALF_OPEN = "half_open"
+
+    def __init__(self, failure_threshold: int = 5, recovery_timeout: float = 30.0):
+        self.failure_threshold = failure_threshold
+        self.recovery_timeout = recovery_timeout
+        self.state = self.CLOSED
+        self.failure_count = 0
+        self.last_failure_time = 0.0
+        self._lock = _threading.Lock()
+
+    def can_execute(self) -> bool:
+        with self._lock:
+            if self.state == self.CLOSED:
+                return True
+            if self.state == self.OPEN:
+                if _time.monotonic() - self.last_failure_time >= self.recovery_timeout:
+                    self.state = self.HALF_OPEN
+                    return True
+                return False
+            # HALF_OPEN: allow one probe
+            return True
+
+    def record_success(self) -> None:
+        with self._lock:
+            self.failure_count = 0
+            self.state = self.CLOSED
+
+    def record_failure(self) -> None:
+        with self._lock:
+            self.failure_count += 1
+            self.last_failure_time = _time.monotonic()
+            if self.failure_count >= self.failure_threshold:
+                self.state = self.OPEN
+
+    @property
+    def is_open(self) -> bool:
+        return self.state == self.OPEN
+
+
+_circuit = _CircuitBreaker(failure_threshold=5, recovery_timeout=30.0)
+
 # Bake in production defaults — RLS-protected publishable key, safe to ship.
 _DEFAULT_SUPABASE_URL = "https://gjinagyyjttyxnaoavnz.supabase.co"
 _DEFAULT_SUPABASE_KEY = "sb_publishable_qwN9PO1L7jUXhhbhhVk2CQ_z1FXG2Qf"
@@ -66,22 +116,31 @@ def _headers(*, prefer: Optional[str] = None, content_profile: bool = True) -> D
 
 
 def _request(method: str, path: str, *, data: Any = None, prefer: Optional[str] = None) -> Any:
+    if not _circuit.can_execute():
+        return {"_error": "circuit breaker open — Supabase temporarily unavailable", "_code": 503}
     url = f"{SUPABASE_URL}/rest/v1/{path}"
     body = json.dumps(data).encode("utf-8") if data else None
     req = Request(url, data=body, method=method, headers=_headers(prefer=prefer))
     try:
         with urlopen(req, timeout=10) as resp:
             raw = resp.read().decode("utf-8")
+            _circuit.record_success()
             return json.loads(raw) if raw.strip() else None
     except HTTPError as e:
         err = e.read().decode("utf-8", errors="replace")
+        # 4xx = client error (not a backend failure), don't trip breaker
+        if 400 <= e.code < 500:
+            _circuit.record_success()
+        else:
+            _circuit.record_failure()
         try:
             err_obj = json.loads(err)
             return {"_error": err_obj.get("message", err[:200]), "_code": e.code}
         except Exception:
             return {"_error": err[:200], "_code": e.code}
-    except URLError as e:
-        return {"_error": str(e.reason), "_code": 0}
+    except (URLError, OSError, TimeoutError) as e:
+        _circuit.record_failure()
+        return {"_error": str(getattr(e, 'reason', e)), "_code": 0}
 
 
 def sb_select(table: str, filters: str = "", order: Optional[str] = None, limit: Optional[int] = None) -> List[Dict]:
@@ -136,32 +195,49 @@ def enable_recording(api_key: str, project_id: str, agent_name: str, session_id:
     _recording_session_id = session_id
 
 
-def sb_rpc(fn_name: str, params: Dict) -> Any:
-    url = f"{SUPABASE_URL}/rest/v1/rpc/{fn_name}"
-    body = json.dumps(params).encode("utf-8")
-    req = Request(url, data=body, method="POST", headers=_headers(content_profile=False))
-    try:
-        with urlopen(req, timeout=10) as resp:
-            raw = resp.read().decode("utf-8")
-            result = json.loads(raw) if raw.strip() else None
-    except HTTPError as e:
-        err = e.read().decode("utf-8", errors="replace")
+def sb_rpc(fn_name: str, params: Dict, *, _max_retries: int = 3) -> Any:
+    import random as _random
+    if not _circuit.can_execute():
+        return {"_error": "circuit breaker open — Supabase temporarily unavailable", "_circuit": "open"}
+    last_err = None
+    for attempt in range(_max_retries):
+        url = f"{SUPABASE_URL}/rest/v1/rpc/{fn_name}"
+        body = json.dumps(params).encode("utf-8")
+        req = Request(url, data=body, method="POST", headers=_headers(content_profile=False))
         try:
-            result = {"_error": json.loads(err).get("message", err[:200])}
-        except Exception:
-            result = {"_error": err[:200]}
-        # Record errors too
-        if _recording_enabled and fn_name not in _SKIP_RECORDING:
-            _bg_record("error", {"rpc": fn_name, "error": str(err)[:200]})
-        return result
-    except URLError as e:
-        return {"_error": str(e.reason)}
-
-    # Auto-record tool calls to session events (hot-reloadable)
+            with urlopen(req, timeout=10) as resp:
+                raw = resp.read().decode("utf-8")
+                result = json.loads(raw) if raw.strip() else None
+            _circuit.record_success()
+            # Auto-record tool calls to session events (hot-reloadable)
+            if _recording_enabled and fn_name not in _SKIP_RECORDING:
+                _bg_record("tool_call", {"rpc": fn_name})
+            return result
+        except HTTPError as e:
+            err = e.read().decode("utf-8", errors="replace")
+            # 4xx errors are not transient — don't retry, don't trip breaker
+            if 400 <= e.code < 500:
+                _circuit.record_success()
+                try:
+                    result = {"_error": json.loads(err).get("message", err[:200])}
+                except Exception:
+                    result = {"_error": err[:200]}
+                if _recording_enabled and fn_name not in _SKIP_RECORDING:
+                    _bg_record("error", {"rpc": fn_name, "error": str(err)[:200]})
+                return result
+            _circuit.record_failure()
+            last_err = err
+        except (URLError, OSError, TimeoutError) as e:
+            _circuit.record_failure()
+            last_err = str(getattr(e, 'reason', e))
+        # Retry with jitter for transient errors (5xx, network)
+        if attempt < _max_retries - 1:
+            delay = (2 ** attempt) + _random.uniform(0, 1)
+            _time.sleep(delay)
+    # All retries exhausted
     if _recording_enabled and fn_name not in _SKIP_RECORDING:
-        _bg_record("tool_call", {"rpc": fn_name})
-
-    return result
+        _bg_record("error", {"rpc": fn_name, "error": str(last_err)[:200], "retries_exhausted": True})
+    return {"_error": str(last_err)[:200] if last_err else "request failed after retries"}
 
 
 def _bg_record(event_type: str, payload: dict):

{meshcode-2.10.13 → meshcode-2.10.15}/meshcode/meshcode_mcp/realtime.py

@@ -53,8 +53,9 @@ class RealtimeListener:
         self.notify_callback = notify_callback
         self.service_role_key = service_role_key
 
-        # Last 100 unread messages — drained by meshcode_check tool
-        self.queue: Deque[Dict] = deque(maxlen=100)
+        # Last 500 unread messages — drained by meshcode_check tool
+        self.queue: Deque[Dict] = deque(maxlen=500)
+        self._overflow_warned = False
         self._task: Optional[asyncio.Task] = None
         # asyncio.Event() in Py3.10+ no longer requires a running loop, but
         # on older Python or certain Windows event-loop policies it can
@@ -99,22 +100,39 @@ class RealtimeListener:
                 pass
 
     async def _run(self) -> None:
-        """Outer loop: reconnect with exponential backoff on disconnect."""
+        """Outer loop: reconnect with exponential backoff on disconnect.
+
+        NEVER gives up — keeps retrying with capped backoff (max 60s).
+        The MCP server must stay alive regardless of Realtime health.
+        """
         backoff = 1
+        consecutive_failures = 0
         while not self._stop.is_set():
             try:
                 await self._connect_and_listen()
                 backoff = 1  # reset on clean disconnect
+                consecutive_failures = 0
             except asyncio.CancelledError:
                 return
             except Exception as e:
-                log.warning(f"Realtime connection error: {e}; reconnecting in {backoff}s")
+                consecutive_failures += 1
+                if consecutive_failures % 10 == 0:
+                    log.error(
+                        f"Realtime: {consecutive_failures} consecutive failures — "
+                        f"still retrying (backoff={backoff}s). Last error: {e}"
+                    )
+                else:
+                    log.warning(
+                        f"Realtime connection error ({consecutive_failures}): "
+                        f"{e}; reconnecting in {backoff}s"
+                    )
+                self._connected = False
                 try:
                     await asyncio.wait_for(self._stop.wait(), timeout=backoff)
                     return  # stop signaled
                 except asyncio.TimeoutError:
                     pass
-                backoff = min(backoff * 2, 30)
+                backoff = min(backoff * 2, 60)
 
     async def _connect_and_listen(self) -> None:
         """Single connection lifecycle: connect, subscribe, listen."""
@@ -155,23 +173,33 @@ class RealtimeListener:
             await ws.send(json.dumps(join_msg))
 
             # Wait for phx_reply to confirm subscription was accepted.
+            # Retry up to 3 times with backoff — transient Supabase latency
+            # can cause one agent to time out while siblings succeed.
             self._subscription_ok = False
-            try:
-                reply_raw = await asyncio.wait_for(ws.recv(), timeout=10.0)
-                reply = json.loads(reply_raw)
-                reply_status = (reply.get("payload") or {}).get("status")
-                if reply_status == "ok":
-                    self._subscription_ok = True
-                    log.info(f"Realtime subscription OK for {self.agent_name} on {topic}")
-                else:
-                    log.error(
-                        f"Realtime subscription FAILED for {self.agent_name}: "
-                        f"status={reply_status} payload={reply.get('payload')}"
-                    )
-            except asyncio.TimeoutError:
-                log.error(f"Realtime subscription TIMEOUT — no phx_reply in 10s for {self.agent_name}")
-            except Exception as e:
-                log.error(f"Realtime subscription error reading phx_reply: {e}")
+            for _sub_attempt in range(3):
+                try:
+                    reply_raw = await asyncio.wait_for(ws.recv(), timeout=15.0)
+                    reply = json.loads(reply_raw)
+                    reply_status = (reply.get("payload") or {}).get("status")
+                    if reply_status == "ok":
+                        self._subscription_ok = True
+                        log.info(f"Realtime subscription OK for {self.agent_name} on {topic}")
+                        break
+                    else:
+                        log.warning(
+                            f"Realtime subscription attempt {_sub_attempt+1} failed for {self.agent_name}: "
+                            f"status={reply_status} payload={reply.get('payload')}"
+                        )
+                except asyncio.TimeoutError:
+                    log.warning(f"Realtime subscription attempt {_sub_attempt+1} TIMEOUT for {self.agent_name}")
+                except Exception as e:
+                    log.warning(f"Realtime subscription attempt {_sub_attempt+1} error: {e}")
+                # Retry: re-send join message after brief backoff
+                if _sub_attempt < 2:
+                    await asyncio.sleep(2 ** _sub_attempt)
+                    await ws.send(json.dumps(join_msg))
+            if not self._subscription_ok:
+                log.error(f"Realtime subscription FAILED after 3 attempts for {self.agent_name}")
 
             # Heartbeat task to keep the connection alive
             heartbeat_task = asyncio.create_task(self._heartbeat(ws))
@@ -228,6 +256,9 @@ class RealtimeListener:
                         "id": record.get("id"),
                         "parent_id": record.get("parent_msg_id"),
                     }
+                    if len(self.queue) >= 400 and not self._overflow_warned:
+                        log.warning(f"Message queue at {len(self.queue)}/500 — risk of dropping messages")
+                        self._overflow_warned = True
                     self.queue.append(enriched)
                     # Wake any meshcode_wait blocked on this event.
                     try:

{meshcode-2.10.13 → meshcode-2.10.15}/meshcode/meshcode_mcp/server.py

@@ -8,11 +8,14 @@ Run with:
     MESHCODE_PROJECT=my-app MESHCODE_AGENT=backend python -m meshcode_mcp serve
 """
 import asyncio
+import atexit
 import json
 import logging
 import os
+import signal
 import sys
 import hashlib as _hashlib
+import traceback as _traceback
 from collections import deque
 from contextlib import asynccontextmanager
 from typing import Any, Dict, List, Optional, Union
@@ -64,16 +67,21 @@ def _agent_color(name: str) -> str:
 
 
 def _mc_log(msg: str, level: str = "info") -> None:
-    """Colored [meshcode-mcp] log line. Uses agent color if available."""
+    """Colored [meshcode-mcp] log line. Uses agent color if available.
+
+    CRITICAL: Must write to stderr, NEVER stdout. MCP protocol uses stdout
+    for JSON-RPC — any non-JSON output to stdout corrupts the stream and
+    causes Claude Code to kill the connection.
+    """
     agent = os.environ.get("MESHCODE_AGENT", "")
     c = _agent_color(agent) if agent else "\033[36m"
     prefix = f"{c}{_ANSI_BOLD}[meshcode-mcp]{_ANSI_RESET}"
     if level == "error":
-        print(f"{prefix} \033[91mERROR:{_ANSI_RESET} {msg}", "warn")
+        print(f"{prefix} \033[91mERROR:{_ANSI_RESET} {msg}", file=sys.stderr)
     elif level == "warn":
-        print(f"{prefix} \033[33mWARNING:{_ANSI_RESET} {msg}", "warn")
+        print(f"{prefix} \033[33mWARNING:{_ANSI_RESET} {msg}", file=sys.stderr)
     else:
-        print(f"{prefix} {c}{msg}{_ANSI_RESET}", "warn")
+        print(f"{prefix} {c}{msg}{_ANSI_RESET}", file=sys.stderr)
 
 
 # ============================================================
@@ -370,34 +378,44 @@ def _get_api_key() -> str:
                 _API_KEY_CACHE = kc_val
                 return kc_val
         except Exception as e:
-            _mc_log(f" keychain lookup failed for profile '{profile}': {e}", "warn")
+            _mc_log(f" keychain lookup failed for profile '{profile}': {e}", file=sys.stderr)
     _API_KEY_CACHE = ""
     return ""
 
 
-# Resolve project_id at startup. Try in order:
+# Resolve project_id at startup with retry+backoff. Try in order:
 #   1. MESHCODE_PROJECT_ID env var (baked by `meshcode setup`, fastest)
 #   2. mc_resolve_project RPC with the user's api_key (security definer, bypasses RLS)
 #   3. Direct SELECT via get_project_id (only works if RLS is open / user is admin)
 _PROJECT_ID: Optional[str] = os.environ.get("MESHCODE_PROJECT_ID") or None
 if not _PROJECT_ID:
-    _api_key = _get_api_key()
-    if _api_key:
-        try:
-            _r = be.sb_rpc("mc_resolve_project", {
-                "p_api_key": _api_key,
-                "p_project_name": PROJECT_NAME,
-            })
-            if isinstance(_r, dict) and _r.get("project_id"):
-                _PROJECT_ID = _r["project_id"]
-            elif isinstance(_r, dict) and _r.get("error"):
-                _mc_log(f" mc_resolve_project: {_r['error']}", "warn")
-        except Exception as _e:
-            _mc_log(f" mc_resolve_project failed: {_e}", "warn")
-if not _PROJECT_ID:
-    _PROJECT_ID = be.get_project_id(PROJECT_NAME)
+    _BOOT_MAX_RETRIES = 3
+    _BOOT_BACKOFF = [2, 5, 10]  # seconds between retries
+    for _boot_attempt in range(_BOOT_MAX_RETRIES):
+        _api_key = _get_api_key()
+        if _api_key:
+            try:
+                _r = be.sb_rpc("mc_resolve_project", {
+                    "p_api_key": _api_key,
+                    "p_project_name": PROJECT_NAME,
+                })
+                if isinstance(_r, dict) and _r.get("project_id"):
+                    _PROJECT_ID = _r["project_id"]
+                    break
+                elif isinstance(_r, dict) and _r.get("error"):
+                    _mc_log(f" mc_resolve_project: {_r['error']}", file=sys.stderr)
+            except Exception as _e:
+                _mc_log(f" mc_resolve_project failed: {_e}", file=sys.stderr)
+        if not _PROJECT_ID:
+            _PROJECT_ID = be.get_project_id(PROJECT_NAME)
+        if _PROJECT_ID:
+            break
+        if _boot_attempt < _BOOT_MAX_RETRIES - 1:
+            _wait = _BOOT_BACKOFF[_boot_attempt]
+            _mc_log(f" project resolution failed (attempt {_boot_attempt+1}/{_BOOT_MAX_RETRIES}), retrying in {_wait}s...", file=sys.stderr)
+            _time.sleep(_wait)
 if not _PROJECT_ID:
-    _mc_log(f"project '{PROJECT_NAME}' not found (check MESHCODE_KEYCHAIN_PROFILE / MESHCODE_API_KEY)", "error")
+    _mc_log(f"project '{PROJECT_NAME}' not found after {_BOOT_MAX_RETRIES} attempts (check MESHCODE_KEYCHAIN_PROFILE / MESHCODE_API_KEY)", "error")
     sys.exit(2)
 
 # Resolve project plan for adaptive features (heartbeat interval, etc.)
@@ -411,7 +429,7 @@ except Exception:
 
 _register_result = be.register_agent(PROJECT_NAME, AGENT_NAME, AGENT_ROLE or "MCP-connected agent", api_key=_get_api_key())
 if isinstance(_register_result, dict) and _register_result.get("error"):
-    _mc_log(f" register failed: {_register_result['error']}", "warn")
+    _mc_log(f" register failed: {_register_result['error']}", file=sys.stderr)
 
 # ── Fetch profile color from dashboard (single source of truth) ──
 try:
@@ -445,7 +463,7 @@ def _flip_status(status: str, task: str = "") -> bool:
         return False
 
 if not _flip_status("idle", ""):
-    _mc_log(f" could not flip status to idle", "warn")
+    _mc_log(f" could not flip status to idle", file=sys.stderr)
 
 
 # ============================================================
@@ -492,16 +510,17 @@ def _schedule_flip(status: str, task: str = "") -> None:
 
 
 def _set_state(state: str, tool: str = "") -> None:
-    """Update the state machine and broadcast to dashboard."""
+    """Update the state machine and broadcast to dashboard. Thread-safe."""
     global _current_state, _current_tool, _last_tool_at, _working_timer
-    # Cancel any pending working→online timer
-    if _working_timer is not None:
-        _working_timer.cancel()
-        _working_timer = None
-    _current_state = state
-    _current_tool = tool
-    if state == "working":
-        _last_tool_at = _time.time()
+    with _flip_lock:
+        # Cancel any pending working→online timer
+        if _working_timer is not None:
+            _working_timer.cancel()
+            _working_timer = None
+        _current_state = state
+        _current_tool = tool
+        if state == "working":
+            _last_tool_at = _time.time()
     _schedule_flip(state, tool)
 
 
@@ -552,13 +571,14 @@ def with_working_status(func):
             except Exception as e:
                 if not skip:
                     _auto_learn_error(name, e, list(kwargs.keys()))
-                raise
+                # NEVER re-raise — return structured error instead of crashing
+                import traceback as _tb
+                _log_crash_to_db("tool_exception", f"{name}: {type(e).__name__}: {e}\n{_tb.format_exc()[-500:]}")
+                return {"error": f"tool {name} failed: {type(e).__name__}: {e}", "_recovered": True}
             finally:
                 if not skip:
                     global _last_tool_at
                     _last_tool_at = _time.time()
-                    # Don't flip to online here — CPU-based detection in heartbeat
-                    # will handle the transition when LLM stops generating
         return awrapper
     else:
         @_functools.wraps(func)
@@ -575,13 +595,14 @@ def with_working_status(func):
             except Exception as e:
                 if not skip:
                     _auto_learn_error(name, e, list(kwargs.keys()))
-                raise
+                # NEVER re-raise — return structured error instead of crashing
+                import traceback as _tb
+                _log_crash_to_db("tool_exception", f"{name}: {type(e).__name__}: {e}\n{_tb.format_exc()[-500:]}")
+                return {"error": f"tool {name} failed: {type(e).__name__}: {e}", "_recovered": True}
             finally:
                 if not skip:
                     global _last_tool_at
                     _last_tool_at = _time.time()
-                    # Don't flip to online here — CPU-based detection in heartbeat
-                    # will handle the transition when LLM stops generating
         return swrapper
 
 
@@ -607,7 +628,7 @@ def _acquire_lease() -> bool:
         })
     except Exception as e:
         # Non-fatal: RPC might not exist on older servers.
-        _mc_log(f"stale-lease pre-clean skipped: {e}", "warn")
+        _mc_log(f"stale-lease pre-clean skipped: {e}", file=sys.stderr)
     for attempt in range(3):
         try:
             r = be.sb_rpc("mc_acquire_agent_lease", {
@@ -659,14 +680,14 @@ def _acquire_lease() -> bool:
                     _mc_log(f"Could not start — agent '{AGENT_NAME}' is running in another window.", "error")
                     _mc_log("Close the other window first, or use a different agent name.", "error")
                     return False
-                _mc_log(f"lease attempt {attempt+1}: {r.get('error')}", "warn")
+                _mc_log(f"lease attempt {attempt+1}: {r.get('error')}", file=sys.stderr)
             else:
                 return True
         except Exception as e:
-            _mc_log(f"lease attempt {attempt+1} failed: {e}", "warn")
+            _mc_log(f"lease attempt {attempt+1} failed: {e}", file=sys.stderr)
         if attempt < 2:
             _time.sleep(2)
-    _mc_log(f" lease failed after 3 attempts — proceeding anyway", "warn")
+    _mc_log(f" lease failed after 3 attempts — proceeding anyway", file=sys.stderr)
     return True
 
 if not _acquire_lease():
@@ -684,7 +705,7 @@ def _boot_diagnostic() -> None:
         be.sb_select("mc_projects", f"id=eq.{_PROJECT_ID}", limit=1)
         checks_passed += 1
     except Exception as e:
-        print(f"[meshcode] BOOT CHECK FAILED: Supabase API unreachable ({e}). Fix: check network/VPN.", "warn")
+        print(f"[meshcode] BOOT CHECK FAILED: Supabase API unreachable ({e}). Fix: check network/VPN.", file=sys.stderr)
 
     # Check 2: Lease valid
     try:
@@ -694,11 +715,11 @@ def _boot_diagnostic() -> None:
             if agent.get("instance_id") == _INSTANCE_ID:
                 checks_passed += 1
             else:
-                print(f"[meshcode] BOOT CHECK FAILED: Lease mismatch — expected {_INSTANCE_ID}, got {agent.get('instance_id')}. Fix: restart agent.", "warn")
+                print(f"[meshcode] BOOT CHECK FAILED: Lease mismatch — expected {_INSTANCE_ID}, got {agent.get('instance_id')}. Fix: restart agent.", file=sys.stderr)
         else:
-            print(f"[meshcode] BOOT CHECK FAILED: Agent '{AGENT_NAME}' not found in project. Fix: register agent first.", "warn")
+            print(f"[meshcode] BOOT CHECK FAILED: Agent '{AGENT_NAME}' not found in project. Fix: register agent first.", file=sys.stderr)
     except Exception as e:
-        print(f"[meshcode] BOOT CHECK FAILED: Could not verify lease ({e}).", "warn")
+        print(f"[meshcode] BOOT CHECK FAILED: Could not verify lease ({e}).", file=sys.stderr)
 
     # Check 3: Heartbeat recent
     try:
@@ -707,7 +728,7 @@ def _boot_diagnostic() -> None:
             if hb:
                 checks_passed += 1
             else:
-                print(f"[meshcode] BOOT CHECK WARNING: No heartbeat recorded yet.", "warn")
+                print(f"[meshcode] BOOT CHECK WARNING: No heartbeat recorded yet.", file=sys.stderr)
         else:
             checks_passed += 1  # skip if no agent data
     except Exception:
@@ -724,9 +745,9 @@ def _boot_diagnostic() -> None:
         checks_passed += 1  # non-critical
 
     if checks_passed == checks_total:
-        print(f"[meshcode] All boot checks passed ({checks_passed}/{checks_total}).", "warn")
+        print(f"[meshcode] All boot checks passed ({checks_passed}/{checks_total}).", file=sys.stderr)
     else:
-        print(f"[meshcode] Boot checks: {checks_passed}/{checks_total} passed. Agent starting anyway.", "warn")
+        print(f"[meshcode] Boot checks: {checks_passed}/{checks_total} passed. Agent starting anyway.", file=sys.stderr)
 
 
 _boot_diagnostic()
@@ -747,6 +768,53 @@ def _release_lease() -> None:
         pass
 
 
+# ── Crash logging + graceful shutdown ──────────────────────────
+_SHUTDOWN_LOGGED = False
+
+
+def _log_crash_to_db(reason: str = "unknown", error_detail: str = "") -> None:
+    """Best-effort crash log to mc_agent_crash_logs table. Non-fatal if table doesn't exist."""
+    global _SHUTDOWN_LOGGED
+    if _SHUTDOWN_LOGGED:
+        return
+    _SHUTDOWN_LOGGED = True
+    try:
+        be.sb_rpc("mc_log_error", {
+            "p_api_key": _get_api_key(),
+            "p_project_id": _PROJECT_ID,
+            "p_agent_name": AGENT_NAME,
+            "p_error_type": reason,
+            "p_error_detail": error_detail[:2000],
+            "p_instance_id": _INSTANCE_ID,
+        })
+    except Exception:
+        # Table may not exist yet — fall back to status update
+        try:
+            be.set_status(_PROJECT_ID, AGENT_NAME, "offline",
+                          f"crashed: {reason[:100]}", api_key=_get_api_key())
+        except Exception:
+            pass
+    _mc_log(f" crash logged: {reason}", file=sys.stderr)
+
+
+def _on_exit() -> None:
+    """atexit handler — release lease and log shutdown."""
+    _log_crash_to_db("process_exit", "atexit handler fired")
+    _release_lease()
+
+
+def _on_signal(signum, frame) -> None:
+    """Signal handler for SIGTERM/SIGINT — clean shutdown."""
+    sig_name = signal.Signals(signum).name if hasattr(signal, 'Signals') else str(signum)
+    _log_crash_to_db("signal", f"Received {sig_name}")
+    _release_lease()
+    sys.exit(128 + signum)
+
+
+atexit.register(_on_exit)
+signal.signal(signal.SIGTERM, _on_signal)
+
+
 # ============================================================
 # Agent identity from Supabase profile (for system instructions)
 # ============================================================
@@ -1037,25 +1105,28 @@ def _heartbeat_thread_fn():
         try:
             be.sb_rpc("mc_heartbeat", {"p_project_id": _PROJECT_ID, "p_agent_name": AGENT_NAME, "p_version": _SDK_VERSION})
 
-            # CPU-based status detection
+            # CPU-based status detection — read shared state under lock
             parent_cpu = _get_parent_cpu()
-            idle_secs = _time.time() - _last_tool_at
+            with _flip_lock:
+                cur_state = _current_state
+                in_wait = _IN_WAIT
+                idle_secs = _time.time() - _last_tool_at
 
-            if _IN_WAIT:
+            if in_wait:
                 # Actually in meshcode_wait right now — listening for messages
-                if _current_state != "waiting":
+                if cur_state != "waiting":
                     _set_state("waiting", "listening for messages")
             elif parent_cpu > 3.0:
                 # LLM is actively generating tokens or streaming
-                if _current_state != "working":
+                if cur_state != "working":
                     _set_state("working", "generating response")
-            elif _current_state == "working":
+            elif cur_state == "working":
                 # LLM just stopped — transition to online (not sleeping)
                 _set_state("online", "")
-            elif _current_state == "online" and idle_secs > 30:
+            elif cur_state == "online" and idle_secs > 30:
                 # Brief idle — show as idle, not sleeping yet
                 _set_state("idle", "idle")
-            elif _current_state == "idle" and idle_secs > 300 and parent_cpu < 2.0 and not _STAY_AWAKE:
+            elif cur_state == "idle" and idle_secs > 300 and parent_cpu < 2.0 and not _STAY_AWAKE:
                 # Extended idle + no CPU activity → sleeping (5 min, not 90s)
                 _set_state("sleeping", "sleeping")
 
@@ -1105,6 +1176,18 @@ async def lifespan(_app):
     )
     await _REALTIME.start()
 
+    # Wait up to 5s for realtime to report connected before proceeding.
+    # Without this, the lifespan yields before the WS is ready, and Claude
+    # Code's handshake can time out on slower network paths — one agent
+    # fails while siblings on the same box succeed.
+    for _rt_check in range(10):
+        if getattr(_REALTIME, '_connected', False):
+            log.info(f"Realtime connected for {AGENT_NAME}")
+            break
+        await asyncio.sleep(0.5)
+    else:
+        log.warning(f"Realtime not connected after 5s for {AGENT_NAME} — continuing with polling fallback")
+
     # IMMEDIATE: send first heartbeat + set online status BEFORE any tool calls.
     # Without this, the agent appears offline for up to 30s after boot.
     for _attempt in range(3):
@@ -1152,16 +1235,19 @@ async def lifespan(_app):
                 })
         except Exception:
             pass  # Never block shutdown
-        log.info("lifespan shutdown — stopping heartbeat + realtime + releasing lease")
-        _heartbeat_stop.set()
-        hb_thread.join(timeout=5)
-        await _REALTIME.stop()
-        # Flip to offline + release lease so the dashboard reflects reality
-        # within seconds (not waiting for the 30s cron to notice).
+        log.info("lifespan shutdown — releasing lease + stopping heartbeat + realtime")
+        # Release lease FIRST — before stopping heartbeat thread.
+        # If heartbeat join times out, the lease is already released so
+        # the agent won't block reconnection.
         try:
             _release_lease()
         except Exception as _e:
             log.warning(f"could not release lease: {_e}")
+        _heartbeat_stop.set()
+        hb_thread.join(timeout=5)
+        if hb_thread.is_alive():
+            log.warning("heartbeat thread did not stop within 5s — lease already released")
+        await _REALTIME.stop()
 
 
 # ============================================================
@@ -1368,9 +1454,9 @@ try:
         elif isinstance(_ls_val, str):
             _LAST_SEEN_TS = _ls_val
         if _LAST_SEEN_TS:
-            print(f"[meshcode] Restored last_seen={_LAST_SEEN_TS} from mesh memory.", "warn")
+            print(f"[meshcode] Restored last_seen={_LAST_SEEN_TS} from mesh memory.", file=sys.stderr)
 except Exception as _e:
-    print(f"[meshcode] Could not restore last_seen: {_e}", "warn")
+    print(f"[meshcode] Could not restore last_seen: {_e}", file=sys.stderr)
 
 
 def _get_pending_tasks_summary() -> Optional[List[Dict[str, str]]]:
@@ -2166,6 +2252,7 @@ def meshcode_scratchpad_set(key: str, value: Any) -> Dict[str, Any]:
     json_value = value if isinstance(value, (dict, list)) else {"_raw": value}
     return be.sb_rpc("mc_scratchpad_set", {
         "p_api_key": api_key,
+        "p_project_id": _PROJECT_ID,
         "p_key": key,
         "p_value": json_value,
         "p_tier": "reference",
@@ -2182,9 +2269,9 @@ def meshcode_scratchpad_get(key: Optional[str] = None) -> Dict[str, Any]:
     """
     api_key = _get_api_key()
     if key:
-        return be.sb_rpc("mc_scratchpad_get", {"p_api_key": api_key, "p_key": key})
+        return be.sb_rpc("mc_scratchpad_get", {"p_api_key": api_key, "p_project_id": _PROJECT_ID, "p_key": key})
     else:
-        return be.sb_rpc("mc_scratchpad_list", {"p_api_key": api_key})
+        return be.sb_rpc("mc_scratchpad_list", {"p_api_key": api_key, "p_project_id": _PROJECT_ID})
 
 
 # ----------------- OBSIDIAN SYNC HELPER -----------------
@@ -2332,6 +2419,7 @@ def meshcode_forget(key: str) -> Dict[str, Any]:
         "p_api_key": api_key,
         "p_agent_name": AGENT_NAME,
         "p_key": key,
+        "p_project_name": PROJECT_NAME,
     })
 
 
@@ -2357,6 +2445,50 @@ def meshcode_recall_search(query: str) -> Dict[str, Any]:
     })
 
 
+# ----------------- HEALTH CHECK -----------------
+
+@mcp.tool()
+def meshcode_health() -> Dict[str, Any]:
+    """Check MCP server health: DB connectivity, Realtime status, circuit breaker state, uptime."""
+    import time as _t
+    health: Dict[str, Any] = {
+        "agent": AGENT_NAME,
+        "project": PROJECT_NAME,
+        "instance_id": _INSTANCE_ID,
+        "sdk_version": _SDK_VERSION,
+    }
+
+    # DB latency check
+    _start = _t.monotonic()
+    try:
+        r = be.sb_select("mc_projects", f"id=eq.{_PROJECT_ID}", limit=1)
+        health["db_latency_ms"] = round((_t.monotonic() - _start) * 1000, 1)
+        health["db_status"] = "ok" if r else "empty"
+    except Exception as e:
+        health["db_latency_ms"] = round((_t.monotonic() - _start) * 1000, 1)
+        health["db_status"] = f"error: {e}"
+
+    # Circuit breaker state
+    health["circuit_breaker"] = {
+        "state": be._circuit.state,
+        "failure_count": be._circuit.failure_count,
+        "threshold": be._circuit.failure_threshold,
+    }
+
+    # Realtime status
+    health["realtime_connected"] = getattr(_rt_state, 'connected', False) if '_rt_state' in dir() else "unknown"
+
+    # Process uptime
+    try:
+        import psutil
+        proc = psutil.Process()
+        health["uptime_seconds"] = round(_t.time() - proc.create_time(), 0)
+    except Exception:
+        health["uptime_seconds"] = "unknown (psutil not available)"
+
+    return health
+
+
 # ----------------- RESOURCES -----------------
 
 @mcp.tool()
@@ -2484,7 +2616,7 @@ def _auto_update() -> None:
         return
 
     # 3. Install the new version (blocking, 60s timeout)
-    print(f"[meshcode] Updating {current} → {latest}...", "warn")
+    print(f"[meshcode] Updating {current} → {latest}...", file=sys.stderr)
     try:
         result = subprocess.run(
             [sys.executable, "-m", "pip", "install", "--upgrade",
@@ -2502,7 +2634,7 @@ def _auto_update() -> None:
         return
 
     # 4. Re-exec to load the new code
-    print(f"[meshcode] Updated to {latest}, restarting...", "warn")
+    print(f"[meshcode] Updated to {latest}, restarting...", file=sys.stderr)
     os.environ["MESHCODE_UPDATED"] = "1"
     try:
         os.execv(sys.executable, [sys.executable] + sys.argv)
@@ -2511,10 +2643,28 @@ def _auto_update() -> None:
 
 
 def run_server():
-    """Start the MCP server on stdio (default for Claude Code)."""
+    """Start the MCP server on stdio (default for Claude Code).
+
+    Wraps mcp.run() with crash recovery: if the event loop dies for any
+    reason, log the crash, release the lease, and exit cleanly instead of
+    leaving the agent in a zombie state.
+    """
     _auto_update()
     print(
         f"[meshcode-mcp] Starting server for {AGENT_NAME}@{PROJECT_NAME}",
         file=sys.stderr,
     )
-    mcp.run()
+    try:
+        mcp.run()
+    except KeyboardInterrupt:
+        _log_crash_to_db("keyboard_interrupt", "User stopped the agent")
+    except SystemExit as e:
+        _log_crash_to_db("system_exit", f"exit code: {e.code}")
+        raise  # re-raise so the process exits with the correct code
+    except Exception as e:
+        import traceback as _tb
+        tb_str = _tb.format_exc()
+        _log_crash_to_db("unhandled_exception", f"{type(e).__name__}: {e}\n{tb_str}")
+        print(f"[meshcode-mcp] FATAL: {e}", file=sys.stderr)
+        print(f"[meshcode-mcp] Stack trace logged to mc_agent_crash_logs", file=sys.stderr)
+        sys.exit(1)

{meshcode-2.10.13 → meshcode-2.10.15}/meshcode.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: meshcode
-Version: 2.10.13
+Version: 2.10.15
 Summary: Real-time communication between AI agents — Supabase-backed CLI
 Author-email: MeshCode <hello@meshcode.io>
 License: MIT

{meshcode-2.10.13 → meshcode-2.10.15}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "meshcode"
-version = "2.10.13"
+version = "2.10.15"
 description = "Real-time communication between AI agents — Supabase-backed CLI"
 readme = "README.md"
 license = {text = "MIT"}