portacode 1.4.35.dev4__tar.gz → 1.4.37.dev0__tar.gz

{portacode-1.4.35.dev4 → portacode-1.4.37.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: portacode
-Version: 1.4.35.dev4
+Version: 1.4.37.dev0
 Summary: Portacode CLI client and SDK
 Home-page: https://github.com/portacode/portacode
 Author: Meena Erian

{portacode-1.4.35.dev4 → portacode-1.4.37.dev0}/portacode/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '1.4.35.dev4'
-__version_tuple__ = version_tuple = (1, 4, 35, 'dev4')
+__version__ = version = '1.4.37.dev0'
+__version_tuple__ = version_tuple = (1, 4, 37, 'dev0')
 
 __commit_id__ = commit_id = None

{portacode-1.4.35.dev4 → portacode-1.4.37.dev0}/portacode/connection/client.py

@@ -17,9 +17,13 @@ from websockets import WebSocketClientProtocol
 from ..keypair import KeyPair
 from .multiplex import Multiplexer
 from ..logging_categories import get_categorized_logger, LogCategory
+from ..exit_codes import AUTH_REJECTED_EXIT_CODE
 
 logger = get_categorized_logger(__name__)
 
+class AuthenticationRejectedError(RuntimeError):
+    """Raised when gateway authentication is rejected for this device key."""
+
 
 class ConnectionManager:
     """Maintain a persistent connection to the Portacode gateway.
@@ -43,6 +47,8 @@ class ConnectionManager:
     CLOCK_SYNC_INITIAL_REQUESTS = 5
     CLOCK_SYNC_TIMEOUT = 20.0
     CLOCK_SYNC_MAX_FAILURES = 3
+    WS_PING_INTERVAL = 60.0
+    WS_PING_TIMEOUT = 20.0
 
     def __init__(self, gateway_url: str, keypair: KeyPair, reconnect_delay: float = 1.0, max_retries: int = None, debug: bool = False):
         self.gateway_url = gateway_url
@@ -85,7 +91,11 @@ class ConnectionManager:
                     logger.warning("Reconnecting in %.1f s (attempt %d)…", LogCategory.CONNECTION, delay, attempt)
                     await asyncio.sleep(delay)
                 logger.info("Connecting to gateway at %s", LogCategory.CONNECTION, self.gateway_url)
-                async with websockets.connect(self.gateway_url) as ws:
+                async with websockets.connect(
+                    self.gateway_url,
+                    ping_interval=self.WS_PING_INTERVAL,
+                    ping_timeout=self.WS_PING_TIMEOUT,
+                ) as ws:
                     # Reset attempt counter after successful connection
                     attempt = 0
 
@@ -118,6 +128,9 @@ class ConnectionManager:
                 logger.warning("Connection error: %s", LogCategory.CONNECTION, exc)
                 # Remove the max_retries limit - keep trying indefinitely
                 # The service manager (systemd) will handle any necessary restarts
+            except AuthenticationRejectedError as exc:
+                logger.error("Authentication rejected by gateway: %s", LogCategory.CONNECTION, exc)
+                sys.exit(AUTH_REJECTED_EXIT_CODE)
             except Exception as exc:
                 # For truly fatal errors (like authentication failures), 
                 # log and exit cleanly so systemd can restart the service
@@ -137,7 +150,7 @@ class ConnectionManager:
             challenge = data["challenge"]
         except Exception:
             # Not a challenge, must be an error
-            raise RuntimeError(f"Gateway rejected authentication: {response}")
+            raise AuthenticationRejectedError(f"Gateway rejected authentication: {response}")
         # Step 3: Sign challenge and send signature
         signature = self.keypair.sign_challenge(challenge)
         signature_b64 = base64.b64encode(signature).decode()
@@ -145,7 +158,7 @@ class ConnectionManager:
         # Step 4: Receive final status
         status = await self.websocket.recv()
         if status != "ok":
-            raise RuntimeError(f"Gateway rejected authentication: {status}")
+            raise AuthenticationRejectedError(f"Gateway rejected authentication: {status}")
         # Print success message in green and show close instructions
         try:
             import click
@@ -206,6 +219,7 @@ class ConnectionManager:
             self._clock_sync_task.cancel()
         self._clock_sync_task = asyncio.create_task(self._clock_sync_loop())
         self._remaining_initial_syncs = self.CLOCK_SYNC_INITIAL_REQUESTS
+        self._clock_sync_failures = 0
 
     async def _stop_clock_sync_task(self) -> None:
         if self._clock_sync_task:
@@ -228,6 +242,9 @@ class ConnectionManager:
                 self._remaining_initial_syncs -= 1
                 if self._remaining_initial_syncs > 0:
                     await asyncio.sleep(self.CLOCK_SYNC_FAST_INTERVAL)
+            while not self._stop_event.is_set():
+                await asyncio.sleep(self.CLOCK_SYNC_INTERVAL)
+                await self._perform_clock_sync()
         except asyncio.CancelledError:
             pass
 

{portacode-1.4.35.dev4 → portacode-1.4.37.dev0}/portacode/connection/terminal.py

@@ -14,9 +14,12 @@ in the handlers directory.
 """
 
 import asyncio
+import faulthandler
 import json
 import logging
 import os
+import sys
+import threading
 import time
 from datetime import datetime, timezone
 from dataclasses import asdict
@@ -389,11 +392,185 @@ class TerminalManager:
         self.mux = mux
         self.debug = debug
         self._last_exposed_services_signature = "__unset__"
+        self._diag_lock = threading.Lock()
+        now_mono = time.monotonic()
+        self._diag_state: Dict[str, Any] = {
+            "heartbeat_monotonic": now_mono,
+            "heartbeat_epoch_s": time.time(),
+            "loop_phase": "init",
+            "active_cmd": None,
+            "active_cmd_started_monotonic": None,
+            "active_cmd_reply_channel": None,
+            "last_completed_cmd": None,
+            "last_completed_cmd_duration_s": None,
+            "queue_stats": {},
+        }
+        self._diag_monitor_enabled = os.getenv("PORTACODE_EVENT_LOOP_BLOCK_MONITOR", "1").lower() not in ("0", "false", "no")
+        self._diag_threshold_s = float(os.getenv("PORTACODE_EVENT_LOOP_BLOCK_THRESHOLD_S", "5"))
+        self._diag_monitor_interval_s = float(os.getenv("PORTACODE_EVENT_LOOP_BLOCK_MONITOR_INTERVAL_S", "0.5"))
+        self._diag_report_interval_s = float(os.getenv("PORTACODE_EVENT_LOOP_BLOCK_REPORT_INTERVAL_S", "30"))
+        self._diag_heartbeat_interval_s = float(os.getenv("PORTACODE_EVENT_LOOP_BLOCK_HEARTBEAT_INTERVAL_S", "0.5"))
+        self._diag_dump_threads = os.getenv("PORTACODE_EVENT_LOOP_BLOCK_DUMP_THREADS", "1").lower() not in ("0", "false", "no")
+        self._diag_monitor_stop = threading.Event()
+        self._diag_monitor_thread: Optional[threading.Thread] = None
+        self._diag_last_report_monotonic = 0.0
+        self._diag_last_report_key: Optional[str] = None
         self._session_manager = None  # Initialize as None first
         self._client_session_manager = ClientSessionManager()  # Initialize client session manager
         self._client_session_manager.set_terminal_manager(self)  # Set reference for cleanup
+        self._start_block_monitor_thread_if_enabled()
         self._set_mux(mux, is_initial=True)
 
+    def _start_block_monitor_thread_if_enabled(self) -> None:
+        if not self._diag_monitor_enabled or self._diag_monitor_thread:
+            return
+        self._diag_monitor_thread = threading.Thread(
+            target=self._event_loop_block_monitor_thread,
+            name="portacode-event-loop-block-monitor",
+            daemon=True,
+        )
+        self._diag_monitor_thread.start()
+
+    def _set_diag_phase(self, phase: str) -> None:
+        now_mono = time.monotonic()
+        now_epoch = time.time()
+        with self._diag_lock:
+            self._diag_state["loop_phase"] = phase
+            self._diag_state["heartbeat_monotonic"] = now_mono
+            self._diag_state["heartbeat_epoch_s"] = now_epoch
+
+    def _set_diag_active_cmd(self, cmd: str, reply_channel: Optional[str]) -> None:
+        now_mono = time.monotonic()
+        now_epoch = time.time()
+        with self._diag_lock:
+            self._diag_state["active_cmd"] = cmd
+            self._diag_state["active_cmd_reply_channel"] = reply_channel
+            self._diag_state["active_cmd_started_monotonic"] = now_mono
+            self._diag_state["heartbeat_monotonic"] = now_mono
+            self._diag_state["heartbeat_epoch_s"] = now_epoch
+
+    def _clear_diag_active_cmd(self, completed_cmd: str, duration_s: float) -> None:
+        now_mono = time.monotonic()
+        now_epoch = time.time()
+        with self._diag_lock:
+            self._diag_state["active_cmd"] = None
+            self._diag_state["active_cmd_reply_channel"] = None
+            self._diag_state["active_cmd_started_monotonic"] = None
+            self._diag_state["last_completed_cmd"] = completed_cmd
+            self._diag_state["last_completed_cmd_duration_s"] = round(duration_s, 4)
+            self._diag_state["heartbeat_monotonic"] = now_mono
+            self._diag_state["heartbeat_epoch_s"] = now_epoch
+
+    def _snapshot_diag_state_for_monitor(self) -> Dict[str, Any]:
+        with self._diag_lock:
+            return dict(self._diag_state)
+
+    def _event_loop_block_monitor_thread(self) -> None:
+        while not self._diag_monitor_stop.wait(self._diag_monitor_interval_s):
+            snapshot = self._snapshot_diag_state_for_monitor()
+            now = time.monotonic()
+            last_hb = float(snapshot.get("heartbeat_monotonic") or now)
+            blocked_for_s = max(now - last_hb, 0.0)
+
+            if blocked_for_s < self._diag_threshold_s:
+                continue
+
+            active_cmd = snapshot.get("active_cmd")
+            queue_stats = snapshot.get("queue_stats") if isinstance(snapshot.get("queue_stats"), dict) else {}
+            control_q = queue_stats.get("control_channel_queue_size")
+            total_mux_q = queue_stats.get("mux_total_queued_frames")
+            has_backlog = (
+                (isinstance(control_q, int) and control_q > 0)
+                or (isinstance(total_mux_q, int) and total_mux_q > 0)
+            )
+            if not active_cmd and not has_backlog:
+                continue
+
+            active_cmd_started = snapshot.get("active_cmd_started_monotonic")
+            active_cmd_age_s = None
+            if active_cmd and isinstance(active_cmd_started, (int, float)):
+                active_cmd_age_s = max(now - float(active_cmd_started), 0.0)
+
+            report_key = f"{snapshot.get('loop_phase')}:{active_cmd}:{snapshot.get('heartbeat_epoch_s')}"
+            if report_key == self._diag_last_report_key and (now - self._diag_last_report_monotonic) < self._diag_report_interval_s:
+                continue
+
+            self._diag_last_report_key = report_key
+            self._diag_last_report_monotonic = now
+
+            payload = {
+                "event": "event_loop_blocking_detected",
+                "blocked_for_s": round(blocked_for_s, 3),
+                "threshold_s": self._diag_threshold_s,
+                "cause_hint": {
+                    "loop_phase": snapshot.get("loop_phase"),
+                    "active_cmd": active_cmd,
+                    "active_cmd_reply_channel": snapshot.get("active_cmd_reply_channel"),
+                    "active_cmd_age_s": round(active_cmd_age_s, 3) if active_cmd_age_s is not None else None,
+                    "last_completed_cmd": snapshot.get("last_completed_cmd"),
+                    "last_completed_cmd_duration_s": snapshot.get("last_completed_cmd_duration_s"),
+                },
+                "queue_stats": queue_stats,
+                "last_heartbeat_epoch_s": snapshot.get("heartbeat_epoch_s"),
+                "monitor_thread_epoch_s": time.time(),
+                "pid": os.getpid(),
+            }
+            logger.error("Event loop stall monitor: %s", json.dumps(payload, sort_keys=True))
+
+            if self._diag_dump_threads:
+                try:
+                    faulthandler.dump_traceback(file=sys.stderr, all_threads=True)
+                except Exception as exc:
+                    logger.error("Event loop stall monitor failed to dump thread traceback: %s", exc)
+
+    def _collect_loop_queue_stats(self) -> Dict[str, Any]:
+        control_q_size = None
+        try:
+            control_q = getattr(self._control_channel, "_incoming", None)
+            if control_q is not None:
+                control_q_size = control_q.qsize()
+        except Exception:
+            control_q_size = None
+
+        mux_channels = getattr(self.mux, "_channels", {}) if self.mux else {}
+        total_queued = 0
+        top_channels = []
+        for channel_id, channel in mux_channels.items():
+            try:
+                incoming_q = getattr(channel, "_incoming", None)
+                size = incoming_q.qsize() if incoming_q is not None else None
+            except Exception:
+                size = None
+            if isinstance(size, int):
+                total_queued += size
+                top_channels.append((str(channel_id), size))
+
+        top_channels.sort(key=lambda item: item[1], reverse=True)
+        return {
+            "control_channel_queue_size": control_q_size,
+            "mux_channel_count": len(mux_channels),
+            "mux_total_queued_frames": total_queued,
+            "mux_top_queues": [{"channel_id": cid, "size": size} for cid, size in top_channels[:5]],
+            "client_sessions_count": len(self._client_session_manager.get_sessions()),
+        }
+
+    async def _diag_heartbeat_loop(self) -> None:
+        while True:
+            try:
+                await asyncio.sleep(self._diag_heartbeat_interval_s)
+                queue_stats = self._collect_loop_queue_stats()
+                now_mono = time.monotonic()
+                now_epoch = time.time()
+                with self._diag_lock:
+                    self._diag_state["heartbeat_monotonic"] = now_mono
+                    self._diag_state["heartbeat_epoch_s"] = now_epoch
+                    self._diag_state["queue_stats"] = queue_stats
+            except asyncio.CancelledError:
+                return
+            except Exception as exc:
+                logger.debug("Event loop heartbeat diagnostics error: %s", exc)
+                await asyncio.sleep(self._diag_heartbeat_interval_s)
+
     # ------------------------------------------------------------------
     # Mux attach/detach helpers (for reconnection resilience)
     # ------------------------------------------------------------------
@@ -469,6 +646,14 @@ class TerminalManager:
             except Exception:
                 pass
         self._exposed_services_task = asyncio.create_task(self._watch_exposed_services())
+
+        # Start passive event-loop diagnostics heartbeat (asyncio thread only).
+        if getattr(self, "_diag_heartbeat_task", None):
+            try:
+                self._diag_heartbeat_task.cancel()
+            except Exception:
+                pass
+        self._diag_heartbeat_task = asyncio.create_task(self._diag_heartbeat_loop())
         
         # For initial connections, request client sessions after control loop starts
         if is_initial:
@@ -531,7 +716,9 @@ class TerminalManager:
         logger.info("terminal_manager: Starting control loop")
         while True:
             try:
+                self._set_diag_phase("control_loop_waiting_for_message")
                 message = await self._control_channel.recv()
+                self._set_diag_phase("control_loop_received_message")
                 logger.debug("terminal_manager: Received message: %s", message)
                 
                 # Older parts of the system may send *raw* str. Ensure dict.
@@ -554,6 +741,8 @@ class TerminalManager:
                     logger.warning("terminal_manager: Missing 'cmd' in control frame: %s", message)
                     continue
                 reply_chan = message.get("reply_channel")
+                cmd_start = time.monotonic()
+                self._set_diag_active_cmd(cmd, reply_chan)
                 
                 logger.info("terminal_manager: Processing command '%s' with reply_channel=%s", cmd, reply_chan)
                 logger.debug("terminal_manager: Full message: %s", message)
@@ -573,6 +762,8 @@ class TerminalManager:
                         asyncio.create_task(self._send_initial_data_to_clients(newly_added_sessions))
                     else:
                         logger.info("terminal_manager: ℹ️ No new sessions to send data to")
+                    self._clear_diag_active_cmd(cmd, time.monotonic() - cmd_start)
+                    self._set_diag_phase("control_loop_completed_client_sessions_update")
                     continue
                 
                 # Dispatch command through registry
@@ -580,9 +771,16 @@ class TerminalManager:
                 if not handled:
                     logger.warning("terminal_manager: Command '%s' was not handled by any handler", cmd)
                     await self._send_error(f"Unknown cmd: {cmd}", reply_chan)
+                self._clear_diag_active_cmd(cmd, time.monotonic() - cmd_start)
+                self._set_diag_phase(f"control_loop_completed_{cmd}")
                     
             except Exception as exc:
                 logger.exception("terminal_manager: Error in control loop: %s", exc)
+                with self._diag_lock:
+                    self._diag_state["active_cmd"] = None
+                    self._diag_state["active_cmd_started_monotonic"] = None
+                    self._diag_state["active_cmd_reply_channel"] = None
+                self._set_diag_phase("control_loop_exception")
                 # Continue processing other messages
                 continue
 

portacode-1.4.37.dev0/portacode/exit_codes.py

@@ -0,0 +1,7 @@
+"""Shared process exit codes used by the Portacode runtime and service managers."""
+
+# Process exited because the gateway rejected device authentication (for example,
+# an unknown/deleted device key). Service supervisors should not auto-restart on
+# this code to avoid endless auth failure loops.
+AUTH_REJECTED_EXIT_CODE = 86
+

{portacode-1.4.35.dev4 → portacode-1.4.37.dev0}/portacode/service.py

@@ -29,6 +29,8 @@ import shutil
 import pwd
 import tempfile
 
+from .exit_codes import AUTH_REJECTED_EXIT_CODE
+
 __all__ = [
     "ServiceManager",
     "get_manager",
@@ -117,8 +119,6 @@ class _SystemdUserService:
         current_shell = os.getenv("SHELL", "/bin/bash")
         
         if self.system_mode:
-            sudo_needed = os.geteuid() != 0
-            prefix = ["sudo"] if sudo_needed else []
             unit = textwrap.dedent(f"""
                 [Unit]
                 Description=Portacode persistent connection (system-wide)
@@ -131,6 +131,7 @@ class _SystemdUserService:
                 Environment=SHELL={current_shell}
                 ExecStart={self.python} -m portacode connect --non-interactive
                 Restart=on-failure
+                RestartPreventExitStatus={AUTH_REJECTED_EXIT_CODE}
                 RestartSec=5
 
                 [Install]
@@ -147,6 +148,7 @@ class _SystemdUserService:
                 Environment=SHELL={current_shell}
                 ExecStart={self.python} -m portacode.cli connect --non-interactive
                 Restart=on-failure
+                RestartPreventExitStatus={AUTH_REJECTED_EXIT_CODE}
                 RestartSec=5
 
                 [Install]
@@ -256,8 +258,6 @@ class _OpenRCService:
             pidfile="/run/portacode.pid"
             directory="{self.home}"
             supervisor=supervise-daemon
-            respawn_delay=5
-            respawn_max=0
 
             depend() {{
                 need net
@@ -289,7 +289,18 @@ class _OpenRCService:
         script = textwrap.dedent(f"""
             #!/bin/sh
             cd "{self.home}"
-            exec "{self.python}" -m portacode connect --non-interactive >> "{self.log_path}" 2>&1
+            while true; do
+                "{self.python}" -m portacode connect --non-interactive >> "{self.log_path}" 2>&1
+                rc="$?"
+                if [ "$rc" -eq {AUTH_REJECTED_EXIT_CODE} ]; then
+                    echo "Portacode authentication rejected; stopping service restart loop." >> "{self.log_path}"
+                    exit 0
+                fi
+                if [ "$rc" -eq 0 ]; then
+                    exit 0
+                fi
+                sleep 5
+            done
         """).lstrip()
         tmp_path = Path(tempfile.gettempdir()) / f"portacode-wrapper-{os.getpid()}"
         tmp_path.write_text(script)
@@ -362,6 +373,8 @@ class _LaunchdService:
             / "Library/LaunchAgents"
             / f"{self.LABEL}.plist"
         )
+        self.script_path = Path.home() / ".local" / "share" / "portacode" / "connect_service.sh"
+        self.log_path = Path.home() / ".local" / "share" / "portacode" / "connect.log"
 
     def _run(self, *args: str) -> subprocess.CompletedProcess[str]:
         cmd = ["launchctl", *args]
@@ -369,6 +382,27 @@ class _LaunchdService:
 
     def install(self) -> None:
         self.plist_path.parent.mkdir(parents=True, exist_ok=True)
+        self.script_path.parent.mkdir(parents=True, exist_ok=True)
+        script = textwrap.dedent(
+            f"""
+            #!/bin/sh
+            cd "$HOME" || exit 1
+            while true; do
+                "{sys.executable}" -m portacode connect --non-interactive >> "{self.log_path}" 2>&1
+                rc="$?"
+                if [ "$rc" -eq {AUTH_REJECTED_EXIT_CODE} ]; then
+                    echo "Portacode authentication rejected; stopping service restart loop." >> "{self.log_path}"
+                    exit 0
+                fi
+                if [ "$rc" -eq 0 ]; then
+                    exit 0
+                fi
+                sleep 5
+            done
+            """
+        ).lstrip()
+        self.script_path.write_text(script)
+        self.script_path.chmod(0o755)
         plist = textwrap.dedent(
             f"""
             <?xml version="1.0" encoding="UTF-8"?>
@@ -378,13 +412,13 @@ class _LaunchdService:
                 <key>Label</key><string>{self.LABEL}</string>
                 <key>ProgramArguments</key>
                 <array>
-                    <string>{sys.executable}</string>
-                    <string>-m</string><string>portacode</string>
-                    <string>connect</string>
-                    <string>--non-interactive</string>
+                    <string>{self.script_path}</string>
                 </array>
                 <key>RunAtLoad</key><true/>
-                <key>KeepAlive</key><true/>
+                <key>KeepAlive</key>
+                <dict>
+                    <key>SuccessfulExit</key><false/>
+                </dict>
             </dict>
             </plist>
             """
@@ -396,6 +430,8 @@ class _LaunchdService:
         self._run("unload", "-w", str(self.plist_path))
         if self.plist_path.exists():
             self.plist_path.unlink()
+        if self.script_path.exists():
+            self.script_path.unlink()
 
     def start(self) -> None:
         self._run("start", self.LABEL)
@@ -451,7 +487,13 @@ class _WindowsTask:
         script = (
             "@echo off\r\n"
             "cd /d %USERPROFILE%\r\n"
+            ":loop\r\n"
             f"{py_cmd} -m portacode connect --non-interactive >> \"%USERPROFILE%\\.local\\share\\portacode\\connect.log\" 2>>&1\r\n"
+            "set \"RC=%ERRORLEVEL%\"\r\n"
+            f"if \"%RC%\"==\"{AUTH_REJECTED_EXIT_CODE}\" exit /b 0\r\n"
+            "if \"%RC%\"==\"0\" exit /b 0\r\n"
+            "timeout /t 5 /nobreak >nul\r\n"
+            "goto loop\r\n"
         )
         self._script_path.write_text(script)
 

{portacode-1.4.35.dev4 → portacode-1.4.37.dev0}/portacode.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: portacode
-Version: 1.4.35.dev4
+Version: 1.4.37.dev0
 Summary: Portacode CLI client and SDK
 Home-page: https://github.com/portacode/portacode
 Author: Meena Erian

{portacode-1.4.35.dev4 → portacode-1.4.37.dev0}/portacode.egg-info/SOURCES.txt

@@ -277,12 +277,12 @@ portacode/__main__.py
 portacode/_version.py
 portacode/cli.py
 portacode/data.py
+portacode/exit_codes.py
 portacode/keypair.py
 portacode/logging_categories.py
 portacode/pairing.py
 portacode/restart.py
 portacode/service.py
-portacode/test_updater.py
 portacode/updater.py
 portacode.egg-info/PKG-INFO
 portacode.egg-info/SOURCES.txt
@@ -396,6 +396,7 @@ todo/issues/portacode_service_silently_down.md
 todo/issues/premature_terminal_exit.md
 todo/issues/project_cpu_hotspots.md
 todo/issues/terminals_exit_upon_starting.md
+todo/issues/websocket_client_silently_dead.md
 todo/issues/wrong_item_classification_on_client_side.md
 tools/generate_play_store_assets.py
 tools/pairing_tester.py

portacode-1.4.37.dev0/todo/issues/websocket_client_silently_dead.md

@@ -0,0 +1,51 @@
+# Issue summary
+
+The SDK is supposed to keep trying to reconnect whenever the websocket connection to the server is dead, and that is what it do most of the time, but in some rare instances, the connection drops in a way where the server shows the the device is offline, but the device logs show that the SDK assums that it is connected. Result is that a manual restart is needed as the device SDK never detects that is it offline/disconnected so the reconnect loop never triggers.
+
+# Issue details and findings
+
+• Most likely root cause: the client got stuck in a half-closed TCP state (CLOSE-WAIT) where the websocket reader stopped consuming bytes, so reconnect logic never
+  triggered.
+
+  What I found (read-only):
+
+  1. Live process is still running since 2026-03-04 08:15:27 UTC with no restart.
+  2. Last Portacode log line is 2026-03-04 08:28:58 UTC; no logs after that.
+  3. The only Portacode network socket is:
+      - 10.10.0.136:41718 -> 94.204.184.54:443 in CLOSE-WAIT
+      - kernel Recv-Q is 177 bytes (unread data still queued)
+      - FD is 6 in process 7061
+  4. Restart policy is Restart=on-failure, but this process never exits, so systemd won’t restart it.
+
+  Code correlation (installed portacode==1.4.35 matches tag v1.4.35 exactly):
+
+  1. Receive loop only reconnects when recv() raises websockets.ConnectionClosed:
+      - /home/user/debo/portacode/portacode/connection/client.py:172:172
+      - /home/user/debo/portacode/portacode/connection/client.py:203:203
+  2. It wraps recv() in asyncio.wait_for(..., timeout=1.0) and cancels it every second on idle:
+      - /home/user/debo/portacode/portacode/connection/client.py:176:176
+  3. Clock-sync-based forced close is only initial burst (5 times), not continuous:
+      - /home/user/debo/portacode/portacode/connection/client.py:231:231
+
+  Inference:
+
+  - You hit a rare wedge where the websocket path stopped draining/closing after peer-side close (or close frame), leaving FD in CLOSE-WAIT.
+  - Because the process didn’t crash and no close exception propagated, reconnect loop never ran, and systemd didn’t intervene.
+  - Limited ongoing heartbeat (only initial clock sync) removes a second safety net once this wedge happens.
+
+  Confidence: medium-high on “stuck reader in CLOSE-WAIT without reconnect”; medium on exact trigger being the per-second wait_for(recv) cancellation race vs a lower-level
+  websockets/SSL edge case.
+
+# Suggestions
+
+1. WebSocket protocol keepalive (Ping/Pong)
+     Enable client-side idle ping at a low frequency (for example every 60-180s only when no traffic). This uses WebSocket control frames, not JSON payloads, and a
+     compliant server stack auto-replies with Pong without custom server code. If pong is missed, client force-closes and reconnects.
+2. TCP keepalive probes
+     Enable and tune socket keepalive (SO_KEEPALIVE, TCP_KEEPIDLE, TCP_KEEPINTVL, TCP_KEEPCNT) on the client connection. This is kernel-level liveness detection with
+     minimal overhead and no app-channel traffic. If peer becomes unreachable/half-dead, TCP eventually errors, allowing client reconnect logic to trigger.
+
+
+# Resolution
+
+Suggestion 1 has been implemented in the commit on which this issue file was added and the issue file is kept to further monitor the issue in case of any future reoccurence. The file may be removed in 3 month later if the issue never reoccured. Scheduled file detetion is 4th of Jun 2026