susops 3.0.0rc3.dev8__tar.gz → 3.0.0rc4.dev2__tar.gz

{susops-3.0.0rc3.dev8 → susops-3.0.0rc4.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: susops
-Version: 3.0.0rc3.dev8
+Version: 3.0.0rc4.dev2
 Summary: SusOps — SSH SOCKS5 proxy manager with PAC server, Textual TUI, and system tray apps
 License: MIT
 Requires-Python: >=3.11

{susops-3.0.0rc3.dev8 → susops-3.0.0rc4.dev2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "susops"
-version = "3.0.0-rc3.dev8"
+version = "3.0.0-rc4.dev2"
 description = "SusOps — SSH SOCKS5 proxy manager with PAC server, Textual TUI, and system tray apps"
 readme = "README.md"
 license = { text = "MIT" }

{susops-3.0.0rc3.dev8 → susops-3.0.0rc4.dev2}/src/susops/client.py

@@ -13,6 +13,7 @@ from __future__ import annotations
 import os
 import subprocess
 import sys
+import threading
 import time
 import urllib.error
 import urllib.request
@@ -22,7 +23,22 @@ from typing import Any
 from susops.core.rpc_protocol import InvocationRequest, InvocationResponse
 
 _WORKSPACE_DEFAULT = Path.home() / ".susops"
-_DAEMON_SPAWN_TIMEOUT = 5.0
+# 15s rather than 5s gives headroom for slow init paths (ssh agent
+# prompts, network probes) on top of the daemon's "publish port first,
+# restore async" startup ordering. Frontends will still detect a truly
+# dead daemon quickly because they poll the port file every 100ms — the
+# timeout only fires for genuine hangs.
+_DAEMON_SPAWN_TIMEOUT = 15.0
+
+# Backoff to avoid pegging CPU and filling logs when the daemon
+# repeatedly exits during startup (e.g. corrupt config, PAC port
+# squatted). On a fast-fail (daemon exits within FAST_FAIL_WINDOW_S of
+# spawn), the next ensure_daemon_running call within BACKOFF_S sleeps
+# until the window expires before retrying.
+_FAST_FAIL_WINDOW_S = 2.0
+_BACKOFF_S = 5.0
+_last_fast_fail: float = 0.0
+_last_fast_fail_lock = threading.Lock()
 
 
 class DaemonUnavailableError(RuntimeError):
@@ -65,17 +81,50 @@ def ensure_daemon_running(workspace: Path = _WORKSPACE_DEFAULT) -> int:
     (PAC port squatted, peer daemon alive, …) is surfaced to the caller
     instead of getting buried under a generic "didn't come up" message.
     """
+    global _last_fast_fail
     if _is_daemon_alive(workspace):
         port = _read_port(workspace)
         if port:
             return port
+        # Daemon claimed its PID file but hasn't published the port yet.
+        # Wait for it instead of spawning a competitor that would race the
+        # O_EXCL claim and exit rc=2 ("another daemon is already running").
+        deadline = time.monotonic() + _DAEMON_SPAWN_TIMEOUT
+        while time.monotonic() < deadline:
+            if not _is_daemon_alive(workspace):
+                break  # died mid-startup, fall through to spawn a new one
+            port = _read_port(workspace)
+            if port:
+                return port
+            time.sleep(0.1)
+        else:
+            raise DaemonUnavailableError(
+                "An existing susops-services daemon is alive but never "
+                "published an RPC port. Try `kill " +
+                (_pid_path(workspace).read_text().strip() or "<pid>") +
+                "` and start again."
+            )
 
+    # If we recently fast-failed a spawn (daemon exited within
+    # _FAST_FAIL_WINDOW_S of starting), wait out the backoff window
+    # before respawning. Prevents tight respawn loops when something is
+    # structurally broken (corrupt config, PAC port squatted, etc.).
+    with _last_fast_fail_lock:
+        since_last = time.monotonic() - _last_fast_fail
+    if since_last < _BACKOFF_S:
+        time.sleep(_BACKOFF_S - since_last)
+
+    # DEVNULL (not PIPE) for stderr — the daemon writes its log to
+    # ~/.susops/logs/susops-services.log directly. Capturing as a pipe
+    # would back up once the daemon's log volume exceeded the kernel
+    # buffer (~64 KB on macOS) and freeze every subsequent log call.
+    spawn_start = time.monotonic()
     proc = subprocess.Popen(
         [sys.executable, "-m", "susops.core.services_daemon",
          "--workspace", str(workspace)],
         stdin=subprocess.DEVNULL,
         stdout=subprocess.DEVNULL,
-        stderr=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
         start_new_session=True,
     )
 
@@ -86,17 +135,29 @@ def ensure_daemon_running(workspace: Path = _WORKSPACE_DEFAULT) -> int:
             if port:
                 return port
         # If the daemon exited (e.g. preflight rejected it), don't keep
-        # polling — surface the reason and bail out immediately.
+        # polling — surface the reason and bail out immediately. The
+        # daemon's log file is the new source of truth for failure
+        # reasons (stderr is no longer piped — see Popen above).
         rc = proc.poll()
         if rc is not None:
+            # Fast fail = structural problem. Record the timestamp so the
+            # next ensure_daemon_running call within _BACKOFF_S sleeps
+            # before retrying.
+            if time.monotonic() - spawn_start < _FAST_FAIL_WINDOW_S:
+                with _last_fast_fail_lock:
+                    _last_fast_fail = time.monotonic()
+            log_path = workspace / "logs" / "susops-services.log"
+            tail = ""
             try:
-                _, err = proc.communicate(timeout=0.5)
-                stderr = err.decode(errors="replace").strip() if err else ""
+                if log_path.exists():
+                    with open(log_path, "r", encoding="utf-8", errors="replace") as f:
+                        lines = f.readlines()[-15:]
+                        tail = "".join(lines).strip()
             except Exception:
-                stderr = ""
+                pass
             msg = (
                     f"Daemon exited during startup (rc={rc})"
-                    + (f":\n{stderr}" if stderr else "")
+                    + (f":\n{tail}" if tail else "")
             )
             raise DaemonUnavailableError(msg)
         time.sleep(0.1)

{susops-3.0.0rc3.dev8 → susops-3.0.0rc4.dev2}/src/susops/core/rpc_server.py

@@ -7,6 +7,7 @@ Anything outside the allowlist below is rejected too — deny-by-default.
 """
 from __future__ import annotations
 
+import asyncio
 import logging
 
 from aiohttp import web
@@ -46,6 +47,8 @@ _ALLOWED_METHODS: set[str] = {
     # Bandwidth
     "get_bandwidth", "get_bandwidth_totals", "get_bandwidth_global",
     "get_bandwidth_history",
+    # Frontend coordination
+    "sse_client_count",
     # Reconnect introspection
     "reconnect_monitor_info",
     # Process introspection
@@ -85,7 +88,15 @@ async def _handle_rpc(request: web.Request) -> web.Response:
         return web.Response(text=resp.to_json(), status=404, content_type="application/json")
 
     try:
-        result = method(*req.args, **req.kwargs)
+        # Method handlers are synchronous — many touch the filesystem or spawn
+        # `ssh -O check` subprocesses (up to 3s each). Running them directly
+        # in the aiohttp event loop blocks every other RPC behind the slow one
+        # and eventually leads to "Daemon RPC failed: timed out". Offload to
+        # the default thread executor so the loop stays responsive.
+        loop = asyncio.get_running_loop()
+        result = await loop.run_in_executor(
+            None, lambda: method(*req.args, **req.kwargs)
+        )
         resp = InvocationResponse(ok=True, result=result)
         return web.Response(text=resp.to_json(), content_type="application/json")
     except Exception as exc:

{susops-3.0.0rc3.dev8 → susops-3.0.0rc4.dev2}/src/susops/core/services_daemon.py

@@ -153,9 +153,28 @@ def main() -> int:
     args = parser.parse_args()
     workspace = Path(args.workspace)
 
-    logging.basicConfig(level=logging.INFO,
-                        format="%(asctime)s [services] %(message)s")
+    # Route logging + raw stderr to a workspace-local file. With stderr
+    # piped from the spawner (client.py used stderr=PIPE) the kernel's
+    # 64 KB pipe buffer would fill within minutes of daemon logging,
+    # blocking every subsequent log call — including ones running on
+    # executor threads serving RPC requests. Result: freeze.
+    log_path = workspace / "logs" / "susops-services.log"
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s [services] %(message)s",
+        handlers=[logging.FileHandler(log_path, mode="a", encoding="utf-8")],
+        force=True,
+    )
     log = logging.getLogger("susops.services")
+    # Anything that bypasses logging (raw `print`, tracebacks) also goes
+    # to the log file rather than the spawner's pipe.
+    try:
+        _stderr_redirect = open(log_path, "a", buffering=1, encoding="utf-8")
+        sys.stderr = _stderr_redirect
+        sys.stdout = _stderr_redirect
+    except Exception:
+        pass
 
     # Install signal handlers BEFORE preflight: SIGTERM arriving in the
     # window between claiming the PID file and entering the try block
@@ -177,7 +196,17 @@ def main() -> int:
         from susops.core.rpc_server import serve
         from susops.facade import SusOpsManager
 
-        mgr = SusOpsManager(workspace=workspace, _enable_background_threads=True)
+        # Defer the SSH/PAC/share probes (_restore_*) until AFTER the RPC
+        # port is bound and the port file is written. Those probes can
+        # block on the ssh agent (e.g. 1Password approval prompt) for
+        # several seconds, and frontends only give up after _DAEMON_SPAWN_TIMEOUT.
+        # By publishing the port first, the daemon is "alive" to frontends
+        # immediately regardless of how slow the agent decides to be.
+        mgr = SusOpsManager(
+            workspace=workspace,
+            _enable_background_threads=True,
+            _skip_restore=True,
+        )
         # CLI `--port` wins. Fall back to the configured `rpc_server_port`,
         # then 0 (auto-allocate). When we auto-allocate, persist the bound
         # port back to config so subsequent spawns reuse it.
@@ -198,6 +227,24 @@ def main() -> int:
         # backoff for several seconds after a fresh daemon spawn.
         sse_port = mgr.ensure_sse_status_server()
 
+        # Now that the daemon is reachable (RPC + SSE), run the deferred
+        # restore work in a background thread so slow ssh-agent prompts
+        # don't keep the daemon process pinned during startup.
+        def _restore_in_background() -> None:
+            try:
+                mgr._restore_pac()
+                if mgr.config.susops_app.restore_shares_on_start:
+                    mgr._restore_shares()
+                mgr._restore_reconnect_monitor()
+            except Exception:
+                log.exception("Background restore failed")
+
+        threading.Thread(
+            target=_restore_in_background,
+            daemon=True,
+            name="susops-restore",
+        ).start()
+
         # Surface daemon-startup details in the in-memory log buffer too so
         # they show up in the TUI Logs tab and the tray Logs window — the
         # `log.info` calls only land on the daemon process's stderr.
@@ -211,13 +258,16 @@ def main() -> int:
             pass
 
         # Idle-shutdown. Exit when the last SSE client disconnects AND there's
-        # no in-flight work (no SSH masters, no shares, no PAC, no watched
-        # connections). Startup gets a small grace so a frontend that calls
-        # ensure_daemon_running() has time to make its first SSE connection
-        # before the periodic check fires.
+        # no in-flight work AND that state holds for _IDLE_CONSECUTIVE_TICKS
+        # consecutive checks. Requiring multiple ticks defends against the
+        # "SSE flap during reconnect → momentary client_count==0 → daemon
+        # exits → frontend respawns it" churn that bit us during heavy
+        # reconnect activity.
         _IDLE_STARTUP_GRACE_S = 3.0
         _IDLE_CHECK_INTERVAL_S = 5.0
+        _IDLE_CONSECUTIVE_TICKS = 3
         startup_time = _time.monotonic()
+        idle_streak = 0  # consecutive ticks where _should_exit() was True
 
         def _should_exit() -> bool:
             if _time.monotonic() - startup_time < _IDLE_STARTUP_GRACE_S:
@@ -229,19 +279,12 @@ def main() -> int:
             return mgr.is_idle()
 
         def _on_clients_changed(count: int) -> None:
-            # Fast path — react immediately when the last SSE client drops.
-            # The periodic check below handles the "no SSE was ever opened"
-            # case (e.g. `susops ps` fires one RPC and exits).
+            # No fast-path exit here anymore — the watcher's consecutive-tick
+            # requirement covers genuine quiescence and rejects flap events
+            # without us having to special-case them.
+            nonlocal idle_streak
             if count > 0:
-                return
-            if _should_exit():
-                msg = "Last client disconnected and no work pending — shutting down"
-                log.info(msg)
-                try:
-                    mgr._log(msg)
-                except Exception:
-                    pass
-                stop_event.set()
+                idle_streak = 0
 
         mgr._status_server.on_clients_changed = _on_clients_changed
         # Surface SSE connect/disconnect in the in-memory log buffer so the
@@ -249,19 +292,26 @@ def main() -> int:
         mgr._status_server.on_log = mgr._log
 
         def _idle_watcher() -> None:
+            nonlocal idle_streak
             while not stop_event.is_set():
                 if stop_event.wait(_IDLE_CHECK_INTERVAL_S):
                     return
                 if _should_exit():
-                    msg = (f"Idle for {_IDLE_CHECK_INTERVAL_S:.0f}s with "
-                           f"no clients — shutting down")
-                    log.info(msg)
-                    try:
-                        mgr._log(msg)
-                    except Exception:
-                        pass
-                    stop_event.set()
-                    return
+                    idle_streak += 1
+                    if idle_streak >= _IDLE_CONSECUTIVE_TICKS:
+                        msg = (
+                            f"Idle for {_IDLE_CHECK_INTERVAL_S * idle_streak:.0f}s "
+                            f"({idle_streak} consecutive checks) — shutting down"
+                        )
+                        log.info(msg)
+                        try:
+                            mgr._log(msg)
+                        except Exception:
+                            pass
+                        stop_event.set()
+                        return
+                else:
+                    idle_streak = 0
 
         threading.Thread(
             target=_idle_watcher, daemon=True, name="susops-idle-watcher",

{susops-3.0.0rc3.dev8 → susops-3.0.0rc4.dev2}/src/susops/core/ssh.py

@@ -282,7 +282,12 @@ def find_master_pid(tag: str, workspace: Path) -> int | None:
 def is_socket_alive(tag: str, workspace: Path) -> bool:
     """Return True if the ControlMaster socket is responsive.
 
-    Uses `ssh -O check` to verify the master is healthy.
+    Uses `ssh -O check` to verify the master is healthy. The 1s timeout
+    is tight on purpose: a healthy local control socket replies in <100ms,
+    and anything longer means the master is stuck (e.g. waiting on agent),
+    in which case we should return False quickly so callers move on. The
+    previous 3s timeout could saturate the daemon's executor pool when
+    multiple callers stalled simultaneously.
     """
     sock = socket_path(tag, workspace)
     if not sock.exists():
@@ -291,7 +296,7 @@ def is_socket_alive(tag: str, workspace: Path) -> bool:
         result = subprocess.run(
             ["ssh", "-O", "check", "-o", f"ControlPath={sock}", "placeholder"],
             capture_output=True,
-            timeout=3,
+            timeout=1,
         )
         return result.returncode == 0
     except (subprocess.TimeoutExpired, FileNotFoundError):

{susops-3.0.0rc3.dev8 → susops-3.0.0rc4.dev2}/src/susops/core/types.py

@@ -81,6 +81,11 @@ class ConnectionStatus:
     pid: Optional[int] = None
     socks_port: int = 0
     enabled: bool = True
+    # ssh master spawned but ControlMaster socket not yet alive — almost
+    # always means "waiting for the user to approve the agent prompt".
+    # Frontends should render this with a distinct glyph (e.g. yellow /
+    # spinner) so it's clear the connection isn't usable yet.
+    pending: bool = False
 
 
 @dataclasses.dataclass(frozen=True)