de-shell 0.2.0__py3-none-any.whl

de_shell/plotting/stream.py

@@ -0,0 +1,172 @@
+"""
+stream.py — refreshing a figure from work that finishes off the main thread.
+
+Every shell app produces frames somewhere other than the asyncio main thread: a
+camera thread, an acquisition runner, a worker computing a result, a background
+fill writing into a buffer. All of them need the same three things, and all
+three had been written separately in each app:
+
+1. **Marshal to the main thread.** Figures may only be touched there.
+2. **Newest wins, with ONE scheduled paint.** A producer faster than the
+   renderer must not queue a callback per frame — the main thread would fall
+   further and further behind while the queue grew, showing ever-staler frames.
+   A single pending slot means a superseded frame is dropped, not backlogged.
+3. **Count what was dropped.** Dropping is correct here, but invisible dropping
+   is how "the display is laggy" becomes unfalsifiable.
+
+There is deliberately **no dask**. A future is any object with
+``add_done_callback`` / ``result`` / ``cancel`` — ``concurrent.futures.Future``
+from ``de_shell.compute.ThreadCompute`` is the common case, and SpyDE's
+distributed adapter satisfies the same protocol without this module knowing.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+from typing import Any, Callable
+
+import numpy as np
+
+log = logging.getLogger(__name__)
+
+
+class FrameStream:
+    """Newest-wins painting into a :class:`~de_shell.plotting.figure.FigureView`.
+
+    Parameters
+    ----------
+    view
+        The figure to paint. Anything with ``show(frame, clim=…) -> bool``.
+    dispatch
+        Schedules a callable on the main thread — ``SessionBase._dispatch_to_main``.
+    on_painted
+        Optional ``fn(frame)`` run on the MAIN thread after a successful paint,
+        for whatever the app hangs off a new frame (stats, counters, a status
+        line).
+    on_error
+        Optional ``fn(exc)`` for a paint or future that failed. Runs on the main
+        thread. Without one, failures are logged and swallowed — a display error
+        must never kill the producer.
+    """
+
+    def __init__(self, view, dispatch: Callable[[Callable[[], None]], None], *,
+                 on_painted: Callable[[np.ndarray], None] | None = None,
+                 on_error: Callable[[Exception], None] | None = None) -> None:
+        self._view = view
+        self._dispatch = dispatch
+        self._on_painted = on_painted
+        self._on_error = on_error
+
+        self._lock = threading.Lock()
+        self._pending: np.ndarray | None = None
+        self._pending_clim: tuple[float, float] | None = None
+        self._scheduled = False
+        self._closed = False
+        #: The future whose result we are still interested in. A newer submission
+        #: supersedes it; its callback then no-ops on this identity check.
+        self._future: Any = None
+
+        self.shown = 0
+        self.dropped = 0
+
+    # ── Submitting ────────────────────────────────────────────────────────────
+
+    def submit(self, frame: np.ndarray, *,
+               clim: tuple[float, float] | None = None) -> None:
+        """Offer a frame from ANY thread. The newest one wins."""
+        if self._closed:
+            return
+        with self._lock:
+            if self._pending is not None:
+                self.dropped += 1
+            self._pending = frame
+            self._pending_clim = clim
+            if self._scheduled:
+                return          # a paint is already on its way; it will take this
+            self._scheduled = True
+        self._dispatch(self._paint_pending)
+
+    def submit_future(self, future, *,
+                      clim: tuple[float, float] | None = None) -> None:
+        """Paint whatever *future* resolves to, when it resolves.
+
+        Supersedes any future still outstanding: the older one is cancelled, and
+        if it was already running its callback no-ops on an identity check
+        rather than painting a frame the user has moved past. That check is the
+        whole latest-wins guarantee for async work — a queued future cancels
+        cleanly, an in-flight one cannot be stopped and must instead be ignored.
+        """
+        if self._closed:
+            return
+        with self._lock:
+            prev, self._future = self._future, future
+        if prev is not None and prev is not future:
+            try:
+                prev.cancel()
+            except Exception as e:
+                log.debug("cancelling superseded frame future failed: %s", e)
+
+        def _done(fut) -> None:
+            with self._lock:
+                if self._future is not fut:
+                    return      # superseded while running — drop it
+                self._future = None
+            try:
+                if fut.cancelled():
+                    return
+                result = fut.result()
+            except Exception as e:
+                log.debug("frame future failed: %s", e)
+                self._fail(e)
+                return
+            if result is not None:
+                self.submit(np.asarray(result), clim=clim)
+
+        try:
+            future.add_done_callback(_done)
+        except Exception as e:
+            log.debug("attaching frame-future callback failed: %s", e)
+            self._fail(e)
+
+    # ── Painting (main thread) ────────────────────────────────────────────────
+
+    def _paint_pending(self) -> None:
+        with self._lock:
+            frame, self._pending = self._pending, None
+            clim, self._pending_clim = self._pending_clim, None
+            self._scheduled = False
+        if frame is None or self._closed:
+            return
+        try:
+            if self._view.show(frame, clim=clim):
+                self.shown += 1
+                if self._on_painted is not None:
+                    self._on_painted(frame)
+        except Exception as e:
+            log.exception("painting frame failed")
+            self._fail(e)
+
+    def _fail(self, exc: Exception) -> None:
+        if self._on_error is None:
+            return
+        # Errors surface on the MAIN thread like paints do, so a handler can
+        # touch UI without each caller having to remember to marshal.
+        self._dispatch(lambda: self._on_error(exc))
+
+    # ── Teardown ──────────────────────────────────────────────────────────────
+
+    def close(self) -> None:
+        """Stop accepting and painting frames. Idempotent.
+
+        Cancels any outstanding future and drops the pending frame, so a
+        producer still winding down cannot paint into a closed window.
+        """
+        with self._lock:
+            self._closed = True
+            self._pending = None
+            fut, self._future = self._future, None
+        if fut is not None:
+            try:
+                fut.cancel()
+            except Exception as e:
+                log.debug("cancelling frame future on close failed: %s", e)

de_shell/process_guard.py

@@ -0,0 +1,190 @@
+"""
+process_guard.py — guarantee Dask workers die with the backend process.
+
+The Python backend is a subprocess of Electron and spawns a Dask LocalCluster
+whose worker/nanny processes are *grandchildren*. If the backend is force-killed
+(Task Manager), crashes, or Electron dies without sending a clean ``quit``, the
+normal ``DaskManager.shutdown()`` never runs and those workers orphan — every
+run leaks ``n_workers`` idle Python processes (observed: ~200 stale python.exe
+holding tens of GB).
+
+The OS-level fix on Windows is a **Job Object** with
+``JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE``: assign this process to the job, and when
+the last handle to the job closes (i.e. when THIS process dies, for ANY reason),
+Windows terminates every process in the job — including all Dask workers spawned
+afterwards, since child processes inherit the job by default.
+
+On POSIX the equivalent is a new session/process-group plus ``prctl`` PDEATHSIG,
+but Dask workers there are reaped by ``shutdown()`` reliably enough; this module
+is a no-op off Windows (returns False) and callers keep the existing teardown.
+"""
+from __future__ import annotations
+
+import logging
+import sys
+
+logger = logging.getLogger(__name__)
+
+
+def unthrottle_windows_timers() -> None:
+    """Ask Windows for real timer interrupts for THIS process (no-op off
+    Windows; idempotent; best-effort).
+
+    MEASURED pathology in the Electron-spawned backend (probe
+    ``_probe_fv_stall.spec.ts`` + ``repro_batch_stall.py``): OS timer waits do
+    not expire on schedule — a ``time.sleep(0.05)`` poll loop slept 15 s and
+    woke only when process I/O arrived; dask's timer-driven scheduler work
+    (task delivery to workers!) froze the same way, so distributed computes
+    only progressed when the user clicked something (stdin I/O). The SAME
+    code run from a console is healthy — Windows withholds timer interrupts
+    from the hidden Electron child (power throttling / timer coalescing).
+
+    Two knobs:
+    * ``SetProcessInformation(ProcessPowerThrottling)`` with EXECUTION_SPEED
+      and IGNORE_TIMER_RESOLUTION control bits CLEARED in the state mask —
+      explicitly opt this process OUT of EcoQoS speed throttling and OUT of
+      "ignore timer-resolution requests" (Win10 1709+).
+    * ``winmm.timeBeginPeriod(1)`` — request 1 ms timer interrupts, which the
+      opt-out above makes Windows actually honor.
+
+    Called at backend startup (app._main) and inside every Dask worker
+    process (dask_manager._WorkerTuningPlugin) — workers are children of this
+    hidden process and inherit the same throttling class.
+    """
+    if sys.platform != "win32":
+        return
+    import ctypes
+    from ctypes import wintypes
+    try:
+        class PROCESS_POWER_THROTTLING_STATE(ctypes.Structure):
+            _fields_ = [("Version", wintypes.ULONG),
+                        ("ControlMask", wintypes.ULONG),
+                        ("StateMask", wintypes.ULONG)]
+
+        PPT_CURRENT_VERSION = 1
+        PPT_EXECUTION_SPEED = 0x1
+        PPT_IGNORE_TIMER_RESOLUTION = 0x4
+        ProcessPowerThrottling = 4
+        state = PROCESS_POWER_THROTTLING_STATE(
+            PPT_CURRENT_VERSION,
+            PPT_EXECUTION_SPEED | PPT_IGNORE_TIMER_RESOLUTION,
+            0,   # both bits cleared = throttling OFF, honor timer resolution
+        )
+        ok = ctypes.windll.kernel32.SetProcessInformation(
+            ctypes.windll.kernel32.GetCurrentProcess(),
+            ProcessPowerThrottling, ctypes.byref(state), ctypes.sizeof(state))
+        logger.info("[timers] power-throttling opt-out: %s",
+                    "ok" if ok else "failed")
+    except Exception as e:
+        logger.info("[timers] power-throttling opt-out unavailable: %s", e)
+    try:
+        res = ctypes.windll.winmm.timeBeginPeriod(1)
+        logger.info("[timers] timeBeginPeriod(1) -> %s (0=ok)", res)
+    except Exception as e:
+        logger.info("[timers] timeBeginPeriod unavailable: %s", e)
+
+
+# Module-level so the job handle lives for the whole process lifetime. If it were
+# a local it would be garbage-collected, closing the handle and (because
+# KILL_ON_JOB_CLOSE) killing us immediately.
+_job_handle = None
+
+
+def install_kill_on_close() -> bool:
+    """Assign the current process to a kill-on-close Job Object (Windows only).
+
+    Returns True if the guard is active, False otherwise (non-Windows, or any
+    failure — the caller should still keep its own ``shutdown()`` path).
+    """
+    global _job_handle
+    if not sys.platform.startswith("win"):
+        return False
+    if _job_handle is not None:
+        return True
+    try:
+        import ctypes
+        from ctypes import wintypes
+
+        kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+
+        # --- function prototypes ---
+        kernel32.CreateJobObjectW.restype = wintypes.HANDLE
+        kernel32.CreateJobObjectW.argtypes = [wintypes.LPVOID, wintypes.LPCWSTR]
+        kernel32.SetInformationJobObject.restype = wintypes.BOOL
+        kernel32.SetInformationJobObject.argtypes = [
+            wintypes.HANDLE, ctypes.c_int, wintypes.LPVOID, wintypes.DWORD,
+        ]
+        kernel32.AssignProcessToJobObject.restype = wintypes.BOOL
+        kernel32.AssignProcessToJobObject.argtypes = [wintypes.HANDLE, wintypes.HANDLE]
+        kernel32.GetCurrentProcess.restype = wintypes.HANDLE
+
+        # JOBOBJECT_EXTENDED_LIMIT_INFORMATION layout
+        class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure):
+            _fields_ = [
+                ("PerProcessUserTimeLimit", ctypes.c_int64),
+                ("PerJobUserTimeLimit", ctypes.c_int64),
+                ("LimitFlags", wintypes.DWORD),
+                ("MinimumWorkingSetSize", ctypes.c_size_t),
+                ("MaximumWorkingSetSize", ctypes.c_size_t),
+                ("ActiveProcessLimit", wintypes.DWORD),
+                ("Affinity", ctypes.c_size_t),
+                ("PriorityClass", wintypes.DWORD),
+                ("SchedulingClass", wintypes.DWORD),
+            ]
+
+        class IO_COUNTERS(ctypes.Structure):
+            _fields_ = [
+                ("ReadOperationCount", ctypes.c_uint64),
+                ("WriteOperationCount", ctypes.c_uint64),
+                ("OtherOperationCount", ctypes.c_uint64),
+                ("ReadTransferCount", ctypes.c_uint64),
+                ("WriteTransferCount", ctypes.c_uint64),
+                ("OtherTransferCount", ctypes.c_uint64),
+            ]
+
+        class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure):
+            _fields_ = [
+                ("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION),
+                ("IoInfo", IO_COUNTERS),
+                ("ProcessMemoryLimit", ctypes.c_size_t),
+                ("JobMemoryLimit", ctypes.c_size_t),
+                ("PeakProcessMemoryUsed", ctypes.c_size_t),
+                ("PeakJobMemoryUsed", ctypes.c_size_t),
+            ]
+
+        JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
+        JobObjectExtendedLimitInformation = 9  # JOBOBJECTINFOCLASS
+
+        job = kernel32.CreateJobObjectW(None, None)
+        if not job:
+            raise ctypes.WinError(ctypes.get_last_error())
+
+        info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION()
+        info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
+        ok = kernel32.SetInformationJobObject(
+            job, JobObjectExtendedLimitInformation,
+            ctypes.byref(info), ctypes.sizeof(info),
+        )
+        if not ok:
+            raise ctypes.WinError(ctypes.get_last_error())
+
+        ok = kernel32.AssignProcessToJobObject(job, kernel32.GetCurrentProcess())
+        if not ok:
+            err = ctypes.get_last_error()
+            # ERROR_ACCESS_DENIED (5): the process is ALREADY in a job that
+            # disallows nesting (e.g. Electron already put us in one, or an older
+            # Windows without nested-job support). In that case the parent's job
+            # governs our lifetime anyway — treat as best-effort success.
+            if err == 5:
+                logger.info("process already in a job object; relying on parent "
+                            "job for worker cleanup")
+                return False
+            raise ctypes.WinError(err)
+
+        _job_handle = job  # keep alive for process lifetime
+        logger.info("Dask workers guarded by kill-on-close Job Object")
+        return True
+    except Exception as e:
+        logger.warning("could not install kill-on-close job object (%s); "
+                       "relying on graceful shutdown for worker cleanup", e)
+        return False

de_shell/session.py

@@ -0,0 +1,211 @@
+"""
+session.py — the app-agnostic half of the backend coordinator.
+
+Every shell app has exactly one Session: the object the frontend talks to
+through IPC and that talks back through ``de_shell.ipc.emit``. Most of what such
+an object does is the same in all three apps — hand out window ids, keep the
+registry of open plots and window controllers, marshal worker results onto the
+asyncio main thread, and persist a little JSON of user settings.
+
+None of that knows what the data is. ``SessionBase`` owns that half; an app
+subclasses it and adds its own (SpyDE: signal trees, the Dask cluster, file
+I/O, the action mixins).
+
+Two things deliberately stay the app's job:
+
+* **Settings location.** Passed in as ``settings_dir``, not derived here. SpyDE
+  reads ``SPYDE_SETTINGS_DIR`` and falls back to ``~/.spyde``; another app has
+  its own directory and its own override variable, and baking a guess into the
+  shell would silently write one app's preferences into another's file.
+* **Shutdown order.** ``shutdown()`` here tears down only what it owns and is
+  safe to call twice. Subclasses override, do their own work, and call
+  ``super().shutdown()`` — ordering between an app's cluster, workers and
+  caches is app knowledge.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Any
+
+log = logging.getLogger(__name__)
+
+
+class SessionBase:
+    """Window registry, main-loop marshalling and the settings store."""
+
+    #: Cap on the persisted recent-files list.
+    MAX_RECENT = 20
+
+    def __init__(self, settings_dir: str) -> None:
+        # ── Window / plot registry ────────────────────────────────────────────
+        self._plots: list[Any] = []           # every open Plot
+        self._next_window_id = 0
+        self._active_window_id: int | None = None   # focused window
+        # window_id -> controller for windows that are NOT registered Plots
+        # (bare `figure` emits). See the WindowController protocol in
+        # de_shell.actions.registry; _forget_window closes + evicts.
+        self._window_controllers: dict[int, object] = {}
+
+        # ── Main-thread marshalling ───────────────────────────────────────────
+        # Set by set_main_loop once the asyncio loop is running. Until then
+        # _dispatch_to_main runs inline, which is what makes a Session usable in
+        # a plain test with no loop at all.
+        self._main_loop = None
+
+        # Set by shutdown() so late work draining on a background thread can't
+        # resurrect anything after teardown.
+        self._closed = False
+
+        # ── Settings ──────────────────────────────────────────────────────────
+        self._settings_path = os.path.join(settings_dir, "settings.json")
+        self._settings: dict[str, Any] = self._load_settings()
+        self._recent_files: list[str] = []
+        try:
+            self._recent_files = list(
+                self._settings.get("recent_files", []))[:self.MAX_RECENT]
+        except Exception as e:
+            log.debug("restoring recent files from settings failed: %s", e)
+        self._update_channel: str = (
+            self._settings.get("update_channel")
+            if self._settings.get("update_channel") in ("stable", "beta")
+            else "stable"
+        )
+
+    # ── Main-thread marshalling ───────────────────────────────────────────────
+
+    def set_main_loop(self, loop) -> None:
+        """Register the main asyncio loop so background workers can marshal their
+        result-apply onto this (main) thread. Call once the loop is running.
+
+        NB the process's frozen-timer pathology (waits only wake on process I/O
+        — see the backend tick in @de/shell-main's backendProcess) is healed by
+        Electron's 0.5 Hz stdin tick; an in-process wake ticker was tried and is
+        useless here because its own sleep freezes the same way.
+        """
+        self._main_loop = loop
+
+    def _dispatch_to_main(self, fn) -> None:
+        """Schedule ``fn()`` on the main asyncio thread.
+
+        Falls back to running inline when no loop is registered yet (early
+        startup, and tests that never start one) — so a worker callback is never
+        silently dropped just because the loop isn't up.
+        """
+        loop = self._main_loop
+        if loop is not None:
+            try:
+                loop.call_soon_threadsafe(fn)
+                return
+            except Exception as e:
+                log.debug("dispatch_to_main failed, running inline: %s", e)
+        fn()
+
+    # ── Window / plot registry ────────────────────────────────────────────────
+
+    def next_window_id(self) -> int:
+        wid = self._next_window_id
+        self._next_window_id += 1
+        return wid
+
+    def register_plot(self, plot) -> None:
+        self._plots.append(plot)
+
+    def unregister_plot(self, plot) -> None:
+        # Identity-based, and removes every occurrence. Kept exactly as it was
+        # when lifted out of SpyDE: `register_plot` does not dedupe, so `remove`
+        # (first match, __eq__-based) would not be equivalent.
+        self._plots = [p for p in self._plots if p is not plot]
+
+    def _plot_by_window_id(self, window_id: int):
+        for p in self._plots:
+            if getattr(p, "window_id", None) == window_id:
+                return p
+        return None
+
+    def register_window_controller(self, window_id: int, controller) -> None:
+        """Give a non-Plot window (a bare `figure` emit) a dispatch + teardown
+        identity. See the WindowController protocol in de_shell.actions.registry;
+        the app's `_forget_window` pops the controller and calls its close()."""
+        self._window_controllers[window_id] = controller
+
+    def controller_by_window_id(self, window_id: int | None):
+        if window_id is None:
+            return None
+        return self._window_controllers.get(window_id)
+
+    # ── Settings & recent files ───────────────────────────────────────────────
+
+    def _load_settings(self) -> dict:
+        try:
+            with open(self._settings_path, encoding="utf-8") as fh:
+                return json.load(fh)
+        except Exception:
+            return {}
+
+    def _save_settings(self) -> None:
+        os.makedirs(os.path.dirname(self._settings_path), exist_ok=True)
+        with open(self._settings_path, "w", encoding="utf-8") as fh:
+            json.dump(self._settings, fh, indent=2)
+
+    def _add_recent(self, path: str) -> None:
+        if path in self._recent_files:
+            self._recent_files.remove(path)
+        self._recent_files.insert(0, path)
+        self._settings["recent_files"] = self._recent_files[:self.MAX_RECENT]
+        try:
+            self._save_settings()
+        except Exception as e:
+            log.debug("saving recent-files settings failed: %s", e)
+
+    def get_recent_files(self) -> list[str]:
+        return list(self._recent_files[:self.MAX_RECENT])
+
+    def set_update_channel(self, channel: str) -> None:
+        """Persist the update channel ('stable' or 'beta') to settings.json.
+
+        Mirrors the choice the Electron main process's autoUpdater actually acts
+        on — kept here too so the preference is visible from the Python side and
+        survives a settings.json inspection independent of Electron's storage.
+        """
+        if channel not in ("stable", "beta"):
+            log.warning("ignoring invalid update_channel %r", channel)
+            return
+        self._update_channel = channel
+        self._settings["update_channel"] = channel
+        try:
+            self._save_settings()
+        except Exception as e:
+            log.debug("saving update_channel setting failed: %s", e)
+
+    # ── First-run welcome tour ────────────────────────────────────────────────
+
+    @property
+    def first_run(self) -> bool:
+        """True until the welcome tour has been opened/dismissed once. Mirrors
+        the ``tutorial_seen`` settings key: absent (never set) => first run."""
+        return not bool(self._settings.get("tutorial_seen", False))
+
+    def mark_tutorial_seen(self) -> None:
+        """Persist that the welcome tour has been shown, so it never auto-opens
+        again. Idempotent — the renderer calls it every time the tour opens."""
+        if self._settings.get("tutorial_seen") is True:
+            return
+        self._settings["tutorial_seen"] = True
+        try:
+            self._save_settings()
+        except Exception as e:
+            log.debug("saving tutorial_seen setting failed: %s", e)
+
+    # ── Shutdown ──────────────────────────────────────────────────────────────
+
+    def shutdown(self) -> None:
+        """Tear down what the base owns. Idempotent.
+
+        Subclasses override, shut their own things down first, then call
+        ``super().shutdown()`` — the ordering between an app's cluster, worker
+        threads and caches is the app's knowledge, not the shell's.
+        """
+        self._closed = True
+        self._window_controllers.clear()

de_shell/timing.py

@@ -0,0 +1,28 @@
+"""
+timing.py — sleeping that actually wakes up in a shell backend.
+
+`time.sleep` is not reliable in this process. The backend is a hidden child of
+the Electron app, and the OS coalesces its timers hard enough that a
+`time.sleep(0.05)` in a poll loop has been measured freezing for **15 seconds**,
+waking only when process I/O arrived — while `threading.Event.wait(timeout)` in
+the same process ticked exactly on schedule, 120 times out of 120.
+
+So every poll loop in a shell app sleeps through `reliable_sleep`. This is the
+same pathology the 0.5 Hz stdin tick in @de/shell-main's backendProcess exists
+for, seen from the Python side.
+"""
+from __future__ import annotations
+
+import threading
+
+# Never set. It exists purely because `Event.wait(timeout)` is scheduled
+# reliably where `time.sleep(timeout)` is not.
+_WAKE = threading.Event()
+
+
+def reliable_sleep(seconds: float) -> None:
+    """Sleep that keeps ticking on the throttled, Electron-spawned backend.
+
+    Use instead of ``time.sleep`` in any loop that must make progress.
+    """
+    _WAKE.wait(seconds)