de-shell 0.2.0__py3-none-any.whl

de_shell/actions/wizard.py

@@ -0,0 +1,115 @@
+"""
+wizard.py — WizardController, the base class for staged-wizard actions.
+
+A *wizard* is the staged action shape (see the shell's actions/registry.py): the
+renderer caret mounts → ``<key>_open`` starts a live preview / controller;
+parameter edits stream in (``<key>_tune`` / ``<key>_set_<param>``); a heavy
+stage runs (``<key>_run``); an optional Commit snapshots the live result into
+a new SignalTree (``<key>_commit`` → :meth:`commit` →
+the app's commit helper); unmount → ``<key>_close`` tears
+everything down.
+
+The controller owns the wizard's state (library / overlay / field / windows)
+instead of a bare dict on the tree, and provides the lifecycle plumbing every
+wizard needs:
+
+* the run/stop **generation guard** (:meth:`guard` / :meth:`still` /
+  :meth:`cancel_inflight`) — see ``lifecycle.bump_generation`` for the React
+  StrictMode contract (open, close, open fired synchronously before any
+  worker lands must leave exactly ONE live controller);
+* **window registration** (:meth:`own_window`) so bare-figure windows the
+  wizard opens are reachable by dispatch and torn down by
+  ``Session._forget_window`` (which calls :meth:`close`);
+* the **worker marshal** (:meth:`run_on_worker`) bound to the session;
+* **overlay replacement** (:meth:`replace_overlay`).
+
+Subclasses override :meth:`remove` (full teardown — MUST be idempotent; guard
+with ``self._closed``) and optionally :meth:`commit`.
+"""
+from __future__ import annotations
+
+import logging
+from typing import Any, Callable
+
+from de_shell.actions.lifecycle import (
+    bump_generation, is_current, replace_tree_attr, run_on_worker as _run_on_worker,
+)
+
+log = logging.getLogger(__name__)
+
+
+class WizardController:
+    #: the wizard's short prefix — "strain" → ``tree._strain_run_gen`` and the
+    #: ``strain_*`` staged-action names.
+    key: str = ""
+
+    #: Declared parameter schema — REQUIRED for every wizard. The same dict
+    #: spec as ``toolbars.yaml parameters:`` / ``Action.parameters`` (``type``
+    #: int/float/bool/enum/file, ``name``, ``default``, ``min``/``max``/
+    #: ``step``, ``choices``, ``tab``, ``extensions``), so any host — the
+    #: Electron caret or an auto-generated notebook form — can render the
+    #: wizard's controls from one source of truth. Resolved host-agnostically
+    #: via ``registry.wizard_parameters(key)``; completeness is enforced by
+    #: ``test_wizard_schemas.py``.
+    parameters: dict = {}
+
+    def __init__(self, session, tree):
+        self.session = session
+        self.tree = tree
+        self._closed = False
+
+    # ── generation guard ──────────────────────────────────────────────────────
+
+    @property
+    def _gen_key(self) -> str:
+        return f"_{self.key}_run_gen"
+
+    def guard(self) -> int:
+        """Open a new run generation (call synchronously in the open handler,
+        BEFORE spawning any worker). Deferred builds check :meth:`still`."""
+        return bump_generation(self.tree, self._gen_key)
+
+    def still(self, gen: int) -> bool:
+        """True if *gen* is still the current run generation."""
+        return is_current(self.tree, self._gen_key, gen)
+
+    def cancel_inflight(self) -> None:
+        """Invalidate any in-flight open (call FIRST in the close handler)."""
+        bump_generation(self.tree, self._gen_key)
+
+    # ── plumbing ──────────────────────────────────────────────────────────────
+
+    def own_window(self, window_id) -> None:
+        """Register this controller for a bare-figure window it opened."""
+        if window_id is None or self.session is None:
+            return
+        reg = getattr(self.session, "register_window_controller", None)
+        if reg is not None:
+            reg(int(window_id), self)
+
+    def run_on_worker(self, work: Callable[[], Any], *, name: str | None = None,
+                      on_done=None, on_error=None) -> None:
+        _run_on_worker(self.session, work, name=name or f"{self.key}-worker",
+                       on_done=on_done, on_error=on_error)
+
+    def replace_overlay(self, attr: str, factory):
+        """Swap ``tree.<attr>`` for a fresh overlay, removing the prior one."""
+        return replace_tree_attr(self.tree, attr, factory)
+
+    # ── lifecycle hooks ───────────────────────────────────────────────────────
+
+    def close(self) -> None:
+        """WindowController protocol — ``Session._forget_window`` calls this
+        when an owned window goes away for any reason. Default: full teardown."""
+        self.remove()
+
+    def remove(self) -> None:
+        """Full teardown of everything the wizard added. MUST be idempotent
+        (guard with ``self._closed``)."""
+        raise NotImplementedError
+
+    def commit(self):
+        """Snapshot the live result into a new SignalTree (the ``<key>_commit``
+        stage) — implement with the app's commit helper.
+        Returns the new tree."""
+        raise NotImplementedError(f"{type(self).__name__} has no commit stage")

de_shell/app.py

@@ -0,0 +1,170 @@
+"""
+app.py — the asyncio backend loop, minus anything app-specific.
+
+Reads JSON messages from stdin (Electron writes them), routes each to the
+session, and lets the session push replies through ``de_shell.ipc.emit``. This
+is the process's main thread: everything that touches a figure has to be
+marshalled back onto it (``SessionBase._dispatch_to_main``).
+
+Usage from an app's ``__main__``::
+
+    from de_shell.app import run
+    run(build_session=lambda: MySession(), app_packages=("de_groundcrew",))
+
+Message routing is fixed for the three the shell owns — ``action``,
+``figure_event``, ``resize``, plus ``quit`` and the ``tick`` no-op (which
+arrives both as its own type and as an action named ``tick``) — and open
+for everything else: pass ``on_message`` to handle app-specific envelopes (SpyDE
+routes its flat ``{"command": "console_*"}`` messages that way).
+
+The `tick` no-op matters more than it looks. Windows throttles timer delivery to
+a hidden child process so hard that its waits can freeze until process I/O
+arrives, so Electron writes a 0.5 Hz tick purely to keep this process
+schedulable. Dropping it as "an unknown message" would log a warning twice a
+second forever.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import sys
+from typing import Callable, Iterable
+
+from de_shell import ipc, log_stream, process_guard
+
+log = logging.getLogger(__name__)
+
+
+def _dispatch_figure_event(msg: dict) -> None:
+    """Forward a frontend interaction event to the anyplotlib figure."""
+    fig_id = msg.get("fig_id")
+    event_json = msg.get("event_json")
+    if fig_id is None or event_json is None:
+        return
+    import anyplotlib._electron as _ael
+    _ael.dispatch_event(fig_id, event_json)
+
+
+def _resize_figure(msg: dict) -> None:
+    """Apply a window resize to the anyplotlib figure layout."""
+    fig_id = msg.get("fig_id")
+    if fig_id is None:
+        return
+    import anyplotlib._electron as _ael
+    w, h = int(msg.get("width", 600)), int(msg.get("height", 400))
+    log.debug("resize figure %s -> %dx%d", fig_id, w, h)
+    _ael.resize_figure(fig_id, w, h)
+
+
+def _install_logging(app_packages: Iterable[str], log_level_env: str) -> None:
+    """Stream logs to the frontend panel, and tee to stderr when asked.
+
+    The tee is what makes a backend failure visible to Playwright: ``emit`` goes
+    down the PLOTAPP stdout channel, which the Electron main process consumes,
+    so without this a backend that dies mid-test dies silently.
+    """
+    level = os.environ.get(log_level_env)
+    if level:
+        handler = logging.StreamHandler(sys.stderr)
+        handler.setLevel(getattr(logging, level.upper(), logging.INFO))
+        handler.setFormatter(
+            logging.Formatter("%(asctime)s %(name)s %(levelname)s %(message)s"))
+        logging.getLogger().addHandler(handler)
+        logging.getLogger().setLevel(getattr(logging, level.upper(), logging.INFO))
+    log_stream.install(level=level or "INFO")
+
+
+async def _main(
+    build_session: Callable[[], object],
+    app_packages: Iterable[str],
+    log_level_env: str,
+    on_message: Callable[[object, dict], bool] | None,
+    on_ready: Callable[[object], None] | None,
+) -> None:
+    ipc.redirect_stray_stdout()
+
+    # FIRST: real timer interrupts. Windows throttles timers for this hidden
+    # Electron child, freezing every timer-driven wait in the process (poll
+    # loops, Event.wait) until I/O arrives — the "it only finishes when you
+    # click" bug. See process_guard.unthrottle_windows_timers.
+    try:
+        process_guard.unthrottle_windows_timers()
+    except Exception as e:
+        log.warning("timer unthrottle failed: %s", e)
+    # Guarantee any worker subprocesses die with this process: a Windows
+    # kill-on-close Job Object makes the OS reap the whole tree however we
+    # exit. Best-effort no-op off Windows.
+    try:
+        process_guard.install_kill_on_close()
+    except Exception as e:
+        log.debug("process guard install failed: %s", e)
+
+    _install_logging(app_packages, log_level_env)
+
+    session = build_session()
+
+    if on_ready is not None:
+        on_ready(session)
+
+    ipc.emit({"type": "ready"})
+
+    loop = asyncio.get_event_loop()
+    # Let background workers marshal their result-applies onto this thread.
+    session.set_main_loop(loop)
+
+    async for msg in ipc.read_messages(loop):
+        msg_type = msg.get("type")
+        try:
+            if msg_type == "action":
+                # The keepalive arrives as an ACTION named tick (that is how
+                # @de/shell-main sends it), so it must be swallowed here, not
+                # left to every app's action table to remember.
+                if msg.get("action") != "tick":
+                    session.dispatch_action(msg)
+            elif msg_type == "figure_event":
+                _dispatch_figure_event(msg)
+            elif msg_type == "resize":
+                _resize_figure(msg)
+            elif msg_type == "tick":
+                pass          # see the module docstring — deliberately silent
+            elif msg_type == "quit":
+                break
+            elif on_message is not None and on_message(session, msg):
+                pass          # the app claimed it
+            else:
+                log.warning("[backend] unknown message type: %s", msg_type)
+        except Exception as e:
+            log.exception("handling %s failed", msg_type)
+            ipc.emit_error(str(e))
+
+    session.shutdown()
+
+
+def run(
+    build_session: Callable[[], object],
+    *,
+    app_packages: Iterable[str] = (),
+    log_level_env: str = "DE_LOG_LEVEL",
+    on_message: Callable[[object, dict], bool] | None = None,
+    on_ready: Callable[[object], None] | None = None,
+) -> None:
+    """Run the backend loop until the frontend says quit.
+
+    Parameters
+    ----------
+    build_session
+        Builds the app's Session. Called once, on the main thread, before the
+        loop starts.
+    app_packages
+        The app's own top-level package names, for log routing.
+    log_level_env
+        Environment variable holding the initial log level. Per-app so two
+        installed apps can be made verbose independently.
+    on_message
+        Handles a message the shell does not know. Return True if claimed.
+    on_ready
+        Runs after the session is built and before ``ready`` is emitted —
+        the place for prewarming and starting background services.
+    """
+    asyncio.run(_main(build_session, app_packages, log_level_env, on_message, on_ready))

de_shell/compute.py

@@ -0,0 +1,103 @@
+"""
+compute.py — the dask-free half of the compute abstraction.
+
+Every shell app needs to run work off the asyncio main thread and marshal the
+result back. Only SpyDE needs a distributed cluster behind that. So the shell
+owns ``ThreadCompute`` — a ``concurrent.futures``-backed submitter — and SpyDE's
+``ComputeBackend`` subclasses it to add the ``dask.distributed`` branch.
+
+The split is load-bearing, not cosmetic: importing this module must never pull in
+dask, because de-groundcrew and de-autopilot are live in-memory apps that do not
+install it.
+"""
+from __future__ import annotations
+
+import concurrent.futures
+import logging
+import threading
+from typing import Callable
+
+log = logging.getLogger(__name__)
+
+
+class SyncFuture:
+    """Immediately-resolved future for already-computed results."""
+
+    def __init__(self, result):
+        self._result = result
+
+    def done(self) -> bool:
+        return True
+
+    def result(self, timeout=None):
+        return self._result
+
+    def cancel(self):
+        return False
+
+    def add_done_callback(self, fn: Callable) -> None:
+        fn(self)
+
+
+class ThreadCompute:
+    """Submit callables to a thread pool, returning ``concurrent.futures.Future``.
+
+    Parameters
+    ----------
+    executor
+        The general-purpose pool. When ``None``, subclasses are expected to
+        provide their own routing (SpyDE's distributed mode does) — the base
+        class then only offers the dedicated interactive-read pool.
+    """
+
+    def __init__(self, executor: concurrent.futures.ThreadPoolExecutor | None = None):
+        self._executor = executor
+        self._lock = threading.Lock()
+        # Dedicated LOCAL pool for interactive frame reads. Created lazily so a
+        # plain threaded app (which already has _executor) never pays for it.
+        self._nav_executor: concurrent.futures.ThreadPoolExecutor | None = None
+
+    def _nav_pool(self) -> concurrent.futures.ThreadPoolExecutor:
+        """The local pool interactive reads run on, built on first use.
+
+        ONE worker, deliberately. ``fut.cancel()`` only takes effect on a QUEUED
+        future — an already-running one runs to completion. With N>1 workers,
+        several superseded reads run concurrently and complete in arbitrary
+        order, so an OLDER frame can land after a newer one and the display jumps
+        backwards while you drag. One worker makes the reads serial, so the only
+        ordering hazard left is a single in-flight read, which the caller's
+        identity check already discards.
+        """
+        with self._lock:
+            if self._nav_executor is None:
+                self._nav_executor = concurrent.futures.ThreadPoolExecutor(
+                    max_workers=1, thread_name_prefix="nav-read")
+            return self._nav_executor
+
+    def shutdown_nav_pool(self) -> None:
+        """Release the local interactive-read pool (Session.shutdown)."""
+        with self._lock:
+            pool, self._nav_executor = self._nav_executor, None
+        if pool is not None:
+            pool.shutdown(wait=False, cancel_futures=True)
+
+    @property
+    def executor(self):
+        """Underlying ThreadPoolExecutor, or None."""
+        return self._executor
+
+    @property
+    def is_distributed(self) -> bool:
+        return False
+
+    def submit(self, fn: Callable, *args, **kwargs) -> concurrent.futures.Future:
+        """Submit a callable, return a concurrent.futures.Future."""
+        if self._executor is None:
+            raise RuntimeError("ThreadCompute has no executor; provide one or override submit()")
+        return self._executor.submit(fn, *args, **kwargs)
+
+    def submit_nav_read(self, fn) -> concurrent.futures.Future:
+        """Run ``fn`` (a no-arg callable returning an ndarray) on the LOCAL
+        interactive-read pool — cancellable, serial, never remote."""
+        pool = self._executor if self._executor is not None else self._nav_pool()
+        return pool.submit(fn)

de_shell/debug_flags.py

@@ -0,0 +1,69 @@
+"""
+debug_flags.py — runtime-toggleable diagnostic switches.
+
+These are cheap boolean flags that gate opt-in instrumentation (e.g. the per-frame
+navigator update profile). Each can be seeded from an env var at import (so it can
+be on from process start) AND toggled live from the UI via the ``set_debug_flag``
+action — no restart needed. Keeping them in ONE module means a single source of
+truth that both the read side (``update_functions``) and the paint side (``plot``)
+read through ``nav_profile_on()``.
+"""
+from __future__ import annotations
+
+import logging
+import os
+
+log = logging.getLogger(__name__)
+
+# Per-frame navigator update profile: one INFO line per move with the stage
+# breakdown (read / dtype / prefetch / lod / levels / transport). Seed from the
+# env so it can be on from startup; toggle live with set_debug_flag("nav_profile").
+_nav_profile: bool = os.environ.get("SPYDE_NAV_PROFILE") == "1"
+
+
+def nav_profile_on() -> bool:
+    """True when per-frame update profiling is active. Read this each frame (it's
+    a cheap module-global lookup) so a live toggle takes effect immediately."""
+    return _nav_profile
+
+
+def set_flag(name: str, value: bool) -> bool:
+    """Set a debug flag by name. Returns the new value. Unknown names are ignored
+    (return False) so a stale UI can't crash the backend."""
+    global _nav_profile
+    if name in ("nav_profile", "profile"):
+        _nav_profile = bool(value)
+        log.info("[debug] nav_profile = %s (per-frame update timing %s)",
+                 _nav_profile, "ON" if _nav_profile else "off")
+        return _nav_profile
+    log.debug("[debug] unknown debug flag %r ignored", name)
+    return False
+
+
+def get_flags() -> dict:
+    """Current flag states — for the UI to reflect the toggle on connect."""
+    return {"nav_profile": _nav_profile}
+
+
+def set_debug_flag(session, plot, payload) -> None:
+    """Staged action: the UI's debug toggle → set a flag + echo the new state.
+
+    Payload: ``{"name": "nav_profile", "value": true|false}``. Echoes
+    ``{"type": "debug_flags", ...}`` so the button can reflect the state, and (when
+    turning nav profiling ON) makes sure INFO records reach the Log panel so the
+    profile lines are actually visible without touching the level dropdown."""
+    name = payload.get("name", "nav_profile")
+    value = bool(payload.get("value", False))
+    set_flag(name, value)
+    if name in ("nav_profile", "profile") and value:
+        # The [NAV/PAINT-PROFILE] lines log at INFO; ensure the handler forwards
+        # INFO (the default is INFO, but a user may have raised it to WARNING).
+        try:
+            from de_shell.log_stream import set_level
+            import logging
+            if logging.getLogger().getEffectiveLevel() > logging.INFO:
+                set_level("INFO")
+        except Exception as e:
+            log.debug("raising log level to INFO for profiling failed: %s", e)
+    from de_shell.ipc import emit
+    emit({"type": "debug_flags", **get_flags()})