de-shell 0.2.0__py3-none-any.whl

de_shell/__init__.py

@@ -0,0 +1,25 @@
+"""
+de_shell — the Python half of the DE desktop-app shell.
+
+The substrate three applications share: SpyDE (offline EM analysis),
+de-groundcrew (live camera/hardware control) and de-autopilot (automated
+acquisition). It answers "how do I be a desktop app with a Python brain and
+pictures in it?" — the asyncio stdin/stdout loop, the PLOTAPP IPC protocol, log
+streaming, the window/figure registry, the action + staged-wizard framework, and
+the anyplotlib plotting wrapper.
+
+It answers nothing about what the data IS. No HyperSpy, no Dask, no
+RosettaSciIO, no pyxem — de-groundcrew and de-autopilot are live, in-memory
+applications and must not acquire those dependencies transitively. That
+constraint is what fixes the boundary, and `tests/test_boundary.py` enforces
+it in a clean subprocess: anything that answers "what is the data and what do
+you do to it?" (the array-cache tiering, the signal tree, the navigator read
+path, the distributed compute branch, every action handler) stays in the app.
+"""
+from __future__ import annotations
+
+#: The package version — the one place it is written. pyproject.toml reads it
+#: (dynamic version) and the release workflow refuses a tag that disagrees.
+__version__ = "0.2.0"
+
+__all__ = ["ipc", "log_stream", "process_guard", "debug_flags", "compute"]

de_shell/actions/context.py

@@ -0,0 +1,62 @@
+"""
+context.py — ActionContext: the adapter passed to action functions.
+
+An action function is handed one of these instead of reaching for the UI. It
+carries the clicked plot, the parameter values the frontend's panel collected
+(forwarded as kwargs), and a per-plot scratch dict for state that must outlive a
+single invocation (an FFT window, a toggle group, a widget the action added).
+
+Everything it touches is duck-typed — ``plot.plot_window``, ``plot.session`` —
+so it does not care what kind of plot or session an app has.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+
+class ActionContext:
+    """The attribute surface an action function is written against."""
+
+    def __init__(self, plot, params: dict[str, Any] | None = None,
+                 action_name: str = ""):
+        self.plot = plot
+        self.params = params or {}
+        self.action_name = action_name
+
+        # Per-plot persistent action state (FFT windows, toggle groups, …).
+        # Stored ON THE PLOT so it survives across action invocations — an
+        # ActionContext is built fresh for each one.
+        if not hasattr(plot, "_action_widgets"):
+            plot._action_widgets = {}
+        self.action_widgets = plot._action_widgets
+
+    # ── Plot / session access ─────────────────────────────────────────────────
+
+    @property
+    def plot_window(self):
+        return self.plot.plot_window
+
+    @property
+    def session(self):
+        return self.plot.session
+
+    # ── Stateful action registration ──────────────────────────────────────────
+
+    def register_action_plot_item(self, action_name: str, item, key: str) -> None:
+        slot = self.action_widgets.setdefault(action_name, {})
+        slot.setdefault("plot_items", {})[key] = item
+
+    def register_action_plot_window(self, action_name: str, plot_window, key: str) -> None:
+        slot = self.action_widgets.setdefault(action_name, {})
+        slot.setdefault("plot_windows", {})[key] = plot_window
+
+    def add_action_widget(self, action_name: str, widget=None, layout=None) -> None:
+        slot = self.action_widgets.setdefault(action_name, {})
+        slot["widget"] = widget
+        slot["layout"] = layout
+
+    def actions(self) -> list:
+        """The toolbar lives in the frontend, so there are no host-side action
+        objects to return. Kept because action code written against the old Qt
+        toolbar still calls it."""
+        return []

de_shell/actions/figure_registry.py

@@ -0,0 +1,53 @@
+"""
+figure_registry.py — per-window keep-alive for bare anyplotlib figures.
+
+Result windows that are NOT registered ``Plot``s emit raw ``figure`` messages
+whose Python-side figure objects must be kept referenced, or their widget
+callbacks are garbage-collected while the window is still open. Historically
+each module kept its own append-only ``_ALIVE`` list, which leaked every figure
+for the process lifetime.
+
+This registry keys the references by ``window_id`` and is evicted from the
+session's ``_forget_window``, so a figure lives exactly as long as its window.
+
+Apps hang their own per-window state off the same eviction via
+:func:`register_evictor`, rather than this module reaching into them — which is
+what it used to do (a hardcoded import of SpyDE's ``actions.views``).
+"""
+from __future__ import annotations
+
+from typing import Any, Callable
+
+_FIGS: dict[int, list[Any]] = {}
+
+#: App callbacks run when a window is forgotten. See `register_evictor`.
+_EVICTORS: list[Callable[[int], None]] = []
+
+
+def register_evictor(fn: Callable[[int], None]) -> None:
+    """Register ``fn(window_id)`` to run whenever a window is forgotten.
+
+    For app state keyed by window id that must die with the window — SpyDE's
+    per-window chip-view arrays, for instance. Registering the same function
+    twice is a no-op, so a module can call this at import without guarding.
+    """
+    if fn not in _EVICTORS:
+        _EVICTORS.append(fn)
+
+
+def keep_alive(window_id: int, fig: Any) -> None:
+    """Keep *fig* referenced until *window_id*'s window is forgotten."""
+    _FIGS.setdefault(int(window_id), []).append(fig)
+
+
+def forget_window(window_id: int) -> None:
+    """Drop every reference held for *window_id*, and run the app's evictors."""
+    wid = int(window_id)
+    _FIGS.pop(wid, None)
+    for fn in _EVICTORS:
+        try:
+            fn(wid)
+        except Exception:
+            # Teardown must not fail: this runs while a window is going away,
+            # and one app's bookkeeping error should not strand the rest.
+            pass

de_shell/actions/lifecycle.py

@@ -0,0 +1,295 @@
+"""
+lifecycle.py — the shared basis set for interactive actions.
+
+Every heavy action in every shell app repeats the same wiring: run the compute
+on a daemon thread and marshal the UI apply back to the asyncio main thread,
+guard against superseded runs (React StrictMode double-mount, rapid re-tune),
+swap a controller/overlay for a newer one, and narrate progress. These helpers
+are the single implementation of those idioms.
+
+What is here is only the part that knows nothing about the data. SpyDE's
+`spyde/actions/lifecycle.py` re-exports all of it alongside its own
+domain lifecycle (the find-vectors attach gap, painting a tree's signal plots,
+the progressive shared-memory fill), so an action keeps importing one module and
+does not have to know which half a given helper came from.
+
+THREADING CONTRACT: UI/figure updates happen on the asyncio main thread only.
+Workers marshal via ``session._dispatch_to_main``; ``de_shell.ipc.emit*`` is safe
+from any thread.
+"""
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from typing import Any, Callable
+
+log = logging.getLogger(__name__)
+
+
+# ── worker-thread marshal ─────────────────────────────────────────────────────
+
+#: Guards the per-session in-flight tally below. One lock for the process: the
+#: critical section is an integer add.
+_WORKER_COUNT_LOCK = threading.Lock()
+
+
+def _count_worker(session, delta: int) -> None:
+    with _WORKER_COUNT_LOCK:
+        try:
+            session._inflight_workers = getattr(
+                session, "_inflight_workers", 0) + delta
+        except Exception:      # a bare stub session that refuses attributes
+            pass
+
+
+def inflight_workers(session) -> int:
+    """How many :func:`run_on_worker` jobs are still running for *session*.
+
+    Background work started here was otherwise invisible: it is a bare daemon
+    thread with no pool and no registry, so nothing could answer "is anything
+    still running?". Anyone needing to know had to pick some VISIBLE side
+    effect of the work and wait for that instead — a proxy that is right until
+    the day the value it watches is set somewhere else too, or set before the
+    work is really done.
+
+    Zero means every job has finished AND handed its ``on_done`` to the
+    dispatcher; it does NOT mean those callbacks have run. Pair it with a
+    round-trip through the loop for that.
+    """
+    with _WORKER_COUNT_LOCK:
+        return int(getattr(session, "_inflight_workers", 0) or 0)
+
+
+def run_on_worker(session, work: Callable[[], Any], *, name: str,
+                  on_done: Callable[[Any], None] | None = None,
+                  on_error: Callable[[Exception], None] | None = None) -> None:
+    """Run ``work()`` on a daemon thread and marshal ``on_done(result)`` back
+    onto the asyncio main thread via ``session._dispatch_to_main``.
+
+    ``on_error(exc)`` runs on the worker thread (it typically just
+    ``emit_error``\\s, which is thread-safe). When *session* can't marshal
+    (``None`` or a bare test stub without ``_dispatch_to_main``) everything
+    runs inline synchronously, so handler tests see the result immediately.
+    """
+    dispatch = getattr(session, "_dispatch_to_main", None)
+    if dispatch is None:
+        try:
+            result = work()
+        except Exception as e:
+            log.exception("%s failed", name)
+            if on_error is not None:
+                on_error(e)
+            return
+        if on_done is not None:
+            on_done(result)
+        return
+
+    def _worker():
+        try:
+            try:
+                result = work()
+            except Exception as e:
+                log.exception("%s failed", name)
+                if on_error is not None:
+                    on_error(e)
+                return
+            if on_done is not None:
+                dispatch(lambda: on_done(result))
+        finally:
+            # AFTER the dispatch, deliberately: a caller that sees the count
+            # reach zero has to be able to conclude the completion callback is
+            # at least QUEUED. Decrementing first would let it conclude that
+            # while `on_done` is still to be handed to the loop.
+            _count_worker(session, -1)
+
+    _count_worker(session, +1)
+    threading.Thread(target=_worker, daemon=True, name=name).start()
+
+
+# ── cancellation (a superseded compute is stopped, not ignored) ───────────────
+
+class ComputeHandle:
+    """The cancellation handle for one dispatched compute.
+
+    ``flag`` is the ``[False]`` stop token the work polls; ``future`` is the
+    future it runs as, when there is one. Constructing a handle registers both
+    on the signal tree, so closing the tree stops the compute.
+
+    A superseded or abandoned compute must be CANCELLED, not left running so its
+    result can be discarded. The generation guard is not a substitute: it drops
+    the result on arrival while the pass keeps reading the dataset, and a pass
+    over the dataset is the most expensive thing the app does.
+
+    A compute with no interruption point — one library call over an array
+    already in memory — can still take a handle. The flag then stops it before
+    it starts and drops a late result, which is all that is available.
+    """
+
+    __slots__ = ("flag", "future", "_tree")
+
+    def __init__(self, tree, future=None):
+        self.flag: list = [False]
+        self.future = future
+        self._tree = tree
+        register = getattr(tree, "register_cancel", None)
+        if register is not None:
+            register(flag=self.flag, future=future)
+
+    @property
+    def stopped(self) -> bool:
+        return bool(self.flag[0])
+
+    def attach(self, future) -> None:
+        """Adopt a future created after the handle, registering it too."""
+        self.future = future
+        register = getattr(self._tree, "register_cancel", None)
+        if register is not None and future is not None:
+            register(future=future)
+
+    def cancel(self) -> None:
+        """Stop the compute and drop it from the tree's registry."""
+        self.flag[0] = True
+        if self.future is not None:
+            try:
+                if not self.future.done():
+                    self.future.cancel()
+            except Exception as e:
+                log.debug("cancelling a superseded compute failed: %s", e)
+        self._unregister()
+
+    def retire(self) -> None:
+        """Drop a finished compute's registration, without marking it stopped.
+
+        Required, or the registry gains an entry per run and a long-lived tree
+        accumulates one for every interaction.
+        """
+        self._unregister()
+
+    def _unregister(self) -> None:
+        unregister = getattr(self._tree, "unregister_cancel", None)
+        if unregister is None:
+            return
+        try:
+            unregister(flag=self.flag, future=self.future)
+        except Exception as e:                               # pragma: no cover
+            log.debug("unregistering a compute failed: %s", e)
+
+
+def supersede(prior: "ComputeHandle | None", tree, future=None) -> ComputeHandle:
+    """Cancel *prior* and return the handle for the compute replacing it."""
+    if prior is not None:
+        prior.cancel()
+    return ComputeHandle(tree, future)
+
+
+# ── generation guard (latest-wins / StrictMode double-mount) ──────────────────
+
+def bump_generation(owner, key: str) -> int:
+    """Bump and return ``owner.<key>`` (an int generation counter).
+
+    The run/stop generation contract: a wizard's *open* handler bumps its
+    ``_<key>_run_gen`` synchronously BEFORE spawning any worker, and every
+    deferred build checks ``is_current`` on arrival; the *close* handler bumps
+    unconditionally FIRST, cancelling any in-flight open. This closes the React
+    StrictMode mount→cleanup→remount race (open, close, open fired synchronously
+    before any worker lands) that otherwise builds two live controllers. Also
+    used per-controller for latest-wins recomputes.
+    """
+    gen = int(getattr(owner, key, 0)) + 1
+    setattr(owner, key, gen)
+    return gen
+
+
+def is_current(owner, key: str, gen: int) -> bool:
+    """True if *gen* is still ``owner.<key>``'s current generation."""
+    return getattr(owner, key, None) == gen
+
+
+# ── controller / overlay replacement ──────────────────────────────────────────
+
+def replace_tree_attr(owner, attr: str, factory: Callable[[], Any] | None):
+    """Replace ``owner.<attr>`` (an overlay/controller) with ``factory()``,
+    removing the prior one first so re-running an action never stacks markers.
+    Pass ``factory=None`` to just remove. Returns the new value (None on a
+    failed attach — logged, not raised)."""
+    old = getattr(owner, attr, None)
+    if old is not None and hasattr(old, "remove"):
+        try:
+            old.remove()
+        except Exception as e:
+            log.debug("removing prior %s failed: %s", attr, e)
+    setattr(owner, attr, None)
+    if factory is None:
+        return None
+    try:
+        new = factory()
+    except Exception as e:
+        log.debug("attaching %s failed: %s", attr, e)
+        new = None
+    setattr(owner, attr, new)
+    return new
+
+
+# ── progress narration ────────────────────────────────────────────────────────
+
+def progress_emitter(prefix: str, *, min_interval: float = 0.5) -> Callable[[int, int], None]:
+    """A throttled ``progress(done, total)`` callback that emits
+    ``"{prefix} {pct}%"`` status lines (always emits the 100% line)."""
+    from de_shell.ipc import emit_status
+    last = [0.0]
+
+    def progress(done, total):
+        if not total:
+            return
+        now = time.monotonic()
+        if done < total and now - last[0] < min_interval:
+            return
+        last[0] = now
+        emit_status(f"{prefix} {int(100 * done / total)}%")
+
+    return progress
+
+
+# ── per-window "Calculating…" overlay ─────────────────────────────────────────
+
+class window_computing:
+    """Context manager: emit ``window_computing`` start/stop around a long
+    compute that paints into ``window_id`` — drives the renderer's floating
+    translucent "Calculating…" chip centered on that plot window.
+
+    ALWAYS emits the matching stop, even on exception — the ``__exit__`` runs
+    unconditionally, so a cancelled or failed compute cannot leave the overlay
+    stuck. ``window_id=None`` is a silent no-op both ways (mirrors
+    ``emit_window_computing``'s own guard) so call sites don't need to
+    special-case an unattached plot.
+
+    Usage::
+
+        with window_computing(nav_plot.window_id):
+            ...long fill...
+
+    or, when the start/stop don't naturally bracket a single call (e.g. a
+    background thread that outlives this function), call ``.start()`` /
+    ``.stop()`` directly and put ``.stop()`` in the thread's own
+    ``try/finally``.
+    """
+
+    def __init__(self, window_id: int | None):
+        self.window_id = window_id
+
+    def start(self) -> None:
+        from de_shell.ipc import emit_window_computing
+        emit_window_computing(self.window_id, True)
+
+    def stop(self) -> None:
+        from de_shell.ipc import emit_window_computing
+        emit_window_computing(self.window_id, False)
+
+    def __enter__(self) -> "window_computing":
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> bool:
+        self.stop()
+        return False

de_shell/actions/registry.py

@@ -0,0 +1,141 @@
+"""
+registry.py — the staged-action registry and the window-controller protocol.
+
+A shell app has exactly TWO dispatch paths for renderer→backend actions; do not
+invent a third:
+
+1. **Toolbar actions** — declared in the app's toolbar config, resolved and
+   invoked by the session with an :class:`~de_shell.actions.context.ActionContext`.
+2. **Staged actions** — the wizard/caret protocol: an action name maps to a
+   ``"module.function"`` dotted path with the uniform ``fn(session, plot,
+   payload)`` signature. Modules are imported LAZILY, so heavy dependencies load
+   on first use rather than at startup — which is most of why the table is
+   dotted strings instead of function references.
+
+Staged-action NAMING CONVENTION (``<key>`` is the wizard's short prefix):
+
+    <key>_open           wizard mounted → start live preview / controller
+    <key>_close          wizard unmounted → tear everything down
+    <key>_tune           debounced live re-tune of preview params
+    <key>_set_<param>    discrete parameter change
+    <key>_run            heavy compute stage (may open a result tree)
+    <key>_commit         snapshot the live result into a new result tree
+
+Wizard-specific extra stages are allowed but must keep the ``<key>_`` prefix.
+
+**The tables live in the app, the mechanism lives here.** An app calls
+:func:`register_staged` (or :func:`register_staged_table`) at import to populate
+the registry, and :func:`register_wizard_schema` to say where each wizard's
+parameter schema is declared. The shell has no business knowing that
+``fit_open`` exists.
+
+WindowController protocol
+-------------------------
+Windows that are NOT registered plots (bare ``figure`` emits) must register a
+*controller* with ``session.register_window_controller(window_id, controller)``
+so dispatch and teardown can reach them. A controller is duck-typed:
+
+    window_id: int                     # the window it drives
+    close() -> None                    # full teardown; called by the session's
+                                       # _forget_window when the window goes
+                                       # away for ANY reason
+    handle_action(name, payload) -> bool   # optional: consume an action aimed
+                                           # at this window; return True if
+                                           # handled
+
+:class:`de_shell.actions.wizard.WizardController` provides a base implementation.
+"""
+from __future__ import annotations
+
+import importlib
+from typing import Callable
+
+#: action name → "module.function". Populated by the app.
+STAGED_HANDLERS: dict[str, str] = {}
+
+
+def register_staged(name: str, dotted_path: str) -> None:
+    """Register a staged action (``fn(session, plot, payload)``) by dotted path."""
+    STAGED_HANDLERS[name] = dotted_path
+
+
+def register_staged_table(table: dict[str, str]) -> None:
+    """Register a whole table at once. Later registrations win, so an app can
+    override a default."""
+    STAGED_HANDLERS.update(table)
+
+
+def resolve_staged(name: str) -> Callable | None:
+    """Lazily import and return the handler for a staged action name."""
+    dotted = STAGED_HANDLERS.get(name)
+    if dotted is None:
+        return None
+    mod, fn = dotted.rsplit(".", 1)
+    return getattr(importlib.import_module(mod), fn)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Wizard parameter schemas — the single host-agnostic lookup
+# ─────────────────────────────────────────────────────────────────────────────
+#
+# Every wizard declares its parameter schema in one place — a `parameters`
+# classattr on its WizardController, or a module-level dict for controller-less
+# wizards — in the same spec as the toolbar config's `parameters:`. This table
+# maps a wizard key to wherever its schema lives, so ANY host (an Electron
+# panel, a notebook form generator, a doc generator) resolves them uniformly.
+
+#: wizard key → (module, attribute), or (`YAML_SCHEMA`, toolbar action title).
+_WIZARD_SCHEMAS: dict[str, tuple[str, str]] = {}
+
+#: Sentinel module name meaning "resolve this from the app's toolbar config".
+YAML_SCHEMA = "__yaml__"
+
+#: Set by `set_yaml_schema_resolver`. Takes a toolbar action title, returns its
+#: `parameters` dict.
+_yaml_resolver: Callable[[str], dict] | None = None
+
+
+def register_wizard_schema(key: str, module: str, attr: str) -> None:
+    """Declare where wizard *key*'s parameter schema lives.
+
+    ``module=YAML_SCHEMA`` means "look *attr* up as a toolbar action title via
+    the registered YAML resolver".
+    """
+    _WIZARD_SCHEMAS[key] = (module, attr)
+
+
+def register_wizard_schemas(table: dict[str, tuple[str, str]]) -> None:
+    """Register a whole schema table at once."""
+    _WIZARD_SCHEMAS.update(table)
+
+
+def set_yaml_schema_resolver(fn: Callable[[str], dict]) -> None:
+    """Install the app's toolbar-config lookup for ``YAML_SCHEMA`` entries."""
+    global _yaml_resolver
+    _yaml_resolver = fn
+
+
+def wizard_parameters(key: str) -> dict:
+    """Return wizard ``key``'s declared parameter schema (a copy).
+
+    The uniform entry point for rendering a wizard's controls in ANY host —
+    same spec as the toolbar config's ``parameters:`` (type/name/default/min/
+    max/step/choices/tab/extensions). Raises ``KeyError`` for unknown keys.
+    """
+    module, attr = _WIZARD_SCHEMAS[key]
+    if module == YAML_SCHEMA:
+        if _yaml_resolver is None:
+            raise RuntimeError(
+                f"wizard {key!r} declares a toolbar-config schema, but no "
+                "resolver is installed — call set_yaml_schema_resolver() at "
+                "app startup."
+            )
+        return _yaml_resolver(attr)
+    obj = getattr(importlib.import_module(module), attr)
+    schema = obj if isinstance(obj, dict) else getattr(obj, "parameters", {})
+    return dict(schema)
+
+
+def wizard_keys() -> list[str]:
+    """All wizard keys with a declared schema."""
+    return list(_WIZARD_SCHEMAS)