plexora 0.0.8__tar.gz → 0.0.9__tar.gz

{plexora-0.0.8/plexora.egg-info → plexora-0.0.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexora
-Version: 0.0.8
+Version: 0.0.9
 Summary: OpenSeadragon-based cellular image viewing and analysis tool.
 License-Expression: LicenseRef-Plexora-Academic-1.0
 Requires-Python: <3.14,>=3.12

{plexora-0.0.8 → plexora-0.0.9}/plexora/connect.py

@@ -222,7 +222,12 @@ def normalize_remote_command(remote_command):
 #: has Plexora is asking for the current one, and a plain install would report
 #: "already satisfied" and leave last year's copy in place -- which is the
 #: exact failure the option exists to end.
-PIP_ARGUMENTS = ("install", "--upgrade", "plexora")
+#: `--progress-bar off` because the install rides the launch's ssh, which asks
+#: for a pty (`-t`) -- and pip with a pty redraws its download bar with
+#: carriage returns, which a line-based log renders as one line growing to
+#: kilobytes. The per-package Collecting/Installing lines still come through,
+#: and they are the progress worth showing.
+PIP_ARGUMENTS = ("install", "--progress-bar", "off", "--upgrade", "plexora")
 
 #: What conda does to a subprocess's output unless told not to: buffers it and
 #: prints the lot when the child exits. Correct for a script whose output is a
@@ -346,23 +351,35 @@ def parse_installed_version(lines):
     return None
 
 
-def install_ssh_argv(target, install_line, *, jump=None, ssh_opts=()):
-    """The ssh that installs Plexora over there and exits.
+#: Printed by the remote shell between the install and the launch. The two
+#: run in ONE ssh -- `pip … && echo … && srun …` -- so the only way to know
+#: pip finished (rather than printed its last routine line) is to say so
+#: ourselves. `&&` is also the failure story: a pip that exits non-zero
+#: short-circuits the chain, the marker never appears, the ssh ends, and
+#: nothing was launched from the half-upgraded environment.
+INSTALL_DONE_MARK = "PLEXORA_INSTALL_DONE"
 
-    No `-t`, unlike every other ssh here. The pty those ask for exists so the
-    remote gets a SIGHUP when this end goes away, which matters for a process
-    meant to outlive the call and not at all for one we wait on -- and it
-    costs something here: pip draws a progress bar with carriage returns when
-    it thinks it has a terminal, and the log this feeds is line-based, so a
-    download would arrive as one line growing to a few kilobytes. Without a
-    pty pip prints plain lines instead. Prompts are unaffected: the askpass
-    relay is what answers them, and it needs no terminal by design.
+
+def parse_install_done(line):
+    """Matcher for the marker line. True on the marker, None otherwise."""
+    return True if line.strip() == INSTALL_DONE_MARK else None
+
+
+def install_prefixed(install_line, launch_line):
+    """One remote command: install, say so, then launch.
+
+    One command because it is one login. The install used to be its own ssh,
+    which was one extra authentication -- and at a site that confirms every
+    login on somebody's phone, "connect" buzzed it twice. Chaining also puts
+    the install literally immediately before the run, in the same shell, so
+    nothing can start from the copy the upgrade was replacing.
+
+    Under a scheduler the chain still runs the install on the LOGIN node --
+    it precedes the `srun` in the same command -- which is where it belongs:
+    the environment is on a shared filesystem, and the allocation should not
+    be spent watching pip download wheels.
     """
-    argv = ["ssh", *_ssh_options(ssh_opts)]
-    if jump:
-        argv += ["-J", jump]
-    argv += [target, install_line]
-    return argv
+    return f"{install_line} && echo {INSTALL_DONE_MARK} && {launch_line}"
 
 
 def remote_command_line(remote_command, port, *, bind_node=False, datasource=None,
@@ -823,24 +840,6 @@ class _Watched:
     def tail(self, count=6):
         return self.lines[-count:]
 
-    def wait_for_exit(self, deadline):
-        """Block until this process ends. Its exit code, or None if it did not.
-
-        The one watcher here that is waited ON rather than merely watched: an
-        install has to have finished before the launch that depends on it
-        starts. `drain` before returning, for the same reason every other
-        reader of `lines` calls it -- the pump thread is still working through
-        what the shell wrote on its way out, and that is where the reason for
-        a non-zero exit lives.
-        """
-        while _now() < deadline:
-            code = self.process.poll()
-            if code is not None:
-                self.drain()
-                return code
-            _sleep(0.2)
-        return self.process.poll()
-
     def stop(self):
         if self.process.poll() is not None:
             return
@@ -1087,14 +1086,13 @@ def _missing_command_hint(remote_command, watched):
 
 
 class _BareInstall:
-    """The four things `_install_plexora` needs, for a caller with no session.
+    """What `_begin_install`/`_await_install` read, for a caller with no session.
 
     `connect_node` drives one ssh by hand -- it predates `NodeSession` and is
-    what `plexora connect --node` still runs -- so there is no object there to
+    what `plexora node connect` still runs -- so there is no object there to
     read `target` and `remote_command` off. This is that object and nothing
-    more: its `watchers` list exists so the install's watcher can be taken
-    back out of it, and its `_phase` is a no-op because a terminal has the
-    echoed lines and no progress list to update.
+    more; its `_phase` is a no-op because a terminal has the echoed lines and
+    no progress list to update.
     """
 
     def __init__(self, target, remote_command, jump, ssh_opts, echo):
@@ -1103,7 +1101,6 @@ class _BareInstall:
         self.jump = jump
         self.ssh_opts = ssh_opts
         self.echo = echo
-        self.watchers = []
         self.install_log = []
         self.installed_version = None
 
@@ -1118,63 +1115,53 @@ class _BareInstall:
         return watched
 
 
-def _install_plexora(session):
-    """Install or update Plexora on the far side, before anything is launched.
-
-    Shared by both session shapes because it is the same act for both: one
-    ssh, run to completion, whose output is the connection's own log. It runs
-    BEFORE the launch and not beside it, which is the whole point of the
-    option -- a viewer started from the copy that was already there, while an
-    upgrade downloads alongside it, is the version the user asked to stop
-    running.
-
-    Under a scheduler it still runs on the target -- the login node -- rather
-    than inside the job. The environment being written to is on a filesystem
-    both machines share, so the compute node gets the same result; and doing
-    it here means the install is not spending an allocation, and not standing
-    between somebody and the announce that says their job finally started.
-
-    The watcher is taken back out of `session.watchers` when it exits. That
-    list is what `alive` is computed from -- `all(w.alive for w in ...)` --
-    so an exited process left in it would report the whole connection dead the
-    moment the install succeeded. Its output has already been echoed and is
-    kept here, which is the part anybody wants afterwards.
-    """
+def _begin_install(session):
+    """Announce the install and hand back the pip line the launch is chained to."""
     line = install_command_line(session.remote_command)
     environment = environment_label(session.remote_command)
     session._phase("installing")
     session.echo(f"  Installing Plexora on {session.target}"
                  + (f" in {environment}" if environment else "")
-                 + "; this can take a few minutes.")
-    watched = session._spawn(
-        install_ssh_argv(session.target, line, jump=session.jump,
-                         ssh_opts=session.ssh_opts),
-        "install",
-    )
-    try:
-        code = watched.wait_for_exit(_now() + INSTALL_TIMEOUT)
-    finally:
-        watched.stop()
-        if watched in session.watchers:
-            session.watchers.remove(watched)
-        if watched in _ACTIVE:
-            _ACTIVE.remove(watched)
-    session.install_log = list(watched.lines)
+                 + " before launching it; this can take a few minutes.")
+    return line
 
-    if code is None:
-        raise ConnectError(
-            f"Installing Plexora on {session.target} did not finish within "
-            f"{INSTALL_TIMEOUT:g}s.\n"
-            + "\n".join(f"    {text}" for text in watched.tail())
-            + "\n\nThe environment may be half-upgraded. Run it by hand over "
-              "there to see where it stopped:\n"
-              f"    {line}\n"
-              "Or turn “Install or update Plexora” off and connect to the "
-              "copy that is already installed."
-        )
-    if code != 0:
-        raise ConnectError(_install_failure(session, line, watched, code))
 
+def _await_install(session, watched, line):
+    """Block until the chained install prints its marker, or explain why not.
+
+    `watched` is the connection's own primary ssh -- install and launch are
+    one process now -- so unlike the old separate-ssh install there is no
+    watcher to clean up afterwards: a marker means the same process is now
+    busy launching, and a death before the marker means the connection failed
+    and the watcher's exit is the correct thing for `alive` to report.
+    """
+    event = watched.events["installed"]
+    deadline = _now() + INSTALL_TIMEOUT
+    # `found`, not the event alone: the pump sets every event when the stream
+    # ends, precisely so waiters do not hang on an announce that is never
+    # coming -- so a set event proves the process finished talking, and only
+    # `found` proves the marker was in what it said.
+    while not watched.found.get("installed"):
+        event.wait(0.2)
+        if watched.found.get("installed"):
+            break
+        if not watched.alive:
+            watched.drain()
+            session.install_log = list(watched.lines)
+            code = watched.process.poll()
+            raise ConnectError(_install_failure(session, line, watched, code))
+        if _now() > deadline:
+            raise ConnectError(
+                f"Installing Plexora on {session.target} did not finish within "
+                f"{INSTALL_TIMEOUT:g}s.\n"
+                + "\n".join(f"    {text}" for text in watched.tail())
+                + "\n\nThe environment may be half-upgraded. Run it by hand "
+                  "over there to see where it stopped:\n"
+                  f"    {line}\n"
+                  "Or turn “Install or update Plexora” off and connect to the "
+                  "copy that is already installed."
+            )
+    session.install_log = list(watched.lines)
     version = parse_installed_version(watched.lines)
     session.installed_version = version
     session.echo(f"  Plexora {version} is installed on {session.target}."
@@ -1311,9 +1298,9 @@ class Session:
         #: one -- the viewer still opens.
         self.data_nodes = []
         self.node_errors = []
-        #: What the install said, and which version it left behind. Kept after
-        #: the process it came from has been taken back out of `watchers` --
-        #: see `_install_plexora` for why it cannot stay there.
+        #: What the install said, and which version it left behind. A copy of
+        #: the primary's lines up to the marker -- the install rides the same
+        #: ssh as the launch (see `install_prefixed`).
         self.install_log = []
         self.installed_version = None
 
@@ -1340,12 +1327,11 @@ class Session:
     def log(self, count=40):
         """Every ssh's recent output, labelled, oldest first.
 
-        The install first and from its own store, because by the time anybody
-        reads this the process it came from has exited and been taken out of
-        `watchers` -- and it is the half of the log somebody diagnosing a
-        failed connection is most likely to be after.
+        Install output needs no special handling any more: the install rides
+        the primary ssh itself, so its lines are in that watcher's tail like
+        everything else the connection printed.
         """
-        lines = [f"[install] {line}" for line in self.install_log[-count:]]
+        lines = []
         for watched in self.watchers:
             lines += [f"[{watched.label}] {line}" for line in watched.tail(count)]
         return lines
@@ -1463,32 +1449,45 @@ class Session:
             matchers["node"] = parse_node_announce
 
         try:
-            # Before the local node and before any launch: this is the step
-            # whose whole purpose is that what starts afterwards is the
-            # version it just installed.
-            if self.install:
-                _install_plexora(self)
+            # The install rides the SAME ssh as the launch, chained ahead of
+            # it -- one login instead of two, and at a Duo site one buzz of
+            # the phone instead of two. `&&` is what keeps the promise the
+            # separate-ssh version made: a failed install launches nothing.
+            install_line = _begin_install(self) if self.install else None
+            if install_line:
+                matchers["installed"] = parse_install_done
 
             local_node = None
             if self.local_node:
                 local_node = self._start_local_node(local_node_serve_port)
 
             if self.srun is None:
+                line = (install_prefixed(install_line, launch)
+                        if install_line else launch)
                 self.primary = self._spawn(
                     direct_ssh_argv(self.target, self.local_port,
-                                    self.remote_port, launch, jump=self.jump,
+                                    self.remote_port, line, jump=self.jump,
                                     ssh_opts=self.ssh_opts,
                                     forwards=forwards, reverse=reverse),
                     "ssh", matchers,
                 )
+                if install_line:
+                    _await_install(self, self.primary, install_line)
+                    # The install spent none of the connection's own budget.
+                    deadline = _now() + self.timeout
                 self._phase("starting")
             else:
+                remote = srun_command_line(self.srun, launch)
+                if install_line:
+                    remote = install_prefixed(install_line, remote)
                 self.primary = self._spawn(
-                    job_ssh_argv(self.target,
-                                 srun_command_line(self.srun, launch),
+                    job_ssh_argv(self.target, remote,
                                  jump=self.jump, ssh_opts=self.ssh_opts),
                     "job", matchers,
                 )
+                if install_line:
+                    _await_install(self, self.primary, install_line)
+                    deadline = _now() + self.timeout
                 self._phase("waiting_for_job")
                 self.node, self.node_port = _wait_for_announce(
                     self.primary, deadline, echo=self.echo)
@@ -1830,7 +1829,7 @@ class NodeSession:
         return bool(self.watchers) and all(w.alive for w in self.watchers)
 
     def log(self, count=40):
-        lines = [f"[install] {line}" for line in self.install_log[-count:]]
+        lines = []
         for watched in self.watchers:
             lines += [f"[{watched.label}] {line}" for line in watched.tail(count)]
         return lines
@@ -1871,30 +1870,44 @@ class NodeSession:
             host=bind)
         matchers = {"node": parse_node_announce}
 
-        # Ahead of the deadline, not inside it: `self.timeout` is how long a
-        # node may take to answer, and an install that ate half of it would
-        # leave a node that was starting normally being called dead.
-        if self.install:
-            _install_plexora(self)
-        deadline = _now() + self.timeout
+        # The install is chained ahead of the launch in the SAME ssh -- one
+        # login, one Duo prompt -- and it has its own budget: `self.timeout`
+        # is how long a node may take to answer, and an install that ate half
+        # of it would leave a node that was starting normally being called
+        # dead. The deadline is therefore taken after the marker.
+        install_line = _begin_install(self) if self.install else None
+        if install_line:
+            matchers["installed"] = parse_install_done
 
         if self.srun is None:
-            self._phase("tunneling")
+            if not install_line:
+                self._phase("tunneling")
+            line = (install_prefixed(install_line, launch)
+                    if install_line else launch)
             self.primary = self._spawn(
                 direct_ssh_argv(self.target, self.local_port, self.remote_port,
-                                launch, jump=self.jump, ssh_opts=self.ssh_opts),
+                                line, jump=self.jump, ssh_opts=self.ssh_opts),
                 "node", matchers)
+            if install_line:
+                _await_install(self, self.primary, install_line)
+                self._phase("tunneling")
         else:
             # No forward on this one: at the moment it opens, the job has not
             # been allocated and there is no host to point a forward at. The
             # announce is what names it, and the tunnel follows.
-            self._phase("waiting_for_job")
-            self.echo("  asking the scheduler for a node to serve the data "
-                      "from; this can wait in the queue.")
+            remote = srun_command_line(self.srun, launch)
+            if install_line:
+                remote = install_prefixed(install_line, remote)
             self.primary = self._spawn(
-                job_ssh_argv(self.target, srun_command_line(self.srun, launch),
+                job_ssh_argv(self.target, remote,
                              jump=self.jump, ssh_opts=self.ssh_opts),
                 "job", matchers)
+            if install_line:
+                _await_install(self, self.primary, install_line)
+            self._phase("waiting_for_job")
+            self.echo("  asking the scheduler for a node to serve the data "
+                      "from; this can wait in the queue.")
+        deadline = _now() + self.timeout
 
         announced = _wait_for_node(self.primary, deadline, echo=self.echo)
         if not announced:
@@ -2133,21 +2146,25 @@ def connect_node(target, serve, *, name=None,
     local, remote = pick_ports(local_port, remote_port)
     name = name or f"{split_target(target)[1]}-node"
 
-    if install:
-        # A stand-in with the four attributes `_install_plexora` reads. This
-        # function drives its ssh by hand rather than through a session object,
-        # and giving it one just to install would be a bigger change than the
-        # shim it would replace.
-        _install_plexora(_BareInstall(target, remote_command, jump, ssh_opts,
-                                      echo))
-
     launch = node_command_line(remote_command, remote, serve,
                                allow_origin=allow_origin, plugins=plugins)
+    matchers = {"node": parse_node_announce}
+    shim = install_line = None
+    if install:
+        # A stand-in with the attributes `_begin_install`/`_await_install`
+        # read. This function drives its ssh by hand rather than through a
+        # session object, and giving it one just to install would be a bigger
+        # change than the shim it replaces.
+        shim = _BareInstall(target, remote_command, jump, ssh_opts, echo)
+        install_line = _begin_install(shim)
+        launch = install_prefixed(install_line, launch)
+        matchers["installed"] = parse_install_done
     argv = direct_ssh_argv(target, local, remote, launch, jump=jump,
                            ssh_opts=ssh_opts)
     echo(f"$ {' '.join(argv)}")
-    watched = _Watched(argv, "node", echo=echo,
-                       matchers={"node": parse_node_announce})
+    watched = _Watched(argv, "node", echo=echo, matchers=matchers)
+    if install_line:
+        _await_install(shim, watched, install_line)
     _ACTIVE.append(watched)
     deadline = _now() + timeout
 

{plexora-0.0.8 → plexora-0.0.9}/plexora/server/node/app.py

@@ -361,6 +361,19 @@ def warm_resources(registry, log=print):
     across a node restart -- so fitting them here would be twenty seconds of
     work for an answer nobody is going to ask this node for.
 
+    **And deliberately not the window SCANS.** An earlier version computed
+    every channel's quantization window here, on the reasoning that a first
+    zoom should not pay for it. Each window is a full-resolution read of a
+    whole channel plane, and a node restored from its manifest can be serving
+    several whole slides -- so on a cluster filesystem that reasoning turned
+    into the node reading a hundred gigabytes, by itself, starting seconds
+    after it announced, and answering nothing -- not even /health -- while it
+    did. Watched live, twice, on two clusters: the panel said "Not answering"
+    about a machine that was busy doing work nobody had asked for yet. Windows
+    are seeded from the persistent store when the last job already paid for
+    them -- a JSON read -- and are otherwise computed on demand, one channel
+    at a time, when a request actually needs that channel.
+
     Sequential, and on one background thread. The reads contend for the same
     file, so racing them wins nothing, and the readers-writer lock means a real
     request can interleave rather than queue behind the whole warm.
@@ -380,8 +393,12 @@ def warm_resources(registry, log=print):
                     continue
                 overview = resource.opened_overview
                 for index in range(len(overview) if overview is not None else 0):
+                    stored = node_api._stored_window(resource, index)
+                    if stored is None:
+                        continue
                     with node_api._reading(resource):
-                        node_api._quantization(resource, index)
+                        node_api._cached(resource, ("qwindow", index),
+                                         lambda window=stored: window)
             except Exception as exc:  # pragma: no cover - unreadable at warm time
                 # Never fatal. A resource that cannot be warmed is one that
                 # will report its own failure when something asks for it, and

{plexora-0.0.8 → plexora-0.0.9/plexora.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: plexora
-Version: 0.0.8
+Version: 0.0.9
 Summary: OpenSeadragon-based cellular image viewing and analysis tool.
 License-Expression: LicenseRef-Plexora-Academic-1.0
 Requires-Python: <3.14,>=3.12

{plexora-0.0.8 → plexora-0.0.9}/pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "plexora"
-version = "0.0.8"
+version = "0.0.9"
 description = "OpenSeadragon-based cellular image viewing and analysis tool."
 readme = "README.md"
 requires-python = ">=3.12,<3.14"