plexora 0.0.1__py3-none-any.whl

plexora/jupyter.py

@@ -0,0 +1,406 @@
+import atexit
+import html
+import os
+import socket
+import subprocess
+import sys
+import time
+import urllib.request
+import webbrowser
+from pathlib import Path
+
+from plexora._url import join_display
+from plexora.datasource import register_datasource, register_anndata_datasource
+from plexora.notebook_env import COLAB, PORT_PLACEHOLDER, colab_origin, resolve_display
+
+
+_SERVERS = {}
+
+
+def _default_data_dir():
+    """Where a notebook viewer keeps its data when the caller names nowhere.
+
+    This used to be `Path(__file__).parent / "data"` -- i.e. inside the
+    installed package. Under a real install that is site-packages: read-only
+    for a system or conda install, destroyed by `pip install -U`, and invisible
+    to `pip uninstall`. The one resolver decides now, exactly as it does for
+    the CLI, so a notebook and a terminal see the same projects.
+    """
+    from plexora import paths
+
+    return paths.data_root()
+
+
+def _free_port():
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+class ServerStartError(RuntimeError):
+    """The sidecar process could not be started, with a reason worth reading."""
+
+
+def _wait_until_ready(port, timeout=30, process=None):
+    """Block until the sidecar answers, or explain why it never will.
+
+    `process` is what makes this useful. Without it a child that died on its
+    first line -- the overwhelmingly common case being a `plexora` that is not
+    importable from the interpreter running the notebook -- was indistinguishable
+    from one that was merely slow, so the notebook cell sat there for the full
+    30 seconds and then reported a timeout, which is the one explanation that
+    was not true.
+    """
+    deadline = time.time() + timeout
+    url = f"http://127.0.0.1:{port}/config"
+    while time.time() < deadline:
+        if process is not None and process.poll() is not None:
+            raise ServerStartError(
+                f"The Plexora server process exited with code "
+                f"{process.returncode} before it was ready.\n"
+                f"The usual cause is that `{sys.executable}` cannot import "
+                f"plexora -- install it into the environment running this "
+                f"kernel, e.g. `pip install -e .` from a source checkout."
+            )
+        try:
+            with urllib.request.urlopen(url, timeout=1) as response:
+                if response.status < 500:
+                    return
+        except Exception:
+            time.sleep(0.25)
+    raise ServerStartError(f"Plexora server did not become ready on port {port}")
+
+
+def _server_key(data_dir, base_url_template, plugins):
+    """What makes two viewers able to share one sidecar.
+
+    `base_url_template`, not the base URL: in proxy mode the URL contains the
+    port, which is freshly chosen on every call, so keying on it meant the
+    lookup could never hit and each `plexora.view()` in a notebook started
+    another server. Harmless when proxying was opt-in and rare; not harmless
+    now that hosted notebooks reach it by default.
+    """
+    return (str(Path(data_dir).expanduser().resolve()), base_url_template, plugins)
+
+
+def _start_server(data_dir, base_url_template, port=None, plugins=None):
+    """Start (or reuse) a sidecar; returns `(port, base_url)`.
+
+    `base_url_template` may contain `{port}`, which is filled in once the port
+    is settled -- the proxied mount path has to name the port it is proxying,
+    but the port is not known until after the cache has been consulted.
+    """
+    key = _server_key(data_dir, base_url_template, plugins)
+    existing = _SERVERS.get(key)
+    if existing and existing.poll() is None:
+        return existing._plexora_port, existing._plexora_base_url
+
+    # One retry, because the gap between _free_port() releasing a port and the
+    # child binding it is a real window on a busy machine, and losing that race
+    # is both plausible and entirely recoverable.
+    attempts = 2 if port is None else 1
+    last = None
+    for attempt in range(attempts):
+        chosen = port or _free_port()
+        base_url = base_url_template.replace(PORT_PLACEHOLDER, str(chosen))
+        try:
+            process = _spawn_server(data_dir, base_url, chosen, plugins)
+        except ServerStartError as exc:
+            last = exc
+            continue
+        process._plexora_port = chosen
+        process._plexora_base_url = base_url
+        _SERVERS[key] = process
+        return chosen, base_url
+    raise last
+
+
+def _spawn_server(data_dir, base_url, port, plugins):
+    resolved_data_dir = str(Path(data_dir).expanduser().resolve())
+    cmd = [
+        sys.executable,
+        "-m",
+        "plexora.server_cli",
+        "--host",
+        "127.0.0.1",
+        "--port",
+        str(port),
+        "--data-dir",
+        resolved_data_dir,
+        "--base-url",
+        base_url,
+        "--notebook-mode",
+    ]
+    if plugins is not None:
+        cmd.extend(["--plugins", plugins])
+    # The FLAG is what actually decides the child's plugins: server_cli.main()
+    # writes it into its own os.environ before `from plexora import app`, which
+    # is in-process and therefore exact. The environment variable below is
+    # belt-and-braces and cannot be the mechanism, because `plugins=""` -- a
+    # deliberate core-only build -- does not survive a process boundary on
+    # Windows at all: setting a variable to "" there deletes it, and the child
+    # would read "unset", which means activate everything.
+    #
+    # The data path has no such constraint either way -- it resolves on demand
+    # -- but is passed both ways so parent and child cannot disagree about
+    # which directory this viewer is for.
+    env = os.environ.copy()
+    env["PLEXORA_DATA_PATH"] = resolved_data_dir
+    env["PLEXORA_BASE_URL"] = base_url
+    env["PLEXORA_NOTEBOOK_MODE"] = "1"
+    if plugins is not None:
+        env["PLEXORA_PLUGINS"] = plugins
+    # No cwd: this used to be the package's parent, which is the repository
+    # root only when running from a checkout and site-packages otherwise.
+    # Nothing the child does is relative to its working directory any more.
+    process = subprocess.Popen(cmd, env=env)
+    try:
+        _wait_until_ready(port, process=process)
+    except ServerStartError:
+        if process.poll() is None:
+            process.terminate()
+        raise
+    return process
+
+
+def _cleanup_servers():
+    for process in _SERVERS.values():
+        if process.poll() is None:
+            process.terminate()
+
+
+atexit.register(_cleanup_servers)
+
+
+class PlexoraViewer:
+    def __init__(
+        self,
+        datasource,
+        data_dir=None,
+        proxy="auto",
+        height=850,
+        width="100%",
+        base_url=None,
+        plugins=None,
+        start=True,
+    ):
+        """`proxy` is one of:
+
+        - "auto" (default) -- look at the environment and decide. Local Jupyter
+          and VS Code Remote get a direct localhost URL exactly as before;
+          JupyterHub, Open OnDemand and Colab get the proxied form they need.
+        - True -- always proxy through the notebook server.
+        - False -- always use a direct 127.0.0.1 URL.
+
+        The default changed from False. That was only ever right when the
+        browser and the kernel were the same machine; anywhere else it produced
+        an iframe pointing at the user's own laptop and rendered blank.
+        """
+        self.datasource = datasource
+        self.data_dir = Path(data_dir or os.environ.get("PLEXORA_DATA_PATH", _default_data_dir())).expanduser().resolve()
+        self.proxy = proxy
+        self.height = height
+        self.width = width
+        self._base_url = base_url
+        # Kept as-is (not truthy-or) so plugins="" -- explicitly core-only --
+        # stays distinguishable from "not passed, use whatever is installed".
+        self.plugins = plugins
+        self._port = None
+        self._display_base = None
+        if start:
+            self.start()
+
+    @classmethod
+    def from_files(
+        cls,
+        name,
+        image,
+        segmentation,
+        features,
+        x,
+        y,
+        id_column="CellID",
+        celltype_column=None,
+        channel_names=None,
+        copy=False,
+        data_dir=None,
+        **viewer_kwargs,
+    ):
+        resolved_data_dir = Path(data_dir or os.environ.get("PLEXORA_DATA_PATH", _default_data_dir())).expanduser().resolve()
+        register_datasource(
+            name=name,
+            image=image,
+            segmentation=segmentation,
+            features=features,
+            x=x,
+            y=y,
+            id_column=id_column,
+            celltype_column=celltype_column,
+            channel_names=channel_names,
+            copy=copy,
+            data_dir=resolved_data_dir,
+        )
+        return cls(datasource=name, data_dir=resolved_data_dir, **viewer_kwargs)
+
+    @classmethod
+    def from_anndata(
+        cls,
+        name,
+        image,
+        features=None,
+        adata=None,
+        segmentation=None,
+        coordinate_source=None,
+        obsm_key=None,
+        x=None,
+        y=None,
+        feature_source="X",
+        layer=None,
+        feature_obs_columns=None,
+        obs_id_field=None,
+        celltype_column=None,
+        subset_by=None,
+        subset_value=None,
+        channel_names=None,
+        copy=False,
+        data_dir=None,
+        **viewer_kwargs,
+    ):
+        resolved_data_dir = Path(data_dir or os.environ.get("PLEXORA_DATA_PATH", _default_data_dir())).expanduser().resolve()
+        register_anndata_datasource(
+            name=name,
+            image=image,
+            features=features,
+            adata=adata,
+            segmentation=segmentation,
+            coordinate_source=coordinate_source,
+            obsm_key=obsm_key,
+            x=x,
+            y=y,
+            feature_source=feature_source,
+            layer=layer,
+            feature_obs_columns=feature_obs_columns,
+            obs_id_field=obs_id_field,
+            celltype_column=celltype_column,
+            subset_by=subset_by,
+            subset_value=subset_value,
+            channel_names=channel_names,
+            copy=copy,
+            data_dir=resolved_data_dir,
+        )
+        return cls(datasource=name, data_dir=resolved_data_dir, **viewer_kwargs)
+
+    def start(self):
+        """Resolve where this viewer lives, then start or reuse a server.
+
+        Resolution happens against the PORT PLACEHOLDER rather than a real
+        port, so a second view() in the same notebook produces the same cache
+        key and reuses the first sidecar instead of spawning another one.
+        """
+        if self._port is not None:
+            return self._port
+        server_base, display_base = resolve_display(self.proxy, self._base_url)
+        self._port, _ = _start_server(
+            self.data_dir, server_base, plugins=self.plugins
+        )
+        # Substituted only now, against whichever port the sidecar actually
+        # ended up on -- which is not necessarily the one this call would have
+        # picked, since it may have reused a server started by an earlier cell.
+        if display_base is COLAB:
+            self._display_base = colab_origin(self._port)
+        else:
+            self._display_base = display_base.replace(
+                PORT_PLACEHOLDER, str(self._port)
+            )
+        return self._port
+
+    @property
+    def url(self):
+        self.start()
+        if self._display_base is None:
+            raise RuntimeError(
+                "Colab did not return a public URL for this port. That needs a "
+                "connected notebook frontend, so it fails under 'Run all' or a "
+                "reconnect. Use viewer.iframe(), which does not, or re-run this "
+                "cell on its own."
+            )
+        return join_display(self._display_base, self.datasource)
+
+    def _colab_iframe(self):
+        """Let Colab's frontend work out the URL, since the kernel could not.
+
+        This is the fallback for `colab_origin()` returning None, and it is not
+        a lesser version of it -- it is the more reliable one. The helper emits
+        Javascript that calls `proxyPort` in the notebook frontend, so it needs
+        no kernel-to-frontend round trip and works under "Run all" and after a
+        reconnect. It is only the fallback because it displays its own output
+        rather than returning a URL, which `.url` and `.open()` need.
+        """
+        from google.colab.output import serve_kernel_port_as_iframe
+
+        return serve_kernel_port_as_iframe(
+            self._port,
+            path=f"/{self.datasource}",
+            width=str(self.width),
+            height=str(self.height),
+        )
+
+    def iframe(self):
+        self.start()
+        if self._display_base is None:
+            return self._colab_iframe()
+        try:
+            from IPython.display import HTML
+        except ImportError:
+            return self._repr_html_()
+        return HTML(self._repr_html_())
+
+    def _ipython_display_(self):
+        """Render when the viewer is the last expression in a cell.
+
+        Defined rather than leaving IPython to find `_repr_html_`, because the
+        Colab fallback cannot be expressed as HTML at all: a <script> in cell
+        output runs inside Colab's sandboxed output frame, where
+        `google.colab.kernel` does not exist. Only frontend Javascript can
+        resolve the port, and only this hook can emit it.
+        """
+        self.start()
+        if self._display_base is None:
+            self._colab_iframe()
+            return
+        from IPython.display import display, HTML
+
+        display(HTML(self._repr_html_()))
+
+    def open(self):
+        """Open the viewer in a browser, where that means anything.
+
+        A hosted notebook's URL is a path on the hub's origin, which this
+        process cannot turn into something webbrowser could open -- and if it
+        could, the browser here is on the wrong machine anyway. Printing it is
+        the honest outcome.
+        """
+        url = self.url
+        if not url.lower().startswith(("http://", "https://")):
+            print(f"Open this under your Jupyter server's address: {url}")
+            return url
+        webbrowser.open(url)
+        return url
+
+    def _repr_html_(self):
+        self.start()
+        if self._display_base is None:
+            # Reachable only when something bypassed _ipython_display_ -- a
+            # message beats an exception raised from inside a repr, which
+            # IPython would show as a traceback about formatting.
+            return (
+                "<pre>Plexora is running, but Colab did not return a public URL "
+                "for it.\nCall viewer.iframe() to display it.</pre>"
+            )
+        src = html.escape(self.url, quote=True)
+        width = html.escape(str(self.width), quote=True)
+        height = int(self.height)
+        return (
+            f'<iframe src="{src}" width="{width}" height="{height}" '
+            'style="border: 0; width: 100%;" allowfullscreen></iframe>'
+        )

plexora/notebook_env.py

@@ -0,0 +1,232 @@
+"""Working out what kind of notebook this is, and therefore what URL works.
+
+`PlexoraViewer` starts a server on 127.0.0.1 and shows it in an iframe. That
+is correct exactly when the browser rendering the notebook is on the same
+machine as the kernel -- local Jupyter, and (because it forwards ports for you)
+VS Code Remote. Everywhere else it produces a URL pointing at the user's own
+laptop, where nothing is listening, and the cell renders a blank box with no
+error anywhere.
+
+"Everywhere else" is most of the interesting places: JupyterHub, Open OnDemand,
+Colab, a plain `jupyter lab` on a workstation the user ssh'd into. Each needs a
+different URL, and each can be recognised from the environment, so the viewer
+asks here instead of making the user pass `proxy=True` after finding out the
+hard way that they needed to.
+
+Two ideas do most of the work:
+
+**A hosted URL is a PATH, not a host.** Under jupyter-server-proxy the viewer
+is reachable at `<notebook prefix>proxy/<port>` on the notebook's OWN origin --
+which is the origin holding the user's auth cookie, and the only one that will
+be allowed to load. So the display base stays path-only and an iframe resolves
+it. Writing in a hostname would be both wrong and unauthenticated.
+
+**Colab is the exception that proves it.** `proxyPort()` returns a whole
+separate `https://…googleusercontent.com` origin, which is why `join_display`
+accepts a full origin at all and why `clean_prefix` refuses one.
+
+Imports only `plexora._url`, and everything that touches an optional
+third-party module does so behind `_module_available` inside a function -- this
+is consulted on the import path of a notebook helper, and a hard dependency on
+`jupyter_server` would make `import plexora` fail on a machine that has no
+Jupyter at all.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+
+from plexora._url import is_full_origin, prefix_with_slash
+
+
+def _module_available(name):
+    """Whether `name` could be imported, without importing it. Never raises.
+
+    `find_spec` alone is not enough, and its failure modes are exactly the ones
+    that turn up in notebooks: it raises ValueError for a module already in
+    sys.modules whose `__spec__` is None (dynamically created modules, some
+    frozen ones, anything a hub's startup code injected), and
+    ModuleNotFoundError when an intermediate package is missing. Every caller
+    here is on the display path of a notebook cell, where an exception would
+    replace the viewer with a traceback about introspection.
+    """
+    if name in sys.modules:
+        return True
+    try:
+        return importlib.util.find_spec(name) is not None
+    except Exception:
+        return False
+
+
+#: Variables that mean "the kernel is not on the machine holding the screen".
+#: The scheduler ones are not padding: Open OnDemand runs Jupyter inside a
+#: batch job and sets no hub variable of its own, so its job variables are the
+#: only evidence available that a direct localhost URL cannot work.
+REMOTE_ENV_VARS = (
+    "SSH_CONNECTION",
+    "SSH_CLIENT",
+    "SSH_TTY",
+    "SLURM_JOB_ID",
+    "PBS_JOBID",
+    "LSB_JOBID",
+)
+
+HUB_ENV_VARS = ("JUPYTERHUB_SERVICE_PREFIX", "JUPYTERHUB_USER", "JUPYTERHUB_API_URL")
+
+#: Stands in for a port that has not been chosen yet, so `resolve_display` can
+#: be asked its question BEFORE a server exists -- which is what lets the
+#: sidecar cache be keyed on the answer. Same spelling as the placeholder
+#: `plexora/proxy.py` hands jupyter-server-proxy, for one convention rather
+#: than two.
+PORT_PLACEHOLDER = "{port}"
+
+#: Returned as the display base when only the Colab FRONTEND can answer, and
+#: only once a real port exists. Every other case resolves to a string that a
+#: port can simply be substituted into; Colab needs a round trip, so it gets a
+#: sentinel and the caller makes that call after starting the server.
+COLAB = "colab"
+
+
+def looks_remote(env=None):
+    env = os.environ if env is None else env
+    return any(env.get(name) for name in REMOTE_ENV_VARS)
+
+
+def in_hub(env=None):
+    env = os.environ if env is None else env
+    return any(env.get(name) for name in HUB_ENV_VARS)
+
+
+def in_colab():
+    return _module_available("google.colab")
+
+
+def colab_origin(port):
+    """The public origin Colab will proxy `port` on, or None.
+
+    None is a normal outcome, not just an error path: `eval_js` runs Javascript
+    in the notebook FRONTEND and waits for an answer, so it needs a browser
+    actually connected to this kernel. Under "Run all", a reconnect, or a
+    headless execution there is nobody to ask, and it hangs or raises. The
+    caller falls back to Colab's own iframe helper, which does not need a
+    round trip.
+    """
+    try:
+        from google.colab.output import eval_js
+    except Exception:
+        return None
+    try:
+        origin = eval_js(f"google.colab.kernel.proxyPort({int(port)})")
+    except Exception:
+        return None
+    if not origin or not is_full_origin(origin):
+        return None
+    return str(origin).rstrip("/")
+
+
+def discover_jupyter_prefix(env=None, echo=print):
+    """The notebook server's base_url (e.g. `/user/me/`), or None.
+
+    JUPYTERHUB_SERVICE_PREFIX is asked first because a hub sets it in the
+    kernel's own environment and it is definitive. Failing that, ask the
+    jupyter_server library which servers are running on this machine -- which
+    is how Open OnDemand is found, since it configures a random prefix per job
+    and advertises it nowhere else.
+    """
+    env = os.environ if env is None else env
+    from_hub = env.get("JUPYTERHUB_SERVICE_PREFIX")
+    if from_hub:
+        return prefix_with_slash(from_hub)
+
+    if not _module_available("jupyter_server"):
+        return None
+    try:
+        from jupyter_server.serverapp import list_running_servers
+
+        servers = [entry for entry in list_running_servers() if entry.get("base_url")]
+    except Exception:
+        return None
+    if not servers:
+        return None
+    if len(servers) > 1:
+        echo(
+            "Plexora found several running Jupyter servers and is assuming "
+            f"{servers[0]['base_url']!r}. Pass base_url=... to plexora.view() "
+            "if that is the wrong one."
+        )
+    return prefix_with_slash(servers[0]["base_url"])
+
+
+def proxy_hint_if_missing(echo=print):
+    """Mention jupyter-server-proxy if it looks absent. Never raises.
+
+    Only a hint, and only ever a hint: on several hubs the kernel runs in a
+    different environment from the notebook server, so the package being
+    missing HERE says nothing about whether the proxy will work THERE. Turning
+    that guess into an exception would break setups that were fine.
+    """
+    if not _module_available("jupyter_server_proxy"):
+        echo(
+            "Note: jupyter-server-proxy does not appear to be installed in this "
+            "environment. If the viewer below does not load, install it in the "
+            "environment running your Jupyter server:\n"
+            "    pip install jupyter-server-proxy"
+        )
+
+
+def resolve_display(proxy="auto", base_url=None, port=PORT_PLACEHOLDER, env=None,
+                    echo=print):
+    """`(server_base_url, display_base)` -- how to mount, and how to reach it.
+
+    The two differ whenever a proxy is involved, and conflating them is the
+    original bug: the sidecar has to generate links under the path the proxy
+    exposes it at, while the notebook has to load that same path against the
+    hub's origin.
+
+    `port` may be `PORT_PLACEHOLDER`, and normally is: the caller wants this
+    answer before it has picked a port, so that the answer can decide whether
+    an already-running sidecar will do. Both returned strings then carry the
+    placeholder through for the caller to substitute. `COLAB` is the one
+    display base that cannot work that way -- see the constant.
+
+    First match wins, and the order encodes what beats what:
+
+    1. An explicit `base_url` is an instruction; nothing overrides it.
+    2. `proxy=False` is also an instruction -- the pre-existing default, kept
+       working verbatim for every notebook that passes it today.
+    3. Colab, whose proxy is a whole origin rather than a path (returns the
+       `COLAB` sentinel; ask `colab_origin(real_port)` once you have one).
+    4. A discoverable notebook prefix, but only with a reason to use it:
+       `proxy=True`, or evidence the kernel is not local.
+    5. Direct localhost. Deliberately last and deliberately not an error --
+       this is plain local Jupyter, and it is also VS Code Remote, which
+       forwards the port itself and would be broken by "helpfully" proxying.
+    """
+    env = os.environ if env is None else env
+
+    if base_url is not None:
+        if is_full_origin(base_url):
+            # A caller who already knows the public origin (a bespoke reverse
+            # proxy, a tunnel they set up) -- the server still mounts at root.
+            return "", str(base_url).rstrip("/")
+        mounted = f"{prefix_with_slash(base_url)}proxy/{port}"
+        return mounted, mounted
+
+    if proxy is False:
+        return "", f"http://127.0.0.1:{port}"
+
+    if in_colab():
+        # Colab maps the whole port onto a subdomain root, so there is no path
+        # to mount under -- the server stays at "/" and only the display
+        # changes.
+        return "", COLAB
+
+    prefix = discover_jupyter_prefix(env, echo=echo)
+    if prefix and (proxy is True or looks_remote(env) or in_hub(env)):
+        proxy_hint_if_missing(echo=echo)
+        mounted = f"{prefix}proxy/{port}"
+        return mounted, mounted
+
+    return "", f"http://127.0.0.1:{port}"