cubevis 0.5.2__py3-none-any.whl

cubevis/readme.rst

@@ -0,0 +1,16 @@
+Visualization for the CASA package
+==================================
+
+This package includes the visualization elements of the CASA package for
+processing data collected from radio telescopes. More information about CASA
+can be found on the `CASA home page <https://casa.nrao.edu/>`_.
+
+cubevis is based on `Bokeh <https://bokeh.org/>`_. It provides visualization
+elements for three different interaction settings:
+
+* Python command line with display to a web browser
+* Jupyter notebook
+* `Electron <https://www.electronjs.org/>`_ app
+
+However not all elements support all interaction settings.
+

cubevis/remote/__init__.py

@@ -0,0 +1,10 @@
+'''Remote Jupyter kernel execution capabilities. These are the dependencies that **may** be installed::
+
+    Successfully installed Send2Trash-1.8.2 anyio-3.7.1 argon2-cffi-23.1.0 argon2-cffi-bindings-21.2.0 babel-2.14.0 beautifulsoup4-4.12.3 bleach-6.1.0 cffi-1.16.0 charset-normalizer-3.3.2 comm-0.2.1 debugpy-1.8.1 defusedxml-0.7.1 deprecation-2.1.0 fastjsonschema-2.19.1 ipykernel-6.29.3 ipython-genutils-0.2.0 json5-0.9.20 jsonschema-4.21.1 jsonschema-specifications-2023.12.1 jupyter-client-6.1.12 jupyter-core-5.7.1 jupyter-packaging-0.12.3 jupyter-server-1.24.0 jupyterlab-3.0.18 jupyterlab-pygments-0.3.0 jupyterlab-server-2.25.3 mistune-3.0.2 nbclassic-0.5.6 nbclient-0.9.0 nbconvert-7.16.2 nbformat-5.9.2 nest-asyncio-1.6.0 notebook-shim-0.2.4 pandocfilters-1.5.1 platformdirs-4.2.0 prometheus-client-0.20.0 psutil-5.9.8 pycparser-2.21 pyzmq-25.1.2 referencing-0.33.0 requests-2.31.0 rpds-py-0.18.0 sniffio-1.3.1 soupsieve-2.5 ssh-ipykernel-interrupt-1.1.2 ssh_ipykernel-1.2.3 terminado-0.18.0 tinycss2-1.2.1 tomlkit-0.12.4 tornado-6.2 urllib3-2.2.1 webencodings-0.5.1 websocket-client-1.7.0
+
+when installing ssh_ipykernel.
+'''
+
+from ._local import *
+from ._remote_kernel import TestProc
+from ._gclean import gclean_local

cubevis/remote/_gclean.py

@@ -0,0 +1,61 @@
+### Remote gclean execution
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../'))
+##>>>import casaremote
+
+# Remote functions needed
+# finalize, update(), reset
+
+# TODO: How to handle dictionaries? """?
+def kwarg_string(*args, **kwargs):
+    ret = []
+    for arg in args:
+        if isinstance(arg, str):
+            ret.append(f"'{arg}'")
+        else:
+            ret.append(f"{arg}")
+    for key, value in kwargs.items():
+        if isinstance(value, str):
+            value = f"'{value}'"
+        ret.append(f"{key}={value}")
+    return ", ".join(ret)
+
+class gclean_local:
+
+    # def _tclean( self, *args, **kwargs ):
+    #     return
+
+    # def _deconvolve( self, *args, **kwargs ):
+    #     return
+    #     # return ( *args, **kwargs )
+
+    def cmds( self ):
+        cmd = casaremote.create_cmd_json("self.gclean_cmds")
+        gclean_id = casaremote.exe_remote_subproc(self.kc, cmd)
+        ret = casaremote.get_subproc_return_val(self.kc, gclean_id)
+        return ret
+
+    def __init__(self, kc, *args, **kwargs):
+        self.kc = kc
+        print("In remote clean init \n")
+        cmd = "t._clean = gclean(" + kwarg_string(*args, **kwargs) + ")"
+        casaremote.exe_and_print(kc, cmd)
+
+    def __next__( self ):
+        cmd = casaremote.create_cmd_json("self.gclean_next")
+        gclean_id = casaremote.exe_remote_subproc(self.kc, cmd)
+        ret = casaremote.get_subproc_return_val(self.kc, gclean_id)
+        return ret
+
+    def restore(self):
+        cmd = casaremote.create_cmd_json("self.gclean_restore")
+        gclean_id = casaremote.exe_remote_subproc(self.kc, cmd)
+        ret = casaremote.get_subproc_return_val(self.kc, gclean_id)
+        return ret
+
+    def has_next(self):
+        cmd = casaremote.create_cmd_json("self.gclean_has_next")
+        gclean_id = casaremote.exe_remote_subproc(self.kc, cmd)
+        ret = casaremote.get_subproc_return_val(self.kc, gclean_id)
+        return ret

cubevis/remote/_local.py

@@ -0,0 +1,287 @@
+'''Functions for local execution to create, tunnel, and connect to remote Jupyter kernels.'''
+import jupyter_client
+##>>>from ssh_ipykernel.kernel import SshKernel
+##>>>from ipykernel import get_connection_info
+from queue import Empty
+import json
+import os
+import asyncio
+import websockets
+import time
+from uuid import uuid4
+from ast import literal_eval
+
+'''Build ssh tunnel forwarding pattern and return list of arguments'''
+def build_ssh_tunnel(host, ports, ip="localhost", verbose=False, quiet=True, ssh_config=""):
+    # Build ssh command with all flags and tunnels
+    # TODO: Handle non-matching remote and local ports
+    ssh_tunnels = []
+    for port in ports:
+        ssh_tunnels += [
+            "-L",
+            "{local_port}:{ip}:{remote_port}".format(
+                local_port=port,
+                ip=ip,
+                remote_port=port,
+            ),
+        ]
+
+    if quiet:
+        kernel_args = ["-q"]
+    elif verbose:
+        kernel_args = ["-v"]
+    else:
+        kernel_args = []
+
+    kernel_args += ["-f", "-N", "-t"]
+    if ssh_config != "":
+        kernel_args += ["-F", str(ssh_config)]
+
+    # Build rest of ssh command and redirect output
+    kernel_args += ssh_tunnels + [host]
+    ssh_tunnel_cmd = "ssh " + " ".join(kernel_args) + " &>/dev/null"
+
+    return ssh_tunnel_cmd
+
+
+'''Tunnel the necessary ports to access a remote jupyter kernel locally'''
+def tunnel_remote_kernel(connect_info_file, host):
+    connect_info_file_local = jupyter_client.find_connection_file(filename=connect_info_file)
+    connect_info_local = json.loads(get_connection_info(connect_info_file_local))
+
+    # Get ports to tunnel
+    tunnel_ports = []
+    for port_name in ['shell_port', 'iopub_port', 'stdin_port', 'control_port', 'hb_port']:
+        tunnel_ports.append(connect_info_local[port_name])
+
+    ssh_tunnel_cmd = build_ssh_tunnel(host, tunnel_ports)
+
+    # # TODO: Check that command was successful
+    os.system(ssh_tunnel_cmd)
+
+'''Start a jupyter console connected to a remote kernel session in the current process'''
+def connect_console(connect_info_file):
+    console_start_cmd = "jupyter-console --existing=" + connect_info_file
+    print("\nConsole start command:\n", console_start_cmd)
+    os.system(console_start_cmd)
+
+'''Start a remote jupyter kernel and return the full path to the connection info file'''
+def start_remote_kernel(kernel_name, sudo=False, timeout=5, env="", backchannel_port=5667):
+    # Get details of selected kernel (if exists)
+    kern_spec_man = jupyter_client.kernelspec.KernelSpecManager()
+    # TODO: Check that kernel name exists
+    ic_kernel = kern_spec_man.get_kernel_spec(kernel_name)
+    connect_info_file_local = ""
+
+    connect_info_file_local = jupyter_client.find_connection_file()
+    connect_info_local = json.loads(get_connection_info(connect_info_file_local))
+
+    # Get remote connection info
+    kernel_args = ic_kernel.argv
+    host = kernel_args[kernel_args.index('--host') + 1]
+    remote_python_path = kernel_args[kernel_args.index('--python') + 1]
+
+    # Create remote kernel information structure
+    kernel = SshKernel(host, connect_info_local, remote_python_path, sudo, timeout, env)
+    kernel.create_remote_connection_info()
+
+    # Build remote kernel launch command
+    sudo = "sudo " if kernel.sudo else ""
+    if kernel.env is not None:
+        env = " ".join(kernel.env)
+    kernel_start_cmd = "{sudo} {env} {python} -m ipykernel_launcher -f {fname}".format(
+        sudo=sudo, env=env, python=kernel.python_full_path, fname=kernel.fname
+    )
+
+    # Build ssh command with all flags and tunnels
+    tunnel_ports = [backchannel_port]
+    for port_name in kernel.remote_ports.keys():
+        tunnel_ports += [kernel.remote_ports[port_name]]
+
+    ssh_tunnel_cmd = build_ssh_tunnel(host,
+                                     tunnel_ports,
+                                     verbose=kernel.verbose,
+                                     quiet=kernel.quiet,
+                                     ssh_config=kernel.ssh_config)
+    # Create tunnels
+    print("\nSSH Tunnel Command: \n", ssh_tunnel_cmd)
+    ret = os.system(ssh_tunnel_cmd)
+    # return kernel
+
+    # Local connection info file location
+    connect_info_file_local_full = os.path.abspath(connect_info_file_local)
+
+    # Create and run command to start remote jupyter kernel
+    remote_kernel_start_command = 'ssh {host} "nohup {cmd} >/dev/null 2>&1 &" &>/dev/null'.format(host=kernel.host, cmd=kernel_start_cmd)
+    print("\nRemote Kernel Start Command: \n", remote_kernel_start_command)
+    os.system(remote_kernel_start_command)
+
+    # Copy the remote kernel connection file to local machine
+    scp_cmd = "scp {host}:{fname} . &>/dev/null".format(host=kernel.host, fname=kernel.fname)
+    print("\nSCP Command:\n", scp_cmd)
+    os.system(scp_cmd)
+    connect_info_file_local_full = os.path.abspath(kernel.fname[5:])
+
+    tunnel_remote_kernel(connect_info_file_local_full, kernel.host)
+
+    # Print command to attach
+    console_start_cmd = "jupyter-console --existing=" + connect_info_file_local_full
+    print("\nConsole start command:\n", console_start_cmd)
+    os.system(ssh_tunnel_cmd)
+
+    return connect_info_file_local_full
+
+
+'''Connect to a remote Jupyter Kernel'''
+def connect_remote(connect_info_file, host):
+    kc = jupyter_client.BlockingKernelClient(connection_file = connect_info_file)
+    kc.load_connection_file()
+    # Start channels if connecting for the first time
+    if(not kc.channels_running):
+        kc.start_channels()
+
+    # Unsure why this is the case
+    try:
+        kc.get_iopub_msg(timeout=1)
+    except Empty:
+        print("Empty IOPub Message Queue...")
+
+    tunnel_remote_kernel(kc.connection_file, host)
+    return kc
+
+'''Print the waiting content of messages on the iopub channel.
+Kernel responds to execution requests with potentially many responses.'''
+def get_io_msgs(kc):
+    state='busy'
+    data={}
+    while state!='idle' and kc.is_alive():
+        try:
+            msg=kc.get_iopub_msg(timeout=1)
+            # print(msg)
+            if not 'content' in msg:
+                continue
+            content = msg['content']
+            print(content)
+            # pretty_print_jupyter(content)
+            if 'data' in content:
+                data=content['data']
+            if 'execution_state' in content:
+                state=content['execution_state']
+        except Empty:
+            pass
+
+'''Helper function for testing'''
+def exe_and_print(kc, cmd):
+    print("\nExecuting: {cmd}".format(cmd=cmd))
+    kc.execute(cmd)
+    get_io_msgs(kc)
+
+'''Execute a command in a subprocess started from a remote Jupyter kernel'''
+def exe_in_subproc(kc, cmd):
+    # Generate a UUID associated with this command and subprocess instance
+    id = str(uuid4())
+    # Put UUID into command JSON
+    cmd['id'] = id
+    exe_and_print("run_subproc_cmd('" + json.dumps(cmd) + "')")
+    return id
+
+'''Helper function to send a remote request to a kernel's backchannel websocket'''
+async def send_remote_request(request, backchannel_port):
+    uri = "ws://localhost:"+str(backchannel_port)
+    print("Remote request URI: " + uri)
+    async with websockets.connect(uri, ping_interval=None) as websocket:
+        await websocket.send(request)
+        response = await websocket.recv()
+        return response
+
+'''Send a remote request to a kernel's backchannel websocket'''
+def remote_request(request, port=5667):
+    print("Sending remote request to port " + str(port) + " with request " + str(request))
+    response = asyncio.get_event_loop().run_until_complete(send_remote_request(request, port))
+    print(f"Response: {response}")
+
+'''Convert arguments to an explicit, stringified representation for remote execution'''
+def kwarg_string(*args, **kwargs):
+    ret = []
+    for arg in args:
+        if isinstance(arg, str):
+            ret.append(f"'{arg}'")
+        else:
+            ret.append(f"{arg}")
+    for key, value in kwargs.items():
+        if isinstance(value, str):
+            value = f"'{value}'"
+        ret.append(f"{key}={value}")
+    return ", ".join(ret)
+
+'''Create a json representation of a remote subprocess command to execute'''
+def create_cmd_json(cmd, *args, **kwargs):
+    json_cmd = {}
+    json_cmd['id'] = str(uuid4())
+    json_cmd['cmd'] = cmd
+    json_cmd['args'] = args
+    json_cmd['kwargs'] = kwargs
+    return json.dumps(json_cmd)
+
+'''Execute a remote command in a subprocess'''
+def exe_remote_subproc(kernel_client, json_cmd):
+    json_cmd = json.loads(json_cmd)
+    id = str(uuid4())
+    json_cmd['id'] = id
+
+    # TODO: Properly check output of subproc status
+    # exe_and_print(kernel_client, "t.subproc_is_free()")
+    # TODO: Figure out how to handle TestProc selection/naming/referencing
+    exe_and_print(kernel_client, "t.add_subproc_cmd('"+str(json.dumps(json_cmd))+"')")
+    return id
+
+'''Properly print kernel-formatted'''
+def pretty_print_jupyter(raw_out):
+    if 'text' in raw_out:
+        print(raw_out['text'])
+
+    if 'data' in raw_out:
+        data = raw_out['data']
+        if 'text/plain' in data:
+            print(data['text/plain'])
+
+'''Get the returned value from remote kernel output'''
+def get_return_val(kc):
+    state='busy'
+    data={}
+    ret = None
+    while state!='idle' and kc.is_alive():
+        try:
+            msg=kc.get_iopub_msg(timeout=1)
+            # print(msg)
+            if not 'content' in msg:
+                continue
+            content = msg['content']
+            print(content)
+            # pretty_print_jupyter(content)
+            if 'data' in content:
+                data=content['data']
+                ret = literal_eval(data['text/plain'])
+            if 'execution_state' in content:
+                state=content['execution_state']
+        except Empty:
+            pass
+    return ret
+
+'''Get the returned value from a remote subprocess'''
+def get_subproc_return_val(kc, id):
+    wait_done(kc, id)
+    kc.execute("t.get_return_val('" + id + "')")
+    ret = get_return_val(kc)
+    print("Return value: " + str(ret))
+    return ret
+
+'''Wait for the remote subprocess identified by 'id' to be finished'''
+def wait_done(kc, id, interval=1):
+    is_running = 1
+
+    while is_running:
+        kc.execute("t.is_running('" + id + "')")
+        is_running = get_return_val(kc)
+        print("Ret in wait done: " + str(is_running))
+        time.sleep(interval)

cubevis/remote/_remote_kernel.py

@@ -0,0 +1,80 @@
+# Functions for long-running process running in a remote jupyter kernel
+
+from multiprocessing import Process, Pipe
+import sys
+import os
+import json
+from time import sleep
+from websockets.server import serve
+import asyncio
+
+class TestProc:
+    _backchannel_task = None # Maintain strong reference to backchannel task
+
+    def __init__(self, port, loop):
+        self.loop = loop
+        self.port = port
+        self.subproc_cmd_pend = []
+
+    async def echo(self, websocket):
+        async for message in websocket:
+            response = json.dumps({"request": message, "response": eval("self."+message)})
+            await websocket.send(response)
+
+    async def backchannel_process(self, port):
+        async with serve(self.echo, "localhost", port, ping_interval=None):
+            await asyncio.Future()  # run forever
+
+    def start_backchannel(self):
+        assert self.loop.is_running()
+        self._backchannel_task = asyncio.create_task(self.backchannel_process(self.port))
+        return
+
+    '''Check subprocess command to see if it's free (1) or busy (0)'''
+    def subproc_is_free(self):
+        if len(self.subproc_cmd_pend) == 0:
+            return 1
+
+        if any(process[1].is_alive() for process in self.subproc_cmd_pend):
+            print("Busy")
+            sleep(1)
+            return 0
+        else:
+            print('All processes done')
+            return 1
+
+    def add_subproc_cmd(self, cmd_json_str):
+        # Check that a subprocess isn't currently running
+        if not self.subproc_is_free():
+            print("Error: Subprocess is still running. Cannot start new process.")
+            return -1
+
+        print(cmd_json_str)
+        cmd_json = json.loads(cmd_json_str)
+        parent_conn, child_conn = Pipe()
+        p = Process(target=eval(cmd_json['cmd']),
+                    args=(child_conn,) + tuple(cmd_json['args']),
+                    kwargs=cmd_json['kwargs'])
+
+        subproc_details = (cmd_json['id'], p, parent_conn)
+        self.subproc_cmd_pend.append(subproc_details)
+        p.start()
+        return
+
+    # Return the requested process' (identified by the UUID) return value
+    def get_return_val(self, id):
+        # Check that id is valid
+        for process in self.subproc_cmd_pend:
+            if id == process[0]:
+                print("placeholder")
+                # Check that process is done
+                # TODO
+
+                # Check parent connection for waiting message
+                # ret_obj = process[3].recv() ???
+
+                # Check that ret_obj valid and that didn't timeout
+                # return ret_obj
+
+        # Else return ID not found
+        return

cubevis/toolbox/__init__.py

@@ -0,0 +1,32 @@
+########################################################################
+#
+# Copyright (C) 2022,2024,2025
+# Associated Universities, Inc. Washington DC, USA.
+#
+# This script is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Library General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This library is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
+# License for more details.
+#
+# You should have received a copy of the GNU Library General Public License
+# along with this library; if not, write to the Free Software Foundation,
+# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
+#
+# Correspondence concerning AIPS++ should be adressed as follows:
+#        Internet email: casa-feedback@nrao.edu.
+#        Postal address: AIPS++ Project Office
+#                        National Radio Astronomy Observatory
+#                        520 Edgemont Road
+#                        Charlottesville, VA 22903-2475 USA
+#
+########################################################################
+'''Common tools used in creating applications.'''
+
+from ._cube import CubeMask
+from ._app_context import AppContext
+from ._region_list import RegionList

cubevis/toolbox/_app_context.py

@@ -0,0 +1,74 @@
+########################################################################
+#
+# Copyright (C) 2024
+# Associated Universities, Inc. Washington DC, USA.
+#
+# This script is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Library General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This library is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
+# License for more details.
+#
+# You should have received a copy of the GNU Library General Public License
+# along with this library; if not, write to the Free Software Foundation,
+# Inc., 675 Massachusetts Ave, Cambridge, MA 02139, USA.
+#
+# Correspondence concerning AIPS++ should be adressed as follows:
+#        Internet email: casa-feedback@nrao.edu.
+#        Postal address: AIPS++ Project Office
+#                        National Radio Astronomy Observatory
+#                        520 Edgemont Road
+#                        Charlottesville, VA 22903-2475 USA
+#
+########################################################################
+from cubevis.bokeh.state import initialize_bokeh
+from tempfile import TemporaryDirectory
+from bokeh.io import output_file
+from os.path import join
+import unicodedata
+import re
+
+class AppContext:
+
+    def _slugify(self, value, allow_unicode=False):
+        """
+        Taken from https://github.com/django/django/blob/master/django/utils/text.py
+        Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
+        dashes to single dashes. Remove characters that aren't alphanumerics,
+        underscores, or hyphens. Convert to lowercase. Also strip leading and
+        trailing whitespace, dashes, and underscores.
+        https://stackoverflow.com/a/295466/2903943
+        """
+        value = str(value)
+        if allow_unicode:
+            value = unicodedata.normalize('NFKC', value)
+        else:
+            value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
+        value = re.sub(r'[^\w\s-]', '', value.lower())
+        return re.sub(r'[-\s]+', '-', value).strip('-_')
+
+    def __init__( self, title, prefix=None, init_bokeh=True ):
+
+        ###
+        ### Setup up Bokeh paths, inject cubevis libraries into Bokeh HTML output
+        ###
+        if init_bokeh:
+            initialize_bokeh( )
+
+        if prefix is None:
+            ## create a prefix from the title
+            prefix = self._slugify(title)[:10]
+        self.__workdir = TemporaryDirectory(prefix=prefix)
+        self.__htmlpath = join( self.__workdir.name, f'''{self._slugify(title)}.html''' )
+        output_file( self.__htmlpath, title=title )
+
+    def __del__( self ):
+        ### remove work directory and its contents
+        self.__workdir.cleanup( )
+
+    def workdir( self ):
+        return self.__workdir.name