latch 2.53.7.dev2__tar.gz → 2.53.8__tar.gz

{latch-2.53.7.dev2/latch.egg-info → latch-2.53.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.53.7.dev2
+Version: 2.53.8
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8
@@ -34,8 +34,6 @@ Requires-Dist: aioconsole==0.6.1
 Requires-Dist: asyncssh==2.13.2
 Requires-Dist: websockets==11.0.3
 Requires-Dist: watchfiles==0.19.0
-Requires-Dist: uvloop==0.19.0
-Requires-Dist: aiohttp==3.9.5
 Provides-Extra: snakemake
 Requires-Dist: snakemake<7.30.2,>=7.18.0; extra == "snakemake"
 Requires-Dist: pulp<2.8,>=2.0; extra == "snakemake"

{latch-2.53.7.dev2 → latch-2.53.8}/latch/ldata/_transfer/upload.py

@@ -71,7 +71,7 @@ def upload(
     dest_data = node_data.data[dest]
 
     if not (dest_data.exists() or dest_data.is_direct_parent()) and not create_parents:
-        raise LatchPathError("no such Latch file or directory", dest, node_data.acc_id)
+        raise LatchPathError("no such Latch file or directory", dest)
 
     dest_is_dir = dest_data.type in {
         LDataNodeType.account_root,

{latch-2.53.7.dev2 → latch-2.53.8}/latch/ldata/type.py

@@ -6,7 +6,7 @@ class LatchPathError(RuntimeError):
     def __init__(
         self,
         message: str,
-        remote_path: str,
+        remote_path: Optional[str] = None,
         acc_id: Optional[str] = None,
     ):
         super().__init__(message)

{latch-2.53.7.dev2 → latch-2.53.8/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.53.7.dev2
+Version: 2.53.8
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8
@@ -34,8 +34,6 @@ Requires-Dist: aioconsole==0.6.1
 Requires-Dist: asyncssh==2.13.2
 Requires-Dist: websockets==11.0.3
 Requires-Dist: watchfiles==0.19.0
-Requires-Dist: uvloop==0.19.0
-Requires-Dist: aiohttp==3.9.5
 Provides-Extra: snakemake
 Requires-Dist: snakemake<7.30.2,>=7.18.0; extra == "snakemake"
 Requires-Dist: pulp<2.8,>=2.0; extra == "snakemake"

{latch-2.53.7.dev2 → latch-2.53.8}/latch.egg-info/SOURCES.txt

@@ -103,18 +103,8 @@ latch_cli/services/workspace.py
 latch_cli/services/cp/__init__.py
 latch_cli/services/cp/autocomplete.py
 latch_cli/services/cp/glob.py
-latch_cli/services/cp/http_utils.py
 latch_cli/services/cp/main.py
 latch_cli/services/cp/utils.py
-latch_cli/services/cp/download/__init__.py
-latch_cli/services/cp/download/main.py
-latch_cli/services/cp/download/worker.py
-latch_cli/services/cp/upload/__init__.py
-latch_cli/services/cp/upload/main.py
-latch_cli/services/cp/upload/worker.py
-latch_cli/services/execute/__init__.py
-latch_cli/services/execute/main.py
-latch_cli/services/execute/utils.py
 latch_cli/services/init/__init__.py
 latch_cli/services/init/init.py
 latch_cli/services/init/__pycache__/__init__.cpython-310.pyc
@@ -161,6 +151,11 @@ latch_cli/services/init/template/README.md
 latch_cli/services/init/template/__init__.py
 latch_cli/services/init/template/task.py
 latch_cli/services/init/template/__pycache__/__init__.cpython-310.pyc
+latch_cli/services/k8s/__init__.py
+latch_cli/services/k8s/attach.py
+latch_cli/services/k8s/execute.py
+latch_cli/services/k8s/utils.py
+latch_cli/services/k8s/ws_utils.py
 latch_cli/services/register/__init__.py
 latch_cli/services/register/constants.py
 latch_cli/services/register/register.py

{latch-2.53.7.dev2 → latch-2.53.8}/latch.egg-info/requires.txt

@@ -23,8 +23,6 @@ aioconsole==0.6.1
 asyncssh==2.13.2
 websockets==11.0.3
 watchfiles==0.19.0
-uvloop==0.19.0
-aiohttp==3.9.5
 
 [pandas]
 pandas>=2.0.0

{latch-2.53.7.dev2 → latch-2.53.8}/latch_cli/main.py

@@ -4,13 +4,15 @@ import os
 import sys
 from pathlib import Path
 from textwrap import dedent
-from typing import Callable, List, Literal, Optional, Tuple, TypeVar, Union
+from typing import Callable, List, Optional, Tuple, TypeVar, Union
 
 import click
 from packaging.version import parse as parse_version
 from typing_extensions import ParamSpec
 
 import latch_cli.click_utils
+from latch.ldata._transfer.progress import Progress as _Progress
+from latch_cli.click_utils import EnumChoice
 from latch_cli.exceptions.handler import CrashHandler
 from latch_cli.services.cp.autocomplete import complete as cp_complete
 from latch_cli.services.cp.autocomplete import remote_complete
@@ -435,7 +437,7 @@ def execute(
 ):
     """Drops the user into an interactive shell from within a task."""
 
-    from latch_cli.services.execute.main import exec
+    from latch_cli.services.k8s.execute import exec
 
     exec(execution_id=execution_id, egn_id=egn_id, container_index=container_index)
 
@@ -691,7 +693,7 @@ LDATA COMMANDS
 @click.option(
     "--progress",
     help="Type of progress information to show while copying",
-    type=click.Choice(["none", "total", "tasks"]),
+    type=EnumChoice(_Progress, case_sensitive=False),
     default="tasks",
     show_default=True,
 )
@@ -703,14 +705,6 @@ LDATA COMMANDS
     default=False,
     show_default=True,
 )
-@click.option(
-    "--force",
-    "-f",
-    help="Don't ask to confirm when overwriting files",
-    is_flag=True,
-    default=False,
-    show_default=True,
-)
 @click.option(
     "--no-glob",
     "-G",
@@ -733,9 +727,8 @@ LDATA COMMANDS
 def cp(
     src: List[str],
     dest: str,
-    progress: Literal["none", "total", "tasks"],
+    progress: _Progress,
     verbose: bool,
-    force: bool,
     no_glob: bool,
     cores: Optional[int] = None,
     chunk_size_mib: Optional[int] = None,
@@ -754,7 +747,6 @@ def cp(
         src,
         dest,
         progress=progress,
-        force=force,
         verbose=verbose,
         expand_globs=not no_glob,
         cores=cores,
@@ -1007,6 +999,19 @@ def generate_entrypoint(
     )
 
 
+@nextflow.command("attach")
+@click.option(
+    "--execution-id", "-e", type=str, help="Optional execution ID to inspect."
+)
+@requires_login
+def attach(execution_id: Optional[str]):
+    """Drops the user into an interactive shell to inspect the workdir of a nextflow execution."""
+
+    from latch_cli.services.k8s.attach import attach
+
+    attach(execution_id)
+
+
 """
 POD COMMANDS
 """

latch-2.53.8/latch_cli/services/cp/main.py

@@ -0,0 +1,106 @@
+from pathlib import Path
+from textwrap import dedent
+from typing import List, Optional
+
+import click
+
+from latch.ldata._transfer.download import download as _download
+from latch.ldata._transfer.progress import Progress
+from latch.ldata._transfer.remote_copy import remote_copy as _remote_copy
+from latch.ldata._transfer.upload import upload as _upload
+from latch.ldata.type import LatchPathError
+from latch_cli.services.cp.glob import expand_pattern
+from latch_cli.utils import human_readable_time, with_si_suffix
+from latch_cli.utils.path import get_path_error, is_remote_path
+
+
+def _copy_and_print(src: str, dst: str, progress: Progress) -> None:
+    _remote_copy(src, dst)
+    if progress != Progress.none:
+        click.echo(dedent(f"""
+            {click.style("Copy Requested.", fg="green")}
+            {click.style("Source: ", fg="blue")}{(src)}
+            {click.style("Destination: ", fg="blue")}{(dst)}"""))
+
+
+def _download_and_print(src: str, dst: Path, progress: Progress, verbose: bool) -> None:
+    if progress != Progress.none:
+        click.secho(f"Downloading {dst.name}", fg="blue")
+    res = _download(src, dst, progress, verbose)
+    if progress != Progress.none:
+        click.echo(dedent(f"""
+			{click.style("Download Complete", fg="green")}
+			{click.style("Time Elapsed: ", fg="blue")}{human_readable_time(res.total_time)}
+			{click.style("Files Downloaded: ", fg="blue")}{res.num_files} ({with_si_suffix(res.total_bytes)})
+			"""))
+
+
+# todo(ayush): come up with a better behavior scheme than unix cp
+def cp(
+    srcs: List[str],
+    dest: str,
+    *,
+    progress: Progress,
+    verbose: bool,
+    expand_globs: bool,
+    cores: Optional[int] = None,
+    chunk_size_mib: Optional[int] = None,
+):
+    if chunk_size_mib is not None and chunk_size_mib < 5:
+        click.secho(
+            "The chunk size specified by --chunk-size-mib must be at least 5. You"
+            f" provided `{chunk_size_mib}`",
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
+    dest_remote = is_remote_path(dest)
+
+    for src in srcs:
+        src_remote = is_remote_path(src)
+
+        try:
+            if src_remote and not dest_remote:
+                if expand_globs:
+                    [
+                        _download_and_print(p, Path(dest), progress, verbose)
+                        for p in expand_pattern(src)
+                    ]
+                else:
+                    _download_and_print(src, Path(dest), progress, verbose)
+            elif not src_remote and dest_remote:
+                if progress != Progress.none:
+                    click.secho(f"Uploading {src}", fg="blue")
+                res = _upload(
+                    src,
+                    dest,
+                    progress=progress,
+                    verbose=verbose,
+                    cores=cores,
+                    chunk_size_mib=chunk_size_mib,
+                )
+                if progress != Progress.none:
+                    click.echo(dedent(f"""
+                        {click.style("Upload Complete", fg="green")}
+                        {click.style("Time Elapsed: ", fg="blue")}{human_readable_time(res.total_time)}
+                        {click.style("Files Uploaded: ", fg="blue")}{res.num_files} ({with_si_suffix(res.total_bytes)})
+                        """))
+            elif src_remote and dest_remote:
+                if expand_globs:
+                    [_copy_and_print(p, dest, progress) for p in expand_pattern(src)]
+                else:
+                    _copy_and_print(src, dest, progress)
+            else:
+                raise ValueError(
+                    dedent(f"""
+                    `latch cp` cannot be used for purely local file copying.
+
+                    Please ensure at least one of your arguments is a remote path (beginning with `latch://`)
+                    """).strip("\n"),
+                )
+        except LatchPathError as e:
+            click.secho(get_path_error(e.remote_path, e.message, e.acc_id), fg="red")
+            raise click.exceptions.Exit(1) from e
+        except Exception as e:
+            click.secho(str(e), fg="red")
+            raise click.exceptions.Exit(1) from e

{latch-2.53.7.dev2 → latch-2.53.8}/latch_cli/services/cp/utils.py

@@ -1,9 +1,8 @@
-import sys
-from typing import Iterable, List, TypedDict, TypeVar
+from typing import List, TypedDict
 
-if sys.version_info >= (3, 9):
+try:
     from functools import cache
-else:
+except ImportError:
     from functools import lru_cache as cache
 
 import gql
@@ -102,7 +101,7 @@ class AccountInfoCurrent(TypedDict):
 
 # todo(taras): support for gcp and azure mounts
 # skipping now due to time. This decision does not
-# influence correctness of the CLI and only
+# influence correcetness of the CLI and only
 # reduces the set of returned autocomplete
 # suggestions
 @cache
@@ -163,20 +162,3 @@ def _get_known_domains_for_account() -> List[str]:
     res.extend(f"{x}.mount" for x in buckets)
 
     return res
-
-
-chunk_batch_size = 3
-
-T = TypeVar("T")
-
-
-def chunked(iter: Iterable[T]) -> Iterable[List[T]]:
-    chunk = []
-    for x in iter:
-        if len(chunk) == chunk_batch_size:
-            yield chunk
-            chunk = []
-
-        chunk.append(x)
-
-    yield chunk

latch-2.53.8/latch_cli/services/k8s/attach.py

@@ -0,0 +1,77 @@
+import asyncio
+import json
+import secrets
+import sys
+from typing import Optional
+from urllib.parse import urljoin, urlparse
+
+import click
+import websockets.client as websockets
+import websockets.exceptions as ws_exceptions
+from latch_sdk_config.latch import NUCLEUS_URL
+
+from latch_cli.utils import get_auth_header
+
+from .utils import get_pvc_info
+from .ws_utils import forward_stdio
+
+
+async def connect(execution_id: str, session_id: str):
+    async with websockets.connect(
+        urlparse(urljoin(NUCLEUS_URL, "/workflows/cli/attach-nf-workdir"))
+        ._replace(scheme="wss")
+        .geturl(),
+        close_timeout=0,
+        extra_headers={"Authorization": get_auth_header()},
+    ) as ws:
+        request = {"execution_id": int(execution_id), "session_id": session_id}
+
+        await ws.send(json.dumps(request))
+        data = await ws.recv()
+
+        msg = ""
+        try:
+            res = json.loads(data)
+            if "error" in res:
+                raise RuntimeError(res["error"])
+        except json.JSONDecodeError:
+            msg = "Unable to connect to pod - internal error."
+        except RuntimeError as e:
+            msg = str(e)
+
+        if msg != "":
+            raise RuntimeError(msg)
+
+        await forward_stdio(ws)
+
+
+def get_session_id():
+    return secrets.token_bytes(8).hex()
+
+
+def attach(execution_id: Optional[str] = None):
+    execution_id = get_pvc_info(execution_id)
+    session_id = get_session_id()
+
+    click.secho(
+        "Attaching to workdir - this may take a few seconds...", dim=True, italic=True
+    )
+
+    import termios
+    import tty
+
+    old_settings_stdin = termios.tcgetattr(sys.stdin.fileno())
+    tty.setraw(sys.stdin)
+
+    msg = ""
+    try:
+        asyncio.run(connect(execution_id, session_id))
+    except ws_exceptions.ConnectionClosedError as e:
+        msg = json.loads(e.reason)["error"]
+    except RuntimeError as e:
+        msg = str(e)
+    finally:
+        termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, old_settings_stdin)
+
+    if msg != "":
+        click.secho(msg, fg="red")

latch-2.53.8/latch_cli/services/k8s/execute.py

@@ -0,0 +1,78 @@
+import asyncio
+import json
+import sys
+from typing import Optional
+from urllib.parse import urljoin, urlparse
+
+import websockets.client as websockets
+from latch_sdk_config.latch import NUCLEUS_URL
+
+from latch_cli.services.k8s.utils import (
+    ContainerNode,
+    EGNNode,
+    ExecutionInfoNode,
+    get_container_info,
+    get_egn_info,
+    get_execution_info,
+)
+from latch_cli.utils import get_auth_header
+
+from .ws_utils import forward_stdio
+
+
+async def connect(egn_info: EGNNode, container_info: Optional[ContainerNode]):
+    async with websockets.connect(
+        urlparse(urljoin(NUCLEUS_URL, "/workflows/cli/shell"))
+        ._replace(scheme="wss")
+        .geturl(),
+        close_timeout=0,
+        extra_headers={"Authorization": get_auth_header()},
+    ) as ws:
+        request = {
+            "egn_id": egn_info["id"],
+            "container_index": (
+                container_info["index"] if container_info is not None else None
+            ),
+        }
+
+        await ws.send(json.dumps(request))
+        data = await ws.recv()
+
+        msg = ""
+        try:
+            res = json.loads(data)
+            if "error" in res:
+                raise RuntimeError(res["error"])
+        except json.JSONDecodeError:
+            msg = "Unable to connect to pod - internal error."
+        except RuntimeError as e:
+            msg = str(e)
+
+        if msg != "":
+            raise RuntimeError(msg)
+
+        await forward_stdio(ws)
+
+
+def exec(
+    execution_id: Optional[str] = None,
+    egn_id: Optional[str] = None,
+    container_index: Optional[int] = None,
+):
+    execution_info: Optional[ExecutionInfoNode] = None
+    if egn_id is None:
+        execution_info = get_execution_info(execution_id)
+
+    egn_info = get_egn_info(execution_info, egn_id)
+    container_info = get_container_info(egn_info, container_index)
+
+    import termios
+    import tty
+
+    old_settings_stdin = termios.tcgetattr(sys.stdin.fileno())
+    tty.setraw(sys.stdin)
+
+    try:
+        asyncio.run(connect(egn_info, container_info))
+    finally:
+        termios.tcsetattr(sys.stdin.fileno(), termios.TCSANOW, old_settings_stdin)

{latch-2.53.7.dev2/latch_cli/services/execute → latch-2.53.8/latch_cli/services/k8s}/utils.py

@@ -9,7 +9,6 @@ from latch_sdk_gql.execute import execute
 
 from latch_cli.click_utils import bold, color
 from latch_cli.menus import select_tui
-from latch_cli.utils import current_workspace
 
 
 # todo(ayush): put this into latch_sdk_gql
@@ -179,35 +178,39 @@ def get_execution_info(execution_id: Optional[str]) -> ExecutionInfoNode:
             """),
             *fragments,
         ),
-        {"createdBy": current_workspace()},
+        {"createdBy": user_config.workspace_id},
     )["runningExecutions"]
 
-    if len(res["nodes"]) == 0:
-        click.secho("You have no executions currently running.", dim=True)
+    nodes = res["nodes"]
+
+    if len(nodes) == 0:
+        click.secho(
+            f"You have no executions currently running.",
+            dim=True,
+        )
         raise click.exceptions.Exit(0)
 
-    if len(res["nodes"]) == 1:
-        execution = res["nodes"][0]
+    if len(nodes) == 1:
+        execution = nodes[0]
         click.secho(
             "Selecting execution"
             f" {color(execution['displayName'])} as it is"
-            " the only"
-            " one currently running in Workspace"
+            " the only one currently running in Workspace"
             f" {color(workspace_str)}.",
         )
 
         return execution
 
     selected_execution = select_tui(
-        "You have multiple executions running in this workspace"
-        f" ({color(workspace_str)}). Which"
-        " execution would you like to inspect?",
+        "You have multiple executions running in"
+        f" this workspace ({color(workspace_str)}). Which execution would you like to"
+        " inspect?",
         [
             {
                 "display_name": f'{x["displayName"]} ({x["workflow"]["displayName"]})',
                 "value": x,
             }
-            for x in res["nodes"]
+            for x in nodes
         ],
         clear_terminal=False,
     )
@@ -368,3 +371,61 @@ def get_container_info(
         raise click.exceptions.Exit(0)
 
     return selected_container_info
+
+
+class Node(TypedDict):
+    id: str
+    displayName: str
+
+
+class nfAvailablePvcs(TypedDict):
+    nodes: List[Node]
+
+
+def get_pvc_info(execution_id: Optional[str]) -> str:
+    if execution_id is not None:
+        return execution_id
+
+    workspace_str: str = user_config.workspace_name or user_config.workspace_id
+
+    res: nfAvailablePvcs = execute(
+        gql.gql("""
+            query NFWorkdirs($wsId: BigInt!) {
+                nfAvailablePvcs(argWsId: $wsId) {
+                    nodes {
+                        id
+                        displayName
+                    }
+                }
+            }
+        """),
+        {"wsId": user_config.workspace_id},
+    )["nfAvailablePvcs"]
+
+    nodes = res["nodes"]
+
+    if len(nodes) == 0:
+        click.secho(
+            f"You have no available workdirs (all have expired).",
+            dim=True,
+        )
+        raise click.exceptions.Exit(0)
+
+    if len(nodes) == 1:
+        execution = nodes[0]
+        click.secho(
+            f"Selecting execution {color(execution['displayName'])} as it is the only"
+            f" one without an expired workDir in Workspace {color(workspace_str)}.",
+        )
+        return execution["id"]
+
+    selected_execution = select_tui(
+        "You have multiple available workDirs in this workspace"
+        f" ({color(workspace_str)}). Which execution would you like to attach to?",
+        options=[{"display_name": x["displayName"], "value": x["id"]} for x in nodes],
+    )
+    if selected_execution is None:
+        click.secho("No execution selected. Exiting.", dim=True)
+        raise click.exceptions.Exit(0)
+
+    return selected_execution