latch 2.53.7__tar.gz → 2.53.7.dev2__tar.gz

latch-2.53.7.dev2/PKG-INFO

@@ -0,0 +1,43 @@
+Metadata-Version: 2.1
+Name: latch
+Version: 2.53.7.dev2
+Summary: The Latch SDK
+Author-email: kenny@latch.bio
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.8,<3.12
+License-File: LICENSE
+Requires-Dist: kubernetes>=24.2.0
+Requires-Dist: pyjwt>=0.2.0
+Requires-Dist: requests>=2.28.1
+Requires-Dist: click>=8.0
+Requires-Dist: docker>=7.1.0
+Requires-Dist: paramiko>=3.2.0
+Requires-Dist: cryptography<43.0.0
+Requires-Dist: scp>=0.14.0
+Requires-Dist: boto3>=1.26.0
+Requires-Dist: tqdm>=4.63.0
+Requires-Dist: lytekit==0.15.12
+Requires-Dist: lytekitplugins-pods==0.6.2
+Requires-Dist: typing-extensions==4.7.1
+Requires-Dist: apscheduler==3.9.1
+Requires-Dist: gql==3.4.0
+Requires-Dist: graphql-core==3.2.3
+Requires-Dist: requests-toolbelt==0.10.1
+Requires-Dist: latch-sdk-gql==0.0.6
+Requires-Dist: latch-sdk-config==0.0.4
+Requires-Dist: python-dateutil>=2.8
+Requires-Dist: GitPython==3.1.40
+Requires-Dist: aioconsole==0.6.1
+Requires-Dist: asyncssh==2.13.2
+Requires-Dist: websockets==11.0.3
+Requires-Dist: watchfiles==0.19.0
+Requires-Dist: uvloop==0.19.0
+Requires-Dist: aiohttp==3.9.5
+Provides-Extra: snakemake
+Requires-Dist: snakemake<7.30.2,>=7.18.0; extra == "snakemake"
+Requires-Dist: pulp<2.8,>=2.0; extra == "snakemake"
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0.0; extra == "pandas"

{latch-2.53.7 → latch-2.53.7.dev2}/latch/ldata/_transfer/upload.py

@@ -71,7 +71,7 @@ def upload(
     dest_data = node_data.data[dest]
 
     if not (dest_data.exists() or dest_data.is_direct_parent()) and not create_parents:
-        raise LatchPathError("no such Latch file or directory", dest)
+        raise LatchPathError("no such Latch file or directory", dest, node_data.acc_id)
 
     dest_is_dir = dest_data.type in {
         LDataNodeType.account_root,

{latch-2.53.7 → latch-2.53.7.dev2}/latch/ldata/type.py

@@ -6,7 +6,7 @@ class LatchPathError(RuntimeError):
     def __init__(
         self,
         message: str,
-        remote_path: Optional[str] = None,
+        remote_path: str,
         acc_id: Optional[str] = None,
     ):
         super().__init__(message)

latch-2.53.7.dev2/latch.egg-info/PKG-INFO

@@ -0,0 +1,43 @@
+Metadata-Version: 2.1
+Name: latch
+Version: 2.53.7.dev2
+Summary: The Latch SDK
+Author-email: kenny@latch.bio
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.8,<3.12
+License-File: LICENSE
+Requires-Dist: kubernetes>=24.2.0
+Requires-Dist: pyjwt>=0.2.0
+Requires-Dist: requests>=2.28.1
+Requires-Dist: click>=8.0
+Requires-Dist: docker>=7.1.0
+Requires-Dist: paramiko>=3.2.0
+Requires-Dist: cryptography<43.0.0
+Requires-Dist: scp>=0.14.0
+Requires-Dist: boto3>=1.26.0
+Requires-Dist: tqdm>=4.63.0
+Requires-Dist: lytekit==0.15.12
+Requires-Dist: lytekitplugins-pods==0.6.2
+Requires-Dist: typing-extensions==4.7.1
+Requires-Dist: apscheduler==3.9.1
+Requires-Dist: gql==3.4.0
+Requires-Dist: graphql-core==3.2.3
+Requires-Dist: requests-toolbelt==0.10.1
+Requires-Dist: latch-sdk-gql==0.0.6
+Requires-Dist: latch-sdk-config==0.0.4
+Requires-Dist: python-dateutil>=2.8
+Requires-Dist: GitPython==3.1.40
+Requires-Dist: aioconsole==0.6.1
+Requires-Dist: asyncssh==2.13.2
+Requires-Dist: websockets==11.0.3
+Requires-Dist: watchfiles==0.19.0
+Requires-Dist: uvloop==0.19.0
+Requires-Dist: aiohttp==3.9.5
+Provides-Extra: snakemake
+Requires-Dist: snakemake<7.30.2,>=7.18.0; extra == "snakemake"
+Requires-Dist: pulp<2.8,>=2.0; extra == "snakemake"
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0.0; extra == "pandas"

{latch-2.53.7 → latch-2.53.7.dev2}/latch.egg-info/SOURCES.txt

@@ -103,15 +103,28 @@ latch_cli/services/workspace.py
 latch_cli/services/cp/__init__.py
 latch_cli/services/cp/autocomplete.py
 latch_cli/services/cp/glob.py
+latch_cli/services/cp/http_utils.py
 latch_cli/services/cp/main.py
 latch_cli/services/cp/utils.py
+latch_cli/services/cp/download/__init__.py
+latch_cli/services/cp/download/main.py
+latch_cli/services/cp/download/worker.py
+latch_cli/services/cp/upload/__init__.py
+latch_cli/services/cp/upload/main.py
+latch_cli/services/cp/upload/worker.py
 latch_cli/services/execute/__init__.py
 latch_cli/services/execute/main.py
 latch_cli/services/execute/utils.py
 latch_cli/services/init/__init__.py
 latch_cli/services/init/init.py
+latch_cli/services/init/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/__pycache__/__init__.cpython-311.pyc
+latch_cli/services/init/__pycache__/__init__.cpython-38.pyc
+latch_cli/services/init/__pycache__/__init__.cpython-39.pyc
+latch_cli/services/init/__pycache__/init.cpython-310.pyc
 latch_cli/services/init/__pycache__/init.cpython-311.pyc
+latch_cli/services/init/__pycache__/init.cpython-38.pyc
+latch_cli/services/init/__pycache__/init.cpython-39.pyc
 latch_cli/services/init/assemble_and_sort/.env
 latch_cli/services/init/assemble_and_sort/LICENSE
 latch_cli/services/init/assemble_and_sort/README.md
@@ -119,18 +132,22 @@ latch_cli/services/init/assemble_and_sort/__init__.py
 latch_cli/services/init/assemble_and_sort/assemble.py
 latch_cli/services/init/assemble_and_sort/sort.py
 latch_cli/services/init/assemble_and_sort/system-requirements.txt
+latch_cli/services/init/assemble_and_sort/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/common/.dockerignore
 latch_cli/services/init/example_conda/__init__.py
 latch_cli/services/init/example_conda/conda_task.py
 latch_cli/services/init/example_conda/environment.yaml
+latch_cli/services/init/example_conda/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_docker/__init__.py
 latch_cli/services/init/example_docker/task.py
+latch_cli/services/init/example_nf_integration/latch_metadata/__pycache__/__init__.cpython-311.pyc
 latch_cli/services/init/example_nfcore/Dockerfile
 latch_cli/services/init/example_nfcore/__init__.py
 latch_cli/services/init/example_nfcore/task.py
 latch_cli/services/init/example_r/__init__.py
 latch_cli/services/init/example_r/environment.R
 latch_cli/services/init/example_r/r_task.py
+latch_cli/services/init/example_r/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_snakemake/Dockerfile
 latch_cli/services/init/example_snakemake/Snakefile
 latch_cli/services/init/example_snakemake/config.yaml
@@ -143,6 +160,7 @@ latch_cli/services/init/template/LICENSE
 latch_cli/services/init/template/README.md
 latch_cli/services/init/template/__init__.py
 latch_cli/services/init/template/task.py
+latch_cli/services/init/template/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/register/__init__.py
 latch_cli/services/register/constants.py
 latch_cli/services/register/register.py

{latch-2.53.7 → latch-2.53.7.dev2}/latch.egg-info/requires.txt

@@ -23,6 +23,8 @@ aioconsole==0.6.1
 asyncssh==2.13.2
 websockets==11.0.3
 watchfiles==0.19.0
+uvloop==0.19.0
+aiohttp==3.9.5
 
 [pandas]
 pandas>=2.0.0

{latch-2.53.7 → latch-2.53.7.dev2}/latch_cli/main.py

@@ -4,15 +4,13 @@ import os
 import sys
 from pathlib import Path
 from textwrap import dedent
-from typing import Callable, List, Optional, Tuple, TypeVar, Union
+from typing import Callable, List, Literal, Optional, Tuple, TypeVar, Union
 
 import click
 from packaging.version import parse as parse_version
 from typing_extensions import ParamSpec
 
 import latch_cli.click_utils
-from latch.ldata._transfer.progress import Progress as _Progress
-from latch_cli.click_utils import EnumChoice
 from latch_cli.exceptions.handler import CrashHandler
 from latch_cli.services.cp.autocomplete import complete as cp_complete
 from latch_cli.services.cp.autocomplete import remote_complete
@@ -693,7 +691,7 @@ LDATA COMMANDS
 @click.option(
     "--progress",
     help="Type of progress information to show while copying",
-    type=EnumChoice(_Progress, case_sensitive=False),
+    type=click.Choice(["none", "total", "tasks"]),
     default="tasks",
     show_default=True,
 )
@@ -705,6 +703,14 @@ LDATA COMMANDS
     default=False,
     show_default=True,
 )
+@click.option(
+    "--force",
+    "-f",
+    help="Don't ask to confirm when overwriting files",
+    is_flag=True,
+    default=False,
+    show_default=True,
+)
 @click.option(
     "--no-glob",
     "-G",
@@ -727,8 +733,9 @@ LDATA COMMANDS
 def cp(
     src: List[str],
     dest: str,
-    progress: _Progress,
+    progress: Literal["none", "total", "tasks"],
     verbose: bool,
+    force: bool,
     no_glob: bool,
     cores: Optional[int] = None,
     chunk_size_mib: Optional[int] = None,
@@ -747,6 +754,7 @@ def cp(
         src,
         dest,
         progress=progress,
+        force=force,
         verbose=verbose,
         expand_globs=not no_glob,
         cores=cores,

latch-2.53.7.dev2/latch_cli/services/cp/download/main.py

@@ -0,0 +1,208 @@
+import asyncio
+import queue
+import shutil
+import time
+from pathlib import Path
+from textwrap import dedent
+from typing import Dict, List, Literal, Optional, TypedDict
+
+import click
+import requests
+import requests.adapters
+import tqdm
+import uvloop
+
+from ....utils import get_auth_header, human_readable_time, with_si_suffix
+from ....utils.path import normalize_path
+from ..glob import expand_pattern
+from .worker import Work, run_workers
+
+http_session = requests.Session()
+
+_adapter = requests.adapters.HTTPAdapter(
+    max_retries=requests.adapters.Retry(
+        status_forcelist=[429, 500, 502, 503, 504],
+        backoff_factor=1,
+        allowed_methods=["GET", "PUT", "POST"],
+    )
+)
+http_session.mount("https://", _adapter)
+http_session.mount("http://", _adapter)
+
+
+class GetSignedUrlData(TypedDict):
+    url: str
+
+
+class GetSignedUrlsRecursiveData(TypedDict):
+    urls: Dict[str, str]
+
+
+def download(
+    srcs: List[str],
+    dest: Path,
+    progress: Literal["none", "total", "tasks"],
+    verbose: bool,
+    force: bool,
+    expand_globs: bool,
+    cores: Optional[int],
+    chunk_size_mib: Optional[int],
+):
+    if cores is None:
+        cores = 4
+    if chunk_size_mib is None:
+        chunk_size_mib = 16
+
+    start = time.monotonic()
+
+    if not dest.parent.exists():
+        click.secho(
+            f"Invalid copy destination {dest}. Parent directory {dest.parent} does"
+            " not exist.",
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
+    if len(srcs) > 1 and not (dest.exists() and dest.is_dir()):
+        click.secho(
+            f"Copy destination {dest} does not exist. Multi-source copies must write to"
+            " a pre-existing directory.",
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
+    from latch.ldata.path import _get_node_data
+    from latch.ldata.type import LDataNodeType
+
+    all_node_data = _get_node_data(*srcs)
+    work_queue = asyncio.Queue[Work]()
+    total = 0
+
+    if expand_globs:
+        new_srcs = []
+        for src in srcs:
+            new_srcs.extend(expand_pattern(src))
+
+        srcs = new_srcs
+
+    # todo(ayush): parallelize
+    for src in srcs:
+        node_data = all_node_data.data[src]
+        normalized = normalize_path(src)
+
+        can_have_children = node_data.type in {
+            LDataNodeType.account_root,
+            LDataNodeType.dir,
+            LDataNodeType.mount,
+            LDataNodeType.mount_gcp,
+            LDataNodeType.mount_azure,
+        }
+
+        if not can_have_children:
+            endpoint = "https://nucleus.latch.bio/ldata/get-signed-url"
+        else:
+            endpoint = "https://nucleus.latch.bio/ldata/get-signed-urls-recursive"
+
+        res = http_session.post(
+            endpoint,
+            headers={"Authorization": get_auth_header()},
+            json={"path": normalized},
+        )
+
+        json = res.json()
+
+        if not can_have_children:
+            gsud: GetSignedUrlData = json["data"]
+            total += 1
+
+            work_dest = dest
+            if dest.exists() and dest.is_dir():
+                work_dest = dest / node_data.name
+
+            if (
+                work_dest.exists()
+                and not force
+                and not click.confirm(
+                    f"Copy destination path {work_dest} already exists and its contents"
+                    " may be overwritten. Proceed?"
+                )
+            ):
+                continue
+
+            try:
+                work_dest.unlink(missing_ok=True)
+                work_queue.put_nowait(Work(gsud["url"], work_dest, chunk_size_mib))
+            except OSError:
+                click.secho(
+                    f"Cannot write file to {work_dest} - directory exists.", fg="red"
+                )
+
+        else:
+            gsurd: GetSignedUrlsRecursiveData = json["data"]
+            total += len(gsurd["urls"])
+
+            work_dest = dest
+            if dest.exists() and not normalized.endswith("/"):
+                work_dest = dest / node_data.name
+
+            if (
+                work_dest.exists()
+                and work_dest.is_dir()
+                and not force
+                and not click.confirm(
+                    f"Copy destination path {work_dest} already exists and its contents"
+                    " may be overwritten. Proceed?"
+                )
+            ):
+                return
+
+            for rel, url in gsurd["urls"].items():
+                res = work_dest / rel
+
+                try:
+                    res.parent.mkdir(exist_ok=True, parents=True)
+                    if res.is_dir():
+                        click.secho(
+                            f"Cannot write file to {work_dest / rel} - directory"
+                            " exists.",
+                            fg="red",
+                        )
+                        continue
+
+                    work_queue.put_nowait(Work(url, work_dest / rel, chunk_size_mib))
+                except (NotADirectoryError, FileExistsError):
+                    click.secho(
+                        f"Cannot write file to {work_dest / rel} - upstream file"
+                        " exists.",
+                        fg="red",
+                    )
+
+    tbar = tqdm.tqdm(
+        total=total,
+        leave=False,
+        colour="green",
+        smoothing=0,
+        unit="B",
+        unit_scale=True,
+        disable=progress == "none",
+    )
+
+    num_workers = min(total, cores)
+    uvloop.install()
+
+    loop = uvloop.new_event_loop()
+    res = loop.run_until_complete(
+        run_workers(work_queue, num_workers, tbar, progress != "none", verbose)
+    )
+
+    total_bytes = sum(res)
+
+    tbar.clear()
+    total_time = time.monotonic() - start
+
+    if progress != "none":
+        click.echo(dedent(f"""\
+            {click.style("Download Complete", fg="green")}
+            {click.style("Time Elapsed:", fg="blue")}     {human_readable_time(total_time)}
+            {click.style("Files Downloaded:", fg="blue")} {total} ({with_si_suffix(total_bytes)})\
+        """))

latch-2.53.7.dev2/latch_cli/services/cp/download/worker.py

@@ -0,0 +1,121 @@
+import asyncio
+import os
+import queue
+import shutil
+import time
+from dataclasses import dataclass
+from http import HTTPStatus
+from pathlib import Path
+from typing import Awaitable, List
+
+import aiohttp
+import tqdm
+import uvloop
+
+from latch_cli.services.cp.utils import chunked
+
+from ....constants import Units
+from ..http_utils import RetryClientSession
+
+
+@dataclass
+class Work:
+    url: str
+    dest: Path
+    chunk_size_mib: int = 5
+
+
+async def download_chunk(
+    sess: aiohttp.ClientSession,
+    url: str,
+    fd: int,
+    index: int,
+    chunk_size: int,
+    pbar: tqdm.tqdm,
+):
+    start = index * chunk_size
+    end = start + chunk_size - 1
+
+    res = await sess.get(url, headers={"Range": f"bytes={start}-{end}"})
+    content = await res.read()
+    pbar.update(os.pwrite(fd, content, start))
+
+
+async def worker(
+    work_queue: asyncio.Queue[Work],
+    tbar: tqdm.tqdm,
+    show_task_progress: bool,
+    print_file_on_completion: bool,
+) -> int:
+    pbar = tqdm.tqdm(
+        total=0,
+        leave=False,
+        smoothing=0,
+        unit="B",
+        unit_scale=True,
+        disable=not show_task_progress,
+    )
+    total_bytes = 0
+
+    try:
+        async with RetryClientSession(read_timeout=90, conn_timeout=10) as sess:
+            while True:
+                try:
+                    work: Work = work_queue.get_nowait()
+                except asyncio.QueueEmpty:
+                    break
+
+                pbar.reset()
+                pbar.desc = work.dest.name
+
+                res = await sess.get(work.url, headers={"Range": "bytes=0-0"})
+
+                # s3 throws a REQUESTED_RANGE_NOT_SATISFIABLE if the file is empty
+                if res.status == 416:
+                    total_size = 0
+                else:
+                    content_range = res.headers["Content-Range"]
+                    total_size = int(content_range.replace("bytes 0-0/", ""))
+
+                assert total_size is not None
+
+                total_bytes += total_size
+                pbar.total = total_size
+
+                chunk_size = work.chunk_size_mib * Units.MiB
+
+                with work.dest.open("wb") as f:
+                    coros: List[Awaitable] = []
+
+                    cur = 0
+                    while cur * chunk_size < total_size:
+                        coros.append(
+                            download_chunk(
+                                sess, work.url, f.fileno(), cur, chunk_size, pbar
+                            )
+                        )
+                        cur += 1
+
+                    await asyncio.gather(*coros)
+
+                if print_file_on_completion:
+                    pbar.write(str(work.dest))
+
+                tbar.update(1)
+
+        return total_bytes
+    finally:
+        pbar.clear()
+
+
+async def run_workers(
+    work_queue: asyncio.Queue[Work],
+    num_workers: int,
+    tbar: tqdm.tqdm,
+    show_task_progress: bool,
+    print_file_on_completion: bool,
+) -> List[int]:
+    return await asyncio.gather(*[
+        worker(work_queue, tbar, show_task_progress, print_file_on_completion)
+        for _ in range(num_workers)
+    ])

latch-2.53.7.dev2/latch_cli/services/cp/http_utils.py

@@ -0,0 +1,91 @@
+import asyncio
+from http import HTTPStatus
+from typing import Awaitable, Callable, Dict, List, Optional
+
+import aiohttp
+import aiohttp.typedefs
+from typing_extensions import ParamSpec
+
+P = ParamSpec("P")
+
+
+class RetriesExhaustedException(RuntimeError): ...
+
+
+class RateLimitExceeded(RuntimeError): ...
+
+
+class RetryClientSession(aiohttp.ClientSession):
+    def __init__(
+        self,
+        status_list: Optional[List[HTTPStatus]] = None,
+        retries: int = 10,
+        backoff: float = 0.1,
+        *args,
+        **kwargs,
+    ):
+
+        self.status_list = (
+            status_list
+            if status_list is not None
+            else [
+                HTTPStatus.TOO_MANY_REQUESTS,  # 429
+                HTTPStatus.INTERNAL_SERVER_ERROR,  # 500
+                HTTPStatus.BAD_GATEWAY,  # 502
+                HTTPStatus.SERVICE_UNAVAILABLE,  # 503
+                HTTPStatus.GATEWAY_TIMEOUT,  # 504
+            ]
+        )
+
+        self.retries = retries
+        self.backoff = backoff
+
+        self.semas: Dict[aiohttp.typedefs.StrOrURL, asyncio.BoundedSemaphore] = {
+            "https://nucleus.latch.bio/ldata/start-upload": asyncio.BoundedSemaphore(2),
+            "https://nucleus.latch.bio/ldata/end-upload": asyncio.BoundedSemaphore(2),
+        }
+
+        super().__init__(*args, **kwargs)
+
+    async def _request(
+        self,
+        method: str,
+        str_or_url: aiohttp.typedefs.StrOrURL,
+        **kwargs,
+    ) -> aiohttp.ClientResponse:
+        sema = self.semas.get(str_or_url)
+
+        error: Optional[Exception] = None
+        last_res: Optional[aiohttp.ClientResponse] = None
+
+        cur = 0
+        while cur < self.retries:
+            if cur > 0:
+                await asyncio.sleep(max(self.backoff * 2**cur, 10))
+
+            cur += 1
+
+            try:
+                if sema is None:
+                    res = await super()._request(method, str_or_url, **kwargs)
+                else:
+                    async with sema:
+                        res = await super()._request(method, str_or_url, **kwargs)
+
+                if res.status in self.status_list:
+                    last_res = res
+                    continue
+
+                return res
+            except Exception as e:
+                error = e
+                continue
+
+        if last_res is not None:
+            return last_res
+
+        if error is not None:
+            raise error
+
+        # we'll never get here but putting here anyway so the type checker is happy
+        raise RetriesExhaustedException