latch 2.39.0.dev32__tar.gz → 2.39.0.dev34__tar.gz

{latch-2.39.0.dev32/latch.egg-info → latch-2.39.0.dev34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev32
+Version: 2.39.0.dev34
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev32 → latch-2.39.0.dev34/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev32
+Version: 2.39.0.dev34
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev32 → latch-2.39.0.dev34}/latch.egg-info/SOURCES.txt

@@ -134,13 +134,7 @@ latch_cli/services/execute/utils.py
 latch_cli/services/init/__init__.py
 latch_cli/services/init/init.py
 latch_cli/services/init/__pycache__/__init__.cpython-310.pyc
-latch_cli/services/init/__pycache__/__init__.cpython-311.pyc
-latch_cli/services/init/__pycache__/__init__.cpython-38.pyc
-latch_cli/services/init/__pycache__/__init__.cpython-39.pyc
 latch_cli/services/init/__pycache__/init.cpython-310.pyc
-latch_cli/services/init/__pycache__/init.cpython-311.pyc
-latch_cli/services/init/__pycache__/init.cpython-38.pyc
-latch_cli/services/init/__pycache__/init.cpython-39.pyc
 latch_cli/services/init/assemble_and_sort/.env
 latch_cli/services/init/assemble_and_sort/LICENSE
 latch_cli/services/init/assemble_and_sort/README.md
@@ -148,26 +142,22 @@ latch_cli/services/init/assemble_and_sort/__init__.py
 latch_cli/services/init/assemble_and_sort/assemble.py
 latch_cli/services/init/assemble_and_sort/sort.py
 latch_cli/services/init/assemble_and_sort/system-requirements.txt
-latch_cli/services/init/assemble_and_sort/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/common/.dockerignore
 latch_cli/services/init/example_conda/__init__.py
 latch_cli/services/init/example_conda/conda_task.py
 latch_cli/services/init/example_conda/environment.yaml
-latch_cli/services/init/example_conda/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_docker/__init__.py
 latch_cli/services/init/example_docker/task.py
 latch_cli/services/init/example_nf_integration/Dockerfile
 latch_cli/services/init/example_nf_integration/main.nf
 latch_cli/services/init/example_nf_integration/workflow.nf
 latch_cli/services/init/example_nf_integration/latch_metadata/__init__.py
-latch_cli/services/init/example_nf_integration/latch_metadata/__pycache__/__init__.cpython-311.pyc
 latch_cli/services/init/example_nfcore/Dockerfile
 latch_cli/services/init/example_nfcore/__init__.py
 latch_cli/services/init/example_nfcore/task.py
 latch_cli/services/init/example_r/__init__.py
 latch_cli/services/init/example_r/environment.R
 latch_cli/services/init/example_r/r_task.py
-latch_cli/services/init/example_r/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_snakemake/Dockerfile
 latch_cli/services/init/example_snakemake/Snakefile
 latch_cli/services/init/example_snakemake/config.yaml
@@ -180,7 +170,6 @@ latch_cli/services/init/template/LICENSE
 latch_cli/services/init/template/README.md
 latch_cli/services/init/template/__init__.py
 latch_cli/services/init/template/task.py
-latch_cli/services/init/template/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/register/__init__.py
 latch_cli/services/register/constants.py
 latch_cli/services/register/register.py

{latch-2.39.0.dev32 → latch-2.39.0.dev34}/latch_cli/extras/nextflow/build.py

@@ -1,12 +1,17 @@
 import glob
+import multiprocessing
 import os
 import shutil
 import subprocess
 import sys
 from collections import OrderedDict, defaultdict
+from concurrent.futures import ProcessPoolExecutor, as_completed, wait
+from ctypes import c_int
+from multiprocessing.managers import SyncManager
 from pathlib import Path
 from textwrap import dedent
 from typing import Dict, List, Optional, cast
+from urllib.parse import urljoin, urlunparse
 
 import boto3
 import click
@@ -301,6 +306,50 @@ def build_from_nextflow_dag(
     wf._nodes = list(node_map.values()) + extra_nodes
 
 
+def _do_download(
+    url: str,
+    output_path: Path,
+    total_count: int,
+    counter,
+    lock,
+):  # todo(ayush): figure out the right type annotation for counter/lock
+    res = tinyrequests.get(url)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_bytes(res.content)
+
+    with lock:
+        counter.value += 1
+        progress_str = f"{counter.value}/{total_count}"
+
+        click.echo("\x1b[0K", nl=False)
+        click.secho(progress_str, dim=True, italic=True, nl=False)
+        click.echo(f"\x1b[{len(progress_str)}D", nl=False)
+
+
+def download_nf_jars(pkg_root: Path):
+    s3_resource = boto3.resource("s3")
+    bucket = s3_resource.Bucket("latch-public")
+
+    objects = list(bucket.objects.filter(Prefix=".nextflow/"))
+
+    click.secho("  Downloading Nextflow binaries: \x1b[?25l", italic=True, nl=False)
+
+    with SyncManager() as man:
+        counter = man.Value(c_int, 0)
+        lock = man.Lock()
+        with ProcessPoolExecutor() as exec:
+            for obj in objects:
+                url = urljoin(
+                    "https://latch-public.s3.us-west-2.amazonaws.com/", obj.key
+                )
+                obj_path = pkg_root / ".latch" / obj.key
+
+                exec.submit(_do_download, url, obj_path, len(objects), counter, lock)
+
+    click.echo("\x1b[0K", nl=False)
+    click.secho("Done. \x1b[?25h", italic=True)
+
+
 # todo(ayush): add versioning system to nf download
 # todo(ayush): allow user to redownload nf anyway via cli option
 def ensure_nf_dependencies(pkg_root: Path, *, force_redownload: bool = False):
@@ -335,32 +384,7 @@ def ensure_nf_dependencies(pkg_root: Path, *, force_redownload: bool = False):
         nf_executable.chmod(0o700)
 
     if not nf_jars.exists():
-        click.secho(
-            "  Downloading Nextflow binaries: \x1b[?25l",
-            italic=True,
-            nl=False,
-        )
-
-        s3_resource = boto3.resource("s3")
-        bucket = s3_resource.Bucket("latch-public")
-
-        objects = list(bucket.objects.filter(Prefix=".nextflow/"))
-
-        # todo(ayush): parallelize
-        for i, obj in enumerate(objects):
-            obj_path = pkg_root / ".latch" / obj.key
-            obj_path.parent.mkdir(parents=True, exist_ok=True)
-
-            bucket.download_file(obj.key, str(obj_path))
-
-            progress_str = f"{i + 1}/{len(objects)}"
-
-            click.echo("\x1b[0K", nl=False)
-            click.secho(progress_str, dim=True, italic=True, nl=False)
-            click.echo(f"\x1b[{len(progress_str)}D", nl=False)
-
-        click.echo("\x1b[0K", nl=False)
-        click.secho("Done. \x1b[?25h", italic=True)
+        download_nf_jars(pkg_root)
 
 
 def build_nf_wf(

{latch-2.39.0.dev32 → latch-2.39.0.dev34}/latch_cli/extras/nextflow/tasks/operator.py

@@ -90,7 +90,7 @@ class NextflowOperatorTask(NextflowBaseTask):
             results.append(
                 reindent(
                     rf"""
-                    {out_name}=out_channels.get("{out_name}", "")
+                    {out_name}=out_channels.get("{out_name}", "[]")
                     """,
                     2,
                 ).rstrip()
@@ -188,17 +188,22 @@ class NextflowOperatorTask(NextflowBaseTask):
 
                     {download_str}
 
-                    subprocess.run(
-                        [{', '.join([f"str({x})" if x.startswith("wf_") else repr(x) for x in run_task_entrypoint])}],
-                        env={{
-                            **os.environ,
-                            "LATCH_CONFIG_DIR_OVERRIDE": str(Path.cwd()),
-                            "LATCH_EXPRESSION": {repr(self.statement)},
-                            "LATCH_RETURN": {repr(json.dumps(self.ret))},
-                            "LATCH_PARAM_VALS": json.dumps(channel_vals),
-                        }},
-                        check=True,
-                    )
+                    try:
+                        subprocess.run(
+                            [{', '.join([f"str({x})" if x.startswith("wf_") else repr(x) for x in run_task_entrypoint])}],
+                            env={{
+                                **os.environ,
+                                "LATCH_CONFIG_DIR_OVERRIDE": str(Path.cwd()),
+                                "LATCH_EXPRESSION": {repr(self.statement)},
+                                "LATCH_RETURN": {repr(json.dumps(self.ret))},
+                                "LATCH_PARAM_VALS": json.dumps(channel_vals),
+                            }},
+                            check=True,
+                        )
+                    except subprocess.CalledProcessError:
+                        log = Path("/root/.nextflow.log").read_text()
+                        print("\n\n\n\n\n" + log)
+                        raise
 
                     out_channels = {{}}
                     files = [Path(f) for f in glob.glob(".latch/task-outputs/*.json")]

{latch-2.39.0.dev32 → latch-2.39.0.dev34}/latch_cli/extras/nextflow/tasks/process.py

@@ -29,7 +29,7 @@ class NextflowProcessTask(NextflowBaseTask):
         wf: NextflowWorkflow,
     ):
         super().__init__(
-            inputs, outputs, id, name, {}, wf, NFTaskType.Process, cpu=16, memory=48
+            inputs, outputs, id, name, {}, wf, NFTaskType.Process, cpu=16, memory=96
         )
 
         self.wf_inputs = {}
@@ -85,7 +85,7 @@ class NextflowProcessTask(NextflowBaseTask):
             results.append(
                 reindent(
                     rf"""
-                    {field.name}=out_channels.get(f"{field.name}", "")
+                    {field.name}=out_channels.get(f"{field.name}", "[]")
                     """,
                     2,
                 ).rstrip()

{latch-2.39.0.dev32 → latch-2.39.0.dev34}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.39.0.dev32",
+    version="v2.39.0.dev34",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),