latch 2.41.3__tar.gz → 2.41.3.dev1__tar.gz

{latch-2.41.3/latch.egg-info → latch-2.41.3.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.41.3
+Version: 2.41.3.dev1
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.41.3 → latch-2.41.3.dev1}/latch/ldata/type.py

@@ -1,8 +1,14 @@
 from enum import Enum
+from typing import Optional
 
 
 class LatchPathError(RuntimeError):
-    def __init__(self, message: str, remote_path: str = None, acc_id: str = None):
+    def __init__(
+        self,
+        message: str,
+        remote_path: Optional[str] = None,
+        acc_id: Optional[str] = None,
+    ):
         super().__init__(message)
         self.message = message
         self.remote_path = remote_path

{latch-2.41.3 → latch-2.41.3.dev1}/latch/resources/tasks.py

@@ -33,9 +33,12 @@ from flytekit import task
 from flytekitplugins.pod import Pod
 from kubernetes.client.models import (
     V1Container,
+    V1PersistentVolumeClaimVolumeSource,
     V1PodSpec,
     V1ResourceRequirements,
     V1Toleration,
+    V1Volume,
+    V1VolumeMount,
 )
 
 from .dynamic import DynamicTaskConfig
@@ -179,6 +182,40 @@ def _get_small_pod() -> Pod:
     )
 
 
+def _get_nextflow_runtime_pod() -> Pod:
+    primary_container = V1Container(name="primary")
+    resources = V1ResourceRequirements(
+        requests={"cpu": "2", "memory": "4Gi", "ephemeral-storage": "100Gi"},
+        limits={"cpu": "2", "memory": "4Gi", "ephemeral-storage": "100Gi"},
+    )
+    primary_container.resources = resources
+    volume_mounts = [V1VolumeMount(mount_path="/nf-workdir", name="nextflow-workdir")]
+    primary_container.volume_mounts = volume_mounts
+
+    return Pod(
+        annotations={
+            "io.kubernetes.cri-o.userns-mode": (
+                "private:uidmapping=0:1048576:65536;gidmapping=0:1048576:65536"
+            )
+        },
+        pod_spec=V1PodSpec(
+            runtime_class_name="sysbox-runc",
+            service_account_name="rahul-test",
+            containers=[primary_container],
+            volumes=[
+                V1Volume(
+                    name="nextflow-workdir",
+                    persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
+                        # this value will be injected by flytepropeller
+                        claim_name="nextflow-pvc-placeholder"
+                    ),
+                )
+            ],
+        ),
+        primary_container_name="primary",
+    )
+
+
 large_gpu_task = functools.partial(task, task_config=_get_large_gpu_pod())
 """This task will get scheduled on a large GPU-enabled node.
 
@@ -314,6 +351,8 @@ small_task = functools.partial(task, task_config=_get_small_pod())
      - False
 """
 
+nextflow_runtime_task = functools.partial(task, task_config=_get_nextflow_runtime_pod())
+
 
 def custom_memory_optimized_task(cpu: int, memory: int):
     """Returns a custom task configuration requesting

{latch-2.41.3 → latch-2.41.3.dev1}/latch/types/metadata.py

@@ -3,11 +3,24 @@ from dataclasses import Field, asdict, dataclass, field
 from enum import Enum
 from pathlib import Path
 from textwrap import indent
-from typing import Any, ClassVar, Dict, List, Optional, Protocol, Tuple, Type, Union
+from typing import (
+    Any,
+    ClassVar,
+    Collection,
+    Dict,
+    Generic,
+    List,
+    Optional,
+    Protocol,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
 
 import click
 import yaml
-from typing_extensions import TypeAlias
+from typing_extensions import Annotated, TypeAlias
 
 from latch_cli.snakemake.config.utils import validate_snakemake_type
 from latch_cli.utils import identifier_suffix_from_str
@@ -16,7 +29,6 @@ from .directory import LatchDir
 from .file import LatchFile
 
 
-@dataclass
 class LatchRule:
     """Class describing a rule that a parameter input must follow"""
 
@@ -388,31 +400,33 @@ class _IsDataclass(Protocol):
 
 
 ParameterType: TypeAlias = Union[
-    Type[None],
-    Type[int],
-    Type[float],
-    Type[str],
-    Type[bool],
-    Type[Enum],
-    Type[_IsDataclass],
-    Type[List["ParameterType"]],
-    Type[LatchFile],
-    Type[LatchDir],
+    None,
+    int,
+    float,
+    str,
+    bool,
+    LatchFile,
+    LatchDir,
+    Enum,
+    _IsDataclass,
+    Collection["ParameterType"],
 ]
 
 
+T = TypeVar("T", bound=ParameterType)
+
+
 @dataclass
-class SnakemakeParameter(LatchParameter):
-    type: Optional[ParameterType] = None
+class SnakemakeParameter(Generic[T], LatchParameter):
+    type: Optional[Type[T]] = None
     """
     The python type of the parameter.
     """
-    # todo(ayush): needs to be typed properly
-    default: Optional[Any] = None
+    default: Optional[T] = None
 
 
 @dataclass
-class SnakemakeFileParameter(SnakemakeParameter):
+class SnakemakeFileParameter(SnakemakeParameter[Union[LatchFile, LatchDir]]):
     """
     Deprecated: use `file_metadata` keyword in `SnakemakeMetadata` instead
     """
@@ -456,6 +470,19 @@ class SnakemakeFileMetadata:
     """
 
 
+@dataclass
+class NextflowParameter(Generic[T], LatchParameter):
+    type: Optional[Type[T]] = None
+    """
+    The python type of the parameter.
+    """
+    default: Optional[T] = None
+
+    def __post_init__(self):
+        if self.samplesheet is True:
+            self.type = Annotated[self.type, "samplesheet"]
+
+
 @dataclass
 class LatchMetadata:
     """Class for organizing workflow metadata
@@ -651,3 +678,21 @@ class SnakemakeMetadata(LatchMetadata):
 
 
 _snakemake_metadata: Optional[SnakemakeMetadata] = None
+
+
+@dataclass
+class NextflowMetadata(LatchMetadata):
+    name: Optional[str] = None
+    parameters: Dict[str, NextflowParameter] = field(default_factory=dict)
+
+    def __post_init__(self):
+        if self.name is None:
+            self.name = f"nf_{identifier_suffix_from_str(self.display_name.lower())}"
+        else:
+            self.name = identifier_suffix_from_str(self.name)
+
+        global _nextflow_metadata
+        _nextflow_metadata = self
+
+
+_nextflow_metadata: Optional[NextflowMetadata] = None

{latch-2.41.3 → latch-2.41.3.dev1/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.41.3
+Version: 2.41.3.dev1
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.41.3 → latch-2.41.3.dev1}/latch.egg-info/SOURCES.txt

@@ -80,6 +80,9 @@ latch_cli/exceptions/cache.py
 latch_cli/exceptions/errors.py
 latch_cli/exceptions/handler.py
 latch_cli/exceptions/traceback.py
+latch_cli/nextflow/__init__.py
+latch_cli/nextflow/dependencies.py
+latch_cli/nextflow/workflow.py
 latch_cli/services/__init__.py
 latch_cli/services/get.py
 latch_cli/services/get_executions.py
@@ -107,7 +110,13 @@ latch_cli/services/execute/utils.py
 latch_cli/services/init/__init__.py
 latch_cli/services/init/init.py
 latch_cli/services/init/__pycache__/__init__.cpython-310.pyc
+latch_cli/services/init/__pycache__/__init__.cpython-311.pyc
+latch_cli/services/init/__pycache__/__init__.cpython-38.pyc
+latch_cli/services/init/__pycache__/__init__.cpython-39.pyc
 latch_cli/services/init/__pycache__/init.cpython-310.pyc
+latch_cli/services/init/__pycache__/init.cpython-311.pyc
+latch_cli/services/init/__pycache__/init.cpython-38.pyc
+latch_cli/services/init/__pycache__/init.cpython-39.pyc
 latch_cli/services/init/assemble_and_sort/.env
 latch_cli/services/init/assemble_and_sort/LICENSE
 latch_cli/services/init/assemble_and_sort/README.md
@@ -115,18 +124,22 @@ latch_cli/services/init/assemble_and_sort/__init__.py
 latch_cli/services/init/assemble_and_sort/assemble.py
 latch_cli/services/init/assemble_and_sort/sort.py
 latch_cli/services/init/assemble_and_sort/system-requirements.txt
+latch_cli/services/init/assemble_and_sort/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/common/.dockerignore
 latch_cli/services/init/example_conda/__init__.py
 latch_cli/services/init/example_conda/conda_task.py
 latch_cli/services/init/example_conda/environment.yaml
+latch_cli/services/init/example_conda/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_docker/__init__.py
 latch_cli/services/init/example_docker/task.py
+latch_cli/services/init/example_nf_integration/latch_metadata/__pycache__/__init__.cpython-311.pyc
 latch_cli/services/init/example_nfcore/Dockerfile
 latch_cli/services/init/example_nfcore/__init__.py
 latch_cli/services/init/example_nfcore/task.py
 latch_cli/services/init/example_r/__init__.py
 latch_cli/services/init/example_r/environment.R
 latch_cli/services/init/example_r/r_task.py
+latch_cli/services/init/example_r/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_snakemake/Dockerfile
 latch_cli/services/init/example_snakemake/Snakefile
 latch_cli/services/init/example_snakemake/config.yaml
@@ -139,6 +152,7 @@ latch_cli/services/init/template/LICENSE
 latch_cli/services/init/template/README.md
 latch_cli/services/init/template/__init__.py
 latch_cli/services/init/template/task.py
+latch_cli/services/init/template/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/register/__init__.py
 latch_cli/services/register/constants.py
 latch_cli/services/register/register.py

{latch-2.41.3 → latch-2.41.3.dev1}/latch_cli/centromere/ctx.py

@@ -59,6 +59,7 @@ class _CentromereCtx:
     default_container: _Container
     workflow_type: WorkflowType
     snakefile: Optional[Path]
+    nf_script: Optional[Path]
 
     latch_register_api_url = config.api.workflow.register
     latch_image_api_url = config.api.workflow.upload_image
@@ -80,6 +81,7 @@ class _CentromereCtx:
         disable_auto_version: bool = False,
         remote: bool = False,
         snakefile: Optional[Path] = None,
+        nf_script: Optional[Path] = None,
         use_new_centromere: bool = False,
     ):
         self.use_new_centromere = use_new_centromere
@@ -93,13 +95,22 @@ class _CentromereCtx:
             self.dkr_repo = config.dkr_repo
             self.pkg_root = pkg_root.resolve()
 
-            if snakefile is None:
-                self.workflow_type = WorkflowType.latchbiosdk
-            else:
+            if snakefile and nf_script:
+                raise ValueError(
+                    "Cannot provide both a snakefile and nextflow script to the"
+                    " register command."
+                )
+            if snakefile is not None:
                 self.workflow_type = WorkflowType.snakemake
                 self.snakefile = snakefile
+            elif nf_script is not None:
+                self.workflow_type = WorkflowType.nextflow
+                self.nf_script = nf_script
+            else:
+                self.workflow_type = WorkflowType.latchbiosdk
 
             self.container_map: Dict[str, _Container] = {}
+
             if self.workflow_type == WorkflowType.latchbiosdk:
                 _import_flyte_objects([self.pkg_root])
                 for entity in FlyteEntities.entities:
@@ -126,7 +137,7 @@ class _CentromereCtx:
                         fg="red",
                     )
                     raise click.exceptions.Exit(1)
-            else:
+            elif self.workflow_type == WorkflowType.snakemake:
                 assert snakefile is not None
 
                 import latch.types.metadata as metadata
@@ -231,6 +242,32 @@ class _CentromereCtx:
                 # todo(kenny): support per container task and custom workflow
                 # name for snakemake
                 self.workflow_name = f"{metadata._snakemake_metadata.name}_jit_register"
+            else:
+                assert nf_script is not None
+
+                import latch.types.metadata as metadata
+
+                from ..services.register.utils import import_module_by_path
+
+                meta = pkg_root / "latch_metadata" / "__init__.py"
+                if meta.exists():
+                    click.echo(f"Using metadata file {click.style(meta, italic=True)}")
+                    import_module_by_path(meta)
+
+                if metadata._nextflow_metadata is None:
+                    click.secho(
+                        dedent("""
+                        Failed to register Nextflow workflow.
+                        Make sure the project root contains a `wf/__init__.py`
+                        with a `NextflowMetadata` object defined.
+                        """),
+                        fg="red",
+                    )
+                    raise click.exceptions.Exit(1)
+
+                self.workflow_name = metadata._nextflow_metadata.name
+
+            assert self.workflow_name is not None
 
             version_file = self.pkg_root / "version"
             try:

{latch-2.41.3 → latch-2.41.3.dev1}/latch_cli/docker_utils/__init__.py

@@ -39,7 +39,7 @@ def get_prologue(
         library_name = '"latch[snakemake]"'
     else:
         library_name = "latch"
-    return [
+    directives = [
         "# DO NOT CHANGE",
         f"from {config.base_image}",
         "",
@@ -68,6 +68,11 @@ def get_prologue(
         f"run pip install {library_name}=={config.latch_version}",
         "run mkdir /opt/latch",
     ]
+    if wf_type == WorkflowType.nextflow:
+        directives.append(
+            "run apt-get update && apt-get install -y default-jre-headless"
+        )
+    return directives
 
 
 def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
@@ -78,8 +83,18 @@ def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
             "",
             "# Latch snakemake workflow entrypoint",
             "# DO NOT CHANGE",
+            "",
             "copy .latch/snakemake_jit_entrypoint.py /root/snakemake_jit_entrypoint.py",
         ]
+    elif wf_type == WorkflowType.nextflow:
+        cmds += [
+            "",
+            "# Latch nextflow workflow entrypoint",
+            "# DO NOT CHANGE",
+            "",
+            "copy .latch/bin/nextflow /root/nextflow",
+            "copy .latch/.nextflow /root/.nextflow",
+        ]
 
     cmds += [
         "",

{latch-2.41.3 → latch-2.41.3.dev1}/latch_cli/main.py

@@ -192,7 +192,15 @@ def init(
     type=bool,
     help="Generate a Dockerfile with arguments needed for Snakemake compatability",
 )
-def dockerfile(pkg_root: str, snakemake: bool = False):
+@click.option(
+    "-n",
+    "--nextflow",
+    is_flag=True,
+    default=False,
+    type=bool,
+    help="Generate a Dockerfile with arguments needed for Nextflow compatability",
+)
+def dockerfile(pkg_root: str, snakemake: bool = False, nextflow: bool = False):
     """Generates a user editable dockerfile for a workflow and saves under `pkg_root/Dockerfile`.
 
     Visit docs.latch.bio to learn more.
@@ -201,6 +209,14 @@ def dockerfile(pkg_root: str, snakemake: bool = False):
     crash_handler.message = "Failed to generate Dockerfile."
     crash_handler.pkg_root = pkg_root
 
+    if snakemake is True and nextflow is True:
+        click.secho(
+            f"Please specify only one workflow type to generate metadata for. Use"
+            f" either `--snakemake` or `--nextflow`.",
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
     from latch_cli.docker_utils import generate_dockerfile
 
     source = Path(pkg_root)
@@ -209,11 +225,14 @@ def dockerfile(pkg_root: str, snakemake: bool = False):
         f"Dockerfile already exists at `{dest}`. Overwrite?"
     ):
         return
+
     workflow_type = WorkflowType.latchbiosdk
     if snakemake is True:
         workflow_type = WorkflowType.snakemake
-    generate_dockerfile(source, dest, wf_type=workflow_type)
+    elif nextflow is True:
+        workflow_type = WorkflowType.nextflow
 
+    generate_dockerfile(source, dest, wf_type=workflow_type)
     click.secho(f"Successfully generated dockerfile `{dest}`", fg="green")
 
 
@@ -412,6 +431,25 @@ def execute(
         " provided."
     ),
 )
+@click.option(
+    "--nf-script",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    default=None,
+    help="Path to a nextflow script to register.",
+)
+@click.option(
+    "--redownload-dependencies",
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Redownload external Nextflow dependencies",
+)
+@click.option(
+    "--execution-profile",
+    type=str,
+    default=None,
+    help="Set execution profile for Nextflow workflow",
+)
 @requires_login
 def register(
     pkg_root: str,
@@ -422,6 +460,9 @@ def register(
     open: bool,
     snakefile: Optional[Path],
     cache_tasks: bool,
+    nf_script: Optional[Path],
+    redownload_dependencies: bool,
+    execution_profile: Optional[str],
 ):
     """Register local workflow code to Latch.
 
@@ -442,6 +483,9 @@ def register(
         skip_confirmation=yes,
         open=open,
         snakefile=snakefile,
+        nf_script=nf_script,
+        nf_redownload_dependencies=redownload_dependencies,
+        nf_execution_profile=execution_profile,
         progress_plain=(docker_progress == "auto" and not sys.stdout.isatty())
         or docker_progress == "plain",
         use_new_centromere=use_new_centromere,

latch-2.41.3.dev1/latch_cli/nextflow/dependencies.py

@@ -0,0 +1,91 @@
+import shutil
+import subprocess
+from concurrent.futures import ProcessPoolExecutor
+from ctypes import c_int
+from multiprocessing.managers import SyncManager
+from pathlib import Path
+from urllib.parse import urljoin
+
+import boto3
+import click
+
+from latch_cli import tinyrequests
+from latch_cli.utils import dedent
+
+
+def _do_download(
+    url: str,
+    output_path: Path,
+    total_count: int,
+    counter,
+    lock,
+):  # todo(ayush): figure out the right type annotation for counter/lock
+    res = tinyrequests.get(url)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_bytes(res.content)
+
+    with lock:
+        counter.value += 1
+        progress_str = f"{counter.value}/{total_count}"
+
+        click.echo("\x1b[0K", nl=False)
+        click.secho(progress_str, dim=True, italic=True, nl=False)
+        click.echo(f"\x1b[{len(progress_str)}D", nl=False)
+
+
+def download_nf_jars(pkg_root: Path):
+    s3_resource = boto3.resource("s3")
+    bucket = s3_resource.Bucket("latch-public")
+
+    objects = list(bucket.objects.filter(Prefix=".nextflow/"))
+
+    click.secho("  Downloading Nextflow binaries: \x1b[?25l", italic=True, nl=False)
+
+    with SyncManager() as man:
+        counter = man.Value(c_int, 0)
+        lock = man.Lock()
+        with ProcessPoolExecutor() as exec:
+            for obj in objects:
+                url = urljoin(
+                    "https://latch-public.s3.us-west-2.amazonaws.com/", obj.key
+                )
+                obj_path = pkg_root / ".latch" / obj.key
+
+                exec.submit(_do_download, url, obj_path, len(objects), counter, lock)
+
+    click.echo("\x1b[0K", nl=False)
+    click.secho("Done. \x1b[?25h", italic=True)
+
+
+def ensure_nf_dependencies(pkg_root: Path, *, force_redownload: bool = False):
+    try:
+        subprocess.run(["java", "--version"], check=True, capture_output=True)
+    except (subprocess.CalledProcessError, FileNotFoundError) as e:
+        click.secho(
+            dedent("""\
+            Java is not installed - this is a requirement to run Nextflow.
+            Please install Java and try again.
+            """),
+            fg="red",
+        )
+        raise click.exceptions.Exit(1) from e
+
+    nf_executable = pkg_root / ".latch" / "bin" / "nextflow"
+    nf_jars = pkg_root / ".latch" / ".nextflow"
+
+    if force_redownload:
+        nf_executable.unlink(missing_ok=True)
+        if nf_jars.exists():
+            shutil.rmtree(nf_jars)
+
+    if not nf_executable.exists():
+        res = tinyrequests.get(
+            "https://latch-public.s3.us-west-2.amazonaws.com/nextflow"
+        )
+        nf_executable.parent.mkdir(parents=True, exist_ok=True)
+
+        nf_executable.write_bytes(res.content)
+        nf_executable.chmod(0o700)
+
+    if not nf_jars.exists():
+        download_nf_jars(pkg_root)