latch 2.41.3__tar.gz → 2.41.3.dev0__tar.gz

{latch-2.41.3/latch.egg-info → latch-2.41.3.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.41.3
+Version: 2.41.3.dev0
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.41.3 → latch-2.41.3.dev0}/latch/ldata/type.py

@@ -1,8 +1,14 @@
 from enum import Enum
+from typing import Optional
 
 
 class LatchPathError(RuntimeError):
-    def __init__(self, message: str, remote_path: str = None, acc_id: str = None):
+    def __init__(
+        self,
+        message: str,
+        remote_path: Optional[str] = None,
+        acc_id: Optional[str] = None,
+    ):
         super().__init__(message)
         self.message = message
         self.remote_path = remote_path

{latch-2.41.3 → latch-2.41.3.dev0}/latch/resources/tasks.py

@@ -33,9 +33,12 @@ from flytekit import task
 from flytekitplugins.pod import Pod
 from kubernetes.client.models import (
     V1Container,
+    V1PersistentVolumeClaimVolumeSource,
     V1PodSpec,
     V1ResourceRequirements,
     V1Toleration,
+    V1Volume,
+    V1VolumeMount,
 )
 
 from .dynamic import DynamicTaskConfig
@@ -179,6 +182,40 @@ def _get_small_pod() -> Pod:
     )
 
 
+def _get_nextflow_runtime_pod() -> Pod:
+    primary_container = V1Container(name="primary")
+    resources = V1ResourceRequirements(
+        requests={"cpu": "2", "memory": "4Gi", "ephemeral-storage": "100Gi"},
+        limits={"cpu": "2", "memory": "4Gi", "ephemeral-storage": "100Gi"},
+    )
+    primary_container.resources = resources
+    volume_mounts = [V1VolumeMount(mount_path="/nf-workdir", name="nextflow-workdir")]
+    primary_container.volume_mounts = volume_mounts
+
+    return Pod(
+        annotations={
+            "io.kubernetes.cri-o.userns-mode": (
+                "private:uidmapping=0:1048576:65536;gidmapping=0:1048576:65536"
+            )
+        },
+        pod_spec=V1PodSpec(
+            runtime_class_name="sysbox-runc",
+            service_account_name="rahul-test",
+            containers=[primary_container],
+            volumes=[
+                V1Volume(
+                    name="nextflow-workdir",
+                    persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
+                        # this value will be injected by flytepropeller
+                        claim_name="nextflow-pvc-placeholder"
+                    ),
+                )
+            ],
+        ),
+        primary_container_name="primary",
+    )
+
+
 large_gpu_task = functools.partial(task, task_config=_get_large_gpu_pod())
 """This task will get scheduled on a large GPU-enabled node.
 
@@ -314,6 +351,8 @@ small_task = functools.partial(task, task_config=_get_small_pod())
      - False
 """
 
+nextflow_runtime_task = functools.partial(task, task_config=_get_nextflow_runtime_pod())
+
 
 def custom_memory_optimized_task(cpu: int, memory: int):
     """Returns a custom task configuration requesting

{latch-2.41.3 → latch-2.41.3.dev0}/latch/types/metadata.py

@@ -3,11 +3,24 @@ from dataclasses import Field, asdict, dataclass, field
 from enum import Enum
 from pathlib import Path
 from textwrap import indent
-from typing import Any, ClassVar, Dict, List, Optional, Protocol, Tuple, Type, Union
+from typing import (
+    Any,
+    ClassVar,
+    Collection,
+    Dict,
+    Generic,
+    List,
+    Optional,
+    Protocol,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
 
 import click
 import yaml
-from typing_extensions import TypeAlias
+from typing_extensions import Annotated, TypeAlias
 
 from latch_cli.snakemake.config.utils import validate_snakemake_type
 from latch_cli.utils import identifier_suffix_from_str
@@ -16,7 +29,6 @@ from .directory import LatchDir
 from .file import LatchFile
 
 
-@dataclass
 class LatchRule:
     """Class describing a rule that a parameter input must follow"""
 
@@ -388,31 +400,33 @@ class _IsDataclass(Protocol):
 
 
 ParameterType: TypeAlias = Union[
-    Type[None],
-    Type[int],
-    Type[float],
-    Type[str],
-    Type[bool],
-    Type[Enum],
-    Type[_IsDataclass],
-    Type[List["ParameterType"]],
-    Type[LatchFile],
-    Type[LatchDir],
+    None,
+    int,
+    float,
+    str,
+    bool,
+    LatchFile,
+    LatchDir,
+    Enum,
+    _IsDataclass,
+    Collection["ParameterType"],
 ]
 
 
+T = TypeVar("T", bound=ParameterType)
+
+
 @dataclass
-class SnakemakeParameter(LatchParameter):
-    type: Optional[ParameterType] = None
+class SnakemakeParameter(Generic[T], LatchParameter):
+    type: Optional[Type[T]] = None
     """
     The python type of the parameter.
     """
-    # todo(ayush): needs to be typed properly
-    default: Optional[Any] = None
+    default: Optional[T] = None
 
 
 @dataclass
-class SnakemakeFileParameter(SnakemakeParameter):
+class SnakemakeFileParameter(SnakemakeParameter[Union[LatchFile, LatchDir]]):
     """
     Deprecated: use `file_metadata` keyword in `SnakemakeMetadata` instead
     """
@@ -456,6 +470,19 @@ class SnakemakeFileMetadata:
     """
 
 
+@dataclass
+class NextflowParameter(Generic[T], LatchParameter):
+    type: Optional[Type[T]] = None
+    """
+    The python type of the parameter.
+    """
+    default: Optional[T] = None
+
+    def __post_init__(self):
+        if self.samplesheet is True:
+            self.type = Annotated[self.type, "samplesheet"]
+
+
 @dataclass
 class LatchMetadata:
     """Class for organizing workflow metadata
@@ -651,3 +678,21 @@ class SnakemakeMetadata(LatchMetadata):
 
 
 _snakemake_metadata: Optional[SnakemakeMetadata] = None
+
+
+@dataclass
+class NextflowMetadata(LatchMetadata):
+    name: Optional[str] = None
+    parameters: Dict[str, NextflowParameter] = field(default_factory=dict)
+
+    def __post_init__(self):
+        if self.name is None:
+            self.name = f"nf_{identifier_suffix_from_str(self.display_name.lower())}"
+        else:
+            self.name = identifier_suffix_from_str(self.name)
+
+        global _nextflow_metadata
+        _nextflow_metadata = self
+
+
+_nextflow_metadata: Optional[NextflowMetadata] = None

{latch-2.41.3 → latch-2.41.3.dev0/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.41.3
+Version: 2.41.3.dev0
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.41.3 → latch-2.41.3.dev0}/latch.egg-info/SOURCES.txt

@@ -107,7 +107,13 @@ latch_cli/services/execute/utils.py
 latch_cli/services/init/__init__.py
 latch_cli/services/init/init.py
 latch_cli/services/init/__pycache__/__init__.cpython-310.pyc
+latch_cli/services/init/__pycache__/__init__.cpython-311.pyc
+latch_cli/services/init/__pycache__/__init__.cpython-38.pyc
+latch_cli/services/init/__pycache__/__init__.cpython-39.pyc
 latch_cli/services/init/__pycache__/init.cpython-310.pyc
+latch_cli/services/init/__pycache__/init.cpython-311.pyc
+latch_cli/services/init/__pycache__/init.cpython-38.pyc
+latch_cli/services/init/__pycache__/init.cpython-39.pyc
 latch_cli/services/init/assemble_and_sort/.env
 latch_cli/services/init/assemble_and_sort/LICENSE
 latch_cli/services/init/assemble_and_sort/README.md
@@ -115,18 +121,22 @@ latch_cli/services/init/assemble_and_sort/__init__.py
 latch_cli/services/init/assemble_and_sort/assemble.py
 latch_cli/services/init/assemble_and_sort/sort.py
 latch_cli/services/init/assemble_and_sort/system-requirements.txt
+latch_cli/services/init/assemble_and_sort/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/common/.dockerignore
 latch_cli/services/init/example_conda/__init__.py
 latch_cli/services/init/example_conda/conda_task.py
 latch_cli/services/init/example_conda/environment.yaml
+latch_cli/services/init/example_conda/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_docker/__init__.py
 latch_cli/services/init/example_docker/task.py
+latch_cli/services/init/example_nf_integration/latch_metadata/__pycache__/__init__.cpython-311.pyc
 latch_cli/services/init/example_nfcore/Dockerfile
 latch_cli/services/init/example_nfcore/__init__.py
 latch_cli/services/init/example_nfcore/task.py
 latch_cli/services/init/example_r/__init__.py
 latch_cli/services/init/example_r/environment.R
 latch_cli/services/init/example_r/r_task.py
+latch_cli/services/init/example_r/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_snakemake/Dockerfile
 latch_cli/services/init/example_snakemake/Snakefile
 latch_cli/services/init/example_snakemake/config.yaml
@@ -139,6 +149,7 @@ latch_cli/services/init/template/LICENSE
 latch_cli/services/init/template/README.md
 latch_cli/services/init/template/__init__.py
 latch_cli/services/init/template/task.py
+latch_cli/services/init/template/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/register/__init__.py
 latch_cli/services/register/constants.py
 latch_cli/services/register/register.py

{latch-2.41.3 → latch-2.41.3.dev0}/latch_cli/centromere/ctx.py

@@ -59,6 +59,7 @@ class _CentromereCtx:
     default_container: _Container
     workflow_type: WorkflowType
     snakefile: Optional[Path]
+    nf_script: Optional[Path]
 
     latch_register_api_url = config.api.workflow.register
     latch_image_api_url = config.api.workflow.upload_image
@@ -80,6 +81,7 @@ class _CentromereCtx:
         disable_auto_version: bool = False,
         remote: bool = False,
         snakefile: Optional[Path] = None,
+        nf_script: Optional[Path] = None,
         use_new_centromere: bool = False,
     ):
         self.use_new_centromere = use_new_centromere
@@ -93,13 +95,22 @@ class _CentromereCtx:
             self.dkr_repo = config.dkr_repo
             self.pkg_root = pkg_root.resolve()
 
-            if snakefile is None:
-                self.workflow_type = WorkflowType.latchbiosdk
-            else:
+            if snakefile and nf_script:
+                raise ValueError(
+                    "Cannot provide both a snakefile and nextflow script to the"
+                    " register command."
+                )
+            if snakefile is not None:
                 self.workflow_type = WorkflowType.snakemake
                 self.snakefile = snakefile
+            elif nf_script is not None:
+                self.workflow_type = WorkflowType.nextflow
+                self.nf_script = nf_script
+            else:
+                self.workflow_type = WorkflowType.latchbiosdk
 
             self.container_map: Dict[str, _Container] = {}
+
             if self.workflow_type == WorkflowType.latchbiosdk:
                 _import_flyte_objects([self.pkg_root])
                 for entity in FlyteEntities.entities:
@@ -126,7 +137,7 @@ class _CentromereCtx:
                         fg="red",
                     )
                     raise click.exceptions.Exit(1)
-            else:
+            elif self.workflow_type == WorkflowType.snakemake:
                 assert snakefile is not None
 
                 import latch.types.metadata as metadata
@@ -231,6 +242,32 @@ class _CentromereCtx:
                 # todo(kenny): support per container task and custom workflow
                 # name for snakemake
                 self.workflow_name = f"{metadata._snakemake_metadata.name}_jit_register"
+            else:
+                assert nf_script is not None
+
+                import latch.types.metadata as metadata
+
+                from ..services.register.utils import import_module_by_path
+
+                meta = pkg_root / "latch_metadata" / "__init__.py"
+                if meta.exists():
+                    click.echo(f"Using metadata file {click.style(meta, italic=True)}")
+                    import_module_by_path(meta)
+
+                if metadata._nextflow_metadata is None:
+                    click.secho(
+                        dedent("""
+                        Failed to register Nextflow workflow.
+                        Make sure the project root contains a `wf/__init__.py`
+                        with a `NextflowMetadata` object defined.
+                        """),
+                        fg="red",
+                    )
+                    raise click.exceptions.Exit(1)
+
+                self.workflow_name = metadata._nextflow_metadata.name
+
+            assert self.workflow_name is not None
 
             version_file = self.pkg_root / "version"
             try:

{latch-2.41.3 → latch-2.41.3.dev0}/latch_cli/docker_utils/__init__.py

@@ -39,7 +39,7 @@ def get_prologue(
         library_name = '"latch[snakemake]"'
     else:
         library_name = "latch"
-    return [
+    directives = [
         "# DO NOT CHANGE",
         f"from {config.base_image}",
         "",
@@ -68,6 +68,11 @@ def get_prologue(
         f"run pip install {library_name}=={config.latch_version}",
         "run mkdir /opt/latch",
     ]
+    if wf_type == WorkflowType.nextflow:
+        directives.append(
+            "run apt-get update && apt-get install -y default-jre-headless"
+        )
+    return directives
 
 
 def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
@@ -78,8 +83,18 @@ def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
             "",
             "# Latch snakemake workflow entrypoint",
             "# DO NOT CHANGE",
+            "",
             "copy .latch/snakemake_jit_entrypoint.py /root/snakemake_jit_entrypoint.py",
         ]
+    elif wf_type == WorkflowType.nextflow:
+        cmds += [
+            "",
+            "# Latch nextflow workflow entrypoint",
+            "# DO NOT CHANGE",
+            "",
+            "copy .latch/bin/nextflow /root/nextflow",
+            "copy .latch/.nextflow /root/.nextflow",
+        ]
 
     cmds += [
         "",

{latch-2.41.3 → latch-2.41.3.dev0}/latch_cli/main.py

@@ -192,7 +192,15 @@ def init(
     type=bool,
     help="Generate a Dockerfile with arguments needed for Snakemake compatability",
 )
-def dockerfile(pkg_root: str, snakemake: bool = False):
+@click.option(
+    "-n",
+    "--nextflow",
+    is_flag=True,
+    default=False,
+    type=bool,
+    help="Generate a Dockerfile with arguments needed for Nextflow compatability",
+)
+def dockerfile(pkg_root: str, snakemake: bool = False, nextflow: bool = False):
     """Generates a user editable dockerfile for a workflow and saves under `pkg_root/Dockerfile`.
 
     Visit docs.latch.bio to learn more.
@@ -201,6 +209,14 @@ def dockerfile(pkg_root: str, snakemake: bool = False):
     crash_handler.message = "Failed to generate Dockerfile."
     crash_handler.pkg_root = pkg_root
 
+    if snakemake is True and nextflow is True:
+        click.secho(
+            f"Please specify only one workflow type to generate metadata for. Use"
+            f" either `--snakemake` or `--nextflow`.",
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
     from latch_cli.docker_utils import generate_dockerfile
 
     source = Path(pkg_root)
@@ -209,11 +225,14 @@ def dockerfile(pkg_root: str, snakemake: bool = False):
         f"Dockerfile already exists at `{dest}`. Overwrite?"
     ):
         return
+
     workflow_type = WorkflowType.latchbiosdk
     if snakemake is True:
         workflow_type = WorkflowType.snakemake
-    generate_dockerfile(source, dest, wf_type=workflow_type)
+    elif nextflow is True:
+        workflow_type = WorkflowType.nextflow
 
+    generate_dockerfile(source, dest, wf_type=workflow_type)
     click.secho(f"Successfully generated dockerfile `{dest}`", fg="green")
 
 
@@ -412,6 +431,25 @@ def execute(
         " provided."
     ),
 )
+@click.option(
+    "--nf-script",
+    type=click.Path(exists=True, dir_okay=False, path_type=Path),
+    default=None,
+    help="Path to a nextflow script to register.",
+)
+@click.option(
+    "--redownload-dependencies",
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Redownload external Nextflow dependencies",
+)
+@click.option(
+    "--execution-profile",
+    type=str,
+    default=None,
+    help="Set execution profile for Nextflow workflow",
+)
 @requires_login
 def register(
     pkg_root: str,
@@ -422,6 +460,9 @@ def register(
     open: bool,
     snakefile: Optional[Path],
     cache_tasks: bool,
+    nf_script: Optional[Path],
+    redownload_dependencies: bool,
+    execution_profile: Optional[str],
 ):
     """Register local workflow code to Latch.
 
@@ -442,6 +483,9 @@ def register(
         skip_confirmation=yes,
         open=open,
         snakefile=snakefile,
+        nf_script=nf_script,
+        nf_redownload_dependencies=redownload_dependencies,
+        nf_execution_profile=execution_profile,
         progress_plain=(docker_progress == "auto" and not sys.stdout.isatty())
         or docker_progress == "plain",
         use_new_centromere=use_new_centromere,

{latch-2.41.3 → latch-2.41.3.dev0}/latch_cli/services/register/register.py

@@ -11,6 +11,7 @@ from typing import Iterable, List, Optional
 import click
 import gql
 import latch_sdk_gql.execute as l_gql
+from flytekit.core.workflow import WorkflowBase
 from scp import SCPClient
 
 from latch.utils import current_workspace, get_workspaces
@@ -198,40 +199,25 @@ def _build_and_serialize(
     dockerfile: Optional[Path] = None,
     *,
     progress_plain: bool = False,
-    cache_tasks: bool = False,
+    sm_jit_wf: Optional[WorkflowBase] = None,
 ):
     assert ctx.pkg_root is not None
 
-    jit_wf = None
-    if ctx.workflow_type == WorkflowType.snakemake:
-        assert ctx.snakefile is not None
-        assert ctx.version is not None
-
-        from ...snakemake.serialize import generate_jit_register_code
-        from ...snakemake.workflow import build_jit_register_wrapper
-
-        jit_wf = build_jit_register_wrapper(cache_tasks)
-        generate_jit_register_code(
-            jit_wf,
-            ctx.pkg_root,
-            ctx.snakefile,
-            ctx.version,
-            image_name,
-            current_workspace(),
-        )
-
     image_build_logs = build_image(ctx, image_name, context_path, dockerfile)
     print_and_write_build_logs(
         image_build_logs, image_name, ctx.pkg_root, progress_plain=progress_plain
     )
 
     if ctx.workflow_type == WorkflowType.snakemake:
-        assert jit_wf is not None
-        assert ctx.dkr_repo is not None
+        from ...snakemake.serialize import (
+            JITRegisterWorkflow,
+            serialize_jit_register_workflow,
+        )
 
-        from ...snakemake.serialize import serialize_jit_register_workflow
+        assert sm_jit_wf is not None and isinstance(sm_jit_wf, JITRegisterWorkflow)
+        assert ctx.dkr_repo is not None
 
-        serialize_jit_register_workflow(jit_wf, tmp_dir, image_name, ctx.dkr_repo)
+        serialize_jit_register_workflow(sm_jit_wf, tmp_dir, image_name, ctx.dkr_repo)
     else:
         serialize_logs, container_id = serialize_pkg_in_container(
             ctx, image_name, tmp_dir
@@ -274,6 +260,9 @@ def register(
     open: bool = False,
     skip_confirmation: bool = False,
     snakefile: Optional[Path] = None,
+    nf_script: Optional[Path] = None,
+    nf_redownload_dependencies: bool = False,
+    nf_execution_profile: Optional[str] = None,
     progress_plain: bool = False,
     cache_tasks: bool = False,
     use_new_centromere: bool = False,
@@ -335,6 +324,7 @@ def register(
         disable_auto_version=disable_auto_version,
         remote=remote,
         snakefile=snakefile,
+        nf_script=nf_script,
         use_new_centromere=use_new_centromere,
     ) as ctx:
         assert ctx.workflow_name is not None, "Unable to determine workflow name"
@@ -372,6 +362,25 @@ def register(
             ])
         )
 
+        click.echo(
+            " ".join([
+                click.style("Workflow type:", fg="bright_blue"),
+                ctx.workflow_type.value,
+            ])
+        )
+        if ctx.workflow_type == WorkflowType.snakemake:
+            click.echo(
+                " ".join(
+                    [click.style("Snakefile:", fg="bright_blue"), str(ctx.snakefile)]
+                )
+            )
+        elif ctx.workflow_type == WorkflowType.nextflow:
+            click.echo(
+                " ".join(
+                    [click.style("NF script:", fg="bright_blue"), str(ctx.nf_script)]
+                )
+            )
+
         if use_new_centromere:
             click.secho("Using experimental registration server.", fg="yellow")
 
@@ -382,6 +391,39 @@ def register(
         else:
             click.secho("Skipping confirmation because of --yes", bold=True)
 
+        sm_jit_wf = None
+        if ctx.workflow_type == WorkflowType.snakemake:
+            assert ctx.snakefile is not None
+            assert ctx.version is not None
+
+            from ...snakemake.serialize import generate_jit_register_code
+            from ...snakemake.workflow import build_jit_register_wrapper
+
+            sm_jit_wf = build_jit_register_wrapper(cache_tasks)
+            generate_jit_register_code(
+                sm_jit_wf,
+                ctx.pkg_root,
+                ctx.snakefile,
+                ctx.version,
+                ctx.default_container.image_name,
+                current_workspace(),
+            )
+        elif ctx.workflow_type == WorkflowType.nextflow:
+            assert ctx.nf_script is not None
+
+            from ...nextflow.dependencies import ensure_nf_dependencies
+            from ...nextflow.workflow import generate_nextflow_workflow
+
+            ensure_nf_dependencies(
+                Path(pkg_root), force_redownload=nf_redownload_dependencies
+            )
+            generate_nextflow_workflow(
+                ctx.pkg_root,
+                ctx.workflow_name,
+                ctx.nf_script,
+                execution_profile=nf_execution_profile,
+            )
+
         click.secho("\nInitializing registration", bold=True)
         transport = None
         scp = None
@@ -416,7 +458,7 @@ def register(
                 td,
                 dockerfile=ctx.default_container.dockerfile,
                 progress_plain=progress_plain,
-                cache_tasks=cache_tasks,
+                sm_jit_wf=sm_jit_wf,
             )
 
             if remote and snakefile is None:

{latch-2.41.3 → latch-2.41.3.dev0}/latch_cli/utils/__init__.py

@@ -384,6 +384,7 @@ class TemporarySSHCredentials:
 class WorkflowType(Enum):
     latchbiosdk = "latchbiosdk"
     snakemake = "snakemake"
+    nextflow = "nextflow"
 
 
 def identifier_suffix_from_str(x: str) -> str: