latch 2.45.2__tar.gz → 2.45.2.dev2__tar.gz

latch-2.45.2.dev2/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.1
+Name: latch
+Version: 2.45.2.dev2
+Summary: The Latch SDK
+Author-email: kenny@latch.bio
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.8,<3.12
+License-File: LICENSE
+Requires-Dist: kubernetes>=24.2.0
+Requires-Dist: pyjwt>=0.2.0
+Requires-Dist: requests>=2.28.1
+Requires-Dist: click>=8.0
+Requires-Dist: docker>=7.1.0
+Requires-Dist: paramiko>=3.2.0
+Requires-Dist: scp>=0.14.0
+Requires-Dist: boto3>=1.26.0
+Requires-Dist: tqdm>=4.63.0
+Requires-Dist: lytekit==0.15.8
+Requires-Dist: lytekitplugins-pods==0.6.2
+Requires-Dist: typing-extensions==4.7.1
+Requires-Dist: apscheduler==3.9.1
+Requires-Dist: gql==3.4.0
+Requires-Dist: graphql-core==3.2.3
+Requires-Dist: requests-toolbelt==0.10.1
+Requires-Dist: latch-sdk-gql==0.0.6
+Requires-Dist: latch-sdk-config==0.0.4
+Requires-Dist: python-dateutil>=2.8
+Requires-Dist: aioconsole==0.6.1
+Requires-Dist: asyncssh==2.13.2
+Requires-Dist: websockets==11.0.3
+Requires-Dist: watchfiles==0.19.0
+Provides-Extra: snakemake
+Requires-Dist: snakemake<7.30.2,>=7.18.0; extra == "snakemake"
+Requires-Dist: pulp<2.8,>=2.0; extra == "snakemake"
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0.0; extra == "pandas"

{latch-2.45.2 → latch-2.45.2.dev2}/latch/ldata/type.py

@@ -1,8 +1,14 @@
 from enum import Enum
+from typing import Optional
 
 
 class LatchPathError(RuntimeError):
-    def __init__(self, message: str, remote_path: str = None, acc_id: str = None):
+    def __init__(
+        self,
+        message: str,
+        remote_path: Optional[str] = None,
+        acc_id: Optional[str] = None,
+    ):
         super().__init__(message)
         self.message = message
         self.remote_path = remote_path

{latch-2.45.2 → latch-2.45.2.dev2}/latch/resources/tasks.py

@@ -34,9 +34,12 @@ from flytekit import task
 from flytekitplugins.pod import Pod
 from kubernetes.client.models import (
     V1Container,
+    V1PersistentVolumeClaimVolumeSource,
     V1PodSpec,
     V1ResourceRequirements,
     V1Toleration,
+    V1Volume,
+    V1VolumeMount,
 )
 
 from latch_cli.constants import Units
@@ -474,3 +477,43 @@ def custom_task(
     return functools.partial(
         task, task_config=_custom_task_config(cpu, memory, storage_gib), timeout=timeout
     )
+
+
+def nextflow_runtime_task(cpu: int, memory: int):
+    primary_container = V1Container(name="primary")
+    resources = V1ResourceRequirements(
+        requests={
+            "cpu": str(cpu),
+            "memory": f"{memory}Gi",
+            "ephemeral-storage": "20Gi",
+        },
+        limits={"cpu": str(cpu), "memory": f"{memory}Gi", "ephemeral-storage": "20Gi"},
+    )
+    primary_container.resources = resources
+    volume_mounts = [V1VolumeMount(mount_path="/nf-workdir", name="nextflow-workdir")]
+    primary_container.volume_mounts = volume_mounts
+
+    task_config = Pod(
+        annotations={
+            "io.kubernetes.cri-o.userns-mode": (
+                "private:uidmapping=0:1048576:65536;gidmapping=0:1048576:65536"
+            )
+        },
+        pod_spec=V1PodSpec(
+            runtime_class_name="sysbox-runc",
+            automount_service_account_token=True,
+            containers=[primary_container],
+            volumes=[
+                V1Volume(
+                    name="nextflow-workdir",
+                    persistent_volume_claim=V1PersistentVolumeClaimVolumeSource(
+                        # this value will be injected by flytepropeller
+                        claim_name="nextflow-pvc-placeholder"
+                    ),
+                )
+            ],
+        ),
+        primary_container_name="primary",
+    )
+
+    return functools.partial(task, task_config=task_config)

{latch-2.45.2 → latch-2.45.2.dev2}/latch/types/directory.py

@@ -114,8 +114,10 @@ class LatchDir(FlyteDirectory):
         self._path_generated = False
 
         if is_valid_url(self.path) and remote_path is None:
+            self._raw_remote_path = str(path)
             self._remote_directory = self.path
         else:
+            self._raw_remote_path = str(remote_path)
             self._remote_directory = None if remote_path is None else str(remote_path)
 
         if kwargs.get("downloader") is not None:

{latch-2.45.2 → latch-2.45.2.dev2}/latch/types/file.py

@@ -77,8 +77,10 @@ class LatchFile(FlyteFile):
         self._path_generated = False
 
         if is_valid_url(self.path) and remote_path is None:
+            self._raw_remote_path = str(path)
             self._remote_path = str(path)
         else:
+            self._raw_remote_path = str(remote_path)
             self._remote_path = None if remote_path is None else str(remote_path)
 
         if kwargs.get("downloader") is not None:

{latch-2.45.2 → latch-2.45.2.dev2}/latch/types/metadata.py

@@ -1,13 +1,32 @@
+import csv
+import functools
 import re
-from dataclasses import Field, asdict, dataclass, field
+from dataclasses import Field, asdict, dataclass, field, fields, is_dataclass
 from enum import Enum
 from pathlib import Path
-from textwrap import indent
-from typing import Any, ClassVar, Dict, List, Optional, Protocol, Tuple, Type, Union
+from textwrap import dedent, indent
+from typing import (
+    Any,
+    Callable,
+    ClassVar,
+    Collection,
+    Dict,
+    Generic,
+    List,
+    Literal,
+    Optional,
+    Protocol,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    get_args,
+    get_origin,
+)
 
 import click
 import yaml
-from typing_extensions import TypeAlias
+from typing_extensions import Annotated, TypeAlias
 
 from latch_cli.snakemake.config.utils import validate_snakemake_type
 from latch_cli.utils import identifier_suffix_from_str
@@ -16,7 +35,6 @@ from .directory import LatchDir
 from .file import LatchFile
 
 
-@dataclass
 class LatchRule:
     """Class describing a rule that a parameter input must follow"""
 
@@ -388,31 +406,33 @@ class _IsDataclass(Protocol):
 
 
 ParameterType: TypeAlias = Union[
-    Type[None],
-    Type[int],
-    Type[float],
-    Type[str],
-    Type[bool],
-    Type[Enum],
-    Type[_IsDataclass],
-    Type[List["ParameterType"]],
-    Type[LatchFile],
-    Type[LatchDir],
+    None,
+    int,
+    float,
+    str,
+    bool,
+    LatchFile,
+    LatchDir,
+    Enum,
+    _IsDataclass,
+    Collection["ParameterType"],
 ]
 
 
+T = TypeVar("T", bound=ParameterType)
+
+
 @dataclass
-class SnakemakeParameter(LatchParameter):
-    type: Optional[ParameterType] = None
+class SnakemakeParameter(Generic[T], LatchParameter):
+    type: Optional[Type[T]] = None
     """
     The python type of the parameter.
     """
-    # todo(ayush): needs to be typed properly
-    default: Optional[Any] = None
+    default: Optional[T] = None
 
 
 @dataclass
-class SnakemakeFileParameter(SnakemakeParameter):
+class SnakemakeFileParameter(SnakemakeParameter[Union[LatchFile, LatchDir]]):
     """
     Deprecated: use `file_metadata` keyword in `SnakemakeMetadata` instead
     """
@@ -456,6 +476,108 @@ class SnakemakeFileMetadata:
     """
 
 
+@dataclass
+class NextflowParameter(Generic[T], LatchParameter):
+    type: Optional[Type[T]] = None
+    """
+    The python type of the parameter.
+    """
+    default: Optional[T] = None
+    """
+    Default value of the parameter
+    """
+
+    samplesheet_type: Literal["csv", "tsv", None] = None
+    """
+    The type of samplesheet to construct from the input parameter.
+
+    Only used if the provided parameter is a samplesheet (samplesheet=True)
+    """
+    samplesheet_constructor: Optional[Callable[[T], Path]] = None
+    """
+    A custom samplesheet constructor.
+
+    Should return the path of the constructed samplesheet. If samplesheet_type is also specified, this takes precedence.
+    Only used if the provided parameter is a samplesheet (samplesheet=True)
+    """
+
+    def __post_init__(self):
+        if not self.samplesheet or self.samplesheet_constructor is not None:
+            return
+
+        t = self.type
+        if get_origin(t) is not list or not is_dataclass(get_args(t)[0]):
+            click.secho(
+                dedent("""\
+                Samplesheets must be a list of dataclasses.
+                """),
+                fg="red",
+            )
+            raise click.exceptions.Exit(1)
+
+        if self.samplesheet_type is not None:
+            delim = "," if self.samplesheet_type == "csv" else "\t"
+            self.samplesheet_constructor = functools.partial(
+                _samplesheet_constructor, t=get_args(self.type)[0], delim=delim
+            )
+            return
+
+        click.secho(
+            dedent("""\
+            A Samplesheet constructor is required for a samplesheet parameter. Please either provide a value for
+            `samplesheet_type` or provide a custom callable to the `samplesheet_constructor` argument.
+            """),
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
+
+DC = TypeVar("DC", bound=_IsDataclass)
+
+
+def _samplesheet_repr(v: Any) -> str:
+    if v is None:
+        return ""
+    if isinstance(v, LatchFile) or isinstance(v, LatchDir):
+        return v.remote_path
+    if isinstance(v, Enum):
+        return getattr(v, "value")
+
+    return str(v)
+
+
+def _samplesheet_constructor(samples: List[DC], t: DC, delim: str = ",") -> Path:
+    samplesheet = Path("samplesheet.csv")
+
+    with open(samplesheet, "w") as f:
+        writer = csv.DictWriter(f, [f.name for f in fields(t)], delimiter=delim)
+        writer.writeheader()
+
+        for sample in samples:
+            row_data = {
+                f.name: _samplesheet_repr(getattr(sample, f.name))
+                for f in fields(sample)
+            }
+            writer.writerow(row_data)
+
+    return samplesheet
+
+
+@dataclass(frozen=True)
+class NextflowRuntimeResources:
+    """Resources for Nextflow runtime tasks"""
+
+    cpus: Optional[int] = 4
+    """
+    Number of CPUs required for the task
+    """
+    memory: Optional[str] = 8
+    """
+    Memory required for the task (e.g. "1 GB")
+    """
+    storage_gib: Optional[int] = 100
+
+
 @dataclass
 class LatchMetadata:
     """Class for organizing workflow metadata
@@ -651,3 +773,37 @@ class SnakemakeMetadata(LatchMetadata):
 
 
 _snakemake_metadata: Optional[SnakemakeMetadata] = None
+
+
+@dataclass
+class NextflowMetadata(LatchMetadata):
+    name: Optional[str] = None
+    """
+    Name of the workflow
+    """
+    parameters: Dict[str, NextflowParameter] = field(default_factory=dict)
+    """
+    A dictionary mapping parameter names (strings) to `NextflowParameter` objects
+    """
+    runtime_resources: NextflowRuntimeResources = field(
+        default_factory=NextflowRuntimeResources
+    )
+    """
+    Resources (cpu/memory/storage) for Nextflow runtime task
+    """
+    output_dir: Optional[LatchDir] = None
+    """
+    Directory to dump Nextflow logs
+    """
+
+    def __post_init__(self):
+        if self.name is None:
+            self.name = f"nf_{identifier_suffix_from_str(self.display_name.lower())}"
+        else:
+            self.name = identifier_suffix_from_str(self.name)
+
+        global _nextflow_metadata
+        _nextflow_metadata = self
+
+
+_nextflow_metadata: Optional[NextflowMetadata] = None

latch-2.45.2.dev2/latch.egg-info/PKG-INFO

@@ -0,0 +1,39 @@
+Metadata-Version: 2.1
+Name: latch
+Version: 2.45.2.dev2
+Summary: The Latch SDK
+Author-email: kenny@latch.bio
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Python: >=3.8,<3.12
+License-File: LICENSE
+Requires-Dist: kubernetes>=24.2.0
+Requires-Dist: pyjwt>=0.2.0
+Requires-Dist: requests>=2.28.1
+Requires-Dist: click>=8.0
+Requires-Dist: docker>=7.1.0
+Requires-Dist: paramiko>=3.2.0
+Requires-Dist: scp>=0.14.0
+Requires-Dist: boto3>=1.26.0
+Requires-Dist: tqdm>=4.63.0
+Requires-Dist: lytekit==0.15.8
+Requires-Dist: lytekitplugins-pods==0.6.2
+Requires-Dist: typing-extensions==4.7.1
+Requires-Dist: apscheduler==3.9.1
+Requires-Dist: gql==3.4.0
+Requires-Dist: graphql-core==3.2.3
+Requires-Dist: requests-toolbelt==0.10.1
+Requires-Dist: latch-sdk-gql==0.0.6
+Requires-Dist: latch-sdk-config==0.0.4
+Requires-Dist: python-dateutil>=2.8
+Requires-Dist: aioconsole==0.6.1
+Requires-Dist: asyncssh==2.13.2
+Requires-Dist: websockets==11.0.3
+Requires-Dist: watchfiles==0.19.0
+Provides-Extra: snakemake
+Requires-Dist: snakemake<7.30.2,>=7.18.0; extra == "snakemake"
+Requires-Dist: pulp<2.8,>=2.0; extra == "snakemake"
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0.0; extra == "pandas"

{latch-2.45.2 → latch-2.45.2.dev2}/latch.egg-info/SOURCES.txt

@@ -80,6 +80,10 @@ latch_cli/exceptions/cache.py
 latch_cli/exceptions/errors.py
 latch_cli/exceptions/handler.py
 latch_cli/exceptions/traceback.py
+latch_cli/nextflow/__init__.py
+latch_cli/nextflow/dependencies.py
+latch_cli/nextflow/utils.py
+latch_cli/nextflow/workflow.py
 latch_cli/services/__init__.py
 latch_cli/services/get.py
 latch_cli/services/get_executions.py
@@ -107,13 +111,7 @@ latch_cli/services/execute/utils.py
 latch_cli/services/init/__init__.py
 latch_cli/services/init/init.py
 latch_cli/services/init/__pycache__/__init__.cpython-310.pyc
-latch_cli/services/init/__pycache__/__init__.cpython-311.pyc
-latch_cli/services/init/__pycache__/__init__.cpython-38.pyc
-latch_cli/services/init/__pycache__/__init__.cpython-39.pyc
 latch_cli/services/init/__pycache__/init.cpython-310.pyc
-latch_cli/services/init/__pycache__/init.cpython-311.pyc
-latch_cli/services/init/__pycache__/init.cpython-38.pyc
-latch_cli/services/init/__pycache__/init.cpython-39.pyc
 latch_cli/services/init/assemble_and_sort/.env
 latch_cli/services/init/assemble_and_sort/LICENSE
 latch_cli/services/init/assemble_and_sort/README.md
@@ -121,22 +119,18 @@ latch_cli/services/init/assemble_and_sort/__init__.py
 latch_cli/services/init/assemble_and_sort/assemble.py
 latch_cli/services/init/assemble_and_sort/sort.py
 latch_cli/services/init/assemble_and_sort/system-requirements.txt
-latch_cli/services/init/assemble_and_sort/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/common/.dockerignore
 latch_cli/services/init/example_conda/__init__.py
 latch_cli/services/init/example_conda/conda_task.py
 latch_cli/services/init/example_conda/environment.yaml
-latch_cli/services/init/example_conda/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_docker/__init__.py
 latch_cli/services/init/example_docker/task.py
-latch_cli/services/init/example_nf_integration/latch_metadata/__pycache__/__init__.cpython-311.pyc
 latch_cli/services/init/example_nfcore/Dockerfile
 latch_cli/services/init/example_nfcore/__init__.py
 latch_cli/services/init/example_nfcore/task.py
 latch_cli/services/init/example_r/__init__.py
 latch_cli/services/init/example_r/environment.R
 latch_cli/services/init/example_r/r_task.py
-latch_cli/services/init/example_r/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_snakemake/Dockerfile
 latch_cli/services/init/example_snakemake/Snakefile
 latch_cli/services/init/example_snakemake/config.yaml
@@ -149,7 +143,6 @@ latch_cli/services/init/template/LICENSE
 latch_cli/services/init/template/README.md
 latch_cli/services/init/template/__init__.py
 latch_cli/services/init/template/task.py
-latch_cli/services/init/template/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/register/__init__.py
 latch_cli/services/register/constants.py
 latch_cli/services/register/register.py

{latch-2.45.2 → latch-2.45.2.dev2}/latch_cli/centromere/ctx.py

@@ -59,6 +59,7 @@ class _CentromereCtx:
     default_container: _Container
     workflow_type: WorkflowType
     snakefile: Optional[Path]
+    nf_script: Optional[Path]
 
     latch_register_api_url = config.api.workflow.register
     latch_image_api_url = config.api.workflow.upload_image
@@ -80,6 +81,7 @@ class _CentromereCtx:
         disable_auto_version: bool = False,
         remote: bool = False,
         snakefile: Optional[Path] = None,
+        nf_script: Optional[Path] = None,
         use_new_centromere: bool = False,
     ):
         self.use_new_centromere = use_new_centromere
@@ -93,13 +95,22 @@ class _CentromereCtx:
             self.dkr_repo = config.dkr_repo
             self.pkg_root = pkg_root.resolve()
 
-            if snakefile is None:
-                self.workflow_type = WorkflowType.latchbiosdk
-            else:
+            if snakefile and nf_script:
+                raise ValueError(
+                    "Cannot provide both a snakefile and nextflow script to the"
+                    " register command."
+                )
+            if snakefile is not None:
                 self.workflow_type = WorkflowType.snakemake
                 self.snakefile = snakefile
+            elif nf_script is not None:
+                self.workflow_type = WorkflowType.nextflow
+                self.nf_script = nf_script
+            else:
+                self.workflow_type = WorkflowType.latchbiosdk
 
             self.container_map: Dict[str, _Container] = {}
+
             if self.workflow_type == WorkflowType.latchbiosdk:
                 _import_flyte_objects([self.pkg_root])
                 for entity in FlyteEntities.entities:
@@ -126,11 +137,7 @@ class _CentromereCtx:
                         fg="red",
                     )
                     raise click.exceptions.Exit(1)
-
-                name_path = pkg_root / latch_constants.pkg_workflow_name
-                if name_path.exists():
-                    self.workflow_name = name_path.read_text().strip()
-            else:
+            elif self.workflow_type == WorkflowType.snakemake:
                 assert snakefile is not None
 
                 import latch.types.metadata as metadata
@@ -142,6 +149,10 @@ class _CentromereCtx:
                 )
                 from ..snakemake.utils import load_snakemake_metadata
 
+                name_path = pkg_root / latch_constants.pkg_workflow_name
+                if name_path.exists():
+                    self.workflow_name = name_path.read_text().strip()
+
                 meta_file = load_snakemake_metadata(pkg_root)
                 if meta_file is not None:
                     click.echo(
@@ -235,6 +246,32 @@ class _CentromereCtx:
                 # todo(kenny): support per container task and custom workflow
                 # name for snakemake
                 self.workflow_name = f"{metadata._snakemake_metadata.name}_jit_register"
+            else:
+                assert nf_script is not None
+
+                import latch.types.metadata as metadata
+
+                from ..services.register.utils import import_module_by_path
+
+                meta = pkg_root / "latch_metadata" / "__init__.py"
+                if meta.exists():
+                    click.echo(f"Using metadata file {click.style(meta, italic=True)}")
+                    import_module_by_path(meta)
+
+                if metadata._nextflow_metadata is None:
+                    click.secho(
+                        dedent("""
+                        Failed to register Nextflow workflow.
+                        Make sure the project root contains a `latch_metadata/__init__.py`
+                        with a `NextflowMetadata` object defined.
+                        """),
+                        fg="red",
+                    )
+                    raise click.exceptions.Exit(1)
+
+                self.workflow_name = metadata._nextflow_metadata.name
+
+            assert self.workflow_name is not None
 
             version_file = self.pkg_root / "version"
             try:

{latch-2.45.2 → latch-2.45.2.dev2}/latch_cli/docker_utils/__init__.py

@@ -39,7 +39,7 @@ def get_prologue(
         library_name = '"latch[snakemake]"'
     else:
         library_name = "latch"
-    return [
+    directives = [
         "# DO NOT CHANGE",
         f"from {config.base_image}",
         "",
@@ -68,6 +68,11 @@ def get_prologue(
         f"run pip install {library_name}=={config.latch_version}",
         "run mkdir /opt/latch",
     ]
+    if wf_type == WorkflowType.nextflow:
+        directives.append(
+            "run apt-get update && apt-get install -y default-jre-headless"
+        )
+    return directives
 
 
 def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
@@ -78,8 +83,18 @@ def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
             "",
             "# Latch snakemake workflow entrypoint",
             "# DO NOT CHANGE",
+            "",
             "copy .latch/snakemake_jit_entrypoint.py /root/snakemake_jit_entrypoint.py",
         ]
+    elif wf_type == WorkflowType.nextflow:
+        cmds += [
+            "",
+            "# Latch nextflow workflow entrypoint",
+            "# DO NOT CHANGE",
+            "",
+            "copy .latch/bin/nextflow /root/nextflow",
+            "copy .latch/.nextflow /root/.nextflow",
+        ]
 
     cmds += [
         "",