latch 2.40.4__tar.gz → 2.40.4.dev0__tar.gz

{latch-2.40.4/latch.egg-info → latch-2.40.4.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.40.4
+Version: 2.40.4.dev0
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.40.4 → latch-2.40.4.dev0}/latch/types/directory.py

@@ -141,6 +141,9 @@ class LatchDir(FlyteDirectory):
 
             super().__init__(self.path, downloader, self._remote_directory)
 
+    def __hash__(self) -> int:
+        return hash(self.path)
+
     def _idempotent_set_path(self):
         if self._path_generated:
             return
@@ -268,6 +271,9 @@ LatchOutputDir = Annotated[
         {"output": True},
     ),
 ]
+
+LatchOutputDir.__name__ = "LatchOutputDir"
+
 """A LatchDir tagged as the output of some workflow.
 
 The Latch Console uses this metadata to avoid checking for existence of the

{latch-2.40.4 → latch-2.40.4.dev0}/latch/types/file.py

@@ -120,6 +120,9 @@ class LatchFile(FlyteFile):
 
             super().__init__(self.path, downloader, self._remote_path)
 
+    def __hash__(self) -> int:
+        return hash(self.path)
+
     def size(self):
         return LPath(self.remote_path).size()
 

{latch-2.40.4 → latch-2.40.4.dev0}/latch/types/metadata.py

@@ -3,16 +3,28 @@ from dataclasses import Field, asdict, dataclass, field
 from enum import Enum
 from pathlib import Path
 from textwrap import indent
-from typing import Any, ClassVar, Dict, List, Optional, Protocol, Tuple, Type, Union
+from typing import (
+    Any,
+    ClassVar,
+    Collection,
+    Dict,
+    Generic,
+    List,
+    Optional,
+    Protocol,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+)
 
 import click
 import yaml
 from typing_extensions import TypeAlias
 
-from latch_cli.snakemake.config.utils import validate_snakemake_type
 from latch_cli.utils import identifier_suffix_from_str
 
-from .directory import LatchDir
+from .directory import LatchDir, LatchOutputDir
 from .file import LatchFile
 
 
@@ -388,31 +400,33 @@ class _IsDataclass(Protocol):
 
 
 ParameterType: TypeAlias = Union[
-    Type[None],
-    Type[int],
-    Type[float],
-    Type[str],
-    Type[bool],
-    Type[Enum],
-    Type[_IsDataclass],
-    Type[List["ParameterType"]],
-    Type[LatchFile],
-    Type[LatchDir],
+    None,
+    int,
+    float,
+    str,
+    bool,
+    LatchFile,
+    LatchDir,
+    Enum,
+    _IsDataclass,
+    Collection["ParameterType"],
 ]
 
 
+T = TypeVar("T", bound=ParameterType)
+
+
 @dataclass
-class SnakemakeParameter(LatchParameter):
-    type: Optional[ParameterType] = None
+class SnakemakeParameter(Generic[T], LatchParameter):
+    type: Optional[Type[T]] = None
     """
     The python type of the parameter.
     """
-    # todo(ayush): needs to be typed properly
-    default: Optional[Any] = None
+    default: Optional[T] = None
 
 
 @dataclass
-class SnakemakeFileParameter(SnakemakeParameter):
+class SnakemakeFileParameter(SnakemakeParameter[Union[LatchFile, LatchDir]]):
     """
     Deprecated: use `file_metadata` keyword in `SnakemakeMetadata` instead
     """
@@ -456,6 +470,29 @@ class SnakemakeFileMetadata:
     """
 
 
+@dataclass
+class NextflowParameter(Generic[T], LatchParameter):
+    type: Optional[Type[T]] = None
+    """
+    The python type of the parameter.
+    """
+    default: Optional[T] = None
+
+
+@dataclass
+class NextflowFileParameter(NextflowParameter[Union[LatchFile, LatchDir]]):
+    path: Optional[Path] = None
+    """
+    The path where the file passed to this parameter will be copied.
+    """
+
+    _download: bool = field(default=True, init=False)
+
+    def __post_init__(self):
+        if self.type is LatchOutputDir:
+            self._download = False
+
+
 @dataclass
 class LatchMetadata:
     """Class for organizing workflow metadata
@@ -571,6 +608,10 @@ class DockerMetadata:
     """
     The name of the Latch Secret that contains the password for the private repository
     """
+    server: Optional[str] = None
+    """
+    The name of a docker server to login to, if relevant (eg. registry.gitlab.com)
+    """
 
 
 @dataclass
@@ -628,6 +669,7 @@ class SnakemakeMetadata(LatchMetadata):
     """
 
     def validate(self):
+        from latch_cli.extras.snakemake.config import validate_snakemake_type
 
         for name, param in self.parameters.items():
             if param.default is None:
@@ -651,3 +693,26 @@ class SnakemakeMetadata(LatchMetadata):
 
 
 _snakemake_metadata: Optional[SnakemakeMetadata] = None
+
+
+@dataclass
+class NextflowMetadata(LatchMetadata):
+    name: Optional[str] = None
+    parameters: Dict[str, NextflowParameter] = field(default_factory=dict)
+    output_directory: Optional[LatchDir] = None
+    docker_metadata: Optional[DockerMetadata] = None
+
+    def __post_init__(self):
+        if self.name is None:
+            self.name = f"nf_{identifier_suffix_from_str(self.display_name.lower())}"
+        else:
+            self.name = identifier_suffix_from_str(self.name)
+
+        if self.output_directory is None:
+            self.output_directory = LatchDir("latch:///Nextflow Outputs/")
+
+        global _nextflow_metadata
+        _nextflow_metadata = self
+
+
+_nextflow_metadata: Optional[NextflowMetadata] = None

{latch-2.40.4 → latch-2.40.4.dev0}/latch/utils.py

@@ -1,3 +1,4 @@
+import os
 from typing import Dict
 
 import gql
@@ -84,17 +85,24 @@ def current_workspace() -> str:
     """
     ws = user_config.workspace_id
     if ws == "":
-        default_ws = execute(
+        res = execute(
             gql.gql("""
                 query DefaultAccountQuery {
                     accountInfoCurrent {
+                        id
                         user {
                             defaultAccount
                         }
                     }
                 }
             """),
-        )["accountInfoCurrent"]["user"]["defaultAccount"]
+        )["accountInfoCurrent"]
+
+        default_ws = res["id"]
+
+        is_local = os.environ.get("FLYTE_INTERNAL_EXECUTION_ID") is None
+        if is_local:
+            default_ws = res["user"]["defaultAccount"]
 
         if default_ws is not None:
             ws = default_ws

{latch-2.40.4 → latch-2.40.4.dev0/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.40.4
+Version: 2.40.4.dev0
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.40.4 → latch-2.40.4.dev0}/latch.egg-info/SOURCES.txt

@@ -80,6 +80,35 @@ latch_cli/exceptions/cache.py
 latch_cli/exceptions/errors.py
 latch_cli/exceptions/handler.py
 latch_cli/exceptions/traceback.py
+latch_cli/extras/__init__.py
+latch_cli/extras/common/__init__.py
+latch_cli/extras/common/serialize.py
+latch_cli/extras/common/utils.py
+latch_cli/extras/common/config/__init__.py
+latch_cli/extras/common/config/parser.py
+latch_cli/extras/common/config/utils.py
+latch_cli/extras/nextflow/__init__.py
+latch_cli/extras/nextflow/build.py
+latch_cli/extras/nextflow/channel.py
+latch_cli/extras/nextflow/config.py
+latch_cli/extras/nextflow/dag.py
+latch_cli/extras/nextflow/file_persistence.py
+latch_cli/extras/nextflow/serialize.py
+latch_cli/extras/nextflow/workflow.py
+latch_cli/extras/nextflow/tasks/__init__.py
+latch_cli/extras/nextflow/tasks/adapters.py
+latch_cli/extras/nextflow/tasks/base.py
+latch_cli/extras/nextflow/tasks/conditional.py
+latch_cli/extras/nextflow/tasks/map.py
+latch_cli/extras/nextflow/tasks/merge.py
+latch_cli/extras/nextflow/tasks/operator.py
+latch_cli/extras/nextflow/tasks/process.py
+latch_cli/extras/snakemake/__init__.py
+latch_cli/extras/snakemake/config.py
+latch_cli/extras/snakemake/serialize.py
+latch_cli/extras/snakemake/single_task_snakemake.py
+latch_cli/extras/snakemake/utils.py
+latch_cli/extras/snakemake/workflow.py
 latch_cli/services/__init__.py
 latch_cli/services/get.py
 latch_cli/services/get_executions.py
@@ -129,6 +158,9 @@ latch_cli/services/init/example_conda/environment.yaml
 latch_cli/services/init/example_conda/__pycache__/__init__.cpython-310.pyc
 latch_cli/services/init/example_docker/__init__.py
 latch_cli/services/init/example_docker/task.py
+latch_cli/services/init/example_nextflow/main.nf
+latch_cli/services/init/example_nextflow/nextflow.config
+latch_cli/services/init/example_nextflow/workflow.nf
 latch_cli/services/init/example_nf_integration/latch_metadata/__pycache__/__init__.cpython-311.pyc
 latch_cli/services/init/example_nfcore/Dockerfile
 latch_cli/services/init/example_nfcore/__init__.py
@@ -159,18 +191,10 @@ latch_cli/services/test_data/ls.py
 latch_cli/services/test_data/remove.py
 latch_cli/services/test_data/upload.py
 latch_cli/services/test_data/utils.py
-latch_cli/snakemake/__init__.py
-latch_cli/snakemake/serialize.py
-latch_cli/snakemake/serialize_utils.py
-latch_cli/snakemake/single_task_snakemake.py
-latch_cli/snakemake/utils.py
-latch_cli/snakemake/workflow.py
-latch_cli/snakemake/config/__init__.py
-latch_cli/snakemake/config/parser.py
-latch_cli/snakemake/config/utils.py
 latch_cli/tui/__init__.py
 latch_cli/utils/__init__.py
 latch_cli/utils/path.py
+latch_cli/utils/workflow.py
 tests/__init__.py
 tests/fixtures.py
 tests/test_ls.py

{latch-2.40.4 → latch-2.40.4.dev0}/latch_cli/auth/oauth2.py

@@ -164,15 +164,13 @@ class OAuth2:
         """
 
         token_url = self.authz_server_host + "/oauth/token"
-        token_body: bytes = json.dumps(
-            {
-                "grant_type": "authorization_code",
-                "client_id": self.client_id,
-                "code_verifier": self.pkce.verifier,
-                "code": auth_code,
-                "redirect_uri": self.redirect_url,
-            }
-        ).encode("utf-8")
+        token_body: bytes = json.dumps({
+            "grant_type": "authorization_code",
+            "client_id": self.client_id,
+            "code_verifier": self.pkce.verifier,
+            "code": auth_code,
+            "redirect_uri": self.redirect_url,
+        }).encode("utf-8")
         token_request = urllib.request.Request(token_url, token_body)
         token_request.add_header("Content-Type", "application/json")
         with urllib.request.urlopen(

{latch-2.40.4 → latch-2.40.4.dev0}/latch_cli/centromere/ctx.py

@@ -1,10 +1,12 @@
+import os
 import re
 import sys
+import time
 import traceback
 from dataclasses import dataclass
 from pathlib import Path
 from textwrap import dedent
-from typing import Dict, Optional, Tuple
+from typing import Dict, Optional, Tuple, cast
 
 import click
 import docker
@@ -51,7 +53,7 @@ class _CentromereCtx:
     dkr_repo: Optional[str] = None
     dkr_client: Optional[docker.APIClient] = None
     ssh_client: Optional[paramiko.SSHClient] = None
-    pkg_root: Optional[Path] = None  # root
+    pkg_root: Path  # root
     disable_auto_version: bool = False
     image_full = None
     version = None
@@ -59,6 +61,7 @@ class _CentromereCtx:
     default_container: _Container
     workflow_type: WorkflowType
     snakefile: Optional[Path]
+    nf_script: Optional[Path]
 
     latch_register_api_url = config.api.workflow.register
     latch_image_api_url = config.api.workflow.upload_image
@@ -80,26 +83,37 @@ class _CentromereCtx:
         disable_auto_version: bool = False,
         remote: bool = False,
         snakefile: Optional[Path] = None,
+        nf_script: Optional[Path] = None,
         use_new_centromere: bool = False,
     ):
         self.use_new_centromere = use_new_centromere
         self.remote = remote
         self.disable_auto_version = disable_auto_version
+        self.pkg_root = pkg_root.resolve()
 
         try:
             self.token = retrieve_or_login()
             self.account_id = current_workspace()
 
             self.dkr_repo = config.dkr_repo
-            self.pkg_root = pkg_root.resolve()
 
-            if snakefile is None:
-                self.workflow_type = WorkflowType.latchbiosdk
-            else:
+            if snakefile and nf_script:
+                raise ValueError(
+                    "Cannot provide both a snakefile and nextflow script to the"
+                    " register command."
+                )
+
+            if snakefile is not None:
                 self.workflow_type = WorkflowType.snakemake
                 self.snakefile = snakefile
+            elif nf_script is not None:
+                self.workflow_type = WorkflowType.nextflow
+                self.nf_script = nf_script
+            else:
+                self.workflow_type = WorkflowType.latchbiosdk
 
             self.container_map: Dict[str, _Container] = {}
+
             if self.workflow_type == WorkflowType.latchbiosdk:
                 _import_flyte_objects([self.pkg_root])
                 for entity in FlyteEntities.entities:
@@ -107,36 +121,40 @@ class _CentromereCtx:
                         self.workflow_name = entity.name
 
                     if isinstance(entity, PythonTask):
-                        if (
-                            hasattr(entity, "dockerfile_path")
-                            and entity.dockerfile_path is not None
-                        ):
+                        dockerfile_path: Optional[Path] = getattr(
+                            entity, "dockerfile_path", None
+                        )
+
+                        if dockerfile_path is not None:
                             self.container_map[entity.name] = _Container(
-                                dockerfile=entity.dockerfile_path,
+                                dockerfile=dockerfile_path,
                                 image_name=self.task_image_name(entity.name),
-                                pkg_dir=entity.dockerfile_path.parent,
+                                pkg_dir=dockerfile_path.parent,
                             )
+
                 if not hasattr(self, "workflow_name"):
                     click.secho(
                         dedent("""
-                                     Unable to locate workflow code. If you
-                                     are a registering a Snakemake project,
-                                     make sure to pass the Snakefile path with
-                                     the --snakefile flag."""),
+                        Unable to locate workflow code.
+
+                        If you are a registering a Snakemake/Nextflow project, make sure to pass the appropriate
+                        script with either the --snakefile or --nf-script flag.
+                        """),
                         fg="red",
                     )
                     raise click.exceptions.Exit(1)
-            else:
+
+            elif self.workflow_type == WorkflowType.snakemake:
                 assert snakefile is not None
 
                 import latch.types.metadata as metadata
 
-                from ..services.register.utils import import_module_by_path
-                from ..snakemake.serialize import (
+                from ..extras.snakemake.serialize import (
                     get_snakemake_metadata_example,
                     snakemake_workflow_extractor,
                 )
-                from ..snakemake.utils import load_snakemake_metadata
+                from ..extras.snakemake.utils import load_snakemake_metadata
+                from ..services.register.utils import import_module_by_path
 
                 meta_file = load_snakemake_metadata(pkg_root)
                 if meta_file is not None:
@@ -151,18 +169,17 @@ class _CentromereCtx:
                     except (ImportError, FileNotFoundError):
                         traceback.print_exc()
                         click.secho(
-                            "\n\n\n"
-                            + "The above error occured when reading "
-                            + "the Snakefile to extract workflow metadata.",
+                            "\n\n\nThe above error occurred when reading the Snakefile"
+                            " to extract workflow metadata.",
                             bold=True,
                             fg="red",
                         )
                         click.secho(
-                            "\nIt is possible to avoid including the Snakefile"
-                            " prior to registration by providing a"
-                            " `latch_metadata.py` file in the workflow root.\nThis"
-                            " way it is not necessary to install dependencies or"
-                            " ensure that Snakemake inputs locally.",
+                            "\nIt is possible to avoid including the Snakefile prior to"
+                            " registration by providing a `latch_metadata.py` file in"
+                            " the workflow root.\nThis way it is not necessary to"
+                            " install dependencies or ensure that Snakemake inputs"
+                            " locally.",
                             fg="red",
                         )
                         click.secho("\nExample ", fg="red", nl=False)
@@ -207,15 +224,14 @@ class _CentromereCtx:
                                 elif system == "Darwin":
                                     res = subprocess.run(["open", new_meta]).returncode
                                 elif system == "Windows":
-                                    import os
-
                                     res = os.system(str(new_meta.resolve()))
                                 else:
                                     res = None
 
                                 if res is not None and res != 0:
                                     click.secho("Failed to open file", fg="red")
-                        sys.exit(1)
+
+                        raise click.exceptions.Exit(1)
 
                 if metadata._snakemake_metadata is None:
                     click.secho(
@@ -232,6 +248,34 @@ class _CentromereCtx:
                 # name for snakemake
                 self.workflow_name = f"{metadata._snakemake_metadata.name}_jit_register"
 
+            else:
+                assert nf_script is not None
+
+                import latch.types.metadata as metadata
+
+                from ..services.register.utils import import_module_by_path
+
+                meta = pkg_root / "latch_metadata" / "__init__.py"
+                if meta.exists():
+                    click.echo(f"Using metadata file {click.style(meta, italic=True)}")
+                    import_module_by_path(meta)
+
+                if metadata._nextflow_metadata is None:
+                    click.secho(
+                        dedent(
+                            "Make sure a `latch_metadata` package exists in the"
+                            " nextflow project root."
+                            "\nYou can generate this package with the"
+                            "`latch generate-metadata --nextflow <pkg_root>` command.",
+                        ),
+                        fg="red",
+                    )
+                    raise click.exceptions.Exit(1)
+
+                self.workflow_name = metadata._nextflow_metadata.name
+
+            assert self.workflow_name is not None
+
             version_file = self.pkg_root / "version"
             try:
                 self.version = version_file.read_text()
@@ -246,7 +290,11 @@ class _CentromereCtx:
             if not self.disable_auto_version:
                 hash = hash_directory(self.pkg_root)
                 self.version = f"{self.version}-{hash[:6]}"
-                click.echo(f"  {self.version}\n")
+
+                if os.environ.get("LATCH_NEW_VERSION_ALWAYS") is not None:
+                    self.version = f"{self.version}-{int(time.monotonic())}"
+
+            click.echo()
 
             if self.nucleus_check_version(self.version, self.workflow_name):
                 click.secho(
@@ -324,6 +372,8 @@ class _CentromereCtx:
 
         from ..utils import identifier_suffix_from_str
 
+        assert self.workflow_name is not None
+
         wf_name = identifier_suffix_from_str(self.workflow_name).lower()
         wf_name = docker_image_name_illegal_pat.sub("_", wf_name)
 

{latch-2.40.4 → latch-2.40.4.dev0}/latch_cli/click_utils.py

@@ -110,6 +110,9 @@ class AnsiCodes:
     bold = "\x1b[1m"
     reset_bold = "\x1b[22m"
 
+    italic = "\x1b[3m"
+    reset_italic = "\x1b[23m"
+
     underline = "\x1b[4m"
     no_underline = "\x1b[24m"
 
@@ -119,13 +122,17 @@ class AnsiCodes:
     url_end = "\x1b]8;;\x1b\\"
 
 
-def bold(s: str) -> str:
+def italic(s: object) -> str:
+    return f"{AnsiCodes.italic}{s}{AnsiCodes.reset_italic}"
+
+
+def bold(s: object) -> str:
     return f"{AnsiCodes.bold}{s}{AnsiCodes.reset_bold}"
 
 
-def underline(s: str) -> str:
+def underline(s: object) -> str:
     return f"{AnsiCodes.underline}{s}{AnsiCodes.no_underline}"
 
 
-def color(s: str, *, color: str = AnsiCodes.color):
+def color(s: object, *, color: str = AnsiCodes.color):
     return f"{color}{s}{AnsiCodes.reset_color}"

{latch-2.40.4 → latch-2.40.4.dev0}/latch_cli/docker_utils/__init__.py

@@ -9,6 +9,7 @@ from typing import List
 import click
 import yaml
 
+from latch_cli.click_utils import color
 from latch_cli.constants import latch_constants
 from latch_cli.utils import WorkflowType
 from latch_cli.workflow_config import LatchWorkflowConfig, create_and_write_config
@@ -37,9 +38,12 @@ def get_prologue(
 ) -> List[str]:
     if wf_type == WorkflowType.snakemake:
         library_name = '"latch[snakemake]"'
+    elif wf_type == WorkflowType.nextflow:
+        library_name = '"latch[nextflow]"'
     else:
         library_name = "latch"
-    return [
+
+    directives = [
         "# DO NOT CHANGE",
         f"from {config.base_image}",
         "",
@@ -68,6 +72,11 @@ def get_prologue(
         f"run pip install {library_name}=={config.latch_version}",
         "run mkdir /opt/latch",
     ]
+    if wf_type == WorkflowType.nextflow:
+        directives.append(
+            "run apt-get update && apt-get install -y default-jre-headless"
+        )
+    return directives
 
 
 def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
@@ -78,8 +87,19 @@ def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
             "",
             "# Latch snakemake workflow entrypoint",
             "# DO NOT CHANGE",
+            "",
             "copy .latch/snakemake_jit_entrypoint.py /root/snakemake_jit_entrypoint.py",
         ]
+    elif wf_type == WorkflowType.nextflow:
+        cmds += [
+            "",
+            "# Latch nextflow workflow entrypoint",
+            "# DO NOT CHANGE",
+            "",
+            "copy .latch/bin/nextflow /root/nextflow",
+            "copy .latch/.nextflow /root/.nextflow",
+            "copy .latch/nf_entrypoint.py /root/nf_entrypoint.py",
+        ]
 
     cmds += [
         "",
@@ -339,18 +359,8 @@ def generate_dockerfile(
         with (pkg_root / latch_constants.pkg_config).open("r") as f:
             config = LatchWorkflowConfig(**json.load(f))
 
-    click.echo(
-        " ".join([
-            click.style("Base image:", fg="bright_blue"),
-            config.base_image,
-        ])
-    )
-    click.echo(
-        " ".join([
-            click.style("Latch SDK version:", fg="bright_blue"),
-            config.latch_version,
-        ])
-    )
+    click.echo(" ".join([color("Base image:"), config.base_image]))
+    click.echo(" ".join([color("Latch SDK version:"), config.latch_version]))
     click.echo()
 
     with outfile.open("w") as f: