latch 2.45.6__tar.gz → 2.45.6.dev2__tar.gz

{latch-2.45.6/latch.egg-info → latch-2.45.6.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.45.6
+Version: 2.45.6.dev2
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.45.6 → latch-2.45.6.dev2}/latch/resources/tasks.py

@@ -479,15 +479,19 @@ def custom_task(
     )
 
 
-def nextflow_runtime_task(cpu: int, memory: int):
+def nextflow_runtime_task(cpu: int, memory: int, storage_gib: int = 50):
     primary_container = V1Container(name="primary")
     resources = V1ResourceRequirements(
         requests={
             "cpu": str(cpu),
             "memory": f"{memory}Gi",
-            "ephemeral-storage": "50Gi",
+            "ephemeral-storage": f"{storage_gib}Gi",
+        },
+        limits={
+            "cpu": str(cpu),
+            "memory": f"{memory}Gi",
+            "ephemeral-storage": f"{storage_gib}Gi",
         },
-        limits={"cpu": str(cpu), "memory": f"{memory}Gi", "ephemeral-storage": "20Gi"},
     )
     primary_container.resources = resources
     volume_mounts = [V1VolumeMount(mount_path="/nf-workdir", name="nextflow-workdir")]

{latch-2.45.6 → latch-2.45.6.dev2}/latch/types/metadata.py

@@ -26,7 +26,7 @@ from typing import (
 
 import click
 import yaml
-from typing_extensions import Annotated, TypeAlias
+from typing_extensions import TypeAlias
 
 from latch_cli.snakemake.config.utils import validate_snakemake_type
 from latch_cli.utils import identifier_suffix_from_str
@@ -794,8 +794,38 @@ class NextflowMetadata(LatchMetadata):
     """
     Directory to dump Nextflow logs
     """
+    about_page_path: Optional[Path] = None
+    """
+    Path to a markdown file containing information about the pipeline - rendered in the About page.
+    """
+
+    def validate(self):
+        if self.about_page_path is not None:
+            if not self.about_page_path.exists():
+                click.secho(
+                    f"Path to about page documentation ({self.about_page_path})"
+                    " doesn't exist.",
+                    fg="red",
+                )
+                raise click.exceptions.Exit(1)
+
+            if not self.about_page_path.is_file():
+                click.secho(
+                    f"About page documentation ({self.about_page_path}) must be a"
+                    " file.",
+                    fg="red",
+                )
+                raise click.exceptions.Exit(1)
+
+    @property
+    def dict(self):
+        d = super().dict
+        del d["__metadata__"]["about_page_path"]
+        return d
 
     def __post_init__(self):
+        self.validate()
+
         if self.name is None:
             if self.display_name is None:
                 click.secho(

{latch-2.45.6 → latch-2.45.6.dev2/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.45.6
+Version: 2.45.6.dev2
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.45.6 → latch-2.45.6.dev2}/latch.egg-info/SOURCES.txt

@@ -83,6 +83,7 @@ latch_cli/exceptions/traceback.py
 latch_cli/nextflow/__init__.py
 latch_cli/nextflow/config.py
 latch_cli/nextflow/dependencies.py
+latch_cli/nextflow/types.py
 latch_cli/nextflow/utils.py
 latch_cli/nextflow/workflow.py
 latch_cli/services/__init__.py

{latch-2.45.6 → latch-2.45.6.dev2}/latch_cli/main.py

@@ -14,6 +14,7 @@ import latch_cli.click_utils
 from latch.ldata._transfer.progress import Progress as _Progress
 from latch_cli.click_utils import EnumChoice
 from latch_cli.exceptions.handler import CrashHandler
+from latch_cli.nextflow.types import NextflowProcessExecutor
 from latch_cli.services.cp.autocomplete import complete as cp_complete
 from latch_cli.services.cp.autocomplete import remote_complete
 from latch_cli.services.init.init import template_flag_to_option
@@ -346,8 +347,17 @@ def generate_metadata(
     default=None,
     help="Set execution profile for Nextflow workflow",
 )
+@click.option(
+    "--process-executor",
+    type=EnumChoice(NextflowProcessExecutor, case_sensitive=False),
+    default=None,
+    help="Set process executor for Nextflow workflow",
+)
 def generate_entrypoint(
-    pkg_root: Path, nf_script: Path, execution_profile: Optional[str]
+    pkg_root: Path,
+    nf_script: Path,
+    execution_profile: Optional[str],
+    process_executor: Optional[NextflowProcessExecutor],
 ):
     """Generate a `wf/entrypoint.py` file from a Nextflow workflow"""
 
@@ -384,6 +394,7 @@ def generate_entrypoint(
         nf_script,
         dest,
         execution_profile=execution_profile,
+        process_executor=process_executor,
     )
 
 
@@ -549,18 +560,23 @@ def execute(
     help="Path to a nextflow script to register.",
 )
 @click.option(
-    "--redownload-dependencies",
-    type=bool,
-    is_flag=True,
-    default=False,
-    help="Redownload external Nextflow dependencies",
+    "--nf-version",
+    type=str,
+    default=None,
+    help="Version of Nextflow runtime to use",
 )
 @click.option(
-    "--execution-profile",
+    "--nf-execution-profile",
     type=str,
     default=None,
     help="Set execution profile for Nextflow workflow",
 )
+@click.option(
+    "--nf-process-executor",
+    type=EnumChoice(NextflowProcessExecutor, case_sensitive=False),
+    default=None,
+    help="Set process executor for Nextflow workflow",
+)
 @requires_login
 def register(
     pkg_root: str,
@@ -572,8 +588,9 @@ def register(
     snakefile: Optional[Path],
     cache_tasks: bool,
     nf_script: Optional[Path],
-    redownload_dependencies: bool,
-    execution_profile: Optional[str],
+    nf_version: Optional[str],
+    nf_execution_profile: Optional[str],
+    nf_process_executor: Optional[NextflowProcessExecutor],
 ):
     """Register local workflow code to Latch.
 
@@ -585,6 +602,21 @@ def register(
     crash_handler.message = "Unable to register workflow."
     crash_handler.pkg_root = pkg_root
 
+    if nf_script is None and (
+        nf_version is not None
+        or nf_execution_profile is not None
+        or nf_process_executor is not None
+    ):
+        click.secho(
+            dedent("""
+            Command Invalid:
+            --nf-version, --execution-profile, and --process-executor flags
+            are only valid when registering a Nextflow workflow.
+            """),
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
     from latch_cli.services.register import register
 
     register(
@@ -595,8 +627,9 @@ def register(
         open=open,
         snakefile=snakefile,
         nf_script=nf_script,
-        nf_redownload_dependencies=redownload_dependencies,
-        nf_execution_profile=execution_profile,
+        nf_version=nf_version,
+        nf_execution_profile=nf_execution_profile,
+        nf_process_executor=nf_process_executor,
         progress_plain=(docker_progress == "auto" and not sys.stdout.isatty())
         or docker_progress == "plain",
         use_new_centromere=use_new_centromere,

{latch-2.45.6 → latch-2.45.6.dev2}/latch_cli/nextflow/config.py

@@ -5,7 +5,7 @@ from typing import Any, Optional, Type
 
 import click
 
-from latch.types.directory import LatchDir
+from latch.types.directory import LatchDir, LatchOutputDir
 from latch.types.file import LatchFile
 from latch_cli.snakemake.config.utils import type_repr
 from latch_cli.snakemake.utils import reindent
@@ -103,7 +103,7 @@ def generate_metadata(
             else:
                 section_title = None
 
-            t = get_param_type(details)
+            t = LatchOutputDir if param == "outdir" else get_param_type(details)
             if param not in required_params:
                 t = Optional[t]
 
@@ -140,7 +140,7 @@ def generate_metadata(
 
             from latch.types.metadata import NextflowParameter
             from latch.types.file import LatchFile
-            from latch.types.directory import LatchDir
+            from latch.types.directory import LatchDir, LatchOutputDir
 
             # Import these into your `__init__.py` file:
             #

{latch-2.45.6 → latch-2.45.6.dev2}/latch_cli/nextflow/dependencies.py

@@ -3,6 +3,7 @@ from concurrent.futures import ProcessPoolExecutor
 from ctypes import c_int
 from multiprocessing.managers import SyncManager
 from pathlib import Path
+from typing import Optional
 from urllib.parse import urljoin
 
 import boto3
@@ -32,12 +33,12 @@ def _do_download(
         click.echo(f"\x1b[{len(progress_str)}D", nl=False)
 
 
-def download_nf_jars(pkg_root: Path):
+def download_nf_jars(pkg_root: Path, nf_version: str):
     s3_resource = boto3.resource("s3")
     s3_resource.meta.client.meta.events.register("choose-signer.s3.*", disable_signing)
     bucket = s3_resource.Bucket("latch-public")
 
-    subdir = "nextflow-v2/"
+    subdir = f"nextflow-v2/{nf_version}/"
     objects = list(bucket.objects.filter(Prefix=f"{subdir}.nextflow/"))
 
     click.secho("  Downloading Nextflow binaries: \x1b[?25l", italic=True, nl=False)
@@ -57,18 +58,42 @@ def download_nf_jars(pkg_root: Path):
     click.secho("Done. \x1b[?25h", italic=True)
 
 
-def ensure_nf_dependencies(pkg_root: Path, *, force_redownload: bool = False):
+def _get_current_version(nf_version_path: Path):
+    if not nf_version_path.exists():
+        return None
+
+    with open(nf_version_path, "r") as f:
+        return f.read().strip()
+
+
+def ensure_nf_dependencies(pkg_root: Path, *, nf_version: Optional[str] = None):
+    nf_version_path = pkg_root / ".latch" / "nextflow_version"
     nf_executable = pkg_root / ".latch" / "bin" / "nextflow"
     nf_jars = pkg_root / ".latch" / ".nextflow"
 
-    if force_redownload:
+    if nf_version is None and nf_version_path.exists():
+        # rahul: if the version file exists locally and there
+        # was no request for update, don't do anything
+        return
+
+    if nf_version is None or nf_version.lower() == "latest":
+        res = tinyrequests.get(
+            "https://latch-public.s3.us-west-2.amazonaws.com/nextflow-v2/LATEST"
+        )
+        res.raise_for_status()
+        nf_version = res.content.decode("utf-8").strip()
+
+    current = _get_current_version(nf_version_path)
+    if current != nf_version:
+        click.secho(f"Updating Nextflow to version {nf_version}", fg="yellow")
+        nf_version_path.unlink(missing_ok=True)
         nf_executable.unlink(missing_ok=True)
         if nf_jars.exists():
             shutil.rmtree(nf_jars)
 
     if not nf_executable.exists():
         res = tinyrequests.get(
-            "https://latch-public.s3.us-west-2.amazonaws.com/nextflow-v2/nextflow"
+            f"https://latch-public.s3.us-west-2.amazonaws.com/nextflow-v2/{nf_version}/nextflow"
         )
         nf_executable.parent.mkdir(parents=True, exist_ok=True)
 
@@ -76,4 +101,7 @@ def ensure_nf_dependencies(pkg_root: Path, *, force_redownload: bool = False):
         nf_executable.chmod(0o700)
 
     if not nf_jars.exists():
-        download_nf_jars(pkg_root)
+        download_nf_jars(pkg_root, nf_version)
+
+    with open(nf_version_path, "w") as f:
+        f.write(nf_version)

latch-2.45.6.dev2/latch_cli/nextflow/types.py

@@ -0,0 +1,6 @@
+from enum import Enum
+
+
+class NextflowProcessExecutor(Enum):
+    local = "local"
+    k8s = "k8s"

{latch-2.45.6 → latch-2.45.6.dev2}/latch_cli/nextflow/workflow.py

@@ -9,6 +9,7 @@ import click
 import latch.types.metadata as metadata
 from latch.types.directory import LatchDir, LatchOutputDir
 from latch.types.file import LatchFile
+from latch_cli.nextflow.types import NextflowProcessExecutor
 from latch_cli.snakemake.config.utils import get_preamble, type_repr
 from latch_cli.snakemake.utils import reindent
 from latch_cli.utils import identifier_from_str, urljoins
@@ -37,6 +38,13 @@ from flytekit.core.annotation import FlyteAnnotation
 
 import latch_metadata
 
+from latch_cli.services.register.utils import import_module_by_path
+
+meta = Path("latch_metadata") / "__init__.py"
+if meta.exists():
+    import_module_by_path(meta)
+
+
 @custom_task(cpu=0.25, memory=0.5, storage_gib=1)
 def initialize() -> str:
     token = os.environ.get("FLYTE_INTERNAL_EXECUTION_ID")
@@ -63,7 +71,7 @@ def initialize() -> str:
 
 {samplesheet_funs}
 
-@nextflow_runtime_task(cpu={cpu}, memory={memory})
+@nextflow_runtime_task(cpu={cpu}, memory={memory}, storage_gib={storage_gib})
 def nextflow_runtime(pvc_name: str, {param_signature}) -> None:
     try:
         shared_dir = Path("/nf-workdir")
@@ -151,6 +159,10 @@ def nextflow_runtime(pvc_name: str, {param_signature}) -> None:
 
 @workflow(metadata._nextflow_metadata)
 def {workflow_func_name}({param_signature_with_defaults}) -> None:
+    \"\"\"
+{docstring}
+    \"\"\"
+
     pvc_name: str = initialize()
     nextflow_runtime(pvc_name=pvc_name, {param_args})
 
@@ -187,18 +199,23 @@ def get_flag(name: str, val: Any) -> List[str]:
         return [flag, str(val)]
 
 
-def generate_nextflow_config(pkg_root: Path):
+def generate_nextflow_config(
+    pkg_root: Path, process_executor: Optional[NextflowProcessExecutor]
+):
+    if process_executor is None:
+        process_executor = NextflowProcessExecutor.k8s
+
     config_path = Path(pkg_root) / "latch.config"
-    config_path.write_text(dedent("""\
-        process {
-            executor = 'k8s'
-        }
+    config_path.write_text(dedent(f"""\
+        process {{
+            executor = '{process_executor.value}'
+        }}
 
-        aws {
-            client {
+        aws {{
+            client {{
                 anonymous = true
-            }
-        }
+            }}
+        }}
         """))
 
     click.secho(f"Nextflow Latch config written to {config_path}", fg="green")
@@ -210,8 +227,9 @@ def generate_nextflow_workflow(
     dest: Path,
     *,
     execution_profile: Optional[str] = None,
+    process_executor: Optional[NextflowProcessExecutor] = None,
 ):
-    generate_nextflow_config(pkg_root)
+    generate_nextflow_config(pkg_root, process_executor)
 
     assert metadata._nextflow_metadata is not None
 
@@ -284,8 +302,19 @@ def generate_nextflow_workflow(
         log_dir = metadata._nextflow_metadata.log_dir._raw_remote_path
     log_dir = urljoins(log_dir, wf_name)
 
+    desc = f"Sample Description"
+    if metadata._nextflow_metadata.about_page_path is not None:
+        desc = metadata._nextflow_metadata.about_page_path.read_text()
+
+    display_name = wf_name
+    if metadata._nextflow_metadata.display_name is not None:
+        display_name = metadata._nextflow_metadata.display_name
+
+    docstring = f"{display_name}\n\n{desc}"
+
     entrypoint = template.format(
         workflow_func_name=identifier_from_str(wf_name),
+        docstring=reindent(docstring, 1),
         nf_script=nf_script.resolve().relative_to(pkg_root.resolve()),
         param_signature_with_defaults=", ".join(
             no_defaults + [f"{name} = {val}" for name, val in defaults]

{latch-2.45.6 → latch-2.45.6.dev2}/latch_cli/services/register/register.py

@@ -15,19 +15,19 @@ from flytekit.core.workflow import WorkflowBase
 from scp import SCPClient
 
 from latch.utils import current_workspace, get_workspaces
-
-from ...centromere.ctx import _CentromereCtx
-from ...centromere.utils import MaybeRemoteDir
-from ...constants import latch_constants
-from ...utils import WorkflowType
-from .constants import ANSI_REGEX, MAX_LINES
-from .utils import (
+from latch_cli.centromere.ctx import _CentromereCtx
+from latch_cli.centromere.utils import MaybeRemoteDir
+from latch_cli.constants import latch_constants
+from latch_cli.nextflow.types import NextflowProcessExecutor
+from latch_cli.services.register.constants import ANSI_REGEX, MAX_LINES
+from latch_cli.services.register.utils import (
     DockerBuildLogItem,
     build_image,
     register_serialized_pkg,
     serialize_pkg_in_container,
     upload_image,
 )
+from latch_cli.utils import WorkflowType
 
 
 def _delete_lines(num: int):
@@ -262,8 +262,9 @@ def register(
     skip_confirmation: bool = False,
     snakefile: Optional[Path] = None,
     nf_script: Optional[Path] = None,
-    nf_redownload_dependencies: bool = False,
+    nf_version: Optional[str] = None,
     nf_execution_profile: Optional[str] = None,
+    nf_process_executor: Optional[NextflowProcessExecutor] = None,
     progress_plain: bool = False,
     cache_tasks: bool = False,
     use_new_centromere: bool = False,
@@ -411,9 +412,7 @@ def register(
             from ...nextflow.dependencies import ensure_nf_dependencies
             from ...nextflow.workflow import generate_nextflow_workflow
 
-            ensure_nf_dependencies(
-                ctx.pkg_root, force_redownload=nf_redownload_dependencies
-            )
+            ensure_nf_dependencies(ctx.pkg_root, nf_version=nf_version)
 
             dest = ctx.pkg_root / "wf" / "entrypoint.py"
             dest.parent.mkdir(exist_ok=True)
@@ -422,6 +421,7 @@ def register(
                 ctx.nf_script,
                 dest,
                 execution_profile=nf_execution_profile,
+                process_executor=nf_process_executor,
             )
 
         click.secho("\nInitializing registration", bold=True)

{latch-2.45.6 → latch-2.45.6.dev2}/latch_cli/snakemake/config/utils.py

@@ -1,11 +1,9 @@
-from dataclasses import asdict, fields, is_dataclass, make_dataclass
+from dataclasses import fields, is_dataclass, make_dataclass
 from enum import Enum
-from textwrap import dedent
 from typing import Any, Dict, List, Optional, Type, Union, get_args, get_origin
 
-import click
 from flytekit.core.annotation import FlyteAnnotation
-from typing_extensions import Annotated, TypeAlias, TypeGuard, TypeVar
+from typing_extensions import Annotated, TypeAlias, TypeGuard
 
 from latch.types.directory import LatchDir
 from latch.types.file import LatchFile

{latch-2.45.6 → latch-2.45.6.dev2}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.45.6",
+    version="v2.45.6.dev2",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),