latch 2.40.4.dev3__tar.gz → 2.40.4.dev5__tar.gz

{latch-2.40.4.dev3/latch.egg-info → latch-2.40.4.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.40.4.dev3
+Version: 2.40.4.dev5
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8
@@ -18,7 +18,7 @@ Requires-Dist: paramiko>=3.2.0
 Requires-Dist: scp>=0.14.0
 Requires-Dist: boto3>=1.26.0
 Requires-Dist: tqdm>=4.63.0
-Requires-Dist: lytekit==0.15.5
+Requires-Dist: lytekit==0.15.6
 Requires-Dist: lytekitplugins-pods==0.6.2
 Requires-Dist: typing-extensions==4.7.1
 Requires-Dist: apscheduler==3.9.1

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch/account.py

@@ -258,12 +258,10 @@ class AccountUpdate:
 
         args = l.ArgumentNode()
         args.name = _name_node("input")
-        args.value = _json_value(
-            {
-                "argOwnerId": self.account.id,
-                "argDisplayNames": display_names,
-            }
-        )
+        args.value = _json_value({
+            "argOwnerId": self.account.id,
+            "argDisplayNames": display_names,
+        })
 
         res.alias = _name_node(f"upd{len(mutations)}")
         res.arguments = tuple([args])
@@ -299,11 +297,9 @@ class AccountUpdate:
 
         args = l.ArgumentNode()
         args.name = _name_node("input")
-        args.value = _json_value(
-            {
-                "argIds": ids,
-            }
-        )
+        args.value = _json_value({
+            "argIds": ids,
+        })
 
         res.alias = _name_node(f"upd{len(mutations)}")
         res.arguments = tuple([args])

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch/registry/project.py

@@ -228,12 +228,10 @@ class ProjectUpdate:
 
         args = l.ArgumentNode()
         args.name = _name_node("input")
-        args.value = _json_value(
-            {
-                "argProjectId": self.project.id,
-                "argDisplayNames": display_names,
-            }
-        )
+        args.value = _json_value({
+            "argProjectId": self.project.id,
+            "argDisplayNames": display_names,
+        })
 
         res.alias = _name_node(f"upd{len(mutations)}")
         res.arguments = tuple([args])
@@ -267,11 +265,9 @@ class ProjectUpdate:
 
         args = l.ArgumentNode()
         args.name = _name_node("input")
-        args.value = _json_value(
-            {
-                "argIds": ids,
-            }
-        )
+        args.value = _json_value({
+            "argIds": ids,
+        })
 
         res.alias = _name_node(f"upd{len(mutations)}")
         res.arguments = tuple([args])

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch/registry/table.py

@@ -496,13 +496,11 @@ class TableUpdate:
 
         args = l.ArgumentNode()
         args.name = _name_node("input")
-        args.value = _json_value(
-            {
-                "argExperimentId": self.table.id,
-                "argNames": names,
-                "argData": _var_node(argDataVar),
-            }
-        )
+        args.value = _json_value({
+            "argExperimentId": self.table.id,
+            "argNames": names,
+            "argData": _var_node(argDataVar),
+        })
 
         res.alias = _name_node(f"upd{len(mutations)}")
         res.arguments = tuple([args])
@@ -537,12 +535,10 @@ class TableUpdate:
 
         args = l.ArgumentNode()
         args.name = _name_node("input")
-        args.value = _json_value(
-            {
-                "argExperimentId": self.table.id,
-                "argNames": names,
-            }
-        )
+        args.value = _json_value({
+            "argExperimentId": self.table.id,
+            "argNames": names,
+        })
 
         res.alias = _name_node(f"upd{len(mutations)}")
         res.arguments = tuple([args])
@@ -690,13 +686,11 @@ class TableUpdate:
 
         args = l.ArgumentNode()
         args.name = _name_node("input")
-        args.value = _json_value(
-            {
-                "argExperimentId": self.table.id,
-                "argKeys": keys,
-                "argTypes": _var_node(argTypesVar),
-            }
-        )
+        args.value = _json_value({
+            "argExperimentId": self.table.id,
+            "argKeys": keys,
+            "argTypes": _var_node(argTypesVar),
+        })
 
         res.alias = _name_node(f"upd{len(mutations)}")
         res.arguments = tuple([args])

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch/types/metadata.py

@@ -553,6 +553,10 @@ class LatchMetadata:
     """ A `LatchAuthor` object that describes the author of the workflow"""
     documentation: Optional[str] = None
     """A link to documentation for the workflow itself"""
+    about_page_markdown: Optional[Path] = None
+    """A path to a markdown file that will be rendered as the workflow About
+    page. If provided, it will override the docstring markdown, if it exists,
+    for SDK workflows."""
     repository: Optional[str] = None
     """A link to the repository where the code for the workflow is hosted"""
     license: str = "MIT"
@@ -577,6 +581,8 @@ class LatchMetadata:
         metadata_dict = asdict(self)
         # remove parameters since that will be handled by each parameters' dict() method
         del metadata_dict["parameters"]
+        # pyyaml is unable to serialize PosixPath objects + unused after registration
+        del metadata_dict["about_page_markdown"]
         metadata_dict["license"] = {"id": self.license}
 
         # flows override all other rendering, so disable them entirely if not provided

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch/verified/deseq2.py

@@ -19,17 +19,13 @@ def deseq2_wf(
     raw_count_table: Optional[
         Annotated[
             LatchFile,
-            FlyteAnnotation(
-                {
-                    "_tmp_hack_deseq2": "counts_table",
-                    "rules": [
-                        {
-                            "regex": r".*\.(csv|tsv|xlsx)$",
-                            "message": "Expected a CSV, TSV, or XLSX file",
-                        }
-                    ],
-                }
-            ),
+            FlyteAnnotation({
+                "_tmp_hack_deseq2": "counts_table",
+                "rules": [{
+                    "regex": r".*\.(csv|tsv|xlsx)$",
+                    "message": "Expected a CSV, TSV, or XLSX file",
+                }],
+            }),
         ]
     ] = None,
     raw_count_tables: List[LatchFile] = [],
@@ -44,17 +40,13 @@ def deseq2_wf(
     conditions_table: Optional[
         Annotated[
             LatchFile,
-            FlyteAnnotation(
-                {
-                    "_tmp_hack_deseq2": "design_matrix",
-                    "rules": [
-                        {
-                            "regex": r".*\.(csv|tsv|xlsx)$",
-                            "message": "Expected a CSV, TSV, or XLSX file",
-                        }
-                    ],
-                }
-            ),
+            FlyteAnnotation({
+                "_tmp_hack_deseq2": "design_matrix",
+                "rules": [{
+                    "regex": r".*\.(csv|tsv|xlsx)$",
+                    "message": "Expected a CSV, TSV, or XLSX file",
+                }],
+            }),
         ]
     ] = None,
     design_matrix_sample_id_column: Optional[
@@ -62,12 +54,10 @@ def deseq2_wf(
     ] = None,
     design_formula: Annotated[
         List[List[str]],
-        FlyteAnnotation(
-            {
-                "_tmp_hack_deseq2": "design_formula",
-                "_tmp_hack_deseq2_allow_clustering": True,
-            }
-        ),
+        FlyteAnnotation({
+            "_tmp_hack_deseq2": "design_formula",
+            "_tmp_hack_deseq2_allow_clustering": True,
+        }),
     ] = [],
     number_of_genes_to_plot: int = 30,
 ) -> LatchDir: ...

{latch-2.40.4.dev3 → latch-2.40.4.dev5/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.40.4.dev3
+Version: 2.40.4.dev5
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8
@@ -18,7 +18,7 @@ Requires-Dist: paramiko>=3.2.0
 Requires-Dist: scp>=0.14.0
 Requires-Dist: boto3>=1.26.0
 Requires-Dist: tqdm>=4.63.0
-Requires-Dist: lytekit==0.15.5
+Requires-Dist: lytekit==0.15.6
 Requires-Dist: lytekitplugins-pods==0.6.2
 Requires-Dist: typing-extensions==4.7.1
 Requires-Dist: apscheduler==3.9.1

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch.egg-info/requires.txt

@@ -7,7 +7,7 @@ paramiko>=3.2.0
 scp>=0.14.0
 boto3>=1.26.0
 tqdm>=4.63.0
-lytekit==0.15.5
+lytekit==0.15.6
 lytekitplugins-pods==0.6.2
 typing-extensions==4.7.1
 apscheduler==3.9.1

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/centromere/utils.py

@@ -2,6 +2,7 @@ import builtins
 import contextlib
 import os
 import random
+import shutil
 import string
 import sys
 import tempfile
@@ -261,6 +262,9 @@ class MaybeRemoteDir:
             return
 
         if self._tempdir is not None and not isinstance(self._tempdir, str):
+            if os.environ.get("LATCH_SAVE_PROTO") is not None:
+                shutil.copytree(Path(self._tempdir.name), Path.cwd() / "protos")
+
             self._tempdir.cleanup()
 
 

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/extras/nextflow/build.py

@@ -50,7 +50,10 @@ def get_node_name(vertex_id: str) -> str:
 
 
 def build_from_nextflow_dag(
-    wf: NextflowWorkflow, *, execution_profile: Optional[str] = None
+    wf: NextflowWorkflow,
+    *,
+    execution_profile: Optional[str] = None,
+    ephemeral_storage_gib: int = 500,
 ):
     global_start_node = Node(
         id=_common_constants.GLOBAL_INPUT_NODE_ID,
@@ -105,18 +108,22 @@ def build_from_nextflow_dag(
             else:
                 input_name = f"channel_{dep.id}"
 
-                output_name = "res"
+                dep_output_name = "res"
                 if len(dep.outputNames) > 0:
                     idx = int(edge.label or "0")
                     input_name = f"{input_name}_{idx}"
-                    output_name = dep.outputNames[idx]
+                    dep_output_name = dep.outputNames[idx]
 
                 if vertex.type == VertexType.Merge:
-                    merge_sources[output_name].append(input_name)
+                    vertex_output_name = "res"
+                    if len(vertex.outputNames) > 0:
+                        vertex_output_name = dep_output_name
+
+                    merge_sources[vertex_output_name].append(input_name)
 
                 task_inputs[input_name] = Optional[str]
 
-                node = NodeOutput(node=node_map[dep.id], var=output_name)
+                node = NodeOutput(node=node_map[dep.id], var=dep_output_name)
 
             task_bindings.append(
                 literals_models.Binding(
@@ -177,6 +184,7 @@ def build_from_nextflow_dag(
                 wf=wf,
                 cpu=vertex.cpu,
                 memory=vertex.memoryBytes,
+                storage_gib=ephemeral_storage_gib,
             )
 
             wf.nextflow_tasks.append(process_task)
@@ -220,9 +228,21 @@ def build_from_nextflow_dag(
                                 ),
                             ).ref
                         ),
-                    )
+                    ),
+                    literals_models.Binding(
+                        var="is_skipped",
+                        binding=literals_models.BindingData(
+                            promise=Promise(
+                                var="is_skipped",
+                                val=NodeOutput(
+                                    node=pre_adapter_node,
+                                    var="is_skipped",
+                                ),
+                            ).ref
+                        ),
+                    ),
                 ],
-                upstream_nodes=[mapped_process_node],
+                upstream_nodes=[mapped_process_node, pre_adapter_node],
                 flyte_entity=post_adapter_task,
             )
 
@@ -396,6 +416,7 @@ def build_nf_wf(
     *,
     redownload_dependencies: bool = False,
     execution_profile: Optional[str] = None,
+    ephemeral_storage_gib: int = 500,
 ) -> NextflowWorkflow:
     ensure_nf_dependencies(pkg_root, force_redownload=redownload_dependencies)
 
@@ -474,7 +495,11 @@ def build_nf_wf(
 
     wf = NextflowWorkflow(pkg_root, nf_script, version, main_dag)
 
-    build_from_nextflow_dag(wf, execution_profile=execution_profile)
+    build_from_nextflow_dag(
+        wf,
+        execution_profile=execution_profile,
+        ephemeral_storage_gib=ephemeral_storage_gib,
+    )
 
     return wf
 

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/extras/nextflow/channel.py

@@ -64,8 +64,16 @@ def get_mapper_inputs(
 NT = TypeVar("NT", bound=NamedTuple)
 
 
-def get_mapper_outputs(adapter_output_cls: Type[NT], mapper_outputs: List[DC]) -> NT:
-    kwargs: Dict[str, str] = {}
+def get_mapper_outputs(
+    adapter_output_cls: Type[NT], mapper_outputs: List[DC], is_skipped: bool
+) -> NT:
+
+    kwargs: Dict[str, Optional[str]] = {}
+    if is_skipped:
+        for f in adapter_output_cls._fields:
+            kwargs[f] = None
+
+        return adapter_output_cls(**kwargs)
 
     if len(mapper_outputs) == 1:
         output = mapper_outputs[0]

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/extras/nextflow/file_persistence.py

@@ -1,5 +1,5 @@
+import json
 import os
-import shutil
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Dict, List, Optional, Set, Union
@@ -230,5 +230,18 @@ def upload_files(channels: Dict[str, List[JSONValue]], outdir: LatchDir):
         local_to_remote[local] = remote
         data.parameter["remote"] = remote
 
+    published_files: List[str] = []
+    try:
+        with open(".latch/published.json", "r") as f:
+            data = f.read()
+        conf = json.loads(data)
+        published_files = conf.get("files", [])
+    except FileNotFoundError:
+        pass
+
+    for file in published_files:
+        relative_path = Path(file).relative_to(Path.home())
+        local_to_remote[file] = urljoins(remote_parent, str(relative_path))
+
     for local, remote in local_to_remote.items():
         _upload(local, remote)

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/extras/nextflow/tasks/adapters.py

@@ -66,7 +66,7 @@ class NextflowProcessPreAdapterTask(NextflowBaseTask):
 
         super().__init__(
             inputs,
-            {"default": List[self.dataclass]},
+            {"default": List[self.dataclass], "is_skipped": bool},
             id,
             name,
             branches,
@@ -96,6 +96,7 @@ class NextflowProcessPreAdapterTask(NextflowBaseTask):
 
             class Res_{self.id}(NamedTuple):
                 default: {type_repr(output_typ)}
+                is_skipped: bool
 
             """,
             0,
@@ -114,7 +115,9 @@ class NextflowProcessPreAdapterTask(NextflowBaseTask):
         return res
 
     def get_fn_return_stmt(self):
-        return reindent(f"return Res_{self.id}(default=result)", 1)
+        return reindent(
+            f"return Res_{self.id}(default=result, is_skipped = not cond)", 1
+        )
 
     def get_fn_code(self, nf_script_path_in_container: Path):
         code_block = self.get_fn_interface()
@@ -173,7 +176,7 @@ class NextflowProcessPostAdapterTask(NextflowBaseTask):
         self.dataclass = dataclass_from_python_params(outputs, id)
 
         super().__init__(
-            {"default": List[self.dataclass]},
+            {"default": List[self.dataclass], "is_skipped": bool},
             outputs,
             id,
             name,
@@ -210,7 +213,8 @@ class NextflowProcessPostAdapterTask(NextflowBaseTask):
             rf"""
             @task(cache=True)
             def {self.name}(
-                default: List[Dataclass_{self.id}]
+                default: List[Dataclass_{self.id}],
+                is_skipped: bool,
             ) -> Res{self.name}:
             """,
             0,
@@ -221,7 +225,7 @@ class NextflowProcessPostAdapterTask(NextflowBaseTask):
     def get_fn_return_stmt(self):
         return reindent(
             rf"""
-            return get_mapper_outputs(Res{self.name}, default)
+            return get_mapper_outputs(Res{self.name}, default, is_skipped)
             """,
             1,
         )

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/extras/nextflow/tasks/base.py

@@ -49,6 +49,7 @@ class NextflowBaseTask(PythonAutoContainerTask[Pod]):
         # ayush: 0.5/1 is too small and stuff starts taking a while to run
         cpu: float = 1,
         memory: float = 2,
+        storage_gib: int = 500,
     ):
         self.id = id
         self.wf = wf
@@ -77,7 +78,9 @@ class NextflowBaseTask(PythonAutoContainerTask[Pod]):
             task_type_version=2,
             name=f"{name}_{id}",
             interface=interface,
-            task_config=custom_task(cpu=cpu, memory=memory).keywords["task_config"],
+            task_config=custom_task(
+                cpu=cpu, memory=memory, storage_gib=storage_gib
+            ).keywords["task_config"],
             task_resolver=NextflowBaseTaskResolver(),
         )
 

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/extras/nextflow/tasks/operator.py

@@ -90,7 +90,7 @@ class NextflowOperatorTask(NextflowBaseTask):
             results.append(
                 reindent(
                     rf"""
-                    {out_name}=out_channels.get("{out_name}", "[]")
+                    {out_name}=out_channels.get("{out_name}")
                     """,
                     2,
                 ).rstrip()

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/extras/nextflow/tasks/process.py

@@ -34,6 +34,7 @@ class NextflowProcessTask(NextflowBaseTask):
         wf: NextflowWorkflow,
         cpu: Optional[int] = None,
         memory: Optional[float] = None,
+        storage_gib: int = 500,
     ):
         super().__init__(
             inputs,
@@ -45,6 +46,7 @@ class NextflowProcessTask(NextflowBaseTask):
             NFTaskType.Process,
             cpu=cpu if cpu is not None else 4,
             memory=memory / 1024 / 1024 / 1024 if memory is not None else 8,
+            storage_gib=storage_gib,
         )
 
         self.wf_inputs = {}
@@ -100,7 +102,7 @@ class NextflowProcessTask(NextflowBaseTask):
             results.append(
                 reindent(
                     rf"""
-                    {field.name}=out_channels.get(f"{field.name}", "[]")
+                    {field.name}=out_channels.get(f"{field.name}")
                     """,
                     2,
                 ).rstrip()

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/extras/nextflow/workflow.py

@@ -2,6 +2,7 @@ from dataclasses import fields, is_dataclass
 from pathlib import Path
 from typing import Any, Dict, List, Tuple, Type
 
+import click
 from flytekit.core.class_based_resolver import ClassStorageTaskResolver
 from flytekit.core.docstring import Docstring
 from flytekit.core.interface import Interface
@@ -48,9 +49,19 @@ class NextflowWorkflow(WorkflowBase, ClassStorageTaskResolver):
 
         self.output_directory = metadata._nextflow_metadata.output_directory
         self.docker_metadata = metadata._nextflow_metadata.docker_metadata
-
+        md_path = metadata._nextflow_metadata.about_page_markdown
+
+        if md_path is not None and md_path.exists():
+            click.secho(f"Rendering workflow About page content from {md_path}")
+            markdown_content = md_path.read_text()
+        else:
+            markdown_content = (
+                "Add markdown content to a file and add the path",
+                "to your workflow metadata to populate this page.\nMore information",
+                "[here](https://wiki.latch.bio/docs/nextflow/quickstart).",
+            )
         docstring = Docstring(
-            f"{metadata._nextflow_metadata.display_name}\n\nSample Description\n\n"
+            f"{metadata._nextflow_metadata.display_name}\n\n{markdown_content}\n\n"
             + str(metadata._nextflow_metadata)
         )
         python_interface = Interface(

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/main.py

@@ -473,6 +473,12 @@ def execute(
     default=None,
     help="Set execution profile for Nextflow workflow",
 )
+@click.option(
+    "--ephemeral-storage-gib",
+    default=500,
+    type=int,
+    help="Input the number of GiB needed of ephemeral storage.",
+)
 @requires_login
 def register(
     pkg_root: str,
@@ -486,6 +492,7 @@ def register(
     nf_script: Optional[Path],
     redownload_dependencies: bool,
     execution_profile: Optional[str],
+    ephemeral_storage_gib: int,
 ):
     """Register local workflow code to Latch.
 
@@ -497,6 +504,22 @@ def register(
     crash_handler.message = "Unable to register workflow."
     crash_handler.pkg_root = pkg_root
 
+    if ephemeral_storage_gib > 4949:
+        click.secho(
+            f"You tried setting ephemeral storage to {ephemeral_storage_gib} GiB. "
+            "This value exceeds the maximum allowed size of 4949 GiB.",
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
+    if ephemeral_storage_gib <= 0:
+        click.secho(
+            f"You tried setting ephemeral storage to {ephemeral_storage_gib} GiB. "
+            "Ephemeral storage value must be a positive integer less than 4949.",
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
     from latch_cli.services.register import register
 
     register(
@@ -513,6 +536,7 @@ def register(
         or docker_progress == "plain",
         use_new_centromere=use_new_centromere,
         cache_tasks=cache_tasks,
+        nf_ephemeral_storage_gib=ephemeral_storage_gib,
     )
 
 

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/latch_cli/services/register/register.py

@@ -280,6 +280,7 @@ def register(
     progress_plain: bool = False,
     cache_tasks: bool = False,
     use_new_centromere: bool = False,
+    nf_ephemeral_storage_gib: int = 500,
 ):
     """Registers a workflow, defined as python code, with Latch.
 
@@ -424,6 +425,7 @@ def register(
                 ctx.nf_script,
                 redownload_dependencies=nf_redownload_dependencies,
                 execution_profile=nf_execution_profile,
+                ephemeral_storage_gib=nf_ephemeral_storage_gib,
             )
             generate_nf_entrypoint(nf_wf, ctx.pkg_root, ctx.nf_script)
 

{latch-2.40.4.dev3 → latch-2.40.4.dev5}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.40.4.dev3",
+    version="v2.40.4.dev5",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),
@@ -30,7 +30,7 @@ setup(
         "scp>=0.14.0",
         "boto3>=1.26.0",
         "tqdm>=4.63.0",
-        "lytekit==0.15.5",
+        "lytekit==0.15.6",
         "lytekitplugins-pods==0.6.2",
         "typing-extensions==4.7.1",
         "apscheduler==3.9.1",