latch 2.39.0.dev21__tar.gz → 2.39.0.dev23__tar.gz

{latch-2.39.0.dev21/latch.egg-info → latch-2.39.0.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev21
+Version: 2.39.0.dev23
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch/types/metadata.py

@@ -661,7 +661,7 @@ class SnakemakeMetadata(LatchMetadata):
     """
 
     def validate(self):
-        from latch_cli.extras.snakemake.config.utils import validate_snakemake_type
+        from latch_cli.extras.snakemake.config import validate_snakemake_type
 
         for name, param in self.parameters.items():
             if param.default is None:

{latch-2.39.0.dev21 → latch-2.39.0.dev23/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev21
+Version: 2.39.0.dev23
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch.egg-info/SOURCES.txt

@@ -85,6 +85,7 @@ latch_cli/extras/common/utils.py
 latch_cli/extras/nextflow/__init__.py
 latch_cli/extras/nextflow/build.py
 latch_cli/extras/nextflow/channel.py
+latch_cli/extras/nextflow/config.py
 latch_cli/extras/nextflow/dag.py
 latch_cli/extras/nextflow/file_persistence.py
 latch_cli/extras/nextflow/serialize.py
@@ -100,13 +101,11 @@ latch_cli/extras/nextflow/tasks/operator.py
 latch_cli/extras/nextflow/tasks/output.py
 latch_cli/extras/nextflow/tasks/process.py
 latch_cli/extras/snakemake/__init__.py
+latch_cli/extras/snakemake/config.py
 latch_cli/extras/snakemake/serialize.py
 latch_cli/extras/snakemake/single_task_snakemake.py
 latch_cli/extras/snakemake/utils.py
 latch_cli/extras/snakemake/workflow.py
-latch_cli/extras/snakemake/config/__init__.py
-latch_cli/extras/snakemake/config/parser.py
-latch_cli/extras/snakemake/config/utils.py
 latch_cli/services/__init__.py
 latch_cli/services/get.py
 latch_cli/services/get_executions.py
@@ -193,6 +192,7 @@ latch_cli/services/test_data/utils.py
 latch_cli/tui/__init__.py
 latch_cli/utils/__init__.py
 latch_cli/utils/path.py
+latch_cli/utils/workflow.py
 tests/__init__.py
 tests/fixtures.py
 tests/test_ls.py

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch_cli/docker_utils/__init__.py

@@ -90,6 +90,7 @@ def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
             "# DO NOT CHANGE",
             "",
             "copy .latch/bin/nextflow /root/nextflow",
+            "copy .latch/.nextflow /root/.nextflow",
             "copy .latch/nf_entrypoint.py /root/nf_entrypoint.py",
         ]
 

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch_cli/extras/common/serialize.py

@@ -15,7 +15,7 @@ from flytekit.core.context_manager import FlyteContextManager
 from flytekit.core.interface import Interface
 from flytekit.core.node import Node
 from flytekit.core.promise import Promise
-from flytekit.core.type_engine import TypeEngine
+from flytekit.core.type_engine import TypeEngine, TypeTransformerFailedError
 from flytekit.core.utils import _dnsify
 from flytekit.core.workflow import WorkflowBase
 from flytekit.models import common as common_models
@@ -89,9 +89,17 @@ def interface_to_parameters(
 
         ctx = FlyteContextManager.current_context()
         if default is not None:
-            default_lv = TypeEngine.to_literal(
-                ctx, default, python_type=interface.inputs[k], expected=v.type
-            )
+            try:
+                default_lv = TypeEngine.to_literal(
+                    ctx, default, python_type=interface.inputs[k], expected=v.type
+                )
+            except TypeTransformerFailedError as e:
+                click.secho(
+                    f"Failed to transform default value for parameter `{k}` to a"
+                    f" literal: {str(e)}",
+                    fg="red",
+                )
+                raise click.exceptions.Exit(1) from e
 
         params[k] = interface_models.Parameter(
             var=v, default=default_lv, required=required

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch_cli/extras/nextflow/build.py

@@ -3,24 +3,27 @@ import os
 import shutil
 import subprocess
 import sys
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 from pathlib import Path
 from textwrap import dedent
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, cast
 
 import boto3
 import click
+import gql
 from flytekit.core import constants as _common_constants
 from flytekit.core.interface import transform_variable_map
 from flytekit.core.node import Node
 from flytekit.core.promise import NodeOutput, Promise
 from flytekit.models import literals as literals_models
+from latch_sdk_gql.execute import execute
 
 from latch_cli import tinyrequests
+from latch_cli.extras.nextflow.tasks.base import NextflowBaseTask, NFTaskType
 
 from ...click_utils import italic
 from ...menus import select_tui
-from ...utils import identifier_from_str
+from ...utils import current_workspace, identifier_from_str
 from ..common.serialize import binding_from_python
 from ..common.utils import reindent
 from .dag import DAG, VertexType
@@ -33,7 +36,6 @@ from .tasks.input import NextflowInputTask
 from .tasks.map import MapContainerTask
 from .tasks.merge import NextflowMergeTask
 from .tasks.operator import NextflowOperatorTask
-from .tasks.output import NextflowOutputTask
 from .tasks.process import NextflowProcessTask
 from .workflow import NextflowWorkflow
 
@@ -84,6 +86,7 @@ def build_from_nextflow_dag(
 
         task_bindings: List[literals_models.Binding] = [*global_wf_input_bindings]
         branches: Dict[str, bool] = {}
+        merge_sources: Dict[str, List[str]] = defaultdict(lambda: [])
         for dep, edge in wf.dag.ancestors()[vertex]:
             if dep.type == VertexType.Conditional:
                 input_name = f"condition_{dep.id}"
@@ -96,11 +99,15 @@ def build_from_nextflow_dag(
                 node = NodeOutput(node=node_map[dep.id], var=f"condition")
             else:
                 input_name = f"channel_{dep.id}"
-                output_name = "res"
 
+                output_name = "res"
                 if len(dep.outputNames) > 0:
                     idx = int(edge.label)
-                    input_name = output_name = dep.outputNames[idx]
+                    input_name = f"{input_name}_{idx}"
+                    output_name = dep.outputNames[idx]
+
+                if vertex.type == VertexType.Merge:
+                    merge_sources[output_name].append(input_name)
 
                 task_inputs[input_name] = Optional[str]
 
@@ -121,6 +128,8 @@ def build_from_nextflow_dag(
             upstream_nodes.append(node_map[dep.id])
 
         node_name = get_node_name(vertex.id)
+        if vertex.type == VertexType.Merge:
+            task_outputs = {k: Optional[str] for k in merge_sources.keys()}
 
         if vertex.type == VertexType.Process:
             pre_adapter_task = NextflowProcessPreAdapterTask(
@@ -256,29 +265,6 @@ def build_from_nextflow_dag(
 
             node_map[vertex.id] = node
 
-        elif vertex.type == VertexType.Output:
-            output_task = NextflowOutputTask(
-                inputs=task_inputs,
-                outputs=task_outputs,
-                name=vertex.label,
-                id=vertex.id,
-                statement=vertex.statement,
-                ret=vertex.ret,
-                branches=branches,
-                wf=wf,
-            )
-            wf.nextflow_tasks.append(output_task)
-
-            node = Node(
-                id=node_name,
-                metadata=output_task.construct_node_metadata(),
-                bindings=task_bindings,
-                upstream_nodes=upstream_nodes,
-                flyte_entity=output_task,
-            )
-
-            node_map[vertex.id] = node
-
         elif vertex.type == VertexType.Merge:
             merge_task = NextflowMergeTask(
                 inputs=task_inputs,
@@ -286,6 +272,7 @@ def build_from_nextflow_dag(
                 name=vertex.label,
                 id=vertex.id,
                 branches=branches,
+                sources=merge_sources,
                 wf=wf,
             )
 
@@ -314,6 +301,9 @@ def build_from_nextflow_dag(
             )
             wf.nextflow_tasks.append(operator_task)
 
+            if vertex.type != "Operator":
+                operator_task.nf_task_type = NFTaskType(vertex.type)
+
             node = Node(
                 id=node_name,
                 metadata=operator_task.construct_node_metadata(),
@@ -391,6 +381,7 @@ def ensure_nf_dependencies(pkg_root: Path, *, force_redownload: bool = False):
 
 def build_nf_wf(
     pkg_root: Path,
+    version: str,
     nf_script: Path,
     *,
     redownload_dependencies: bool = False,
@@ -471,7 +462,7 @@ def build_nf_wf(
 
             raise click.exceptions.Exit(0)
 
-    wf = NextflowWorkflow(nf_script, main_dag)
+    wf = NextflowWorkflow(nf_script, version, main_dag)
 
     build_from_nextflow_dag(wf, execution_profile=execution_profile)
 
@@ -507,6 +498,7 @@ def generate_nf_entrypoint(
         from latch_cli.extras.nextflow.file_persistence import download_files, stage_for_output, upload_files
         from latch_cli.extras.nextflow.channel import get_mapper_inputs, get_boolean_value, get_mapper_outputs
         from latch_cli.utils import check_exists_and_rename, get_parameter_json_value, urljoins
+        from latch_cli.utils.workflow import _override_task_status
 
         from latch.resources.tasks import custom_task
         from latch.types.directory import LatchDir, LatchOutputDir

latch-2.39.0.dev23/latch_cli/extras/nextflow/config.py

@@ -0,0 +1,158 @@
+import json
+import os
+import subprocess
+from pathlib import Path
+from typing import List
+
+import click
+
+from ...utils import best_effort_display_name, identifier_from_str
+from ..common.config.parser import parse_config, write_metadata
+from ..common.config.utils import JSONValue, get_preamble, type_repr
+from ..common.utils import reindent
+from .build import ensure_nf_dependencies
+
+nextflow_metadata = """
+from latch.types.metadata import NextflowMetadata, LatchAuthor, EnvironmentConfig
+from latch.types.directory import LatchDir
+
+from .parameters import generated_parameters
+
+NextflowMetadata(
+    name="Your Workflow Name",
+    display_name="Your Workflow Name",
+    author=LatchAuthor(
+        name="Your Name",
+    ),
+    parameters=generated_parameters,
+    output_directory=LatchDir("latch:///your_output_directory"),
+)
+
+"""
+
+
+def generate_nf_metadata(
+    config_path: Path,
+    *,
+    skip_confirmation: bool = False,
+    generate_defaults: bool = False,
+    infer_files: bool = False,
+):
+    if not config_path.is_dir():
+        click.secho(
+            f"Nextflow config path must point to the package root directory.",
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
+    ensure_nf_dependencies(config_path)
+
+    env = {
+        **os.environ,
+        # read NF binaries from `.latch/.nextflow` instead of system
+        "NXF_HOME": str(config_path / ".latch" / ".nextflow"),
+        # don't display version mismatch warning
+        "NXF_DISABLE_CHECK_LATEST": "true",
+        # don't emit .nextflow.log files
+        "NXF_LOG_FILE": "/dev/null",
+    }
+
+    config_file = ".latch/nf-config.yaml"
+    try:
+        subprocess.run(
+            [
+                ".latch/bin/nextflow",
+                "config",
+                "-format",
+                "json",
+                "-output",
+                config_file,
+            ],
+            check=True,
+            env=env if os.environ.get("LATCH_NEXTFLOW_DEV") is not None else None,
+            cwd=config_path,
+        )
+    except subprocess.CalledProcessError as e:
+        click.secho(
+            "Failed to read config from nextflow package.",
+            fg="red",
+        )
+        raise click.exceptions.Exit(1) from e
+
+    config_file = config_path / config_file
+
+    try:
+        config: JSONValue = json.loads(config_file.read_text())
+    except json.JSONDecodeError as e:
+        click.secho(
+            reindent(
+                f"""
+                Error loading config from {config_file}:
+
+                {e}
+                """,
+                0,
+            ),
+            fg="red",
+        )
+        raise click.exceptions.Exit(1) from e
+
+    parsed = parse_config(config.get("params", {}), infer_files=infer_files)
+
+    preambles: List[str] = []
+    params: List[str] = []
+
+    for k, (typ, (val, default)) in parsed.items():
+        preambles.append(get_preamble(typ))
+
+        param_str = reindent(
+            f"""\
+            {repr(identifier_from_str(k))}: NextflowParameter(
+                display_name={repr(best_effort_display_name(k))},
+                type={type_repr(typ)},
+            __default__),""",
+            0,
+        )
+
+        default_str = ""
+        if generate_defaults and default is not None:
+            default_str = f"    default={repr(default)},\n"
+
+        param_str = param_str.replace("__default__", default_str)
+
+        param_str = reindent(param_str, 1)
+        params.append(param_str)
+
+    params_file_str = (
+        reindent(
+            r"""
+            from dataclasses import dataclass
+            import typing
+            import typing_extensions
+
+            from flytekit.core.annotation import FlyteAnnotation
+
+            from latch.types.metadata import NextflowParameter
+            from latch.types.file import LatchFile
+            from latch.types.directory import LatchDir
+
+            __preambles__
+
+            # Import these into your `__init__.py` file:
+            #
+            # from .parameters import generated_parameters, file_metadata
+
+            generated_parameters = {
+            __params__
+            }
+
+            """,
+            0,
+        )
+        .replace("__preambles__", "".join(preambles))
+        .replace("__params__", "\n".join(params))
+    )
+
+    write_metadata(
+        nextflow_metadata, params_file_str, skip_confirmation=skip_confirmation
+    )

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch_cli/extras/nextflow/dag.py

@@ -24,7 +24,6 @@ class VertexType(str, Enum):
     Conditional = "Conditional"
     Merge = "Merge"
     Input = "Input"
-    Output = "Output"
 
 
 @dataclass(frozen=True)

latch-2.39.0.dev23/latch_cli/extras/nextflow/serialize.py

@@ -0,0 +1,60 @@
+from typing import cast
+
+import gql
+from latch_sdk_gql.execute import execute
+
+from latch_cli.extras.common.serialize import serialize
+from latch_cli.extras.nextflow.tasks.base import NextflowBaseTask
+from latch_cli.extras.nextflow.workflow import NextflowWorkflow
+from latch_cli.utils import current_workspace
+
+
+def serialize_nf(
+    nf_wf: NextflowWorkflow,
+    output_dir: str,
+    image_name: str,
+    dkr_repo: str,
+):
+    serialize(nf_wf, output_dir, image_name, dkr_repo, write_spec=True)
+
+
+def add_task_metadata(wf: NextflowWorkflow):
+    node_names = []
+    task_types = []
+
+    for node in wf.nodes:
+        task = cast(NextflowBaseTask, node.flyte_entity)
+
+        node_names.append(node.id)
+        task_types.append(task.nf_task_type)
+
+    execute(
+        gql.gql("""
+            mutation NFBulkInsertMut(
+                $nodeNames: [String]!
+                $ownerId: BigInt!
+                $types: [String]!
+                $version: String!
+                $workflowName: String!
+            ) {
+                bulkInsertNextflowNodeInfo(
+                    input: {
+                        ownerId: $ownerId
+                        workflowName: $workflowName
+                        version: $version
+                        nodeNames: $nodeNames
+                        types: $types
+                    }
+                ) {
+                    clientMutationId
+                }
+            }
+        """),
+        {
+            "ownerId": current_workspace(),
+            "version": wf.version,
+            "workflowName": wf.name,
+            "nodeNames": node_names,
+            "types": task_types,
+        },
+    )

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch_cli/extras/nextflow/tasks/adapters.py

@@ -7,7 +7,7 @@ from latch.types.metadata import ParameterType, _IsDataclass
 
 from ...common.utils import reindent, type_repr
 from ..workflow import NextflowWorkflow
-from .base import NextflowBaseTask
+from .base import NextflowBaseTask, NFTaskType
 
 
 def dataclass_from_python_params(
@@ -65,7 +65,13 @@ class NextflowProcessPreAdapterTask(NextflowBaseTask):
         self.dataclass = dataclass_from_python_params(inputs, id)
 
         super().__init__(
-            inputs, {"default": List[self.dataclass]}, id, name, branches, wf
+            inputs,
+            {"default": List[self.dataclass]},
+            id,
+            name,
+            branches,
+            wf,
+            NFTaskType.Adapter,
         )
 
     def get_fn_interface(self):
@@ -110,7 +116,7 @@ class NextflowProcessPreAdapterTask(NextflowBaseTask):
     def get_fn_return_stmt(self):
         return reindent(f"return Res_{self.id}(default=result)", 1)
 
-    def get_fn_code(self, nf_path_in_container: str):
+    def get_fn_code(self, nf_script_path_in_container: Path):
         code_block = self.get_fn_interface()
         code_block += self.get_fn_conditions()
 
@@ -142,6 +148,10 @@ class NextflowProcessPreAdapterTask(NextflowBaseTask):
             rf"""
             else:
                 print("TASK SKIPPED")
+                try:
+                    _override_task_status(status="SKIPPED")
+                except Exception as e:
+                    print(f"Failed to override task status: {{e}}")
                 result = []
 
             """,
@@ -162,7 +172,15 @@ class NextflowProcessPostAdapterTask(NextflowBaseTask):
     ):
         self.dataclass = dataclass_from_python_params(outputs, id)
 
-        super().__init__({"default": List[self.dataclass]}, outputs, id, name, {}, wf)
+        super().__init__(
+            {"default": List[self.dataclass]},
+            outputs,
+            id,
+            name,
+            {},
+            wf,
+            NFTaskType.Adapter,
+        )
 
     def get_fn_interface(self):
         res = ""

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch_cli/extras/nextflow/tasks/base.py

@@ -1,4 +1,5 @@
 import importlib
+from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Mapping, Type, cast
 
@@ -25,6 +26,16 @@ from ...common.utils import reindent
 from ..workflow import NextflowWorkflow
 
 
+class NFTaskType(str, Enum):
+    Process = "Process"
+    Adapter = "Adapter"
+    Operator = "Operator"
+    SubWorkflow = "SubWorkflow"
+    Generator = "Generator"
+    Conditional = "Conditional"
+    Merge = "Merge"
+
+
 class NextflowBaseTask(PythonAutoContainerTask[Pod]):
     def __init__(
         self,
@@ -34,12 +45,14 @@ class NextflowBaseTask(PythonAutoContainerTask[Pod]):
         name: str,
         branches: Dict[str, bool],
         wf: NextflowWorkflow,
+        nf_task_type: NFTaskType,
         # todo(ayush): expose / infer these somehow
         cpu: int = 4,
         memory: int = 8,
     ):
         self.id = id
         self.wf = wf
+        self.nf_task_type = nf_task_type
 
         interface = Interface(inputs, outputs, docstring=None)
         self._python_inputs = inputs

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch_cli/extras/nextflow/tasks/conditional.py

@@ -1,6 +1,7 @@
 from typing import Dict, List, Mapping, Optional, Type
 
 from latch.types.metadata import ParameterType
+from latch_cli.extras.nextflow.tasks.base import NFTaskType
 
 from ...common.utils import reindent
 from ..workflow import NextflowWorkflow
@@ -29,23 +30,12 @@ class NextflowConditionalTask(NextflowOperatorTask):
             wf,
         )
 
-    def get_fn_return_stmt(self):
-        results: List[str] = []
-        for out_name, out_type in self._python_outputs.items():
-            results.append(
-                reindent(
-                    rf"""
-                    {out_name}=json.loads(out_channels.get("{out_name}", "true"))
-                    """,
-                    2,
-                ).rstrip()
-            )
-
-        return_str = ",\n".join(results)
+        self.nf_task_type = NFTaskType.Conditional
 
+    def get_fn_return_stmt(self):
         return reindent(
             rf"""
-            res = out_channels.get({repr(out_name)})
+            res = out_channels.get("condition")
 
             if res is not None:
                 res = get_boolean_value(res)
@@ -53,4 +43,4 @@ class NextflowConditionalTask(NextflowOperatorTask):
             return Res{self.name}(condition=res)
             """,
             1,
-        ).replace("__return_str__", return_str)
+        )

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch_cli/extras/nextflow/tasks/input.py

@@ -53,6 +53,10 @@ class NextflowInputTask(NextflowOperatorTask):
                 res = {channel_input}
             else:
                 print("TASK SKIPPED")
+                try:
+                    _override_task_status(status="SKIPPED")
+                except Exception as e:
+                    print(f"Failed to override task status: {{e}}")
                 res = None
 
             """,

{latch-2.39.0.dev21 → latch-2.39.0.dev23}/latch_cli/extras/nextflow/tasks/map.py

@@ -18,7 +18,7 @@ from flytekit.models.interface import Variable
 from flytekit.models.task import Container, K8sPod, Sql
 from flytekitplugins.pod.task import Pod
 
-from .base import NextflowBaseTaskResolver
+from .base import NextflowBaseTaskResolver, NFTaskType
 from .process import NextflowProcessTask
 
 
@@ -38,6 +38,8 @@ class MapContainerTask(PythonAutoContainerTask[Pod]):
             container_task.python_interface
         )
 
+        self.nf_task_type = NFTaskType.Process
+
         super().__init__(
             name=name,
             interface=collection_interface,