latch 2.39.0.dev27__tar.gz → 2.39.0.dev29__tar.gz

{latch-2.39.0.dev27/latch.egg-info → latch-2.39.0.dev29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev27
+Version: 2.39.0.dev29
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev27 → latch-2.39.0.dev29/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev27
+Version: 2.39.0.dev29
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev27 → latch-2.39.0.dev29}/latch.egg-info/SOURCES.txt

@@ -97,11 +97,9 @@ latch_cli/extras/nextflow/tasks/__init__.py
 latch_cli/extras/nextflow/tasks/adapters.py
 latch_cli/extras/nextflow/tasks/base.py
 latch_cli/extras/nextflow/tasks/conditional.py
-latch_cli/extras/nextflow/tasks/input.py
 latch_cli/extras/nextflow/tasks/map.py
 latch_cli/extras/nextflow/tasks/merge.py
 latch_cli/extras/nextflow/tasks/operator.py
-latch_cli/extras/nextflow/tasks/output.py
 latch_cli/extras/nextflow/tasks/process.py
 latch_cli/extras/snakemake/__init__.py
 latch_cli/extras/snakemake/config.py

{latch-2.39.0.dev27 → latch-2.39.0.dev29}/latch_cli/extras/nextflow/build.py

@@ -33,7 +33,6 @@ from .tasks.adapters import (
     NextflowProcessPreAdapterTask,
 )
 from .tasks.conditional import NextflowConditionalTask
-from .tasks.input import NextflowInputTask
 from .tasks.map import MapContainerTask
 from .tasks.merge import NextflowMergeTask
 from .tasks.operator import NextflowOperatorTask
@@ -165,7 +164,8 @@ def build_from_nextflow_dag(
                 name=identifier_from_str(vertex.label),
                 statement=vertex.statement,
                 ret=vertex.ret,
-                import_path=Path(vertex.module),
+                script_path=Path(vertex.subWorkflowPath),
+                calling_subwf_name=vertex.subWorkflowName,
                 process_name=vertex.label,
                 unaliased=vertex.unaliased,
                 execution_profile=execution_profile,
@@ -223,43 +223,24 @@ def build_from_nextflow_dag(
 
         elif vertex.type in VertexType.Conditional:
             conditional_task = NextflowConditionalTask(
-                task_inputs,
-                vertex.id,
-                f"conditional_{vertex.label}",
-                vertex.statement,
-                vertex.ret,
-                branches,
-                wf,
-            )
-            wf.nextflow_tasks.append(conditional_task)
-
-            node = Node(
-                id=node_name,
-                metadata=conditional_task.construct_node_metadata(),
-                bindings=task_bindings,
-                upstream_nodes=upstream_nodes,
-                flyte_entity=conditional_task,
-            )
-
-            node_map[vertex.id] = node
-
-        elif vertex.type == VertexType.Input:
-            input_task = NextflowInputTask(
                 inputs=task_inputs,
-                outputs=task_outputs,
-                name=vertex.label,
                 id=vertex.id,
+                name=f"conditional_{vertex.label}",
+                statement=vertex.statement,
+                ret=vertex.ret,
                 branches=branches,
+                script_path=Path(vertex.subWorkflowPath),
+                calling_subwf_name=vertex.subWorkflowName,
                 wf=wf,
             )
-            wf.nextflow_tasks.append(input_task)
+            wf.nextflow_tasks.append(conditional_task)
 
             node = Node(
                 id=node_name,
-                metadata=input_task.construct_node_metadata(),
+                metadata=conditional_task.construct_node_metadata(),
                 bindings=task_bindings,
                 upstream_nodes=upstream_nodes,
-                flyte_entity=input_task,
+                flyte_entity=conditional_task,
             )
 
             node_map[vertex.id] = node
@@ -272,6 +253,8 @@ def build_from_nextflow_dag(
                 id=vertex.id,
                 branches=branches,
                 sources=merge_sources,
+                script_path=Path(vertex.subWorkflowPath),
+                calling_subwf_name=vertex.subWorkflowName,
                 wf=wf,
             )
 
@@ -296,6 +279,8 @@ def build_from_nextflow_dag(
                 statement=vertex.statement,
                 ret=vertex.ret,
                 branches=branches,
+                script_path=Path(vertex.subWorkflowPath),
+                calling_subwf_name=vertex.subWorkflowName,
                 wf=wf,
             )
             wf.nextflow_tasks.append(operator_task)
@@ -461,7 +446,7 @@ def build_nf_wf(
 
             raise click.exceptions.Exit(0)
 
-    wf = NextflowWorkflow(nf_script, version, main_dag)
+    wf = NextflowWorkflow(pkg_root, nf_script, version, main_dag)
 
     build_from_nextflow_dag(wf, execution_profile=execution_profile)
 

{latch-2.39.0.dev27 → latch-2.39.0.dev29}/latch_cli/extras/nextflow/dag.py

@@ -36,6 +36,8 @@ class Vertex:
     outputNames: List[str] = field(hash=False)
     module: str
     unaliased: str
+    subWorkflowName: str
+    subWorkflowPath: str
 
 
 @dataclass(frozen=True)
@@ -55,6 +57,8 @@ class _VertexContentJson(TypedDict):
     outputNames: List[str]
     module: str
     unaliased: str
+    subWorkflowName: str
+    subWorkflowPath: str
 
 
 class _VertexJson(TypedDict):

{latch-2.39.0.dev27 → latch-2.39.0.dev29}/latch_cli/extras/nextflow/tasks/conditional.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Dict, List, Mapping, Optional, Type
 
 from latch.types.metadata import ParameterType
@@ -17,6 +18,8 @@ class NextflowConditionalTask(NextflowOperatorTask):
         statement: str,
         ret: List[str],
         branches: Dict[str, bool],
+        script_path: Path,
+        calling_subwf_name: str,
         wf: NextflowWorkflow,
     ):
         super().__init__(
@@ -27,6 +30,8 @@ class NextflowConditionalTask(NextflowOperatorTask):
             statement,
             ret,
             branches,
+            script_path,
+            calling_subwf_name,
             wf,
         )
 

{latch-2.39.0.dev27 → latch-2.39.0.dev29}/latch_cli/extras/nextflow/tasks/merge.py

@@ -18,6 +18,8 @@ class NextflowMergeTask(NextflowOperatorTask):
         name: str,
         branches: Dict[str, bool],
         sources: Dict[str, List[str]],
+        script_path: Path,
+        calling_subwf_name: str,
         wf: NextflowWorkflow,
     ):
         super().__init__(
@@ -28,6 +30,8 @@ class NextflowMergeTask(NextflowOperatorTask):
             "",
             [],
             branches,
+            script_path,
+            calling_subwf_name,
             wf,
         )
 

{latch-2.39.0.dev27 → latch-2.39.0.dev29}/latch_cli/extras/nextflow/tasks/operator.py

@@ -21,10 +21,14 @@ class NextflowOperatorTask(NextflowBaseTask):
         statement: str,
         ret: List[str],
         branches: Dict[str, bool],
+        script_path: Path,
+        calling_subwf_name: str,
         wf: NextflowWorkflow,
     ):
         self.statement = statement
         self.ret = ret
+        self.script_path = script_path
+        self.calling_subwf_name = calling_subwf_name
 
         super().__init__(inputs, outputs, id, name, branches, wf, NFTaskType.Operator)
 
@@ -171,6 +175,11 @@ class NextflowOperatorTask(NextflowBaseTask):
             download_str = rf"""download_files(channel_vals, LatchDir({repr(self.wf.output_directory.remote_path)}))"""
             upload_str = rf"""upload_files({{k: json.loads(v) for k, v in out_channels.items()}}, LatchDir({repr(self.wf.output_directory.remote_path)}))"""
 
+        if self.script_path.resolve() != self.wf.nf_script.resolve():
+            stem = self.script_path.resolve().relative_to(self.wf.pkg_root.resolve())
+            run_task_entrypoint[2] = str(Path("/root") / stem)
+            run_task_entrypoint.extend(["-entry", self.calling_subwf_name])
+
         code_block += reindent(
             rf"""
                     channel_vals = [{", ".join([f"json.loads({x})" for x in self.channel_inputs])}]

{latch-2.39.0.dev27 → latch-2.39.0.dev29}/latch_cli/extras/nextflow/tasks/process.py

@@ -21,14 +21,15 @@ class NextflowProcessTask(NextflowBaseTask):
         name: str,
         statement: str,
         ret: List[str],
-        import_path: Path,
+        script_path: Path,
+        calling_subwf_name: str,
         process_name: str,
         unaliased: str,
         execution_profile: Optional[str],
         wf: NextflowWorkflow,
     ):
         super().__init__(
-            inputs, outputs, id, name, {}, wf, NFTaskType.Process, cpu=16, memory=32
+            inputs, outputs, id, name, {}, wf, NFTaskType.Process, cpu=16, memory=48
         )
 
         self.wf_inputs = {}
@@ -50,7 +51,8 @@ class NextflowProcessTask(NextflowBaseTask):
 
         self.statement = statement
         self.ret = ret
-        self.import_path = import_path
+        self.script_path = script_path
+        self.calling_subwf_name = calling_subwf_name
         self.process_name = process_name
         self.unaliased = unaliased
         self.execution_profile = execution_profile
@@ -152,19 +154,10 @@ class NextflowProcessTask(NextflowBaseTask):
                     1,
                 )
 
-        include_str = ""
-        if self.import_path.resolve() != self.wf.nf_script.resolve():
-            include_meta = {}
-            stem = str(
-                self.import_path.resolve().relative_to(
-                    self.wf.nf_script.parent.resolve()
-                )
-            )
-            include_meta["path"] = f"./{stem}"
-            include_meta["alias"] = self.process_name
-            include_meta["name"] = self.unaliased
-
-            include_str = json.dumps(include_meta)
+        if self.script_path.resolve() != self.wf.nf_script.resolve():
+            stem = self.script_path.resolve().relative_to(self.wf.pkg_root.resolve())
+            run_task_entrypoint[2] = str(Path("/root") / stem)
+            run_task_entrypoint.extend(["-entry", self.calling_subwf_name])
 
         code_block += reindent(
             rf"""
@@ -178,7 +171,6 @@ class NextflowProcessTask(NextflowBaseTask):
                     [{','.join([f"str({x})" if x.startswith("wf_") else repr(x) for x in run_task_entrypoint])}],
                     env={{
                         **os.environ,
-                        "LATCH_INCLUDE_META": {repr(include_str)},
                         "LATCH_EXPRESSION": {repr(self.statement)},
                         "LATCH_RETURN": {repr(json.dumps(self.ret))},
                         "LATCH_PARAM_VALS": json.dumps(channel_vals),
@@ -188,9 +180,7 @@ class NextflowProcessTask(NextflowBaseTask):
             except subprocess.CalledProcessError:
                 log = Path("/root/.nextflow.log").read_text()
                 print("\n\n\n\n\n" + log)
-
-                import time
-                time.sleep(10000)
+                raise
 
             out_channels = {{}}
             files = [Path(f) for f in glob.glob(".latch/task-outputs/*.json")]

{latch-2.39.0.dev27 → latch-2.39.0.dev29}/latch_cli/extras/nextflow/workflow.py

@@ -42,7 +42,7 @@ def _get_flags_to_params(key: str, t: Type, flags: Dict[str, str]):
 
 
 class NextflowWorkflow(WorkflowBase, ClassStorageTaskResolver):
-    def __init__(self, nf_script: Path, version: str, dag: DAG):
+    def __init__(self, pkg_root: Path, nf_script: Path, version: str, dag: DAG):
         assert metadata._nextflow_metadata is not None
         assert metadata._nextflow_metadata.output_directory is not None
 
@@ -91,6 +91,7 @@ class NextflowWorkflow(WorkflowBase, ClassStorageTaskResolver):
 
         self.nextflow_tasks: List[NextflowBaseTask] = []
 
+        self.pkg_root = pkg_root
         self.nf_script = nf_script
         self.version = version
         self.dag = dag

{latch-2.39.0.dev27 → latch-2.39.0.dev29}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.39.0.dev27",
+    version="v2.39.0.dev29",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),

latch-2.39.0.dev27/latch_cli/extras/nextflow/tasks/input.py

@@ -1,67 +0,0 @@
-from pathlib import Path
-from typing import Dict, List, Mapping, Type
-
-from latch.types.metadata import ParameterType
-
-from ....extras.common.utils import reindent
-from ..workflow import NextflowWorkflow
-from .operator import NextflowOperatorTask
-
-
-class NextflowInputTask(NextflowOperatorTask):
-    def __init__(
-        self,
-        inputs: Mapping[str, Type[ParameterType]],
-        outputs: Mapping[str, Type[ParameterType]],
-        id: str,
-        name: str,
-        branches: Dict[str, bool],
-        wf: NextflowWorkflow,
-    ):
-        super().__init__(inputs, outputs, id, name, "", [], branches, wf)
-
-        assert len(self.channel_inputs) == 1, (
-            self.channel_inputs,
-            self.conditional_inputs,
-            self.wf_inputs,
-        )
-
-    def get_fn_return_stmt(self):
-        results: List[str] = []
-        for out_name in self._python_outputs.keys():
-            results.append(reindent(rf"{out_name}=res", 2).rstrip())
-
-        return_str = ",\n".join(results)
-
-        return reindent(
-            rf"""
-                return Res{self.name}(
-            __return_str__
-                )
-            """,
-            0,
-        ).replace("__return_str__", return_str)
-
-    def get_fn_code(self, nf_script_path_in_container: Path):
-        code_block = self.get_fn_interface()
-        code_block += self.get_fn_conditions()
-
-        channel_input = self.channel_inputs.popitem()[0]
-
-        code_block += reindent(
-            rf"""
-                res = {channel_input}
-            else:
-                print("TASK SKIPPED")
-                try:
-                    _override_task_status(status="SKIPPED")
-                except Exception as e:
-                    print(f"Failed to override task status: {{e}}")
-                res = None
-
-            """,
-            1,
-        )
-
-        code_block += self.get_fn_return_stmt()
-        return code_block

latch-2.39.0.dev27/latch_cli/extras/nextflow/tasks/output.py

@@ -1,86 +0,0 @@
-import json
-from pathlib import Path
-from typing import Dict, List, Mapping, Type
-
-from latch.types.metadata import ParameterType
-
-from ....extras.common.utils import reindent
-from ..workflow import NextflowWorkflow
-from .operator import NextflowOperatorTask
-
-
-class NextflowOutputTask(NextflowOperatorTask):
-    def __init__(
-        self,
-        inputs: Mapping[str, Type[ParameterType]],
-        outputs: Mapping[str, Type[ParameterType]],
-        id: str,
-        name: str,
-        statement: str,
-        ret: List[str],
-        branches: Dict[str, bool],
-        wf: NextflowWorkflow,
-    ):
-        super().__init__(
-            inputs,
-            outputs,
-            id,
-            name,
-            statement,
-            ret,
-            branches,
-            wf,
-        )
-
-    def get_fn_return_stmt(self):
-        assert len(self._python_outputs) == len(
-            self.channel_inputs
-        ), f"{self.name, self._python_outputs, self.channel_inputs}"
-
-        results: List[str] = []
-        for out_name, param_name in zip(
-            self._python_outputs.keys(), self.channel_inputs.keys()
-        ):
-            results.append(
-                reindent(rf"{out_name}=({param_name} if cond else None)", 2).rstrip()
-            )
-
-        return_str = ",\n".join(results)
-
-        return reindent(
-            rf"""
-                return Res{self.name}(
-            __return_str__
-                )
-            """,
-            0,
-        ).replace("__return_str__", return_str)
-
-    def get_fn_conditions(self):
-        res: List[str] = []
-        for k in self.conditional_inputs.keys():
-            res.append(f"({k} == {self.branches[k]})")
-        for k in self.channel_inputs.keys():
-            res.append(f"({k} is not None)")
-
-        if len(res) == 0:
-            return reindent(
-                f"""\
-                cond = True
-                """,
-                1,
-            )
-
-        return reindent(
-            f"""\
-            cond = ({' and '.join(res)})
-            """,
-            1,
-        )
-
-    def get_fn_code(self, nf_script_path_in_container: Path):
-        code_block = self.get_fn_interface()
-        code_block += self.get_fn_conditions()
-        code_block += self.get_fn_return_stmt()
-
-        return code_block