latch 2.39.0.dev19__tar.gz → 2.39.0.dev21__tar.gz

{latch-2.39.0.dev19/latch.egg-info → latch-2.39.0.dev21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev19
+Version: 2.39.0.dev21
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev19 → latch-2.39.0.dev21/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev19
+Version: 2.39.0.dev21
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev19 → latch-2.39.0.dev21}/latch_cli/extras/nextflow/build.py

@@ -42,7 +42,9 @@ def get_node_name(vertex_id: str) -> str:
     return f"n{vertex_id}"
 
 
-def build_from_nextflow_dag(wf: NextflowWorkflow):
+def build_from_nextflow_dag(
+    wf: NextflowWorkflow, *, execution_profile: Optional[str] = None
+):
     global_start_node = Node(
         id=_common_constants.GLOBAL_INPUT_NODE_ID,
         metadata=None,
@@ -158,6 +160,7 @@ def build_from_nextflow_dag(wf: NextflowWorkflow):
                 import_path=Path(vertex.module),
                 process_name=vertex.label,
                 unaliased=vertex.unaliased,
+                execution_profile=execution_profile,
                 wf=wf,
             )
 
@@ -391,6 +394,7 @@ def build_nf_wf(
     nf_script: Path,
     *,
     redownload_dependencies: bool = False,
+    execution_profile: Optional[str] = None,
 ) -> NextflowWorkflow:
     ensure_nf_dependencies(pkg_root, force_redownload=redownload_dependencies)
 
@@ -469,7 +473,7 @@ def build_nf_wf(
 
     wf = NextflowWorkflow(nf_script, main_dag)
 
-    build_from_nextflow_dag(wf)
+    build_from_nextflow_dag(wf, execution_profile=execution_profile)
 
     return wf
 
@@ -501,7 +505,7 @@ def generate_nf_entrypoint(
 
         from flytekit.extras.persistence import LatchPersistence
         from latch_cli.extras.nextflow.file_persistence import download_files, stage_for_output, upload_files
-        from latch_cli.extras.nextflow.channel import get_mapper_inputs, get_boolean_value
+        from latch_cli.extras.nextflow.channel import get_mapper_inputs, get_boolean_value, get_mapper_outputs
         from latch_cli.utils import check_exists_and_rename, get_parameter_json_value, urljoins
 
         from latch.resources.tasks import custom_task

{latch-2.39.0.dev19 → latch-2.39.0.dev21}/latch_cli/extras/nextflow/channel.py

@@ -1,9 +1,9 @@
 import json
-from typing import Dict, List, Optional, Type, TypeVar
+from typing import Dict, List, NamedTuple, Optional, Type, TypeVar
 
 from latch.types.metadata import _IsDataclass
 
-T = TypeVar("T", bound=_IsDataclass)
+DC = TypeVar("DC", bound=_IsDataclass)
 
 
 def get_boolean_value(channel_value: str) -> bool:
@@ -29,10 +29,10 @@ def get_boolean_value(channel_value: str) -> bool:
 
 
 def get_mapper_inputs(
-    cls: Type[T],
+    cls: Type[DC],
     wf_inputs: Dict[str, object],
     channel_inputs: Dict[str, str],
-) -> List[T]:
+) -> List[DC]:
     value_channels = {}
     queue_channels = {}
 
@@ -49,7 +49,7 @@ def get_mapper_inputs(
     if min_len == float("inf"):
         min_len = 1
 
-    res: List[T] = []
+    res: List[DC] = []
     for i in range(min_len):
         kwargs = {**wf_inputs, **value_channels}
 
@@ -59,3 +59,31 @@ def get_mapper_inputs(
         res.append(cls(**kwargs))
 
     return res
+
+
+NT = TypeVar("NT", bound=NamedTuple)
+
+
+def get_mapper_outputs(adapter_output_cls: Type[NT], mapper_outputs: List[DC]) -> NT:
+    kwargs: Dict[str, str] = {}
+
+    if len(mapper_outputs) == 1:
+        output = mapper_outputs[0]
+        for f in adapter_output_cls._fields:
+            kwargs[f] = getattr(output, f)
+
+        return adapter_output_cls(**kwargs)
+
+    channels: Dict[str, List] = {}
+    for f in adapter_output_cls._fields:
+        channels[f] = []
+
+    for output in mapper_outputs:
+        for f in adapter_output_cls._fields:
+            channel = getattr(output, f)
+            channels[f].extend(json.loads(channel))
+
+    for f in adapter_output_cls._fields:
+        kwargs[f] = json.dumps(channels[f])
+
+    return adapter_output_cls(**kwargs)

{latch-2.39.0.dev19 → latch-2.39.0.dev21}/latch_cli/extras/nextflow/file_persistence.py

@@ -104,11 +104,11 @@ def download_files(
 ):
     paths: List[Path] = []
     for channel in channels:
-        if type(channel) != list:
-            continue
-
-        for param in channel:
-            _extract_paths(param, paths)
+        if type(channel) == dict and "value" in channel:
+            _extract_paths(channel["value"], paths)
+        elif type(channel) == list:
+            for param in channel:
+                _extract_paths(param, paths)
 
     remote = _get_remote(outdir)
 
@@ -178,11 +178,11 @@ def _upload(local: Path, remote: str):
 def upload_files(channels: Dict[str, List[JSONValue]], outdir: LatchDir):
     paths: List[Path] = []
     for channel in channels.values():
-        if type(channel) != list:
-            continue
-
-        for param in channel:
-            _extract_paths(param, paths)
+        if type(channel) == dict and "value" in channel:
+            _extract_paths(channel["value"], paths)
+        elif type(channel) == list:
+            for param in channel:
+                _extract_paths(param, paths)
 
     remote = _get_remote(outdir)
 

{latch-2.39.0.dev19 → latch-2.39.0.dev21}/latch_cli/extras/nextflow/tasks/adapters.py

@@ -216,9 +216,7 @@ class NextflowProcessPostAdapterTask(NextflowBaseTask):
 
         return reindent(
             rf"""
-            return Res{self.name}(
-            __return_str__
-            )
+            return get_mapper_outputs(Res{self.name}, default)
             """,
             1,
         ).replace("__return_str__", return_str)

{latch-2.39.0.dev19 → latch-2.39.0.dev21}/latch_cli/extras/nextflow/tasks/conditional.py

@@ -48,7 +48,7 @@ class NextflowConditionalTask(NextflowOperatorTask):
             res = out_channels.get({repr(out_name)})
 
             if res is not None:
-                res = json.loads(res)[0]["boolean"]
+                res = get_boolean_value(res)
 
             return Res{self.name}(condition=res)
             """,

{latch-2.39.0.dev19 → latch-2.39.0.dev21}/latch_cli/extras/nextflow/tasks/process.py

@@ -1,7 +1,7 @@
 import json
 from dataclasses import fields, is_dataclass
 from pathlib import Path
-from typing import List, Mapping, Type
+from typing import List, Mapping, Optional, Type
 
 from latch.types.directory import LatchDir
 from latch.types.file import LatchFile
@@ -24,6 +24,7 @@ class NextflowProcessTask(NextflowBaseTask):
         import_path: Path,
         process_name: str,
         unaliased: str,
+        execution_profile: Optional[str],
         wf: NextflowWorkflow,
     ):
         super().__init__(inputs, outputs, id, name, {}, wf, cpu=16, memory=32)
@@ -50,6 +51,7 @@ class NextflowProcessTask(NextflowBaseTask):
         self.import_path = import_path
         self.process_name = process_name
         self.unaliased = unaliased
+        self.execution_profile = execution_profile
 
     def get_fn_interface(self):
         input_name, input_t = list(self._python_inputs.items())[0]
@@ -112,6 +114,9 @@ class NextflowProcessTask(NextflowBaseTask):
             str(nf_script_path_in_container),
         ]
 
+        if self.execution_profile is not None:
+            run_task_entrypoint.extend(["-profile", self.execution_profile])
+
         for flag, val in self.wf.flags_to_params.items():
             run_task_entrypoint.extend([flag, str(val)])
 
@@ -166,17 +171,24 @@ class NextflowProcessTask(NextflowBaseTask):
 
             download_files(channel_vals, LatchDir({repr(self.wf.output_directory.remote_path)}))
 
-            subprocess.run(
-                [{','.join([f"str({x})" if x.startswith("wf_") else repr(x) for x in run_task_entrypoint])}],
-                env={{
-                    **os.environ,
-                    "LATCH_INCLUDE_META": {repr(include_str)},
-                    "LATCH_EXPRESSION": {repr(self.statement)},
-                    "LATCH_RETURN": {repr(json.dumps(self.ret))},
-                    "LATCH_PARAM_VALS": json.dumps(channel_vals),
-                }},
-                check=True,
-            )
+            try:
+                subprocess.run(
+                    [{','.join([f"str({x})" if x.startswith("wf_") else repr(x) for x in run_task_entrypoint])}],
+                    env={{
+                        **os.environ,
+                        "LATCH_INCLUDE_META": {repr(include_str)},
+                        "LATCH_EXPRESSION": {repr(self.statement)},
+                        "LATCH_RETURN": {repr(json.dumps(self.ret))},
+                        "LATCH_PARAM_VALS": json.dumps(channel_vals),
+                    }},
+                    check=True,
+                )
+            except subprocess.CalledProcessError:
+                log = Path("/root/.nextflow.log").read_text()
+                print("\n\n\n\n\n" + log)
+
+                import time
+                time.sleep(10000)
 
             out_channels = {{}}
             files = [Path(f) for f in glob.glob(".latch/task-outputs/*.json")]

{latch-2.39.0.dev19 → latch-2.39.0.dev21}/latch_cli/main.py

@@ -405,6 +405,12 @@ def execute(
     default=False,
     help="Redownload external Nextflow dependencies",
 )
+@click.option(
+    "--execution-profile",
+    type=str,
+    default=None,
+    help="Set execution profile for Nextflow workflow",
+)
 def register(
     pkg_root: str,
     disable_auto_version: bool,
@@ -416,6 +422,7 @@ def register(
     cache_tasks: bool,
     nf_script: Optional[Path],
     redownload_dependencies: bool,
+    execution_profile: Optional[str],
 ):
     """Register local workflow code to Latch.
 
@@ -438,6 +445,7 @@ def register(
         snakefile=snakefile,
         nf_script=nf_script,
         nf_redownload_dependencies=redownload_dependencies,
+        nf_execution_profile=execution_profile,
         progress_plain=(docker_progress == "auto" and not sys.stdout.isatty())
         or docker_progress == "plain",
         use_new_centromere=use_new_centromere,

{latch-2.39.0.dev19 → latch-2.39.0.dev21}/latch_cli/services/register/register.py

@@ -275,6 +275,7 @@ def register(
     snakefile: Optional[Path] = None,
     nf_script: Optional[Path] = None,
     nf_redownload_dependencies: bool = False,
+    nf_execution_profile: Optional[str] = None,
     progress_plain: bool = False,
     cache_tasks: bool = False,
     use_new_centromere: bool = False,
@@ -420,6 +421,7 @@ def register(
                 ctx.pkg_root,
                 ctx.nf_script,
                 redownload_dependencies=nf_redownload_dependencies,
+                execution_profile=nf_execution_profile,
             )
             generate_nf_entrypoint(nf_wf, ctx.pkg_root, ctx.nf_script)
 

{latch-2.39.0.dev19 → latch-2.39.0.dev21}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.39.0.dev19",
+    version="v2.39.0.dev21",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),