latch 2.39.0.dev18__tar.gz → 2.39.0.dev20__tar.gz

{latch-2.39.0.dev18/latch.egg-info → latch-2.39.0.dev20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev18
+Version: 2.39.0.dev20
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev18 → latch-2.39.0.dev20/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev18
+Version: 2.39.0.dev20
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev18 → latch-2.39.0.dev20}/latch_cli/extras/nextflow/build.py

@@ -42,7 +42,9 @@ def get_node_name(vertex_id: str) -> str:
     return f"n{vertex_id}"
 
 
-def build_from_nextflow_dag(wf: NextflowWorkflow):
+def build_from_nextflow_dag(
+    wf: NextflowWorkflow, *, execution_profile: Optional[str] = None
+):
     global_start_node = Node(
         id=_common_constants.GLOBAL_INPUT_NODE_ID,
         metadata=None,
@@ -158,6 +160,7 @@ def build_from_nextflow_dag(wf: NextflowWorkflow):
                 import_path=Path(vertex.module),
                 process_name=vertex.label,
                 unaliased=vertex.unaliased,
+                execution_profile=execution_profile,
                 wf=wf,
             )
 
@@ -391,6 +394,7 @@ def build_nf_wf(
     nf_script: Path,
     *,
     redownload_dependencies: bool = False,
+    execution_profile: Optional[str] = None,
 ) -> NextflowWorkflow:
     ensure_nf_dependencies(pkg_root, force_redownload=redownload_dependencies)
 
@@ -469,7 +473,7 @@ def build_nf_wf(
 
     wf = NextflowWorkflow(nf_script, main_dag)
 
-    build_from_nextflow_dag(wf)
+    build_from_nextflow_dag(wf, execution_profile=execution_profile)
 
     return wf
 
@@ -501,7 +505,7 @@ def generate_nf_entrypoint(
 
         from flytekit.extras.persistence import LatchPersistence
         from latch_cli.extras.nextflow.file_persistence import download_files, stage_for_output, upload_files
-        from latch_cli.extras.nextflow.channel import get_mapper_inputs
+        from latch_cli.extras.nextflow.channel import get_mapper_inputs, get_boolean_value
         from latch_cli.utils import check_exists_and_rename, get_parameter_json_value, urljoins
 
         from latch.resources.tasks import custom_task

{latch-2.39.0.dev18 → latch-2.39.0.dev20}/latch_cli/extras/nextflow/channel.py

@@ -1,11 +1,33 @@
 import json
-from typing import Dict, List, Type, TypeVar
+from typing import Dict, List, Optional, Type, TypeVar
 
 from latch.types.metadata import _IsDataclass
 
 T = TypeVar("T", bound=_IsDataclass)
 
 
+def get_boolean_value(channel_value: str) -> bool:
+    og_error: Optional[Exception] = None
+
+    try:
+        payload = json.loads(channel_value)
+
+        if type(payload) is list:
+            return payload[0]["boolean"]
+        elif type(payload) is dict:
+            return payload["value"]["boolean"]
+    except Exception as e:
+        og_error = e
+
+    error = ValueError(
+        f"Unable to extract boolean value for conditional: {channel_value}"
+    )
+    if og_error is not None:
+        raise error from og_error
+
+    raise error
+
+
 def get_mapper_inputs(
     cls: Type[T],
     wf_inputs: Dict[str, object],

{latch-2.39.0.dev18 → latch-2.39.0.dev20}/latch_cli/extras/nextflow/file_persistence.py

@@ -104,11 +104,11 @@ def download_files(
 ):
     paths: List[Path] = []
     for channel in channels:
-        if type(channel) != list:
-            continue
-
-        for param in channel:
-            _extract_paths(param, paths)
+        if type(channel) == dict and "value" in channel:
+            _extract_paths(channel["value"], paths)
+        elif type(channel) == list:
+            for param in channel:
+                _extract_paths(param, paths)
 
     remote = _get_remote(outdir)
 
@@ -178,11 +178,11 @@ def _upload(local: Path, remote: str):
 def upload_files(channels: Dict[str, List[JSONValue]], outdir: LatchDir):
     paths: List[Path] = []
     for channel in channels.values():
-        if type(channel) != list:
-            continue
-
-        for param in channel:
-            _extract_paths(param, paths)
+        if type(channel) == dict and "value" in channel:
+            _extract_paths(channel["value"], paths)
+        elif type(channel) == list:
+            for param in channel:
+                _extract_paths(param, paths)
 
     remote = _get_remote(outdir)
 

{latch-2.39.0.dev18 → latch-2.39.0.dev20}/latch_cli/extras/nextflow/tasks/conditional.py

@@ -48,7 +48,7 @@ class NextflowConditionalTask(NextflowOperatorTask):
             res = out_channels.get({repr(out_name)})
 
             if res is not None:
-                res = json.loads(res)[0]["boolean"]
+                res = get_boolean_value(res)
 
             return Res{self.name}(condition=res)
             """,

{latch-2.39.0.dev18 → latch-2.39.0.dev20}/latch_cli/extras/nextflow/tasks/operator.py

@@ -135,7 +135,6 @@ class NextflowOperatorTask(NextflowBaseTask):
                     """,
                     2,
                 )
-
             elif is_blob_type(typ):
                 code_block += reindent(
                     f"""

{latch-2.39.0.dev18 → latch-2.39.0.dev20}/latch_cli/extras/nextflow/tasks/process.py

@@ -1,7 +1,7 @@
 import json
 from dataclasses import fields, is_dataclass
 from pathlib import Path
-from typing import List, Mapping, Type
+from typing import List, Mapping, Optional, Type
 
 from latch.types.directory import LatchDir
 from latch.types.file import LatchFile
@@ -24,6 +24,7 @@ class NextflowProcessTask(NextflowBaseTask):
         import_path: Path,
         process_name: str,
         unaliased: str,
+        execution_profile: Optional[str],
         wf: NextflowWorkflow,
     ):
         super().__init__(inputs, outputs, id, name, {}, wf, cpu=16, memory=32)
@@ -50,6 +51,7 @@ class NextflowProcessTask(NextflowBaseTask):
         self.import_path = import_path
         self.process_name = process_name
         self.unaliased = unaliased
+        self.execution_profile = execution_profile
 
     def get_fn_interface(self):
         input_name, input_t = list(self._python_inputs.items())[0]
@@ -112,6 +114,9 @@ class NextflowProcessTask(NextflowBaseTask):
             str(nf_script_path_in_container),
         ]
 
+        if self.execution_profile is not None:
+            run_task_entrypoint.extend(["-profile", self.execution_profile])
+
         for flag, val in self.wf.flags_to_params.items():
             run_task_entrypoint.extend([flag, str(val)])
 
@@ -138,9 +143,8 @@ class NextflowProcessTask(NextflowBaseTask):
                 code_block += reindent(
                     f"""
                     if {k} is not None:
-                        {k}_p = {k}._create_imposters()
-                        check_exists_and_rename({k}_p, Path("/root") / {k}_p.name)
-                        wf_paths[{k}] = Path("/root") / {k}_p.name
+                        {k}_p = Path("/root/").resolve() # superhack
+                        wf_paths[{k}] = {k}_p
 
                     """,
                     1,
@@ -167,17 +171,24 @@ class NextflowProcessTask(NextflowBaseTask):
 
             download_files(channel_vals, LatchDir({repr(self.wf.output_directory.remote_path)}))
 
-            subprocess.run(
-                [{','.join([f"str({x})" if x.startswith("wf_") else repr(x) for x in run_task_entrypoint])}],
-                env={{
-                    **os.environ,
-                    "LATCH_INCLUDE_META": {repr(include_str)},
-                    "LATCH_EXPRESSION": {repr(self.statement)},
-                    "LATCH_RETURN": {repr(json.dumps(self.ret))},
-                    "LATCH_PARAM_VALS": json.dumps(channel_vals),
-                }},
-                check=True,
-            )
+            try:
+                subprocess.run(
+                    [{','.join([f"str({x})" if x.startswith("wf_") else repr(x) for x in run_task_entrypoint])}],
+                    env={{
+                        **os.environ,
+                        "LATCH_INCLUDE_META": {repr(include_str)},
+                        "LATCH_EXPRESSION": {repr(self.statement)},
+                        "LATCH_RETURN": {repr(json.dumps(self.ret))},
+                        "LATCH_PARAM_VALS": json.dumps(channel_vals),
+                    }},
+                    check=True,
+                )
+            except subprocess.CalledProcessError:
+                log = Path("/root/.nextflow.log").read_text()
+                print("\n\n\n\n\n" + log)
+
+                import time
+                time.sleep(10000)
 
             out_channels = {{}}
             files = [Path(f) for f in glob.glob(".latch/task-outputs/*.json")]

{latch-2.39.0.dev18 → latch-2.39.0.dev20}/latch_cli/main.py

@@ -405,6 +405,12 @@ def execute(
     default=False,
     help="Redownload external Nextflow dependencies",
 )
+@click.option(
+    "--execution-profile",
+    type=str,
+    default=None,
+    help="Set execution profile for Nextflow workflow",
+)
 def register(
     pkg_root: str,
     disable_auto_version: bool,
@@ -416,6 +422,7 @@ def register(
     cache_tasks: bool,
     nf_script: Optional[Path],
     redownload_dependencies: bool,
+    execution_profile: Optional[str],
 ):
     """Register local workflow code to Latch.
 
@@ -438,6 +445,7 @@ def register(
         snakefile=snakefile,
         nf_script=nf_script,
         nf_redownload_dependencies=redownload_dependencies,
+        nf_execution_profile=execution_profile,
         progress_plain=(docker_progress == "auto" and not sys.stdout.isatty())
         or docker_progress == "plain",
         use_new_centromere=use_new_centromere,

{latch-2.39.0.dev18 → latch-2.39.0.dev20}/latch_cli/services/register/register.py

@@ -275,6 +275,7 @@ def register(
     snakefile: Optional[Path] = None,
     nf_script: Optional[Path] = None,
     nf_redownload_dependencies: bool = False,
+    nf_execution_profile: Optional[str] = None,
     progress_plain: bool = False,
     cache_tasks: bool = False,
     use_new_centromere: bool = False,
@@ -420,6 +421,7 @@ def register(
                 ctx.pkg_root,
                 ctx.nf_script,
                 redownload_dependencies=nf_redownload_dependencies,
+                execution_profile=nf_execution_profile,
             )
             generate_nf_entrypoint(nf_wf, ctx.pkg_root, ctx.nf_script)
 

{latch-2.39.0.dev18 → latch-2.39.0.dev20}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.39.0.dev18",
+    version="v2.39.0.dev20",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),