latch 2.45.2.dev2__tar.gz → 2.45.2.dev4__tar.gz

{latch-2.45.2.dev2/latch.egg-info → latch-2.45.2.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.45.2.dev2
+Version: 2.45.2.dev4
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.45.2.dev2 → latch-2.45.2.dev4/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.45.2.dev2
+Version: 2.45.2.dev4
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.45.2.dev2 → latch-2.45.2.dev4}/latch_cli/docker_utils/__init__.py

@@ -78,24 +78,6 @@ def get_prologue(
 def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
     cmds: list[str] = []
 
-    if wf_type == WorkflowType.snakemake:
-        cmds += [
-            "",
-            "# Latch snakemake workflow entrypoint",
-            "# DO NOT CHANGE",
-            "",
-            "copy .latch/snakemake_jit_entrypoint.py /root/snakemake_jit_entrypoint.py",
-        ]
-    elif wf_type == WorkflowType.nextflow:
-        cmds += [
-            "",
-            "# Latch nextflow workflow entrypoint",
-            "# DO NOT CHANGE",
-            "",
-            "copy .latch/bin/nextflow /root/nextflow",
-            "copy .latch/.nextflow /root/.nextflow",
-        ]
-
     cmds += [
         "",
         "# Latch workflow registration metadata",
@@ -324,14 +306,66 @@ def infer_commands(pkg_root: Path) -> List[DockerCmdBlock]:
     return commands
 
 
+def copy_file_commands(wf_type: WorkflowType) -> List[DockerCmdBlock]:
+    cmd = []
+
+    cmd += [
+        "",
+        "# Copy workflow data (use .dockerignore to skip files)",
+        "",
+        "copy . /root/",
+    ]
+
+    if wf_type == WorkflowType.snakemake:
+        cmd += [
+            "",
+            "# Latch snakemake workflow entrypoint",
+            "# DO NOT CHANGE",
+            "",
+            "copy .latch/snakemake_jit_entrypoint.py /root/snakemake_jit_entrypoint.py",
+        ]
+    elif wf_type == WorkflowType.nextflow:
+        cmd += [
+            "",
+            "# Latch nextflow workflow entrypoint",
+            "# DO NOT CHANGE",
+            "",
+            "run ln -s /root/.latch/bin/nextflow /root/nextflow",
+            "run ln -s /root/.latch/.nextflow /root/.nextflow",
+        ]
+
+    return cmd
+
+
+def generate_dockerignore(pkg_root: Path, *, wf_type: WorkflowType) -> None:
+    if wf_type != WorkflowType.nextflow:
+        return
+
+    dest = Path(pkg_root) / ".dockerignore"
+    if dest.exists() and not click.confirm(
+        f".dockerignore already exists at `{dest}`. Overwrite?"
+    ):
+        return
+
+    with Path(".dockerignore").open("w") as f:
+        f.write(".git/\n")
+        f.write(".github/\n")
+        f.write(".nextflow*\n")
+        f.write("work/\n")
+        f.write("results/\n")
+        f.write(".nextflow.log*\n")
+
+    click.secho(f"Successfully generated .dockerignore `{dest}`", fg="green")
+
+
 def generate_dockerfile(
-    pkg_root: Path, outfile: Path, *, wf_type: WorkflowType = WorkflowType.latchbiosdk
+    pkg_root: Path, *, wf_type: WorkflowType = WorkflowType.latchbiosdk
 ) -> None:
     """Generate a best effort Dockerfile from files in the workflow directory.
 
     Args:
         pkg_root: A path to a workflow directory.
-        outfile: The path to write the generated Dockerfile.
+        dest: The path to write the generated Dockerfile.
         wf_type: The type of workflow (eg. snakemake) the Dockerfile is for
 
     Example:
@@ -342,6 +376,11 @@ def generate_dockerfile(
             #   ├── Dockerfile
             #   └── ...
     """
+    dest = pkg_root / "Dockerfile"
+    if dest.exists() and not click.confirm(
+        f"Dockerfile already exists at `{dest}`. Overwrite?"
+    ):
+        return
 
     click.secho("Generating Dockerfile", bold=True)
     try:
@@ -368,7 +407,7 @@ def generate_dockerfile(
     )
     click.echo()
 
-    with outfile.open("w") as f:
+    with dest.open("w") as f:
         f.write("\n".join(get_prologue(config, wf_type)) + "\n\n")
 
         commands = infer_commands(pkg_root)
@@ -381,8 +420,7 @@ def generate_dockerfile(
 
             block.write_block(f)
 
-        f.write("# Copy workflow data (use .dockerignore to skip files)\n")
-        f.write("copy . /root/\n\n")
+        f.write("\n".join(copy_file_commands(wf_type)) + "\n\n")
 
         for block in commands:
             if block.order != DockerCmdBlockOrder.postcopy:
@@ -392,6 +430,8 @@ def generate_dockerfile(
 
         f.write("\n".join(get_epilogue(wf_type)) + "\n")
 
+    click.secho(f"Successfully generated dockerfile `{dest}`", fg="green")
+
 
 def get_default_dockerfile(pkg_root: Path, *, wf_type: WorkflowType):
     default_dockerfile = pkg_root / "Dockerfile"

{latch-2.45.2.dev2 → latch-2.45.2.dev4}/latch_cli/main.py

@@ -217,14 +217,7 @@ def dockerfile(pkg_root: str, snakemake: bool = False, nextflow: bool = False):
         )
         raise click.exceptions.Exit(1)
 
-    from latch_cli.docker_utils import generate_dockerfile
-
-    source = Path(pkg_root)
-    dest = source / "Dockerfile"
-    if dest.exists() and not click.confirm(
-        f"Dockerfile already exists at `{dest}`. Overwrite?"
-    ):
-        return
+    from latch_cli.docker_utils import generate_dockerfile, generate_dockerignore
 
     workflow_type = WorkflowType.latchbiosdk
     if snakemake is True:
@@ -232,8 +225,9 @@ def dockerfile(pkg_root: str, snakemake: bool = False, nextflow: bool = False):
     elif nextflow is True:
         workflow_type = WorkflowType.nextflow
 
-    generate_dockerfile(source, dest, wf_type=workflow_type)
-    click.secho(f"Successfully generated dockerfile `{dest}`", fg="green")
+    source = Path(pkg_root)
+    generate_dockerfile(source, wf_type=workflow_type)
+    generate_dockerignore(source, wf_type=workflow_type)
 
 
 @main.command("generate-metadata")

{latch-2.45.2.dev2 → latch-2.45.2.dev4}/latch_cli/nextflow/dependencies.py

@@ -1,5 +1,4 @@
 import shutil
-import subprocess
 from concurrent.futures import ProcessPoolExecutor
 from ctypes import c_int
 from multiprocessing.managers import SyncManager
@@ -8,9 +7,9 @@ from urllib.parse import urljoin
 
 import boto3
 import click
+from botocore.handlers import disable_signing
 
 from latch_cli import tinyrequests
-from latch_cli.utils import dedent
 
 
 def _do_download(
@@ -35,6 +34,7 @@ def _do_download(
 
 def download_nf_jars(pkg_root: Path):
     s3_resource = boto3.resource("s3")
+    s3_resource.meta.client.meta.events.register("choose-signer.s3.*", disable_signing)
     bucket = s3_resource.Bucket("latch-public")
 
     subdir = "nextflow-v2/"
@@ -51,8 +51,6 @@ def download_nf_jars(pkg_root: Path):
                     "https://latch-public.s3.us-west-2.amazonaws.com/", obj.key
                 )
                 obj_path = pkg_root / ".latch" / obj.key[len(subdir) :]
-                print(obj_path)
-
                 exec.submit(_do_download, url, obj_path, len(objects), counter, lock)
 
     click.echo("\x1b[0K", nl=False)

{latch-2.45.2.dev2 → latch-2.45.2.dev4}/latch_cli/nextflow/workflow.py

@@ -60,47 +60,56 @@ def initialize() -> str:
 
 {preambles}
 
-{samplesheet_constructors}
+{samplesheet_funs}
 
 @nextflow_runtime_task(cpu={cpu}, memory={memory})
 def nextflow_runtime(pvc_name: str, {param_signature}) -> None:
     try:
         shared_dir = Path("/nf-workdir")
 
+{samplesheet_constructors}
+
         shutil.copytree(
             Path("/root"),
             shared_dir,
-            ignore=lambda src, names: ["latch", ".latch"],
+            ignore=lambda src, names: ["latch", ".latch", "nextflow", ".nextflow", "work", "results"],
             ignore_dangling_symlinks=True,
             dirs_exist_ok=True,
         )
 
+        cmd = [
+            "/root/nextflow",
+            "run",
+            str(shared_dir / "{nf_script}"),
+            "-work-dir",
+            str(shared_dir),
+            "-profile",
+            "{execution_profile}",
+            "-process.executor",
+            "k8s",
+{params_to_flags}
+        ]
+
+        print("Launching Nextflow Runtime")
+        print(cmd, flush=True)
         env = {{
             **os.environ,
             "NXF_HOME": "/root/.nextflow",
             "K8_STORAGE_CLAIM_NAME": pvc_name,
         }}
         subprocess.run(
-            [
-                "/root/nextflow",
-                "run",
-                str(shared_dir / "{nf_script}"),
-                "-work-dir",
-                str(shared_dir),
-                "-profile",
-                "{execution_profile}",
-                "-process.executor",
-                "k8s",
-{params_to_flags}
-            ],
+            cmd,
             env=env,
             check=True,
+            cwd=str(shared_dir),
         )
     except subprocess.CalledProcessError:
         remote = LPath(urljoins("{remote_output_dir}", _get_execution_name(), "nextflow.log"))
         print()
         print(f"Uploading .nextflow.log to {{remote.path}}")
-        remote.upload_from(Path("/root/.nextflow.log"))
+        remote.upload_from(shared_dir / ".nextflow.log")
+        import time
+        time.sleep(10 * 60)
         raise
     finally:
         token = os.environ.get("FLYTE_INTERNAL_EXECUTION_ID")
@@ -172,7 +181,8 @@ def generate_nextflow_workflow(
     flags = []
     defaults: List[Tuple[str, str]] = []
     no_defaults: List[str] = []
-    preambles: List[str] = []
+    preambles: set[str] = set()
+    samplesheet_funs: List[str] = []
     samplesheet_constructors: List[str] = []
     for param_name, param in parameters.items():
         sig = f"{param_name}: {type_repr(param.type)}"
@@ -195,7 +205,7 @@ def generate_nextflow_workflow(
             no_defaults.append(sig)
 
         if param.samplesheet:
-            samplesheet_constructors.append(
+            samplesheet_funs.append(
                 reindent(
                     f"""
                     {param_name}_construct_samplesheet = metadata._nextflow_metadata.parameters[{repr(param_name)}].samplesheet_constructor
@@ -204,19 +214,26 @@ def generate_nextflow_workflow(
                 ),
             )
 
+            samplesheet_constructors.append(
+                reindent(
+                    f"{param_name}_samplesheet ="
+                    f" {param_name}_construct_samplesheet({param_name})",
+                    2,
+                ),
+            )
+
             flags.append(
                 reindent(
-                    f"*get_flag({repr(param_name)},"
-                    f" {param_name}_construct_samplesheet({param_name}))",
-                    4,
+                    f"*get_flag({repr(param_name)}, {param_name}_samplesheet)",
+                    2,
                 )
             )
         else:
             flags.append(reindent(f"*get_flag({repr(param_name)}, {param_name})", 4))
 
         preamble = get_preamble(param.type)
-        if len(preamble) > 0:
-            preambles.append(preamble)
+        if len(preamble) > 0 and preamble not in preambles:
+            preambles.add(preamble)
 
     if metadata._nextflow_metadata.output_dir is None:
         output_dir = "latch:///nextflow_outputs"
@@ -238,7 +255,8 @@ def generate_nextflow_workflow(
         execution_profile=(
             execution_profile if execution_profile is not None else "standard"
         ),
-        preambles="\n\n".join(preambles),
+        preambles="\n\n".join(list(preambles)),
+        samplesheet_funs="\n".join(samplesheet_funs),
         samplesheet_constructors="\n".join(samplesheet_constructors),
         cpu=resources.cpus,
         memory=resources.memory,

{latch-2.45.2.dev2 → latch-2.45.2.dev4}/latch_cli/snakemake/config/utils.py

@@ -221,10 +221,7 @@ def type_repr(t: Type, *, add_namespace: bool = False) -> str:
         args = get_args(t)
 
         if len(args) != 2 or args[1] is not type(None):
-            raise ValueError(
-                "Union types other than Optional are not yet supported in Snakemake"
-                " workflows."
-            )
+            raise ValueError("Union types other than Optional are not yet supported")
 
         return f"typing.Optional[{type_repr(args[0], add_namespace=add_namespace)}]"
 

{latch-2.45.2.dev2 → latch-2.45.2.dev4}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.45.2.dev2",
+    version="v2.45.2.dev4",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),