latch 2.45.2.dev3__tar.gz → 2.45.2.dev5__tar.gz

{latch-2.45.2.dev3/latch.egg-info → latch-2.45.2.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.45.2.dev3
+Version: 2.45.2.dev5
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.45.2.dev3 → latch-2.45.2.dev5/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.45.2.dev3
+Version: 2.45.2.dev5
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.45.2.dev3 → latch-2.45.2.dev5}/latch_cli/docker_utils/__init__.py

@@ -78,24 +78,6 @@ def get_prologue(
 def get_epilogue(wf_type: WorkflowType = WorkflowType.latchbiosdk) -> List[str]:
     cmds: list[str] = []
 
-    if wf_type == WorkflowType.snakemake:
-        cmds += [
-            "",
-            "# Latch snakemake workflow entrypoint",
-            "# DO NOT CHANGE",
-            "",
-            "copy .latch/snakemake_jit_entrypoint.py /root/snakemake_jit_entrypoint.py",
-        ]
-    elif wf_type == WorkflowType.nextflow:
-        cmds += [
-            "",
-            "# Latch nextflow workflow entrypoint",
-            "# DO NOT CHANGE",
-            "",
-            "copy .latch/bin/nextflow /root/nextflow",
-            "copy .latch/.nextflow /root/.nextflow",
-        ]
-
     cmds += [
         "",
         "# Latch workflow registration metadata",
@@ -324,14 +306,66 @@ def infer_commands(pkg_root: Path) -> List[DockerCmdBlock]:
     return commands
 
 
+def copy_file_commands(wf_type: WorkflowType) -> List[DockerCmdBlock]:
+    cmd = []
+
+    cmd += [
+        "",
+        "# Copy workflow data (use .dockerignore to skip files)",
+        "",
+        "copy . /root/",
+    ]
+
+    if wf_type == WorkflowType.snakemake:
+        cmd += [
+            "",
+            "# Latch snakemake workflow entrypoint",
+            "# DO NOT CHANGE",
+            "",
+            "copy .latch/snakemake_jit_entrypoint.py /root/snakemake_jit_entrypoint.py",
+        ]
+    elif wf_type == WorkflowType.nextflow:
+        cmd += [
+            "",
+            "# Latch nextflow workflow entrypoint",
+            "# DO NOT CHANGE",
+            "",
+            "run ln -s /root/.latch/bin/nextflow /root/nextflow",
+            "run ln -s /root/.latch/.nextflow /root/.nextflow",
+        ]
+
+    return cmd
+
+
+def generate_dockerignore(pkg_root: Path, *, wf_type: WorkflowType) -> None:
+    if wf_type != WorkflowType.nextflow:
+        return
+
+    dest = Path(pkg_root) / ".dockerignore"
+    if dest.exists() and not click.confirm(
+        f".dockerignore already exists at `{dest}`. Overwrite?"
+    ):
+        return
+
+    with Path(".dockerignore").open("w") as f:
+        f.write(".git/\n")
+        f.write(".github/\n")
+        f.write(".nextflow*\n")
+        f.write("work/\n")
+        f.write("results/\n")
+        f.write(".nextflow.log*\n")
+
+    click.secho(f"Successfully generated .dockerignore `{dest}`", fg="green")
+
+
 def generate_dockerfile(
-    pkg_root: Path, outfile: Path, *, wf_type: WorkflowType = WorkflowType.latchbiosdk
+    pkg_root: Path, *, wf_type: WorkflowType = WorkflowType.latchbiosdk
 ) -> None:
     """Generate a best effort Dockerfile from files in the workflow directory.
 
     Args:
         pkg_root: A path to a workflow directory.
-        outfile: The path to write the generated Dockerfile.
+        dest: The path to write the generated Dockerfile.
         wf_type: The type of workflow (eg. snakemake) the Dockerfile is for
 
     Example:
@@ -342,6 +376,11 @@ def generate_dockerfile(
             #   ├── Dockerfile
             #   └── ...
     """
+    dest = pkg_root / "Dockerfile"
+    if dest.exists() and not click.confirm(
+        f"Dockerfile already exists at `{dest}`. Overwrite?"
+    ):
+        return
 
     click.secho("Generating Dockerfile", bold=True)
     try:
@@ -368,7 +407,7 @@ def generate_dockerfile(
     )
     click.echo()
 
-    with outfile.open("w") as f:
+    with dest.open("w") as f:
         f.write("\n".join(get_prologue(config, wf_type)) + "\n\n")
 
         commands = infer_commands(pkg_root)
@@ -381,8 +420,7 @@ def generate_dockerfile(
 
             block.write_block(f)
 
-        f.write("# Copy workflow data (use .dockerignore to skip files)\n")
-        f.write("copy . /root/\n\n")
+        f.write("\n".join(copy_file_commands(wf_type)) + "\n\n")
 
         for block in commands:
             if block.order != DockerCmdBlockOrder.postcopy:
@@ -392,6 +430,8 @@ def generate_dockerfile(
 
         f.write("\n".join(get_epilogue(wf_type)) + "\n")
 
+    click.secho(f"Successfully generated dockerfile `{dest}`", fg="green")
+
 
 def get_default_dockerfile(pkg_root: Path, *, wf_type: WorkflowType):
     default_dockerfile = pkg_root / "Dockerfile"

{latch-2.45.2.dev3 → latch-2.45.2.dev5}/latch_cli/main.py

@@ -217,14 +217,7 @@ def dockerfile(pkg_root: str, snakemake: bool = False, nextflow: bool = False):
         )
         raise click.exceptions.Exit(1)
 
-    from latch_cli.docker_utils import generate_dockerfile
-
-    source = Path(pkg_root)
-    dest = source / "Dockerfile"
-    if dest.exists() and not click.confirm(
-        f"Dockerfile already exists at `{dest}`. Overwrite?"
-    ):
-        return
+    from latch_cli.docker_utils import generate_dockerfile, generate_dockerignore
 
     workflow_type = WorkflowType.latchbiosdk
     if snakemake is True:
@@ -232,8 +225,9 @@ def dockerfile(pkg_root: str, snakemake: bool = False, nextflow: bool = False):
     elif nextflow is True:
         workflow_type = WorkflowType.nextflow
 
-    generate_dockerfile(source, dest, wf_type=workflow_type)
-    click.secho(f"Successfully generated dockerfile `{dest}`", fg="green")
+    source = Path(pkg_root)
+    generate_dockerfile(source, wf_type=workflow_type)
+    generate_dockerignore(source, wf_type=workflow_type)
 
 
 @main.command("generate-metadata")

{latch-2.45.2.dev3 → latch-2.45.2.dev5}/latch_cli/nextflow/dependencies.py

@@ -1,5 +1,4 @@
 import shutil
-import subprocess
 from concurrent.futures import ProcessPoolExecutor
 from ctypes import c_int
 from multiprocessing.managers import SyncManager
@@ -8,9 +7,9 @@ from urllib.parse import urljoin
 
 import boto3
 import click
+from botocore.handlers import disable_signing
 
 from latch_cli import tinyrequests
-from latch_cli.utils import dedent
 
 
 def _do_download(
@@ -35,6 +34,7 @@ def _do_download(
 
 def download_nf_jars(pkg_root: Path):
     s3_resource = boto3.resource("s3")
+    s3_resource.meta.client.meta.events.register("choose-signer.s3.*", disable_signing)
     bucket = s3_resource.Bucket("latch-public")
 
     subdir = "nextflow-v2/"
@@ -51,8 +51,6 @@ def download_nf_jars(pkg_root: Path):
                     "https://latch-public.s3.us-west-2.amazonaws.com/", obj.key
                 )
                 obj_path = pkg_root / ".latch" / obj.key[len(subdir) :]
-                print(obj_path)
-
                 exec.submit(_do_download, url, obj_path, len(objects), counter, lock)
 
     click.echo("\x1b[0K", nl=False)

{latch-2.45.2.dev3 → latch-2.45.2.dev5}/latch_cli/nextflow/workflow.py

@@ -72,29 +72,33 @@ def nextflow_runtime(pvc_name: str, {param_signature}) -> None:
         shutil.copytree(
             Path("/root"),
             shared_dir,
-            ignore=lambda src, names: ["latch", ".latch", "nextflow", ".nextflow"],
+            ignore=lambda src, names: ["latch", ".latch", "nextflow", ".nextflow", "work", "results"],
             ignore_dangling_symlinks=True,
             dirs_exist_ok=True,
         )
 
+        cmd = [
+            "/root/nextflow",
+            "run",
+            str(shared_dir / "{nf_script}"),
+            "-work-dir",
+            str(shared_dir),
+            "-profile",
+            "{execution_profile}",
+            "-process.executor",
+            "k8s",
+{params_to_flags}
+        ]
+
+        print("Launching Nextflow Runtime")
+        print(cmd, flush=True)
         env = {{
             **os.environ,
             "NXF_HOME": "/root/.nextflow",
             "K8_STORAGE_CLAIM_NAME": pvc_name,
         }}
         subprocess.run(
-            [
-                "/root/nextflow",
-                "run",
-                str(shared_dir / "{nf_script}"),
-                "-work-dir",
-                str(shared_dir),
-                "-profile",
-                "{execution_profile}",
-                "-process.executor",
-                "k8s",
-{params_to_flags}
-            ],
+            cmd,
             env=env,
             check=True,
             cwd=str(shared_dir),
@@ -104,6 +108,8 @@ def nextflow_runtime(pvc_name: str, {param_signature}) -> None:
         print()
         print(f"Uploading .nextflow.log to {{remote.path}}")
         remote.upload_from(shared_dir / ".nextflow.log")
+        import time
+        time.sleep(10 * 60)
         raise
     finally:
         token = os.environ.get("FLYTE_INTERNAL_EXECUTION_ID")
@@ -219,7 +225,7 @@ def generate_nextflow_workflow(
             flags.append(
                 reindent(
                     f"*get_flag({repr(param_name)}, {param_name}_samplesheet)",
-                    4,
+                    2,
                 )
             )
         else:

{latch-2.45.2.dev3 → latch-2.45.2.dev5}/latch_cli/snakemake/config/utils.py

@@ -221,10 +221,7 @@ def type_repr(t: Type, *, add_namespace: bool = False) -> str:
         args = get_args(t)
 
         if len(args) != 2 or args[1] is not type(None):
-            raise ValueError(
-                "Union types other than Optional are not yet supported in Snakemake"
-                " workflows."
-            )
+            raise ValueError("Union types other than Optional are not yet supported")
 
         return f"typing.Optional[{type_repr(args[0], add_namespace=add_namespace)}]"
 

{latch-2.45.2.dev3 → latch-2.45.2.dev5}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.45.2.dev3",
+    version="v2.45.2.dev5",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),