latch 2.45.2.dev1__tar.gz → 2.45.2.dev3__tar.gz

{latch-2.45.2.dev1/latch.egg-info → latch-2.45.2.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.45.2.dev1
+Version: 2.45.2.dev3
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.45.2.dev1 → latch-2.45.2.dev3}/latch/types/metadata.py

@@ -1,21 +1,27 @@
+import csv
+import functools
 import re
-from dataclasses import Field, asdict, dataclass, field
+from dataclasses import Field, asdict, dataclass, field, fields, is_dataclass
 from enum import Enum
 from pathlib import Path
-from textwrap import indent
+from textwrap import dedent, indent
 from typing import (
     Any,
+    Callable,
     ClassVar,
     Collection,
     Dict,
     Generic,
     List,
+    Literal,
     Optional,
     Protocol,
     Tuple,
     Type,
     TypeVar,
     Union,
+    get_args,
+    get_origin,
 )
 
 import click
@@ -477,10 +483,84 @@ class NextflowParameter(Generic[T], LatchParameter):
     The python type of the parameter.
     """
     default: Optional[T] = None
+    """
+    Default value of the parameter
+    """
+
+    samplesheet_type: Literal["csv", "tsv", None] = None
+    """
+    The type of samplesheet to construct from the input parameter.
+
+    Only used if the provided parameter is a samplesheet (samplesheet=True)
+    """
+    samplesheet_constructor: Optional[Callable[[T], Path]] = None
+    """
+    A custom samplesheet constructor.
+
+    Should return the path of the constructed samplesheet. If samplesheet_type is also specified, this takes precedence.
+    Only used if the provided parameter is a samplesheet (samplesheet=True)
+    """
 
     def __post_init__(self):
-        if self.samplesheet is True:
-            self.type = Annotated[self.type, "samplesheet"]
+        if not self.samplesheet or self.samplesheet_constructor is not None:
+            return
+
+        t = self.type
+        if get_origin(t) is not list or not is_dataclass(get_args(t)[0]):
+            click.secho(
+                dedent("""\
+                Samplesheets must be a list of dataclasses.
+                """),
+                fg="red",
+            )
+            raise click.exceptions.Exit(1)
+
+        if self.samplesheet_type is not None:
+            delim = "," if self.samplesheet_type == "csv" else "\t"
+            self.samplesheet_constructor = functools.partial(
+                _samplesheet_constructor, t=get_args(self.type)[0], delim=delim
+            )
+            return
+
+        click.secho(
+            dedent("""\
+            A Samplesheet constructor is required for a samplesheet parameter. Please either provide a value for
+            `samplesheet_type` or provide a custom callable to the `samplesheet_constructor` argument.
+            """),
+            fg="red",
+        )
+        raise click.exceptions.Exit(1)
+
+
+DC = TypeVar("DC", bound=_IsDataclass)
+
+
+def _samplesheet_repr(v: Any) -> str:
+    if v is None:
+        return ""
+    if isinstance(v, LatchFile) or isinstance(v, LatchDir):
+        return v.remote_path
+    if isinstance(v, Enum):
+        return getattr(v, "value")
+
+    return str(v)
+
+
+def _samplesheet_constructor(samples: List[DC], t: DC, delim: str = ",") -> Path:
+    samplesheet = Path("samplesheet.csv")
+
+    with open(samplesheet, "w") as f:
+        writer = csv.DictWriter(f, [f.name for f in fields(t)], delimiter=delim)
+        writer.writeheader()
+
+        for sample in samples:
+            row_data = {
+                f.name: _samplesheet_repr(getattr(sample, f.name))
+                for f in fields(sample)
+            }
+            writer.writerow(row_data)
+
+    return samplesheet
 
 
 @dataclass(frozen=True)

{latch-2.45.2.dev1 → latch-2.45.2.dev3/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.45.2.dev1
+Version: 2.45.2.dev3
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.45.2.dev1 → latch-2.45.2.dev3}/latch_cli/nextflow/workflow.py

@@ -1,5 +1,4 @@
-import re
-from dataclasses import fields, is_dataclass
+from dataclasses import asdict, fields, is_dataclass
 from enum import Enum
 from pathlib import Path
 from typing import Any, List, Optional, Tuple
@@ -37,15 +36,15 @@ from flytekit.core.annotation import FlyteAnnotation
 
 import latch_metadata
 
-
 @custom_task(cpu=0.25, memory=0.5, storage_gib=1)
 def initialize() -> str:
     token = os.environ.get("FLYTE_INTERNAL_EXECUTION_ID")
     if token is None:
         raise RuntimeError("failed to get execution token")
 
-    print("Provisioning shared storage volume...")
     headers = {{"Authorization": f"Latch-Execution-Token {{token}}"}}
+
+    print("Provisioning shared storage volume... ", end="")
     resp = requests.post(
         "http://nf-dispatcher-service.flyte.svc.cluster.local/provision-storage",
         headers=headers,
@@ -54,21 +53,26 @@ def initialize() -> str:
         }}
     )
     resp.raise_for_status()
+    print("Done.")
+
     return resp.json()["name"]
 
 
 {preambles}
 
+{samplesheet_funs}
 
 @nextflow_runtime_task(cpu={cpu}, memory={memory})
 def nextflow_runtime(pvc_name: str, {param_signature}) -> None:
     try:
         shared_dir = Path("/nf-workdir")
 
+{samplesheet_constructors}
+
         shutil.copytree(
             Path("/root"),
             shared_dir,
-            ignore=lambda src, names: ["latch", ".latch"],
+            ignore=lambda src, names: ["latch", ".latch", "nextflow", ".nextflow"],
             ignore_dangling_symlinks=True,
             dirs_exist_ok=True,
         )
@@ -80,22 +84,26 @@ def nextflow_runtime(pvc_name: str, {param_signature}) -> None:
         }}
         subprocess.run(
             [
-                "/root/.latch/bin/nextflow",
+                "/root/nextflow",
                 "run",
                 str(shared_dir / "{nf_script}"),
                 "-work-dir",
                 str(shared_dir),
                 "-profile",
                 "{execution_profile}",
+                "-process.executor",
+                "k8s",
 {params_to_flags}
             ],
             env=env,
             check=True,
+            cwd=str(shared_dir),
         )
     except subprocess.CalledProcessError:
         remote = LPath(urljoins("{remote_output_dir}", _get_execution_name(), "nextflow.log"))
+        print()
         print(f"Uploading .nextflow.log to {{remote.path}}")
-        remote.upload_from(Path("/root/.nextflow.log"))
+        remote.upload_from(shared_dir / ".nextflow.log")
         raise
     finally:
         token = os.environ.get("FLYTE_INTERNAL_EXECUTION_ID")
@@ -165,12 +173,11 @@ def generate_nextflow_workflow(
     resources = metadata._nextflow_metadata.runtime_resources
 
     flags = []
-    for param_name, param in parameters.items():
-        flags.append(reindent(f"*get_flag({repr(param_name)}, {param_name})", 3))
-
     defaults: List[Tuple[str, str]] = []
     no_defaults: List[str] = []
-    preambles: List[str] = []
+    preambles: set[str] = set()
+    samplesheet_funs: List[str] = []
+    samplesheet_constructors: List[str] = []
     for param_name, param in parameters.items():
         sig = f"{param_name}: {type_repr(param.type)}"
         if param.default is not None:
@@ -191,9 +198,36 @@ def generate_nextflow_workflow(
         else:
             no_defaults.append(sig)
 
+        if param.samplesheet:
+            samplesheet_funs.append(
+                reindent(
+                    f"""
+                    {param_name}_construct_samplesheet = metadata._nextflow_metadata.parameters[{repr(param_name)}].samplesheet_constructor
+                    """,
+                    0,
+                ),
+            )
+
+            samplesheet_constructors.append(
+                reindent(
+                    f"{param_name}_samplesheet ="
+                    f" {param_name}_construct_samplesheet({param_name})",
+                    2,
+                ),
+            )
+
+            flags.append(
+                reindent(
+                    f"*get_flag({repr(param_name)}, {param_name}_samplesheet)",
+                    4,
+                )
+            )
+        else:
+            flags.append(reindent(f"*get_flag({repr(param_name)}, {param_name})", 4))
+
         preamble = get_preamble(param.type)
-        if len(preamble) > 0:
-            preambles.append(preamble)
+        if len(preamble) > 0 and preamble not in preambles:
+            preambles.add(preamble)
 
     if metadata._nextflow_metadata.output_dir is None:
         output_dir = "latch:///nextflow_outputs"
@@ -215,7 +249,9 @@ def generate_nextflow_workflow(
         execution_profile=(
             execution_profile if execution_profile is not None else "standard"
         ),
-        preambles="\n\n".join(preambles),
+        preambles="\n\n".join(list(preambles)),
+        samplesheet_funs="\n".join(samplesheet_funs),
+        samplesheet_constructors="\n".join(samplesheet_constructors),
         cpu=resources.cpus,
         memory=resources.memory,
         storage_gib=resources.storage_gib,
@@ -227,6 +263,5 @@ def generate_nextflow_workflow(
     entrypoint_path.write_text(entrypoint)
 
     click.secho(
-        f"Nextflow workflow written to {pkg_root / 'wf' / 'entrypoint.py'}",
-        fg="green",
+        f"Nextflow workflow written to {pkg_root / 'wf' / 'entrypoint.py'}", fg="green"
     )

{latch-2.45.2.dev1 → latch-2.45.2.dev3}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.45.2.dev1",
+    version="v2.45.2.dev3",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),