latch 2.39.0.dev24__tar.gz → 2.39.0.dev26__tar.gz

{latch-2.39.0.dev24/latch.egg-info → latch-2.39.0.dev26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev24
+Version: 2.39.0.dev26
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch/types/metadata.py

@@ -696,6 +696,8 @@ class NextflowMetadata(LatchMetadata):
     def __post_init__(self):
         if self.name is None:
             self.name = f"nf_{identifier_suffix_from_str(self.display_name.lower())}"
+        else:
+            self.name = identifier_suffix_from_str(self.name)
 
         if self.output_directory is None:
             self.output_directory = LatchDir("latch:///Nextflow Outputs/")

{latch-2.39.0.dev24 → latch-2.39.0.dev26/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev24
+Version: 2.39.0.dev26
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch.egg-info/SOURCES.txt

@@ -82,6 +82,9 @@ latch_cli/extras/__init__.py
 latch_cli/extras/common/__init__.py
 latch_cli/extras/common/serialize.py
 latch_cli/extras/common/utils.py
+latch_cli/extras/common/config/__init__.py
+latch_cli/extras/common/config/parser.py
+latch_cli/extras/common/config/utils.py
 latch_cli/extras/nextflow/__init__.py
 latch_cli/extras/nextflow/build.py
 latch_cli/extras/nextflow/channel.py

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/docker_utils/__init__.py

@@ -9,6 +9,7 @@ from typing import List
 import click
 import yaml
 
+from latch_cli.click_utils import color
 from latch_cli.constants import latch_constants
 from latch_cli.utils import WorkflowType
 from latch_cli.workflow_config import LatchWorkflowConfig, create_and_write_config
@@ -352,18 +353,8 @@ def generate_dockerfile(
         with (pkg_root / latch_constants.pkg_config).open("r") as f:
             config = LatchWorkflowConfig(**json.load(f))
 
-    click.echo(
-        " ".join([
-            click.style("Base image:", fg="bright_blue"),
-            config.base_image,
-        ])
-    )
-    click.echo(
-        " ".join([
-            click.style("Latch SDK version:", fg="bright_blue"),
-            config.latch_version,
-        ])
-    )
+    click.echo(" ".join([color("Base image:"), config.base_image]))
+    click.echo(" ".join([color("Latch SDK version:"), config.latch_version]))
     click.echo()
 
     with outfile.open("w") as f:

latch-2.39.0.dev26/latch_cli/extras/common/config/parser.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+from typing import Dict, Tuple, Type, TypeVar
+
+import click
+
+from ..utils import reindent
+from .utils import JSONValue, parse_type, parse_value
+
+T = TypeVar("T")
+
+
+def parse_config(
+    config: JSONValue,
+    *,
+    infer_files: bool = False,
+) -> Dict[str, Tuple[Type[T], T]]:
+    parsed: Dict[str, Type] = {}
+    for k, v in config.items():
+        try:
+            typ = parse_type(v, k, infer_files=infer_files)
+        except ValueError as e:
+            click.secho(
+                f"WARNING: Skipping parameter {k}. Failed to parse type: {e}.",
+                fg="yellow",
+            )
+            continue
+        val, default = parse_value(typ, v)
+
+        parsed[k] = (typ, (val, default))
+
+    return parsed
+
+
+def write_metadata(
+    metadata: str,
+    params: str,
+    *,
+    skip_confirmation: bool = False,
+) -> None:
+    metadata_root = Path("latch_metadata")
+    if metadata_root.is_file():
+        if not click.confirm("A file exists at `latch_metadata`. Delete it?"):
+            raise click.exceptions.Exit(0)
+
+        metadata_root.unlink()
+
+    metadata_root.mkdir(exist_ok=True)
+
+    metadata_path = metadata_root / Path("__init__.py")
+    old_metadata_path = Path("latch_metadata.py")
+
+    if old_metadata_path.exists() and not metadata_path.exists():
+        if click.confirm(
+            "Found legacy `latch_metadata.py` file in current directory. This is"
+            " deprecated and will be ignored in future releases. Move to"
+            " `latch_metadata/__init__.py`? (This will not change file contents)"
+        ):
+            old_metadata_path.rename(metadata_path)
+    elif old_metadata_path.exists() and metadata_path.exists():
+        click.secho(
+            "Warning: Found both `latch_metadata.py` and"
+            " `latch_metadata/__init__.py` in current directory."
+            " `latch_metadata.py` will be ignored.",
+            fg="yellow",
+        )
+
+    if not metadata_path.exists():
+        metadata_path.write_text(reindent(metadata, 0))
+        click.secho("Generated `latch_metadata/__init__.py`.", fg="green")
+
+    params_path = metadata_root / Path("parameters.py")
+    if (
+        params_path.exists()
+        and not skip_confirmation
+        and not click.confirm(
+            "File `latch_metadata/parameters.py` already exists. Overwrite?"
+        )
+    ):
+        raise click.exceptions.Exit(0)
+
+    params_path.write_text(reindent(params, 0))
+    click.secho("Generated `latch_metadata/parameters.py`.", fg="green")

latch-2.39.0.dev26/latch_cli/extras/common/config/utils.py

@@ -0,0 +1,235 @@
+from dataclasses import fields, is_dataclass, make_dataclass
+from typing import Any, Dict, List, Optional, Type, Union, get_args, get_origin
+
+from flytekit.core.annotation import FlyteAnnotation
+from typing_extensions import Annotated, TypeAlias, TypeGuard
+
+from latch.types.directory import LatchDir
+from latch.types.file import LatchFile
+from latch_cli.utils import identifier_from_str
+
+from ..utils import is_primitive_type, is_primitive_value, type_repr
+
+JSONValue: TypeAlias = Union[int, str, bool, float, None, List["JSONValue"], "JSONDict"]
+JSONDict: TypeAlias = Dict[str, "JSONValue"]
+
+# ayush: yoinked from console
+valid_extensions = {
+    "bed",
+    "vcf",
+    "css",
+    "csv",
+    "gif",
+    "png",
+    "pdf",
+    "webp",
+    "xhtml",
+    "xlsx",
+    "xml",
+    "py",
+    "log",
+    "json",
+    "gz",
+    "mmtf",
+    "deseqreport",
+    "sam",
+    "bam",
+    "cram",
+    "tsv",
+    "tab",
+    "sf",
+    "txt",
+    "text",
+    "license",
+    "readme",
+    "r",
+    "rscript",
+    "md",
+    "markdown",
+    "markdn",
+    "mdown",
+    "htm",
+    "html",
+    "ipynb",
+    "jpeg",
+    "jpg",
+    "jif",
+    "jpe",
+    "jfif",
+    "js",
+    "mjs",
+    "es",
+    "ts",
+    "jsx",
+    "tsx",
+    "svg",
+    "svgz",
+    "fasta",
+    "fna",
+    "fa",
+    "ffn",
+    "faa",
+    "frn",
+    "fastq",
+    "fq",
+    "pdb",
+    "pdb1",
+    "ent",
+    "brk",
+    "ml2",
+    "mol2",
+    "sy2",
+    "hdf",
+    "h4",
+    "hdf4",
+    "he4",
+    "h5",
+    "hdf5",
+    "he5",
+    "h5ad",
+    "yaml",
+    "yml",
+}
+
+
+def parse_type(
+    v: JSONValue, name: Optional[str] = None, *, infer_files: bool = False
+) -> Type:
+    if v is None:
+        return str
+
+    if infer_files and isinstance(v, str):
+        if any([v.endswith(ext) for ext in valid_extensions]):
+            return LatchFile
+        elif v.endswith("/"):
+            return LatchDir
+
+    if is_primitive_value(v):
+        return type(v)
+
+    if isinstance(v, list):
+        parsed_types = tuple(
+            parse_type(
+                x,
+                name,
+                infer_files=infer_files,
+            )
+            for x in v
+        )
+
+        if len(set(parsed_types)) != 1:
+            raise ValueError(
+                "Generic Lists are not supported - please"
+                f" ensure that all elements in {name} are of the same type",
+            )
+        typ = parsed_types[0]
+        if typ in {LatchFile, LatchDir}:
+            return Annotated[List[typ], FlyteAnnotation({"size": len(v)})]
+        return List[typ]
+
+    assert isinstance(v, dict)
+
+    if name is None:
+        name = "SnakemakeRecord"
+
+    fields: Dict[str, Type] = {}
+    for k, x in v.items():
+        fields[identifier_from_str(k)] = parse_type(
+            x,
+            k,
+            infer_files=infer_files,
+        )
+
+    return make_dataclass(identifier_from_str(name), fields.items())
+
+
+# returns raw value and generated default
+def parse_value(t: Type, v: JSONValue):
+    if v is None:
+        return None, None
+
+    if get_origin(t) is Annotated:
+        args = get_args(t)
+        assert len(args) > 0
+        return parse_value(args[0], v)
+
+    if t in {LatchFile, LatchDir}:
+        # ayush: autogenerated defaults don't make sense for files/dirs since their
+        # value in the config is their local path
+        return v, None
+
+    if is_primitive_value(v):
+        return v, v
+
+    if isinstance(v, list):
+        assert get_origin(t) is list
+
+        args = get_args(t)
+        assert len(args) > 0
+
+        sub_type = args[0]
+        res = [parse_value(sub_type, x) for x in v]
+        return [x[0] for x in res], [x[1] for x in res]
+
+    assert isinstance(v, dict), v
+    assert is_dataclass(t), t
+
+    ret = {}
+    defaults = {}
+    fs = {identifier_from_str(f.name): f for f in fields(t)}
+
+    for k, x in v.items():
+        sanitized = identifier_from_str(k)
+        assert sanitized in fs, sanitized
+        val, default = parse_value(fs[sanitized].type, x)
+        ret[sanitized] = val
+        defaults[sanitized] = default
+
+    return t(**ret), t(**defaults)
+
+
+def is_list_type(typ: Type) -> TypeGuard[Type[List]]:
+    return get_origin(typ) is list
+
+
+def dataclass_repr(typ: Type, *, make_optionals: bool = False) -> str:
+    assert is_dataclass(typ)
+
+    lines = ["@dataclass", f"class {typ.__name__}:"]
+    for f in fields(typ):
+        t = f.type
+        if make_optionals:
+            t = Optional[t]
+
+        lines.append(f"    {f.name}: {type_repr(t)}")
+
+    return "\n".join(lines) + "\n\n\n"
+
+
+def get_preamble(typ: Type, *, make_optionals: bool = False) -> str:
+    if get_origin(typ) is Annotated:
+        args = get_args(typ)
+        assert len(args) > 0
+        return get_preamble(args[0], make_optionals=make_optionals)
+
+    if is_primitive_type(typ) or typ in {LatchFile, LatchDir}:
+        return ""
+
+    if get_origin(typ) in {Union, list}:
+        return "".join(
+            [get_preamble(t, make_optionals=make_optionals) for t in get_args(typ)]
+        )
+
+    assert is_dataclass(typ), typ
+
+    preambles = []
+    for f in fields(typ):
+        t = f.type
+        if make_optionals:
+            t = Optional[f.type]
+
+        preambles.append(get_preamble(t, make_optionals=make_optionals))
+
+    preamble = "".join(preambles)
+
+    return "".join([preamble, dataclass_repr(typ, make_optionals=make_optionals)])

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/extras/common/serialize.py

@@ -1,7 +1,7 @@
 import sys
 from dataclasses import fields
 from pathlib import Path
-from typing import Any, Dict, Optional, Type, TypeVar, Union, get_args
+from typing import Any, Dict, List, Optional, Type, TypeVar, Union, get_args
 
 import click
 from flyteidl.admin.launch_plan_pb2 import LaunchPlan as _idl_admin_LaunchPlan
@@ -292,7 +292,7 @@ RegistrableEntity = Union[
 ]
 
 
-def should_register_with_admin(entity: RegistrableEntity) -> bool:
+def should_register_with_admin(entity: FlyteSerializableModel) -> bool:
     return isinstance(entity, get_args(RegistrableEntity))
 
 
@@ -327,15 +327,29 @@ def serialize(
     )
     admin_lp = get_serializable_launch_plan(lp, settings, registrable_entity_cache)
 
-    registrable_entities = [
-        x.to_flyte_idl()
-        for x in list(
-            filter(should_register_with_admin, list(registrable_entity_cache.values()))
-        )
-        + [admin_lp]
-    ]
+    click.secho(
+        "\nBuilding flyte models: \x1b[?25l",
+        nl=False,
+    )
+
+    registrable_entities = []
+    i = 0
+    for x in list(registrable_entity_cache.values()) + [admin_lp]:
+        progress_str = f"{i + 1}/{len(registrable_entity_cache) + 1}"
+
+        click.echo("\x1b[0K", nl=False)
+        click.secho(progress_str, dim=True, italic=True, nl=False)
+        click.echo(f"\x1b[{len(progress_str)}D", nl=False)
+
+        i += 1
+
+        if not should_register_with_admin(x):
+            continue
+
+        registrable_entities.append(x.to_flyte_idl())
 
     click.secho("\nSerializing workflow entities", bold=True)
+
     persist_registrable_entities(registrable_entities, output_dir)
 
     if not write_spec:

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/extras/common/utils.py

@@ -3,7 +3,7 @@ from typing import Type, Union, get_args, get_origin
 
 from typing_extensions import Annotated, TypeGuard
 
-from latch.types.directory import LatchDir
+from latch.types.directory import LatchDir, LatchOutputDir
 from latch.types.file import LatchFile
 
 
@@ -17,7 +17,7 @@ def reindent(x: str, level: int) -> str:
 def is_primitive_type(
     typ: Type,
 ) -> TypeGuard[Union[Type[None], Type[str], Type[bool], Type[int], Type[float]]]:
-    return typ in {Type[None], str, bool, int, float}
+    return typ in {type(None), str, bool, int, float}
 
 
 def is_primitive_value(val: object) -> TypeGuard[Union[None, str, bool, int, float]]:
@@ -34,6 +34,13 @@ def is_blob_type(typ: Type) -> TypeGuard[Union[Type[LatchFile], Type[LatchDir]]]
     return typ in {LatchFile, LatchDir}
 
 
+def is_downloadable_blob_type(typ: Type):
+    if not is_blob_type(typ):
+        return False
+
+    return typ is not LatchOutputDir
+
+
 def type_repr(t: Type, *, add_namespace: bool = False) -> str:
     if getattr(t, "__name__", None) == "NoneType":
         return "None"

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/build.py

@@ -19,6 +19,7 @@ from flytekit.models import literals as literals_models
 from latch_sdk_gql.execute import execute
 
 from latch_cli import tinyrequests
+from latch_cli.extras.common.config.utils import get_preamble
 from latch_cli.extras.nextflow.tasks.base import NextflowBaseTask, NFTaskType
 
 from ...click_utils import italic
@@ -102,7 +103,7 @@ def build_from_nextflow_dag(
 
                 output_name = "res"
                 if len(dep.outputNames) > 0:
-                    idx = int(edge.label)
+                    idx = int(edge.label or "0")
                     input_name = f"{input_name}_{idx}"
                     output_name = dep.outputNames[idx]
 
@@ -128,8 +129,6 @@ def build_from_nextflow_dag(
             upstream_nodes.append(node_map[dep.id])
 
         node_name = get_node_name(vertex.id)
-        if vertex.type == VertexType.Merge:
-            task_outputs = {k: Optional[str] for k in merge_sources.keys()}
 
         if vertex.type == VertexType.Process:
             pre_adapter_task = NextflowProcessPreAdapterTask(
@@ -513,6 +512,9 @@ def generate_nf_entrypoint(
         0,
     )
 
+    for t in wf.python_interface.inputs.values():
+        preamble += get_preamble(t, make_optionals=True)
+
     nf_script_path_in_container = nf_script.resolve().relative_to(pkg_root.resolve())
 
     entrypoint_code = [preamble]

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/config.py

@@ -2,7 +2,7 @@ import json
 import os
 import subprocess
 from pathlib import Path
-from typing import List
+from typing import List, Optional
 
 import click
 
@@ -37,6 +37,7 @@ def generate_nf_metadata(
     skip_confirmation: bool = False,
     generate_defaults: bool = False,
     infer_files: bool = False,
+    make_optionals: bool = True,
 ):
     if not config_path.is_dir():
         click.secho(
@@ -103,7 +104,10 @@ def generate_nf_metadata(
     params: List[str] = []
 
     for k, (typ, (val, default)) in parsed.items():
-        preambles.append(get_preamble(typ))
+        if make_optionals:
+            typ = Optional[typ]
+
+        preambles.append(get_preamble(typ, make_optionals=make_optionals))
 
         param_str = reindent(
             f"""\

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/serialize.py

@@ -15,7 +15,7 @@ def serialize_nf(
     image_name: str,
     dkr_repo: str,
 ):
-    serialize(nf_wf, output_dir, image_name, dkr_repo, write_spec=True)
+    serialize(nf_wf, output_dir, image_name, dkr_repo)
 
 
 def add_task_metadata(wf: NextflowWorkflow):

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/tasks/base.py

@@ -47,8 +47,8 @@ class NextflowBaseTask(PythonAutoContainerTask[Pod]):
         wf: NextflowWorkflow,
         nf_task_type: NFTaskType,
         # todo(ayush): expose / infer these somehow
-        cpu: int = 4,
-        memory: int = 8,
+        cpu: float = 0.5,
+        memory: float = 1,
     ):
         self.id = id
         self.wf = wf

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/tasks/operator.py

@@ -130,7 +130,7 @@ class NextflowOperatorTask(NextflowBaseTask):
                     if {k} is not None:
                         {k}_p = Path({k}).resolve()
                         check_exists_and_rename({k}_p, Path("/root") / {k}_p.name)
-                        wf_paths[{k}] = Path("/root") / {k}_p.name
+                        wf_paths["{k}"] = Path("/root") / {k}_p.name
 
                     """,
                     2,
@@ -140,7 +140,7 @@ class NextflowOperatorTask(NextflowBaseTask):
                     f"""
                     if {k} is not None:
                         {k}_p = Path("/root/").resolve() # superhack
-                        wf_paths[{k}] = {k}_p
+                        wf_paths["{k}"] = {k}_p
 
                     """,
                     2,

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/tasks/process.py

@@ -136,7 +136,7 @@ class NextflowProcessTask(NextflowBaseTask):
                     if {k} is not None:
                         {k}_p = Path({k}).resolve()
                         check_exists_and_rename({k}_p, Path("/root") / {k}_p.name)
-                        wf_paths[{k}] = Path("/root") / {k}_p.name
+                        wf_paths["{k}"] = Path("/root") / {k}_p.name
 
                     """,
                     1,
@@ -146,7 +146,7 @@ class NextflowProcessTask(NextflowBaseTask):
                     f"""
                     if {k} is not None:
                         {k}_p = Path("/root/").resolve() # superhack
-                        wf_paths[{k}] = {k}_p
+                        wf_paths["{k}"] = {k}_p
 
                     """,
                     1,

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/workflow.py

@@ -1,5 +1,6 @@
+from dataclasses import fields, is_dataclass
 from pathlib import Path
-from typing import List
+from typing import Any, Dict, List, Tuple, Type
 
 from flytekit.core.class_based_resolver import ClassStorageTaskResolver
 from flytekit.core.docstring import Docstring
@@ -13,10 +14,32 @@ from flytekit.core.workflow import (
 from flytekit.exceptions import scopes as exception_scopes
 
 from latch.types import metadata
-from latch.types.metadata import NextflowFileParameter
+from latch_cli.extras.common.utils import is_blob_type, is_downloadable_blob_type
 
 from .dag import DAG
 
+# def _get_flattened_inputs(
+#     key: str, t: Type, val: Any, inputs: Dict[str, Tuple[Type, Any]]
+# ):
+#     if not is_dataclass(t):
+#         inputs[key] = (t, val)
+#         return
+
+#     for f in fields(t):
+#         v = val
+#         if val is not None:
+#             v = getattr()
+
+
+def _get_flags_to_params(key: str, t: Type, flags: Dict[str, str]):
+    if is_blob_type(t):
+        flags[f"--{key}"] = f"wf_paths['wf_{key}']"
+    elif is_dataclass(t):
+        for f in fields(t):
+            _get_flags_to_params(f"{key}.{f.name}", f.type, flags)
+    else:
+        flags[f"--{key}"] = f"wf_{key}"
+
 
 class NextflowWorkflow(WorkflowBase, ClassStorageTaskResolver):
     def __init__(self, nf_script: Path, version: str, dag: DAG):
@@ -39,19 +62,17 @@ class NextflowWorkflow(WorkflowBase, ClassStorageTaskResolver):
             docstring=docstring,
         )
 
-        self.flags_to_params = {
-            f"--{k}": (
-                f"wf_paths[wf_{k}]"
-                if isinstance(v, NextflowFileParameter)
-                else f"wf_{k}"
-            )
-            for k, v in metadata._nextflow_metadata.parameters.items()
-        }
+        self.flattened_inputs = {}
+
+        self.flags_to_params = {}
+        for k, v in metadata._nextflow_metadata.parameters.items():
+            assert v.type is not None
+            _get_flags_to_params(k, v.type, self.flags_to_params)
 
         self.downloadable_params = {
-            k: str(v.path)
+            k
             for k, v in metadata._nextflow_metadata.parameters.items()
-            if isinstance(v, NextflowFileParameter) and v._download
+            if is_downloadable_blob_type(v.type)
         }
 
         name = metadata._nextflow_metadata.name

{latch-2.39.0.dev24 → latch-2.39.0.dev26}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.39.0.dev24",
+    version="v2.39.0.dev26",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),