latch 2.39.0.dev25__tar.gz → 2.39.0.dev26__tar.gz

{latch-2.39.0.dev25/latch.egg-info → latch-2.39.0.dev26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev25
+Version: 2.39.0.dev26
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch/types/metadata.py

@@ -696,6 +696,8 @@ class NextflowMetadata(LatchMetadata):
     def __post_init__(self):
         if self.name is None:
             self.name = f"nf_{identifier_suffix_from_str(self.display_name.lower())}"
+        else:
+            self.name = identifier_suffix_from_str(self.name)
 
         if self.output_directory is None:
             self.output_directory = LatchDir("latch:///Nextflow Outputs/")

{latch-2.39.0.dev25 → latch-2.39.0.dev26/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev25
+Version: 2.39.0.dev26
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/docker_utils/__init__.py

@@ -9,6 +9,7 @@ from typing import List
 import click
 import yaml
 
+from latch_cli.click_utils import color
 from latch_cli.constants import latch_constants
 from latch_cli.utils import WorkflowType
 from latch_cli.workflow_config import LatchWorkflowConfig, create_and_write_config
@@ -352,18 +353,8 @@ def generate_dockerfile(
         with (pkg_root / latch_constants.pkg_config).open("r") as f:
             config = LatchWorkflowConfig(**json.load(f))
 
-    click.echo(
-        " ".join([
-            click.style("Base image:", fg="bright_blue"),
-            config.base_image,
-        ])
-    )
-    click.echo(
-        " ".join([
-            click.style("Latch SDK version:", fg="bright_blue"),
-            config.latch_version,
-        ])
-    )
+    click.echo(" ".join([color("Base image:"), config.base_image]))
+    click.echo(" ".join([color("Latch SDK version:"), config.latch_version]))
     click.echo()
 
     with outfile.open("w") as f:

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/common/config/utils.py

@@ -188,81 +188,48 @@ def parse_value(t: Type, v: JSONValue):
     return t(**ret), t(**defaults)
 
 
-def is_primitive_type(
-    typ: Type,
-) -> TypeGuard[Union[Type[None], Type[str], Type[bool], Type[int], Type[float]]]:
-    return typ in {Type[None], str, bool, int, float}
-
-
-def is_primitive_value(val: object) -> TypeGuard[Union[None, str, bool, int, float]]:
-    return is_primitive_type(type(val))
-
-
 def is_list_type(typ: Type) -> TypeGuard[Type[List]]:
     return get_origin(typ) is list
 
 
-def type_repr(t: Type, *, add_namespace: bool = False) -> str:
-    if is_primitive_type(t) or t in {LatchFile, LatchDir}:
-        return t.__name__
-
-    if get_origin(t) is None:
-        return f"{'latch_metadata.' if add_namespace else ''}{t.__name__}"
-
-    if get_origin(t) is list:
-        args = get_args(t)
-        if len(args) > 0:
-            return f"typing.List[{type_repr(args[0], add_namespace=add_namespace)}]"
-
-        return "typing.List"
-
-    if get_origin(t) is Union:
-        args = get_args(t)
-
-        if len(args) != 2 or args[1] is not type(None):
-            raise ValueError(
-                "Union types other than Optional are not yet supported in Snakemake"
-                " workflows."
-            )
-
-        return f"typing.Optional[{type_repr(args[0], add_namespace=add_namespace)}]"
-
-    if get_origin(t) is Annotated:
-        args = get_args(t)
-        assert len(args) > 1
-        assert isinstance(args[1], FlyteAnnotation)
-        return (
-            f"typing_extensions.Annotated[{type_repr(args[0], add_namespace=add_namespace)},"
-            f" FlyteAnnotation({repr(args[1].data)})]"
-        )
-
-    return t.__name__
-
-
-def dataclass_repr(typ: Type) -> str:
+def dataclass_repr(typ: Type, *, make_optionals: bool = False) -> str:
     assert is_dataclass(typ)
 
     lines = ["@dataclass", f"class {typ.__name__}:"]
     for f in fields(typ):
-        lines.append(f"    {f.name}: {type_repr(f.type)}")
+        t = f.type
+        if make_optionals:
+            t = Optional[t]
+
+        lines.append(f"    {f.name}: {type_repr(t)}")
 
     return "\n".join(lines) + "\n\n\n"
 
 
-def get_preamble(typ: Type) -> str:
+def get_preamble(typ: Type, *, make_optionals: bool = False) -> str:
     if get_origin(typ) is Annotated:
         args = get_args(typ)
         assert len(args) > 0
-        return get_preamble(args[0])
+        return get_preamble(args[0], make_optionals=make_optionals)
 
     if is_primitive_type(typ) or typ in {LatchFile, LatchDir}:
         return ""
 
     if get_origin(typ) in {Union, list}:
-        return "".join([get_preamble(t) for t in get_args(typ)])
+        return "".join(
+            [get_preamble(t, make_optionals=make_optionals) for t in get_args(typ)]
+        )
 
     assert is_dataclass(typ), typ
 
-    preamble = "".join([get_preamble(f.type) for f in fields(typ)])
+    preambles = []
+    for f in fields(typ):
+        t = f.type
+        if make_optionals:
+            t = Optional[f.type]
+
+        preambles.append(get_preamble(t, make_optionals=make_optionals))
+
+    preamble = "".join(preambles)
 
-    return "".join([preamble, dataclass_repr(typ)])
+    return "".join([preamble, dataclass_repr(typ, make_optionals=make_optionals)])

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/common/serialize.py

@@ -1,7 +1,7 @@
 import sys
 from dataclasses import fields
 from pathlib import Path
-from typing import Any, Dict, Optional, Type, TypeVar, Union, get_args
+from typing import Any, Dict, List, Optional, Type, TypeVar, Union, get_args
 
 import click
 from flyteidl.admin.launch_plan_pb2 import LaunchPlan as _idl_admin_LaunchPlan
@@ -292,7 +292,7 @@ RegistrableEntity = Union[
 ]
 
 
-def should_register_with_admin(entity: RegistrableEntity) -> bool:
+def should_register_with_admin(entity: FlyteSerializableModel) -> bool:
     return isinstance(entity, get_args(RegistrableEntity))
 
 
@@ -327,15 +327,29 @@ def serialize(
     )
     admin_lp = get_serializable_launch_plan(lp, settings, registrable_entity_cache)
 
-    registrable_entities = [
-        x.to_flyte_idl()
-        for x in list(
-            filter(should_register_with_admin, list(registrable_entity_cache.values()))
-        )
-        + [admin_lp]
-    ]
+    click.secho(
+        "\nBuilding flyte models: \x1b[?25l",
+        nl=False,
+    )
+
+    registrable_entities = []
+    i = 0
+    for x in list(registrable_entity_cache.values()) + [admin_lp]:
+        progress_str = f"{i + 1}/{len(registrable_entity_cache) + 1}"
+
+        click.echo("\x1b[0K", nl=False)
+        click.secho(progress_str, dim=True, italic=True, nl=False)
+        click.echo(f"\x1b[{len(progress_str)}D", nl=False)
+
+        i += 1
+
+        if not should_register_with_admin(x):
+            continue
+
+        registrable_entities.append(x.to_flyte_idl())
 
     click.secho("\nSerializing workflow entities", bold=True)
+
     persist_registrable_entities(registrable_entities, output_dir)
 
     if not write_spec:

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/common/utils.py

@@ -3,7 +3,7 @@ from typing import Type, Union, get_args, get_origin
 
 from typing_extensions import Annotated, TypeGuard
 
-from latch.types.directory import LatchDir
+from latch.types.directory import LatchDir, LatchOutputDir
 from latch.types.file import LatchFile
 
 
@@ -17,7 +17,7 @@ def reindent(x: str, level: int) -> str:
 def is_primitive_type(
     typ: Type,
 ) -> TypeGuard[Union[Type[None], Type[str], Type[bool], Type[int], Type[float]]]:
-    return typ in {Type[None], str, bool, int, float}
+    return typ in {type(None), str, bool, int, float}
 
 
 def is_primitive_value(val: object) -> TypeGuard[Union[None, str, bool, int, float]]:
@@ -34,6 +34,13 @@ def is_blob_type(typ: Type) -> TypeGuard[Union[Type[LatchFile], Type[LatchDir]]]
     return typ in {LatchFile, LatchDir}
 
 
+def is_downloadable_blob_type(typ: Type):
+    if not is_blob_type(typ):
+        return False
+
+    return typ is not LatchOutputDir
+
+
 def type_repr(t: Type, *, add_namespace: bool = False) -> str:
     if getattr(t, "__name__", None) == "NoneType":
         return "None"

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/build.py

@@ -19,6 +19,7 @@ from flytekit.models import literals as literals_models
 from latch_sdk_gql.execute import execute
 
 from latch_cli import tinyrequests
+from latch_cli.extras.common.config.utils import get_preamble
 from latch_cli.extras.nextflow.tasks.base import NextflowBaseTask, NFTaskType
 
 from ...click_utils import italic
@@ -102,7 +103,7 @@ def build_from_nextflow_dag(
 
                 output_name = "res"
                 if len(dep.outputNames) > 0:
-                    idx = int(edge.label)
+                    idx = int(edge.label or "0")
                     input_name = f"{input_name}_{idx}"
                     output_name = dep.outputNames[idx]
 
@@ -128,8 +129,6 @@ def build_from_nextflow_dag(
             upstream_nodes.append(node_map[dep.id])
 
         node_name = get_node_name(vertex.id)
-        if vertex.type == VertexType.Merge:
-            task_outputs = {k: Optional[str] for k in merge_sources.keys()}
 
         if vertex.type == VertexType.Process:
             pre_adapter_task = NextflowProcessPreAdapterTask(
@@ -513,6 +512,9 @@ def generate_nf_entrypoint(
         0,
     )
 
+    for t in wf.python_interface.inputs.values():
+        preamble += get_preamble(t, make_optionals=True)
+
     nf_script_path_in_container = nf_script.resolve().relative_to(pkg_root.resolve())
 
     entrypoint_code = [preamble]

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/config.py

@@ -2,7 +2,7 @@ import json
 import os
 import subprocess
 from pathlib import Path
-from typing import List
+from typing import List, Optional
 
 import click
 
@@ -37,6 +37,7 @@ def generate_nf_metadata(
     skip_confirmation: bool = False,
     generate_defaults: bool = False,
     infer_files: bool = False,
+    make_optionals: bool = True,
 ):
     if not config_path.is_dir():
         click.secho(
@@ -103,7 +104,10 @@ def generate_nf_metadata(
     params: List[str] = []
 
     for k, (typ, (val, default)) in parsed.items():
-        preambles.append(get_preamble(typ))
+        if make_optionals:
+            typ = Optional[typ]
+
+        preambles.append(get_preamble(typ, make_optionals=make_optionals))
 
         param_str = reindent(
             f"""\

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/serialize.py

@@ -15,7 +15,7 @@ def serialize_nf(
     image_name: str,
     dkr_repo: str,
 ):
-    serialize(nf_wf, output_dir, image_name, dkr_repo, write_spec=True)
+    serialize(nf_wf, output_dir, image_name, dkr_repo)
 
 
 def add_task_metadata(wf: NextflowWorkflow):

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/tasks/base.py

@@ -47,8 +47,8 @@ class NextflowBaseTask(PythonAutoContainerTask[Pod]):
         wf: NextflowWorkflow,
         nf_task_type: NFTaskType,
         # todo(ayush): expose / infer these somehow
-        cpu: int = 4,
-        memory: int = 8,
+        cpu: float = 0.5,
+        memory: float = 1,
     ):
         self.id = id
         self.wf = wf

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/tasks/operator.py

@@ -130,7 +130,7 @@ class NextflowOperatorTask(NextflowBaseTask):
                     if {k} is not None:
                         {k}_p = Path({k}).resolve()
                         check_exists_and_rename({k}_p, Path("/root") / {k}_p.name)
-                        wf_paths[{k}] = Path("/root") / {k}_p.name
+                        wf_paths["{k}"] = Path("/root") / {k}_p.name
 
                     """,
                     2,
@@ -140,7 +140,7 @@ class NextflowOperatorTask(NextflowBaseTask):
                     f"""
                     if {k} is not None:
                         {k}_p = Path("/root/").resolve() # superhack
-                        wf_paths[{k}] = {k}_p
+                        wf_paths["{k}"] = {k}_p
 
                     """,
                     2,

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/tasks/process.py

@@ -136,7 +136,7 @@ class NextflowProcessTask(NextflowBaseTask):
                     if {k} is not None:
                         {k}_p = Path({k}).resolve()
                         check_exists_and_rename({k}_p, Path("/root") / {k}_p.name)
-                        wf_paths[{k}] = Path("/root") / {k}_p.name
+                        wf_paths["{k}"] = Path("/root") / {k}_p.name
 
                     """,
                     1,
@@ -146,7 +146,7 @@ class NextflowProcessTask(NextflowBaseTask):
                     f"""
                     if {k} is not None:
                         {k}_p = Path("/root/").resolve() # superhack
-                        wf_paths[{k}] = {k}_p
+                        wf_paths["{k}"] = {k}_p
 
                     """,
                     1,

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/latch_cli/extras/nextflow/workflow.py

@@ -1,5 +1,6 @@
+from dataclasses import fields, is_dataclass
 from pathlib import Path
-from typing import List
+from typing import Any, Dict, List, Tuple, Type
 
 from flytekit.core.class_based_resolver import ClassStorageTaskResolver
 from flytekit.core.docstring import Docstring
@@ -13,10 +14,32 @@ from flytekit.core.workflow import (
 from flytekit.exceptions import scopes as exception_scopes
 
 from latch.types import metadata
-from latch.types.metadata import NextflowFileParameter
+from latch_cli.extras.common.utils import is_blob_type, is_downloadable_blob_type
 
 from .dag import DAG
 
+# def _get_flattened_inputs(
+#     key: str, t: Type, val: Any, inputs: Dict[str, Tuple[Type, Any]]
+# ):
+#     if not is_dataclass(t):
+#         inputs[key] = (t, val)
+#         return
+
+#     for f in fields(t):
+#         v = val
+#         if val is not None:
+#             v = getattr()
+
+
+def _get_flags_to_params(key: str, t: Type, flags: Dict[str, str]):
+    if is_blob_type(t):
+        flags[f"--{key}"] = f"wf_paths['wf_{key}']"
+    elif is_dataclass(t):
+        for f in fields(t):
+            _get_flags_to_params(f"{key}.{f.name}", f.type, flags)
+    else:
+        flags[f"--{key}"] = f"wf_{key}"
+
 
 class NextflowWorkflow(WorkflowBase, ClassStorageTaskResolver):
     def __init__(self, nf_script: Path, version: str, dag: DAG):
@@ -39,19 +62,17 @@ class NextflowWorkflow(WorkflowBase, ClassStorageTaskResolver):
             docstring=docstring,
         )
 
-        self.flags_to_params = {
-            f"--{k}": (
-                f"wf_paths[wf_{k}]"
-                if isinstance(v, NextflowFileParameter)
-                else f"wf_{k}"
-            )
-            for k, v in metadata._nextflow_metadata.parameters.items()
-        }
+        self.flattened_inputs = {}
+
+        self.flags_to_params = {}
+        for k, v in metadata._nextflow_metadata.parameters.items():
+            assert v.type is not None
+            _get_flags_to_params(k, v.type, self.flags_to_params)
 
         self.downloadable_params = {
-            k: str(v.path)
+            k
             for k, v in metadata._nextflow_metadata.parameters.items()
-            if isinstance(v, NextflowFileParameter) and v._download
+            if is_downloadable_blob_type(v.type)
         }
 
         name = metadata._nextflow_metadata.name

{latch-2.39.0.dev25 → latch-2.39.0.dev26}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.39.0.dev25",
+    version="v2.39.0.dev26",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),