latch 2.39.0.dev23__tar.gz → 2.39.0.dev25__tar.gz

{latch-2.39.0.dev23/latch.egg-info → latch-2.39.0.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev23
+Version: 2.39.0.dev25
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev23 → latch-2.39.0.dev25/latch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: latch
-Version: 2.39.0.dev23
+Version: 2.39.0.dev25
 Summary: The Latch SDK
 Author-email: kenny@latch.bio
 Classifier: Programming Language :: Python :: 3.8

{latch-2.39.0.dev23 → latch-2.39.0.dev25}/latch.egg-info/SOURCES.txt

@@ -82,6 +82,9 @@ latch_cli/extras/__init__.py
 latch_cli/extras/common/__init__.py
 latch_cli/extras/common/serialize.py
 latch_cli/extras/common/utils.py
+latch_cli/extras/common/config/__init__.py
+latch_cli/extras/common/config/parser.py
+latch_cli/extras/common/config/utils.py
 latch_cli/extras/nextflow/__init__.py
 latch_cli/extras/nextflow/build.py
 latch_cli/extras/nextflow/channel.py

latch-2.39.0.dev25/latch_cli/extras/common/config/parser.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+from typing import Dict, Tuple, Type, TypeVar
+
+import click
+
+from ..utils import reindent
+from .utils import JSONValue, parse_type, parse_value
+
+T = TypeVar("T")
+
+
+def parse_config(
+    config: JSONValue,
+    *,
+    infer_files: bool = False,
+) -> Dict[str, Tuple[Type[T], T]]:
+    parsed: Dict[str, Type] = {}
+    for k, v in config.items():
+        try:
+            typ = parse_type(v, k, infer_files=infer_files)
+        except ValueError as e:
+            click.secho(
+                f"WARNING: Skipping parameter {k}. Failed to parse type: {e}.",
+                fg="yellow",
+            )
+            continue
+        val, default = parse_value(typ, v)
+
+        parsed[k] = (typ, (val, default))
+
+    return parsed
+
+
+def write_metadata(
+    metadata: str,
+    params: str,
+    *,
+    skip_confirmation: bool = False,
+) -> None:
+    metadata_root = Path("latch_metadata")
+    if metadata_root.is_file():
+        if not click.confirm("A file exists at `latch_metadata`. Delete it?"):
+            raise click.exceptions.Exit(0)
+
+        metadata_root.unlink()
+
+    metadata_root.mkdir(exist_ok=True)
+
+    metadata_path = metadata_root / Path("__init__.py")
+    old_metadata_path = Path("latch_metadata.py")
+
+    if old_metadata_path.exists() and not metadata_path.exists():
+        if click.confirm(
+            "Found legacy `latch_metadata.py` file in current directory. This is"
+            " deprecated and will be ignored in future releases. Move to"
+            " `latch_metadata/__init__.py`? (This will not change file contents)"
+        ):
+            old_metadata_path.rename(metadata_path)
+    elif old_metadata_path.exists() and metadata_path.exists():
+        click.secho(
+            "Warning: Found both `latch_metadata.py` and"
+            " `latch_metadata/__init__.py` in current directory."
+            " `latch_metadata.py` will be ignored.",
+            fg="yellow",
+        )
+
+    if not metadata_path.exists():
+        metadata_path.write_text(reindent(metadata, 0))
+        click.secho("Generated `latch_metadata/__init__.py`.", fg="green")
+
+    params_path = metadata_root / Path("parameters.py")
+    if (
+        params_path.exists()
+        and not skip_confirmation
+        and not click.confirm(
+            "File `latch_metadata/parameters.py` already exists. Overwrite?"
+        )
+    ):
+        raise click.exceptions.Exit(0)
+
+    params_path.write_text(reindent(params, 0))
+    click.secho("Generated `latch_metadata/parameters.py`.", fg="green")

latch-2.39.0.dev25/latch_cli/extras/common/config/utils.py

@@ -0,0 +1,268 @@
+from dataclasses import fields, is_dataclass, make_dataclass
+from typing import Any, Dict, List, Optional, Type, Union, get_args, get_origin
+
+from flytekit.core.annotation import FlyteAnnotation
+from typing_extensions import Annotated, TypeAlias, TypeGuard
+
+from latch.types.directory import LatchDir
+from latch.types.file import LatchFile
+from latch_cli.utils import identifier_from_str
+
+from ..utils import is_primitive_type, is_primitive_value, type_repr
+
+JSONValue: TypeAlias = Union[int, str, bool, float, None, List["JSONValue"], "JSONDict"]
+JSONDict: TypeAlias = Dict[str, "JSONValue"]
+
+# ayush: yoinked from console
+valid_extensions = {
+    "bed",
+    "vcf",
+    "css",
+    "csv",
+    "gif",
+    "png",
+    "pdf",
+    "webp",
+    "xhtml",
+    "xlsx",
+    "xml",
+    "py",
+    "log",
+    "json",
+    "gz",
+    "mmtf",
+    "deseqreport",
+    "sam",
+    "bam",
+    "cram",
+    "tsv",
+    "tab",
+    "sf",
+    "txt",
+    "text",
+    "license",
+    "readme",
+    "r",
+    "rscript",
+    "md",
+    "markdown",
+    "markdn",
+    "mdown",
+    "htm",
+    "html",
+    "ipynb",
+    "jpeg",
+    "jpg",
+    "jif",
+    "jpe",
+    "jfif",
+    "js",
+    "mjs",
+    "es",
+    "ts",
+    "jsx",
+    "tsx",
+    "svg",
+    "svgz",
+    "fasta",
+    "fna",
+    "fa",
+    "ffn",
+    "faa",
+    "frn",
+    "fastq",
+    "fq",
+    "pdb",
+    "pdb1",
+    "ent",
+    "brk",
+    "ml2",
+    "mol2",
+    "sy2",
+    "hdf",
+    "h4",
+    "hdf4",
+    "he4",
+    "h5",
+    "hdf5",
+    "he5",
+    "h5ad",
+    "yaml",
+    "yml",
+}
+
+
+def parse_type(
+    v: JSONValue, name: Optional[str] = None, *, infer_files: bool = False
+) -> Type:
+    if v is None:
+        return str
+
+    if infer_files and isinstance(v, str):
+        if any([v.endswith(ext) for ext in valid_extensions]):
+            return LatchFile
+        elif v.endswith("/"):
+            return LatchDir
+
+    if is_primitive_value(v):
+        return type(v)
+
+    if isinstance(v, list):
+        parsed_types = tuple(
+            parse_type(
+                x,
+                name,
+                infer_files=infer_files,
+            )
+            for x in v
+        )
+
+        if len(set(parsed_types)) != 1:
+            raise ValueError(
+                "Generic Lists are not supported - please"
+                f" ensure that all elements in {name} are of the same type",
+            )
+        typ = parsed_types[0]
+        if typ in {LatchFile, LatchDir}:
+            return Annotated[List[typ], FlyteAnnotation({"size": len(v)})]
+        return List[typ]
+
+    assert isinstance(v, dict)
+
+    if name is None:
+        name = "SnakemakeRecord"
+
+    fields: Dict[str, Type] = {}
+    for k, x in v.items():
+        fields[identifier_from_str(k)] = parse_type(
+            x,
+            k,
+            infer_files=infer_files,
+        )
+
+    return make_dataclass(identifier_from_str(name), fields.items())
+
+
+# returns raw value and generated default
+def parse_value(t: Type, v: JSONValue):
+    if v is None:
+        return None, None
+
+    if get_origin(t) is Annotated:
+        args = get_args(t)
+        assert len(args) > 0
+        return parse_value(args[0], v)
+
+    if t in {LatchFile, LatchDir}:
+        # ayush: autogenerated defaults don't make sense for files/dirs since their
+        # value in the config is their local path
+        return v, None
+
+    if is_primitive_value(v):
+        return v, v
+
+    if isinstance(v, list):
+        assert get_origin(t) is list
+
+        args = get_args(t)
+        assert len(args) > 0
+
+        sub_type = args[0]
+        res = [parse_value(sub_type, x) for x in v]
+        return [x[0] for x in res], [x[1] for x in res]
+
+    assert isinstance(v, dict), v
+    assert is_dataclass(t), t
+
+    ret = {}
+    defaults = {}
+    fs = {identifier_from_str(f.name): f for f in fields(t)}
+
+    for k, x in v.items():
+        sanitized = identifier_from_str(k)
+        assert sanitized in fs, sanitized
+        val, default = parse_value(fs[sanitized].type, x)
+        ret[sanitized] = val
+        defaults[sanitized] = default
+
+    return t(**ret), t(**defaults)
+
+
+def is_primitive_type(
+    typ: Type,
+) -> TypeGuard[Union[Type[None], Type[str], Type[bool], Type[int], Type[float]]]:
+    return typ in {Type[None], str, bool, int, float}
+
+
+def is_primitive_value(val: object) -> TypeGuard[Union[None, str, bool, int, float]]:
+    return is_primitive_type(type(val))
+
+
+def is_list_type(typ: Type) -> TypeGuard[Type[List]]:
+    return get_origin(typ) is list
+
+
+def type_repr(t: Type, *, add_namespace: bool = False) -> str:
+    if is_primitive_type(t) or t in {LatchFile, LatchDir}:
+        return t.__name__
+
+    if get_origin(t) is None:
+        return f"{'latch_metadata.' if add_namespace else ''}{t.__name__}"
+
+    if get_origin(t) is list:
+        args = get_args(t)
+        if len(args) > 0:
+            return f"typing.List[{type_repr(args[0], add_namespace=add_namespace)}]"
+
+        return "typing.List"
+
+    if get_origin(t) is Union:
+        args = get_args(t)
+
+        if len(args) != 2 or args[1] is not type(None):
+            raise ValueError(
+                "Union types other than Optional are not yet supported in Snakemake"
+                " workflows."
+            )
+
+        return f"typing.Optional[{type_repr(args[0], add_namespace=add_namespace)}]"
+
+    if get_origin(t) is Annotated:
+        args = get_args(t)
+        assert len(args) > 1
+        assert isinstance(args[1], FlyteAnnotation)
+        return (
+            f"typing_extensions.Annotated[{type_repr(args[0], add_namespace=add_namespace)},"
+            f" FlyteAnnotation({repr(args[1].data)})]"
+        )
+
+    return t.__name__
+
+
+def dataclass_repr(typ: Type) -> str:
+    assert is_dataclass(typ)
+
+    lines = ["@dataclass", f"class {typ.__name__}:"]
+    for f in fields(typ):
+        lines.append(f"    {f.name}: {type_repr(f.type)}")
+
+    return "\n".join(lines) + "\n\n\n"
+
+
+def get_preamble(typ: Type) -> str:
+    if get_origin(typ) is Annotated:
+        args = get_args(typ)
+        assert len(args) > 0
+        return get_preamble(args[0])
+
+    if is_primitive_type(typ) or typ in {LatchFile, LatchDir}:
+        return ""
+
+    if get_origin(typ) in {Union, list}:
+        return "".join([get_preamble(t) for t in get_args(typ)])
+
+    assert is_dataclass(typ), typ
+
+    preamble = "".join([get_preamble(f.type) for f in fields(typ)])
+
+    return "".join([preamble, dataclass_repr(typ)])

{latch-2.39.0.dev23 → latch-2.39.0.dev25}/setup.py

@@ -13,7 +13,7 @@ if cur_ver < (3, 8) or cur_ver > (3, 11):
 
 setup(
     name="latch",
-    version="v2.39.0.dev23",
+    version="v2.39.0.dev25",
     author_email="kenny@latch.bio",
     description="The Latch SDK",
     packages=find_packages(),