flyte 0.1.0__py3-none-any.whl → 0.2.0a0__py3-none-any.whl

flyte/_utils/lazy_module.py

@@ -0,0 +1,54 @@
+import importlib.util
+import sys
+import types
+
+
+class _LazyModule(types.ModuleType):
+    """
+    `lazy_module` returns an instance of this class if the module is not found in the python environment.
+    """
+
+    def __init__(self, module_name: str):
+        super().__init__(module_name)
+        self._module_name = module_name
+
+    def __getattribute__(self, attr):
+        raise ImportError(f"Module {object.__getattribute__(self, '_module_name')} is not yet installed.")
+
+
+def is_imported(module_name):
+    """
+    This function is used to check if a module has been imported by the regular import.
+    Return false if module is lazy imported and not used yet.
+    """
+    return (
+        module_name in sys.modules
+        and object.__getattribute__(lazy_module(module_name), "__class__").__name__ != "_LazyModule"
+    )
+
+
+def lazy_module(fullname):
+    """
+    This function is used to lazily import modules.  It is used in the following way:
+    .. code-block:: python
+        from flytekit.lazy_import import lazy_module
+        sklearn = lazy_module("sklearn")
+        sklearn.svm.SVC()
+    :param Text fullname: The full name of the module to import
+    """
+    if fullname in sys.modules:
+        return sys.modules[fullname]
+    # https://docs.python.org/3/library/importlib.html#implementing-lazy-imports
+    spec = importlib.util.find_spec(fullname)
+    if spec is None or spec.loader is None:
+        # Return a lazy module if the module is not found in the python environment,
+        # so that we can raise a proper error when the user tries to access an attribute in the module.
+        # The reason to do this is because importlib.util.LazyLoader still requires
+        # the module to be installed even if you don't use it.
+        return _LazyModule(fullname)
+    loader = importlib.util.LazyLoader(spec.loader)
+    spec.loader = loader
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[fullname] = module
+    loader.exec_module(module)
+    return module

flyte/_utils/org_discovery.py

@@ -0,0 +1,57 @@
+def hostname_from_url(url: str) -> str:
+    """Parse a URL and return the hostname part."""
+
+    # Handle dns:/// format specifically (gRPC convention)
+    if url.startswith("dns:///"):
+        return url[7:]  # Skip the "dns:///" prefix
+
+    # Handle standard URL formats
+    import urllib.parse
+
+    parsed = urllib.parse.urlparse(url)
+    return parsed.netloc or parsed.path.lstrip("/").rsplit("/")[0]
+
+
+def org_from_endpoint(endpoint: str | None) -> str | None:
+    """
+    Extracts the organization from the endpoint URL. The organization is assumed to be the first part of the domain.
+    This is temporary until we have a proper organization discovery mechanism through APIs.
+
+    :param endpoint: The endpoint URL
+    :return: The organization name or None if not found
+    """
+    if not endpoint:
+        return None
+
+    hostname = hostname_from_url(endpoint)
+    domain_parts = hostname.split(".")
+    if len(domain_parts) > 2:
+        # Assuming the organization is the first part of the domain
+        return domain_parts[0]
+    return None
+
+
+def sanitize_endpoint(endpoint: str | None) -> str | None:
+    """
+    Sanitize the endpoint URL by ensuring it has a valid scheme.
+    :param endpoint: The endpoint URL to sanitize
+    :return: Sanitized endpoint URL or None if the input was None
+    """
+    if not endpoint:
+        return None
+    if "://" not in endpoint:
+        endpoint = f"dns:///{endpoint}"
+    else:
+        if endpoint.startswith("https://"):
+            # If the endpoint starts with dns:///, we assume it's a gRPC endpoint
+            endpoint = f"dns:///{endpoint[8:]}"
+        elif endpoint.startswith("http://"):
+            # If the endpoint starts with http://, we assume it's a REST endpoint
+            endpoint = f"dns:///{endpoint[7:]}"
+        elif not endpoint.startswith("dns:///"):
+            raise RuntimeError(
+                f"Invalid endpoint {endpoint}, expected format is "
+                f"dns:///<hostname> or https://<hostname> or http://<hostname>"
+            )
+    endpoint = endpoint.removesuffix("/")
+    return endpoint

flyte/_utils/uv_script_parser.py

@@ -0,0 +1,49 @@
+import pathlib
+import re
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+import toml
+
+
+@dataclass
+class ToolUVConfig:
+    exclude_newer: Optional[str] = None
+
+
+@dataclass
+class UVScriptMetadata:
+    requires_python: Optional[str] = None
+    dependencies: List[str] = field(default_factory=list)
+    tool: Optional[Dict[str, ToolUVConfig]] = None
+
+
+def _extract_uv_metadata_block(text: str) -> str | None:
+    pattern = re.compile(r"# /// script\s*(.*?)# ///", re.DOTALL)
+    match = pattern.search(text)
+    if not match:
+        return None
+    lines = [line.lstrip("# ").rstrip() for line in match.group(1).splitlines()]
+    return "\n".join(lines)
+
+
+def parse_uv_script_file(path: pathlib.Path) -> UVScriptMetadata:
+    if not path.exists() or not path.is_file():
+        raise FileNotFoundError(f"File not found: {path}")
+
+    text = path.read_text(encoding="utf-8")
+    raw_header = _extract_uv_metadata_block(text)
+    if raw_header is None:
+        raise ValueError("No uv metadata block found")
+
+    try:
+        data = toml.loads(raw_header)
+    except toml.TomlDecodeError as e:
+        raise ValueError(f"Invalid TOML in metadata block: {e}")
+
+    tool_data = data.get("tool", {}).get("uv", {})
+    return UVScriptMetadata(
+        requires_python=data.get("requires-python"),
+        dependencies=data.get("dependencies", []),
+        tool={"uv": ToolUVConfig(exclude_newer=tool_data.get("exclude-newer"))} if tool_data else None,
+    )

flyte/_version.py

@@ -0,0 +1,21 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+else:
+    VERSION_TUPLE = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+
+__version__ = version = '0.2.0a0'
+__version_tuple__ = version_tuple = (0, 2, 0, 'a0')

flyte/cli/__init__.py

@@ -0,0 +1,3 @@
+from flyte.cli.main import main
+
+__all__ = ["main"]

flyte/cli/_abort.py

@@ -0,0 +1,28 @@
+import rich_click as click
+from rich.console import Console
+
+from flyte.cli import _common as common
+
+
+@click.group(name="abort")
+def abort():
+    """
+    Abort an ongoing process.
+    """
+
+
+@abort.command(cls=common.CommandBase)
+@click.argument("run-name", type=str, required=True)
+@click.pass_obj
+def run(cfg: common.CLIConfig, run_name: str, project: str | None = None, domain: str | None = None):
+    """
+    Abort a run.
+    """
+    from flyte.remote import Run
+
+    cfg.init(project=project, domain=domain)
+    r = Run.get(name=run_name)
+    if r:
+        console = Console()
+        r.abort()
+        console.print(f"Run '{run_name}' has been aborted.")

flyte/cli/_common.py

@@ -0,0 +1,337 @@
+from __future__ import annotations
+
+import importlib.util
+import logging
+import os
+import sys
+from abc import abstractmethod
+from dataclasses import dataclass, replace
+from pathlib import Path
+from types import MappingProxyType, ModuleType
+from typing import Any, Dict, Iterable, List, Optional
+
+import rich.box
+import rich.repr
+import rich_click as click
+from rich.console import Console
+from rich.panel import Panel
+from rich.table import Table
+from rich.traceback import Traceback
+
+import flyte.errors
+from flyte._logging import logger
+from flyte.config import Config
+
+PREFERRED_BORDER_COLOR = "dim cyan"
+PREFERRED_ACCENT_COLOR = "bold #FFD700"
+HEADER_STYLE = f"{PREFERRED_ACCENT_COLOR} on black"
+PANELS = False
+
+PROJECT_OPTION = click.Option(
+    param_decls=["-p", "--project"],
+    required=False,
+    type=str,
+    default=None,
+    help="Project to which this command applies.",
+    show_default=True,
+)
+
+DOMAIN_OPTION = click.Option(
+    param_decls=["-d", "--domain"],
+    required=False,
+    type=str,
+    default=None,
+    help="Domain to which this command applies.",
+    show_default=True,
+)
+
+DRY_RUN_OPTION = click.Option(
+    param_decls=["--dry-run", "--dryrun"],
+    required=False,
+    type=bool,
+    is_flag=True,
+    default=False,
+    help="Dry run. Do not actually call the backend service.",
+    show_default=True,
+)
+
+
+def _common_options() -> List[click.Option]:
+    """
+    Common options that will be added to all commands and groups that inherit from CommandBase or GroupBase.
+    """
+    return [PROJECT_OPTION, DOMAIN_OPTION]
+
+
+# This is global state for the CLI, it is manipulated by the main command
+
+
+@rich.repr.auto
+@dataclass(frozen=True)
+class CLIConfig:
+    """
+    This is the global state for the CLI. It is manipulated by the main command.
+    """
+
+    config: Config
+    ctx: click.Context
+    log_level: int | None = logging.ERROR
+    endpoint: str | None = None
+    insecure: bool = False
+    org: str | None = None
+
+    def replace(self, **kwargs) -> CLIConfig:
+        """
+        Replace the global state with a new one.
+        """
+        return replace(self, **kwargs)
+
+    def init(self, project: str | None = None, domain: str | None = None):
+        from flyte.config._config import TaskConfig
+
+        task_cfg = TaskConfig(
+            org=self.org or self.config.task.org,
+            project=project or self.config.task.project,
+            domain=domain or self.config.task.domain,
+        )
+
+        kwargs: Dict[str, Any] = {}
+        if self.endpoint:
+            kwargs["endpoint"] = self.endpoint
+        if self.insecure is not None:
+            kwargs["insecure"] = self.insecure
+        platform_cfg = self.config.platform.replace(**kwargs)
+
+        updated_config = self.config.with_params(platform_cfg, task_cfg)
+
+        flyte.init_from_config(updated_config, log_level=self.log_level)
+
+
+class InvokeBaseMixin:
+    """
+    Mixin to catch grpc.RpcError, flyte.RpcError, other errors and other exceptions
+    and raise them as gclick.ClickException.
+    """
+
+    def invoke(self, ctx):
+        import grpc
+
+        try:
+            return super().invoke(ctx)  # type: ignore
+        except grpc.aio.AioRpcError as e:
+            if e.code() == grpc.StatusCode.UNAUTHENTICATED:
+                raise click.ClickException(f"Authentication failed. Please check your credentials. {e.details()}")
+            if e.code() == grpc.StatusCode.NOT_FOUND:
+                raise click.ClickException(f"Requested object NOT FOUND. Please check your input. Error: {e.details()}")
+            if e.code() == grpc.StatusCode.ALREADY_EXISTS:
+                raise click.ClickException("Resource already exists.")
+            if e.code() == grpc.StatusCode.INTERNAL:
+                raise click.ClickException(f"Internal server error: {e.details()}")
+            if e.code() == grpc.StatusCode.UNAVAILABLE:
+                raise click.ClickException(
+                    f"Service is currently unavailable. Please try again later. Error: {e.details()}"
+                )
+            if e.code() == grpc.StatusCode.PERMISSION_DENIED:
+                raise click.ClickException(f"Permission denied. Please check your access rights. Error: {e.details()}")
+            if e.code() == grpc.StatusCode.INVALID_ARGUMENT:
+                raise click.ClickException(f"Invalid argument provided. Please check your input. Error: {e.details()}")
+            raise click.ClickException(f"RPC error invoking command: {e!s}") from e
+        except flyte.errors.InitializationError as e:
+            raise click.ClickException(f"Initialization failed. Pass remote config for CLI. (Reason: {e})")
+        except flyte.errors.BaseRuntimeError as e:
+            raise click.ClickException(f"{e.kind} failure, {e.code}. {e}") from e
+        except click.exceptions.Exit as e:
+            # This is a normal exit, do nothing
+            raise e
+        except click.exceptions.NoArgsIsHelpError:
+            # Do not raise an error if no arguments are passed, just show the help message.
+            # https://github.com/pallets/click/pull/1489
+            return None
+        except Exception as e:
+            if ctx.obj and ctx.obj.log_level and ctx.obj.log_level <= logging.DEBUG:
+                # If the user has requested verbose output, print the full traceback
+                console = Console()
+                console.print(Traceback.from_exception(type(e), e, e.__traceback__))
+            raise click.ClickException(f"Error invoking command: {e}") from e
+
+
+class CommandBase(InvokeBaseMixin, click.RichCommand):
+    """
+    Base class for all commands, that adds common options to all commands if enabled.
+    """
+
+    common_options_enabled = True
+
+    def __init__(self, *args, **kwargs):
+        if "params" not in kwargs:
+            kwargs["params"] = []
+        if self.common_options_enabled:
+            kwargs["params"].extend(_common_options())
+        super().__init__(*args, **kwargs)
+
+
+class GroupBase(InvokeBaseMixin, click.RichGroup):
+    """
+    Base class for all commands, that adds common options to all commands if enabled.
+    """
+
+    common_options_enabled = True
+
+    def __init__(self, *args, **kwargs):
+        if "params" not in kwargs:
+            kwargs["params"] = []
+        if self.common_options_enabled:
+            kwargs["params"].extend(_common_options())
+        super().__init__(*args, **kwargs)
+
+
+class GroupBaseNoOptions(GroupBase):
+    common_options_enabled = False
+
+
+def get_option_from_metadata(metadata: MappingProxyType) -> click.Option:
+    return metadata["click.option"]
+
+
+def key_value_callback(_: Any, param: str, values: List[str]) -> Optional[Dict[str, str]]:
+    """
+    Callback for click to parse key-value pairs.
+    """
+    if not values:
+        return None
+    result = {}
+    for v in values:
+        if "=" not in v:
+            raise click.BadParameter(f"Expected key-value pair of the form key=value, got {v}")
+        k, v_ = v.split("=", 1)
+        result[k.strip()] = v_.strip()
+    return result
+
+
+class ObjectsPerFileGroup(GroupBase):
+    """
+    Group that creates a command for each object in a python file.
+    """
+
+    def __init__(self, filename: Path | None = None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if filename is None:
+            raise ValueError("filename must be provided")
+        if not filename.exists():
+            raise click.ClickException(f"{filename} does not exists")
+        self.filename = filename
+        self._objs: Dict[str, Any] | None = None
+
+    @abstractmethod
+    def _filter_objects(self, module: ModuleType) -> Dict[str, Any]:
+        """
+        Filter the objects in the module to only include the ones we want to expose.
+        """
+        raise NotImplementedError
+
+    @property
+    def objs(self) -> Dict[str, Any]:
+        if self._objs is not None:
+            return self._objs
+
+        module_name = os.path.splitext(os.path.basename(self.filename))[0]
+        module_path = os.path.dirname(os.path.abspath(self.filename))
+
+        spec = importlib.util.spec_from_file_location(module_name, self.filename)
+        if spec is None or spec.loader is None:
+            raise click.ClickException(f"Could not load module {module_name} from {self.filename}")
+
+        module = importlib.util.module_from_spec(spec)
+        sys.modules[module_name] = module
+
+        sys.path.append(module_path)
+        spec.loader.exec_module(module)
+
+        self._objs = self._filter_objects(module)
+        if not self._objs:
+            raise click.ClickException(f"No objects found in {self.filename}")
+        return self._objs
+
+    def list_commands(self, ctx):
+        m = list(self.objs.keys())
+        return sorted(m)
+
+    @abstractmethod
+    def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command: ...
+
+    def get_command(self, ctx, obj_name):
+        obj = self.objs[obj_name]
+        return self._get_command_for_obj(ctx, obj_name, obj)
+
+
+class FileGroup(GroupBase):
+    """
+    Group that creates a command for each file in the current directory that is not __init__.py.
+    """
+
+    common_options_enabled = False
+
+    def __init__(
+        self,
+        *args,
+        directory: Path | None = None,
+        **kwargs,
+    ):
+        if "params" not in kwargs:
+            kwargs["params"] = []
+        super().__init__(*args, **kwargs)
+        self._files = None
+        self._dir = directory
+
+    @property
+    def files(self):
+        if self._files is None:
+            directory = self._dir or Path(".").absolute()
+            self._files = [os.fspath(p) for p in directory.glob("*.py") if p.name != "__init__.py"]
+        return self._files
+
+    def list_commands(self, ctx):
+        return self.files
+
+    def get_command(self, ctx, filename):
+        raise NotImplementedError
+
+
+def get_table(title: str, vals: Iterable[Any]) -> Table:
+    """
+    Get a table from a list of values.
+    """
+    table = Table(
+        title=title,
+        box=rich.box.SQUARE_DOUBLE_HEAD,
+        header_style=HEADER_STYLE,
+        show_header=True,
+        border_style=PREFERRED_BORDER_COLOR,
+    )
+    headers = None
+    has_rich_repr = False
+    for p in vals:
+        if hasattr(p, "__rich_repr__"):
+            has_rich_repr = True
+        elif not isinstance(p, (list, tuple)):
+            raise ValueError("Expected a list or tuple of values, or an object with __rich_repr__ method.")
+        o = list(p.__rich_repr__()) if has_rich_repr else p
+        if headers is None:
+            headers = [k for k, _ in o]
+            for h in headers:
+                table.add_column(h.capitalize())
+        table.add_row(*[str(v) for _, v in o])
+    return table
+
+
+def get_panel(title: str, renderable: Any) -> Panel:
+    """
+    Get a panel from a list of values.
+    """
+    if not PANELS:
+        return renderable
+    return Panel.fit(
+        renderable,
+        title=f"[{PREFERRED_ACCENT_COLOR}]{title}[/{PREFERRED_ACCENT_COLOR}]",
+        border_style=PREFERRED_BORDER_COLOR,
+    )

flyte/cli/_create.py

@@ -0,0 +1,145 @@
+from pathlib import Path
+from typing import Any, Dict, get_args
+
+import rich_click as click
+
+import flyte.cli._common as common
+from flyte.remote import SecretTypes
+
+
+@click.group(name="create")
+def create():
+    """
+    Create resources in a Flyte deployment.
+    """
+
+
+@create.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=True)
+@click.argument("value", type=str, required=False)
+@click.option("--from-file", type=click.Path(exists=True), help="Path to the file with the binary secret.")
+@click.option(
+    "--type", type=click.Choice(get_args(SecretTypes)), default="regular", help="Type of the secret.", show_default=True
+)
+@click.pass_obj
+def secret(
+    cfg: common.CLIConfig,
+    name: str,
+    value: str | bytes | None = None,
+    from_file: str | None = None,
+    type: SecretTypes = "regular",
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Create a new secret. The name of the secret is required. For example:
+
+    ```bash
+    $ flyte create secret my_secret --value my_value
+    ```
+
+    If `--from-file` is specified, the value will be read from the file instead of being provided directly:
+
+    ```bash
+    $ flyte create secret my_secret --from-file /path/to/secret_file
+    ```
+
+    The `--type` option can be used to create specific types of secrets.
+    Either `regular` or `image_pull` can be specified.
+    Secrets intended to access container images should be specified as `image_pull`.
+    Other secrets should be specified as `regular`.
+    If no type is specified, `regular` is assumed.
+
+    ```bash
+    $ flyte create secret my_secret --type image_pull
+    ```
+    """
+    from flyte.remote import Secret
+
+    cfg.init(project, domain)
+    if from_file:
+        with open(from_file, "rb") as f:
+            value = f.read()
+    Secret.create(name=name, value=value, type=type)
+
+
+@create.command(cls=common.CommandBase)
+@click.option("--endpoint", type=str, help="Endpoint of the Flyte backend.")
+@click.option("--insecure", is_flag=True, help="Use an insecure connection to the Flyte backend.")
+@click.option(
+    "--org",
+    type=str,
+    required=False,
+    help="Organization to use. This will override the organization in the configuration file.",
+)
+@click.option(
+    "-o",
+    "--output",
+    type=click.Path(exists=False, writable=True),
+    default=Path.cwd() / "config.yaml",
+    help="Path to the output directory where the configuration will be saved. Defaults to current directory.",
+    show_default=True,
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    default=False,
+    help="Force overwrite of the configuration file if it already exists.",
+    show_default=True,
+)
+def config(
+    output: str,
+    endpoint: str | None = None,
+    insecure: bool = False,
+    org: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+    force: bool = False,
+):
+    """
+    Creates a configuration file for Flyte CLI.
+    If the `--output` option is not specified, it will create a file named `config.yaml` in the current directory.
+    If the file already exists, it will raise an error unless the `--force` option is used.
+    """
+    import yaml
+
+    from flyte._utils import org_from_endpoint, sanitize_endpoint
+
+    output_path = Path(output)
+
+    if output_path.exists() and not force:
+        force = click.confirm(f"Overwrite [{output_path}]?", default=False)
+        if not force:
+            click.echo(f"Will not overwrite the existing config file at {output_path}")
+            return
+
+    admin: Dict[str, Any] = {}
+    if endpoint:
+        endpoint = sanitize_endpoint(endpoint)
+        admin["endpoint"] = endpoint
+    if insecure:
+        admin["insecure"] = insecure
+
+    if not org and endpoint:
+        org = org_from_endpoint(endpoint)
+
+    task: Dict[str, str] = {}
+    if org:
+        task["org"] = org
+    if project:
+        task["project"] = project
+    if domain:
+        task["domain"] = domain
+
+    if not admin and not task:
+        raise click.BadParameter("At least one of --endpoint or --org must be provided.")
+
+    with open(output_path, "w") as f:
+        d: Dict[str, Any] = {}
+        if admin:
+            d["admin"] = admin
+        if task:
+            d["task"] = task
+        yaml.dump(d, f)
+
+    click.echo(f"Config file written to {output_path}")