flyte 0.0.1b3__py3-none-any.whl → 0.2.0a0__py3-none-any.whl

flyte/_utils/async_cache.py

@@ -0,0 +1,139 @@
+import asyncio
+import time
+from collections import OrderedDict
+from typing import Awaitable, Callable, Dict, Generic, Optional, TypeVar
+
+K = TypeVar("K")
+V = TypeVar("V")
+
+
+class AsyncLRUCache(Generic[K, V]):
+    """
+    A high-performance async-compatible LRU cache.
+
+    Examples:
+    ```python
+    # Create a cache instance
+    cache = AsyncLRUCache[str, dict](maxsize=100)
+
+    async def fetch_data(user_id: str) -> dict:
+        # Define the expensive operation as a local function
+        async def get_user_data():
+            await asyncio.sleep(1)  # Simulating network/DB delay
+            return {"id": user_id, "name": f"User {user_id}"}
+
+    # Use the cache
+    return await cache.get(f"user:{user_id}", get_user_data)
+    ```
+    This cache can be used from async coroutines and handles concurrent access safely.
+    """
+
+    def __init__(self, maxsize: int = 128, ttl: Optional[float] = None):
+        """
+        Initialize the async LRU cache.
+
+        Args:
+            maxsize: Maximum number of items to keep in the cache
+            ttl: Time-to-live for cache entries in seconds, or None for no expiration
+        """
+        self._cache: OrderedDict[K, tuple[V, float]] = OrderedDict()
+        self._maxsize = maxsize
+        self._ttl = ttl
+        self._locks: Dict[K, asyncio.Lock] = {}
+        self._access_lock = asyncio.Lock()
+
+    async def get(self, key: K, value_func: Callable[[], V | Awaitable[V]]) -> V:
+        """
+        Get a value from the cache, computing it if necessary.
+
+        Args:
+            key: The cache key
+            value_func: Function or coroutine to compute the value if not cached
+
+        Returns:
+            The cached or computed value
+        """
+        # Fast path: check if key exists and is not expired
+        if key in self._cache:
+            value, timestamp = self._cache[key]
+            if self._ttl is None or time.time() - timestamp < self._ttl:
+                # Move the accessed item to the end (most recently used)
+                async with self._access_lock:
+                    self._cache.move_to_end(key)
+                return value
+
+        # Slow path: compute the value
+        # Get or create a lock for this key to prevent redundant computation
+        async with self._access_lock:
+            lock = self._locks.get(key)
+            if lock is None:
+                lock = asyncio.Lock()
+                self._locks[key] = lock
+
+        async with lock:
+            # Check again in case another coroutine computed the value while we waited
+            if key in self._cache:
+                value, timestamp = self._cache[key]
+                if self._ttl is None or time.time() - timestamp < self._ttl:
+                    async with self._access_lock:
+                        self._cache.move_to_end(key)
+                    return value
+
+            # Compute the value
+            if asyncio.iscoroutinefunction(value_func):
+                value = await value_func()
+            else:
+                value = value_func()  # type: ignore
+
+            # Store in cache
+            async with self._access_lock:
+                self._cache[key] = (value, time.time())
+                # Evict least recently used items if needed
+                while len(self._cache) > self._maxsize:
+                    self._cache.popitem(last=False)
+                # Clean up the lock
+                self._locks.pop(key, None)
+
+            return value
+
+    async def set(self, key: K, value: V) -> None:
+        """
+        Explicitly set a value in the cache.
+
+        Args:
+            key: The cache key
+            value: The value to cache
+        """
+        async with self._access_lock:
+            self._cache[key] = (value, time.time())
+            # Evict least recently used items if needed
+            while len(self._cache) > self._maxsize:
+                self._cache.popitem(last=False)
+
+    async def invalidate(self, key: K) -> None:
+        """Remove a specific key from the cache."""
+        async with self._access_lock:
+            self._cache.pop(key, None)
+
+    async def clear(self) -> None:
+        """Clear the entire cache."""
+        async with self._access_lock:
+            self._cache.clear()
+            self._locks.clear()
+
+    async def contains(self, key: K) -> bool:
+        """Check if a key exists in the cache and is not expired."""
+        if key not in self._cache:
+            return False
+
+        if self._ttl is None:
+            return True
+
+        _, timestamp = self._cache[key]
+        return time.time() - timestamp < self._ttl
+
+
+# Example usage:
+"""
+
+"""

flyte/_utils/coro_management.py

@@ -21,5 +21,3 @@ async def run_coros(*coros: typing.Coroutine, return_when: str = asyncio.FIRST_C
         err = t.exception()
         if err:
             raise err
-        else:
-            print(f"Task result: {t.result()}")

flyte/_utils/helpers.py

@@ -1,6 +1,7 @@
 import os
 import string
 import typing
+from contextlib import contextmanager
 from pathlib import Path
 
 
@@ -51,7 +52,33 @@ def base36_encode(byte_data: bytes) -> str:
     return "".join(reversed(base36))
 
 
-# does not work at all in the setuptools case. see old flytekit editable installs
+def _iter_editable():
+    """
+    Yield (project_name, source_path) for every editable distribution
+    visible to the current interpreter
+    """
+    import json
+    import pathlib
+    from importlib.metadata import distributions
+
+    for dist in distributions():
+        # PEP-610 / PEP-660 (preferred, wheel-style editables)
+        direct = dist.read_text("direct_url.json")
+        if direct:
+            data = json.loads(direct)
+            if data.get("dir_info", {}).get("editable"):  # spec key
+                # todo: will need testing on windows
+                yield dist.metadata["Name"], pathlib.Path(data["url"][7:])  # strip file://
+                continue
+
+        # Legacy setuptools-develop / pip-e (egg-link)
+        for file in dist.files or ():  # importlib.metadata 3.8+
+            if file.suffix == ".egg-link":
+                with open(dist.locate_file(file), "r") as f:
+                    line = f.readline()
+                    yield dist.metadata["Name"], pathlib.Path(line.strip())
+
+
 def get_cwd_editable_install() -> typing.Optional[Path]:
     """
     This helper function is incomplete since it hasn't been tested with all the package managers out there,
@@ -64,28 +91,13 @@ def get_cwd_editable_install() -> typing.Optional[Path]:
 
     :return:
     """
-    import site
 
     from flyte._logging import logger
 
-    egg_links = [Path(p) for p in Path(site.getsitepackages()[0]).glob("*.egg-link")]
-    pth_files = [Path(p) for p in Path(site.getsitepackages()[0]).glob("*.pth")]
-
-    if not egg_links and not pth_files:
-        logger.debug("No editable installs found.")
-        return None
-
     editable_installs = []
-    egg_links.extend(pth_files)
-    for file in egg_links:
-        with open(file, "r") as f:
-            line = f.readline()
-            if line:
-                # Check if the first line is a directory
-                p = Path(line)
-                if p.is_dir():
-                    editable_installs.append(p)
-    logger.debug(f"Editable installs: {editable_installs}")
+    for name, path in _iter_editable():
+        logger.debug(f"Detected editable install: {name} at {path}")
+        editable_installs.append(path)
 
     # check to see if the current working directory is in any of the editable installs
     # including if the current folder is the root folder, one level up from the src and contains
@@ -106,3 +118,17 @@ def get_cwd_editable_install() -> typing.Optional[Path]:
                     return install  # note we want the install folder, not the parent
 
     return None
+
+
+@contextmanager
+def _selector_policy():
+    import asyncio
+
+    original_policy = asyncio.get_event_loop_policy()
+    try:
+        if os.name == "nt" and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
+            asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+
+        yield
+    finally:
+        asyncio.set_event_loop_policy(original_policy)

flyte/_utils/org_discovery.py

@@ -0,0 +1,57 @@
+def hostname_from_url(url: str) -> str:
+    """Parse a URL and return the hostname part."""
+
+    # Handle dns:/// format specifically (gRPC convention)
+    if url.startswith("dns:///"):
+        return url[7:]  # Skip the "dns:///" prefix
+
+    # Handle standard URL formats
+    import urllib.parse
+
+    parsed = urllib.parse.urlparse(url)
+    return parsed.netloc or parsed.path.lstrip("/").rsplit("/")[0]
+
+
+def org_from_endpoint(endpoint: str | None) -> str | None:
+    """
+    Extracts the organization from the endpoint URL. The organization is assumed to be the first part of the domain.
+    This is temporary until we have a proper organization discovery mechanism through APIs.
+
+    :param endpoint: The endpoint URL
+    :return: The organization name or None if not found
+    """
+    if not endpoint:
+        return None
+
+    hostname = hostname_from_url(endpoint)
+    domain_parts = hostname.split(".")
+    if len(domain_parts) > 2:
+        # Assuming the organization is the first part of the domain
+        return domain_parts[0]
+    return None
+
+
+def sanitize_endpoint(endpoint: str | None) -> str | None:
+    """
+    Sanitize the endpoint URL by ensuring it has a valid scheme.
+    :param endpoint: The endpoint URL to sanitize
+    :return: Sanitized endpoint URL or None if the input was None
+    """
+    if not endpoint:
+        return None
+    if "://" not in endpoint:
+        endpoint = f"dns:///{endpoint}"
+    else:
+        if endpoint.startswith("https://"):
+            # If the endpoint starts with dns:///, we assume it's a gRPC endpoint
+            endpoint = f"dns:///{endpoint[8:]}"
+        elif endpoint.startswith("http://"):
+            # If the endpoint starts with http://, we assume it's a REST endpoint
+            endpoint = f"dns:///{endpoint[7:]}"
+        elif not endpoint.startswith("dns:///"):
+            raise RuntimeError(
+                f"Invalid endpoint {endpoint}, expected format is "
+                f"dns:///<hostname> or https://<hostname> or http://<hostname>"
+            )
+    endpoint = endpoint.removesuffix("/")
+    return endpoint

flyte/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.0.1b3'
-__version_tuple__ = version_tuple = (0, 0, 1, 'b3')
+__version__ = version = '0.2.0a0'
+__version_tuple__ = version_tuple = (0, 2, 0, 'a0')

flyte/cli/__init__.py

@@ -0,0 +1,3 @@
+from flyte.cli.main import main
+
+__all__ = ["main"]

flyte/cli/_abort.py

@@ -0,0 +1,28 @@
+import rich_click as click
+from rich.console import Console
+
+from flyte.cli import _common as common
+
+
+@click.group(name="abort")
+def abort():
+    """
+    Abort an ongoing process.
+    """
+
+
+@abort.command(cls=common.CommandBase)
+@click.argument("run-name", type=str, required=True)
+@click.pass_obj
+def run(cfg: common.CLIConfig, run_name: str, project: str | None = None, domain: str | None = None):
+    """
+    Abort a run.
+    """
+    from flyte.remote import Run
+
+    cfg.init(project=project, domain=domain)
+    r = Run.get(name=run_name)
+    if r:
+        console = Console()
+        r.abort()
+        console.print(f"Run '{run_name}' has been aborted.")

flyte/{_cli → cli}/_common.py

@@ -11,22 +11,28 @@ from types import MappingProxyType, ModuleType
 from typing import Any, Dict, Iterable, List, Optional
 
 import rich.box
+import rich.repr
 import rich_click as click
+from rich.console import Console
 from rich.panel import Panel
 from rich.table import Table
+from rich.traceback import Traceback
 
 import flyte.errors
+from flyte._logging import logger
+from flyte.config import Config
 
 PREFERRED_BORDER_COLOR = "dim cyan"
 PREFERRED_ACCENT_COLOR = "bold #FFD700"
 HEADER_STYLE = f"{PREFERRED_ACCENT_COLOR} on black"
+PANELS = False
 
 PROJECT_OPTION = click.Option(
     param_decls=["-p", "--project"],
     required=False,
     type=str,
-    default="default",
-    help="Project to operate on",
+    default=None,
+    help="Project to which this command applies.",
     show_default=True,
 )
 
@@ -34,8 +40,8 @@ DOMAIN_OPTION = click.Option(
     param_decls=["-d", "--domain"],
     required=False,
     type=str,
-    default="development",
-    help="Domain to operate on",
+    default=None,
+    help="Domain to which this command applies.",
     show_default=True,
 )
 
@@ -45,14 +51,14 @@ DRY_RUN_OPTION = click.Option(
     type=bool,
     is_flag=True,
     default=False,
-    help="Dry run, do not actually call the backend service.",
+    help="Dry run. Do not actually call the backend service.",
     show_default=True,
 )
 
 
 def _common_options() -> List[click.Option]:
     """
-    Common options for that will be added to all commands and groups that inherit from CommandBase or GroupBase.
+    Common options that will be added to all commands and groups that inherit from CommandBase or GroupBase.
     """
     return [PROJECT_OPTION, DOMAIN_OPTION]
 
@@ -60,16 +66,19 @@ def _common_options() -> List[click.Option]:
 # This is global state for the CLI, it is manipulated by the main command
 
 
+@rich.repr.auto
 @dataclass(frozen=True)
 class CLIConfig:
     """
     This is the global state for the CLI. It is manipulated by the main command.
     """
 
+    config: Config
+    ctx: click.Context
     log_level: int | None = logging.ERROR
     endpoint: str | None = None
     insecure: bool = False
-    org_override: str | None = None
+    org: str | None = None
 
     def replace(self, **kwargs) -> CLIConfig:
         """
@@ -78,22 +87,30 @@ class CLIConfig:
         return replace(self, **kwargs)
 
     def init(self, project: str | None = None, domain: str | None = None):
-        import flyte
-
-        flyte.init(
-            endpoint=self.endpoint,
-            insecure=self.insecure,
-            org=self.org_override,
-            project=project,
-            domain=domain,
-            log_level=self.log_level,
+        from flyte.config._config import TaskConfig
+
+        task_cfg = TaskConfig(
+            org=self.org or self.config.task.org,
+            project=project or self.config.task.project,
+            domain=domain or self.config.task.domain,
         )
 
+        kwargs: Dict[str, Any] = {}
+        if self.endpoint:
+            kwargs["endpoint"] = self.endpoint
+        if self.insecure is not None:
+            kwargs["insecure"] = self.insecure
+        platform_cfg = self.config.platform.replace(**kwargs)
+
+        updated_config = self.config.with_params(platform_cfg, task_cfg)
+
+        flyte.init_from_config(updated_config, log_level=self.log_level)
+
 
 class InvokeBaseMixin:
     """
-    Mixin to catch grpc.RpcError, flyte.RpcError, other errors and other exceptions and raise them as
-     gclick.ClickException.
+    Mixin to catch grpc.RpcError, flyte.RpcError, other errors and other exceptions
+    and raise them as gclick.ClickException.
     """
 
     def invoke(self, ctx):
@@ -108,10 +125,33 @@ class InvokeBaseMixin:
                 raise click.ClickException(f"Requested object NOT FOUND. Please check your input. Error: {e.details()}")
             if e.code() == grpc.StatusCode.ALREADY_EXISTS:
                 raise click.ClickException("Resource already exists.")
+            if e.code() == grpc.StatusCode.INTERNAL:
+                raise click.ClickException(f"Internal server error: {e.details()}")
+            if e.code() == grpc.StatusCode.UNAVAILABLE:
+                raise click.ClickException(
+                    f"Service is currently unavailable. Please try again later. Error: {e.details()}"
+                )
+            if e.code() == grpc.StatusCode.PERMISSION_DENIED:
+                raise click.ClickException(f"Permission denied. Please check your access rights. Error: {e.details()}")
+            if e.code() == grpc.StatusCode.INVALID_ARGUMENT:
+                raise click.ClickException(f"Invalid argument provided. Please check your input. Error: {e.details()}")
             raise click.ClickException(f"RPC error invoking command: {e!s}") from e
-        except flyte.errors.InitializationError:
-            raise click.ClickException("Initialize the CLI with a remote configuration. For example, pass --endpoint")
+        except flyte.errors.InitializationError as e:
+            raise click.ClickException(f"Initialization failed. Pass remote config for CLI. (Reason: {e})")
+        except flyte.errors.BaseRuntimeError as e:
+            raise click.ClickException(f"{e.kind} failure, {e.code}. {e}") from e
+        except click.exceptions.Exit as e:
+            # This is a normal exit, do nothing
+            raise e
+        except click.exceptions.NoArgsIsHelpError:
+            # Do not raise an error if no arguments are passed, just show the help message.
+            # https://github.com/pallets/click/pull/1489
+            return None
         except Exception as e:
+            if ctx.obj and ctx.obj.log_level and ctx.obj.log_level <= logging.DEBUG:
+                # If the user has requested verbose output, print the full traceback
+                console = Console()
+                console.print(Traceback.from_exception(type(e), e, e.__traceback__))
             raise click.ClickException(f"Error invoking command: {e}") from e
 
 
@@ -173,8 +213,10 @@ class ObjectsPerFileGroup(GroupBase):
     Group that creates a command for each object in a python file.
     """
 
-    def __init__(self, filename: Path, *args, **kwargs):
+    def __init__(self, filename: Path | None = None, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        if filename is None:
+            raise ValueError("filename must be provided")
         if not filename.exists():
             raise click.ClickException(f"{filename} does not exists")
         self.filename = filename
@@ -267,12 +309,18 @@ def get_table(title: str, vals: Iterable[Any]) -> Table:
         border_style=PREFERRED_BORDER_COLOR,
     )
     headers = None
+    has_rich_repr = False
     for p in vals:
+        if hasattr(p, "__rich_repr__"):
+            has_rich_repr = True
+        elif not isinstance(p, (list, tuple)):
+            raise ValueError("Expected a list or tuple of values, or an object with __rich_repr__ method.")
+        o = list(p.__rich_repr__()) if has_rich_repr else p
         if headers is None:
-            headers = [k for k, _ in p.__rich_repr__()]
+            headers = [k for k, _ in o]
             for h in headers:
                 table.add_column(h.capitalize())
-        table.add_row(*[str(v) for _, v in p.__rich_repr__()])
+        table.add_row(*[str(v) for _, v in o])
     return table
 
 
@@ -280,6 +328,8 @@ def get_panel(title: str, renderable: Any) -> Panel:
     """
     Get a panel from a list of values.
     """
+    if not PANELS:
+        return renderable
     return Panel.fit(
         renderable,
         title=f"[{PREFERRED_ACCENT_COLOR}]{title}[/{PREFERRED_ACCENT_COLOR}]",

flyte/cli/_create.py

@@ -0,0 +1,145 @@
+from pathlib import Path
+from typing import Any, Dict, get_args
+
+import rich_click as click
+
+import flyte.cli._common as common
+from flyte.remote import SecretTypes
+
+
+@click.group(name="create")
+def create():
+    """
+    Create resources in a Flyte deployment.
+    """
+
+
+@create.command(cls=common.CommandBase)
+@click.argument("name", type=str, required=True)
+@click.argument("value", type=str, required=False)
+@click.option("--from-file", type=click.Path(exists=True), help="Path to the file with the binary secret.")
+@click.option(
+    "--type", type=click.Choice(get_args(SecretTypes)), default="regular", help="Type of the secret.", show_default=True
+)
+@click.pass_obj
+def secret(
+    cfg: common.CLIConfig,
+    name: str,
+    value: str | bytes | None = None,
+    from_file: str | None = None,
+    type: SecretTypes = "regular",
+    project: str | None = None,
+    domain: str | None = None,
+):
+    """
+    Create a new secret. The name of the secret is required. For example:
+
+    ```bash
+    $ flyte create secret my_secret --value my_value
+    ```
+
+    If `--from-file` is specified, the value will be read from the file instead of being provided directly:
+
+    ```bash
+    $ flyte create secret my_secret --from-file /path/to/secret_file
+    ```
+
+    The `--type` option can be used to create specific types of secrets.
+    Either `regular` or `image_pull` can be specified.
+    Secrets intended to access container images should be specified as `image_pull`.
+    Other secrets should be specified as `regular`.
+    If no type is specified, `regular` is assumed.
+
+    ```bash
+    $ flyte create secret my_secret --type image_pull
+    ```
+    """
+    from flyte.remote import Secret
+
+    cfg.init(project, domain)
+    if from_file:
+        with open(from_file, "rb") as f:
+            value = f.read()
+    Secret.create(name=name, value=value, type=type)
+
+
+@create.command(cls=common.CommandBase)
+@click.option("--endpoint", type=str, help="Endpoint of the Flyte backend.")
+@click.option("--insecure", is_flag=True, help="Use an insecure connection to the Flyte backend.")
+@click.option(
+    "--org",
+    type=str,
+    required=False,
+    help="Organization to use. This will override the organization in the configuration file.",
+)
+@click.option(
+    "-o",
+    "--output",
+    type=click.Path(exists=False, writable=True),
+    default=Path.cwd() / "config.yaml",
+    help="Path to the output directory where the configuration will be saved. Defaults to current directory.",
+    show_default=True,
+)
+@click.option(
+    "--force",
+    is_flag=True,
+    default=False,
+    help="Force overwrite of the configuration file if it already exists.",
+    show_default=True,
+)
+def config(
+    output: str,
+    endpoint: str | None = None,
+    insecure: bool = False,
+    org: str | None = None,
+    project: str | None = None,
+    domain: str | None = None,
+    force: bool = False,
+):
+    """
+    Creates a configuration file for Flyte CLI.
+    If the `--output` option is not specified, it will create a file named `config.yaml` in the current directory.
+    If the file already exists, it will raise an error unless the `--force` option is used.
+    """
+    import yaml
+
+    from flyte._utils import org_from_endpoint, sanitize_endpoint
+
+    output_path = Path(output)
+
+    if output_path.exists() and not force:
+        force = click.confirm(f"Overwrite [{output_path}]?", default=False)
+        if not force:
+            click.echo(f"Will not overwrite the existing config file at {output_path}")
+            return
+
+    admin: Dict[str, Any] = {}
+    if endpoint:
+        endpoint = sanitize_endpoint(endpoint)
+        admin["endpoint"] = endpoint
+    if insecure:
+        admin["insecure"] = insecure
+
+    if not org and endpoint:
+        org = org_from_endpoint(endpoint)
+
+    task: Dict[str, str] = {}
+    if org:
+        task["org"] = org
+    if project:
+        task["project"] = project
+    if domain:
+        task["domain"] = domain
+
+    if not admin and not task:
+        raise click.BadParameter("At least one of --endpoint or --org must be provided.")
+
+    with open(output_path, "w") as f:
+        d: Dict[str, Any] = {}
+        if admin:
+            d["admin"] = admin
+        if task:
+            d["task"] = task
+        yaml.dump(d, f)
+
+    click.echo(f"Config file written to {output_path}")