flyte 0.2.0b5__py3-none-any.whl → 0.2.0b8__py3-none-any.whl

flyte/config/_reader.py

@@ -0,0 +1,207 @@
+import os
+import pathlib
+import typing
+from dataclasses import dataclass
+from functools import lru_cache
+from os import getenv
+from pathlib import Path
+
+import yaml
+
+from flyte._logging import logger
+
+# This is the default config file name for flyte
+FLYTECTL_CONFIG_ENV_VAR = "FLYTECTL_CONFIG"
+UCTL_CONFIG_ENV_VAR = "UCTL_CONFIG"
+
+
+@dataclass
+class YamlConfigEntry(object):
+    """
+    Creates a record for the config entry.
+    Args:
+        switch: dot-delimited string that should match flytectl args. Leaving it as dot-delimited instead of a list
+          of strings because it's easier to maintain alignment with flytectl.
+        config_value_type: Expected type of the value
+    """
+
+    switch: str
+    config_value_type: typing.Type = str
+
+    def get_env_name(self) -> str:
+        var_name = self.switch.upper().replace(".", "_")
+        return f"FLYTE_{var_name}"
+
+    def read_from_env(self, transform: typing.Optional[typing.Callable] = None) -> typing.Optional[typing.Any]:
+        """
+        Reads the config entry from environment variable, the structure of the env var is current
+        ``FLYTE_{SECTION}_{OPTION}`` all upper cased. We will change this in the future.
+        :return:
+        """
+        env = self.get_env_name()
+        v = os.environ.get(env, None)
+        if v is None:
+            return None
+        return transform(v) if transform else v
+
+    def read_from_file(
+        self, cfg: "ConfigFile", transform: typing.Optional[typing.Callable] = None
+    ) -> typing.Optional[typing.Any]:
+        if not cfg:
+            return None
+        try:
+            v = cfg.get(self)
+            if isinstance(v, bool) or bool(v is not None and v):
+                return transform(v) if transform else v
+        except Exception:
+            ...
+
+        return None
+
+
+@dataclass
+class ConfigEntry(object):
+    """
+    A top level Config entry holder, that holds multiple different representations of the config.
+    Legacy means the INI style config files. YAML support is for the flytectl config file, which is there by default
+    when flytectl starts a sandbox
+    """
+
+    yaml_entry: YamlConfigEntry
+    transform: typing.Optional[typing.Callable[[str], typing.Any]] = None
+
+    def read(self, cfg: typing.Optional["ConfigFile"] = None) -> typing.Optional[typing.Any]:
+        """
+        Reads the config Entry from the various sources in the following order,
+        #. First try to read from the relevant environment variable,
+        #. If missing, then try to read from the legacy config file, if one was parsed.
+        #. If missing, then try to read from the yaml file.
+
+        The constructor for ConfigFile currently does not allow specification of both the ini and yaml style formats.
+
+        :param cfg:
+        :return:
+        """
+        from_env = self.yaml_entry.read_from_env(self.transform)
+        if from_env is not None:
+            return from_env
+        if cfg and cfg.yaml_config and self.yaml_entry:
+            return self.yaml_entry.read_from_file(cfg, self.transform)
+
+        return None
+
+
+class ConfigFile(object):
+    def __init__(self, location: str):
+        """
+        Load the config from this location
+        """
+        self._location = location
+        self._yaml_config = self._read_yaml_config(location)
+
+    @property
+    def path(self) -> pathlib.Path:
+        """
+        Returns the path to the config file.
+        :return: Path to the config file
+        """
+        return pathlib.Path(self._location)
+
+    @staticmethod
+    def _read_yaml_config(location: str) -> typing.Optional[typing.Dict[str, typing.Any]]:
+        with open(location, "r") as fh:
+            try:
+                yaml_contents = yaml.safe_load(fh)
+                return yaml_contents
+            except yaml.YAMLError as exc:
+                logger.warning(f"Error {exc} reading yaml config file at {location}, ignoring...")
+                return None
+
+    def _get_from_yaml(self, c: YamlConfigEntry) -> typing.Any:
+        keys = c.switch.split(".")  # flytectl switches are dot delimited
+        d = typing.cast(typing.Dict[str, typing.Any], self.yaml_config)
+        try:
+            for k in keys:
+                d = d[k]
+            return d
+        except KeyError:
+            return None
+
+    def get(self, c: YamlConfigEntry) -> typing.Any:
+        return self._get_from_yaml(c)
+
+    @property
+    def yaml_config(self) -> typing.Dict[str, typing.Any] | None:
+        return self._yaml_config
+
+
+def resolve_config_path() -> pathlib.Path | None:
+    """
+    Config is read from the following locations in order of precedence:
+    1. ./config.yaml if it exists
+    2. `UCTL_CONFIG` environment variable
+    3. `FLYTECTL_CONFIG` environment variable
+    4. ~/.union/config.yaml if it exists
+    5. ~/.flyte/config.yaml if it exists
+    """
+    current_location_config = Path("config.yaml")
+    if current_location_config.exists():
+        return current_location_config
+    logger.debug("No ./config.yaml found, returning None")
+
+    uctl_path_from_env = getenv(UCTL_CONFIG_ENV_VAR, None)
+    if uctl_path_from_env:
+        return pathlib.Path(uctl_path_from_env)
+    logger.debug("No UCTL_CONFIG environment variable found, checking FLYTECTL_CONFIG")
+
+    flytectl_path_from_env = getenv(FLYTECTL_CONFIG_ENV_VAR, None)
+    if flytectl_path_from_env:
+        return pathlib.Path(flytectl_path_from_env)
+    logger.debug("No FLYTECTL_CONFIG environment variable found, checking default locations")
+
+    home_dir_union_config = Path(Path.home(), ".union", "config.yaml")
+    if home_dir_union_config.exists():
+        return home_dir_union_config
+    logger.debug("No ~/.union/config.yaml found, checking current directory")
+
+    home_dir_flytectl_config = Path(Path.home(), ".flyte", "config.yaml")
+    if home_dir_flytectl_config.exists():
+        return home_dir_flytectl_config
+    logger.debug("No ~/.flyte/config.yaml found, checking current directory")
+
+    return None
+
+
+@lru_cache
+def get_config_file(c: typing.Union[str, ConfigFile, None]) -> ConfigFile | None:
+    """
+    Checks if the given argument is a file or a configFile and returns a loaded configFile else returns None
+    """
+    if "PYTEST_VERSION" in os.environ:
+        # Use default local config in the pytest environment
+        return None
+    if isinstance(c, str):
+        logger.debug(f"Using specified config file at {c}")
+        return ConfigFile(c)
+    elif isinstance(c, ConfigFile):
+        return c
+    config_path = resolve_config_path()
+    if config_path:
+        return ConfigFile(str(config_path))
+    return None
+
+
+def read_file_if_exists(filename: typing.Optional[str], encoding=None) -> typing.Optional[str]:
+    """
+    Reads the contents of the file if passed a path. Otherwise, returns None.
+
+    :param filename: The file path to load
+    :param encoding: The encoding to use when reading the file.
+    :return: The contents of the file as a string or None.
+    """
+    if not filename:
+        return None
+
+    file = pathlib.Path(filename)
+    logger.debug(f"Reading file contents from [{file}] with current directory [{os.getcwd()}].")
+    return file.read_text(encoding=encoding)

flyte/extras/_container.py

@@ -215,7 +215,7 @@ class ContainerTask(TaskTemplate):
                 output_dict[k] = self._convert_output_val_to_correct_type(output_val, output_type)
         return output_dict
 
-    def execute(self, **kwargs) -> Any:
+    async def execute(self, **kwargs) -> Any:
         try:
             import docker
         except ImportError:

flyte/remote/_logs.py

@@ -49,6 +49,7 @@ class AsyncLogViewer:
         name: str = "Logs",
         show_ts: bool = False,
         filter_system: bool = False,
+        panel: bool = False,
     ):
         self.console = Console()
         self.log_source = log_source
@@ -58,12 +59,15 @@ class AsyncLogViewer:
         self.show_ts = show_ts
         self.total_lines = 0
         self.filter_flyte = filter_system
+        self.panel = panel
 
-    def _render(self) -> Panel:
+    def _render(self) -> Panel | Text:
         log_text = Text()
         for line in self.lines:
             log_text.append(line)
-        return Panel(log_text, title=self.name, border_style="yellow")
+        if self.panel:
+            return Panel(log_text, title=self.name, border_style="yellow")
+        return log_text
 
     async def run(self):
         with Live(self._render(), refresh_per_second=20, console=self.console) as live:
@@ -139,6 +143,7 @@ class Logs:
         show_ts: bool = False,
         raw: bool = False,
         filter_system: bool = False,
+        panel: bool = False,
     ):
         """
         Create a log viewer for a given action ID and attempt.
@@ -149,6 +154,7 @@ class Logs:
         :param show_ts: Whether to show timestamps in the logs.
         :param raw: if True, return the raw log lines instead of a viewer.
         :param filter_system: Whether to filter log lines based on system logs.
+        :param panel: Whether to use a panel for the log viewer. only applicable if raw is False.
         """
         if attempt < 1:
             raise ValueError("Attempt number must be greater than 0.")
@@ -165,5 +171,6 @@ class Logs:
             show_ts=show_ts,
             name=f"{action_id.run.name}:{action_id.name} ({attempt})",
             filter_system=filter_system,
+            panel=panel,
         )
         await viewer.run()

flyte/remote/_run.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import asyncio
 from dataclasses import dataclass, field
 from datetime import datetime, timedelta, timezone
-from typing import AsyncGenerator, AsyncIterator, Iterator, Literal, Tuple, Union, cast
+from typing import AsyncGenerator, AsyncIterator, Iterator, List, Literal, Tuple, Union, cast
 
 import grpc
 import rich.repr
@@ -46,33 +46,42 @@ def _action_time_phase(action: run_definition_pb2.Action | run_definition_pb2.Ac
         yield "error", action.error_info if action.HasField("error_info") else "NA"
 
 
-def _action_rich_repr(action: run_definition_pb2.Action, root: bool = False) -> rich.repr.Result:
+def _action_rich_repr(action: run_definition_pb2.Action) -> rich.repr.Result:
     """
     Rich representation of the action.
     """
-    yield "run-name", action.id.run.name
+    yield "run", action.id.run.name
+    if action.metadata.HasField("task"):
+        yield "task", action.metadata.task.id.name
+        yield "type", "task"
     yield "name", action.id.name
     yield from _action_time_phase(action)
-    yield "task", action.metadata.task.id.name
-    if not root:
-        yield "group", action.metadata.group
-        yield "parent", action.metadata.parent
+    yield "group", action.metadata.group
+    yield "parent", action.metadata.parent
+    yield "attempts", action.status.attempts
 
 
-def _action_details_rich_repr(action: run_definition_pb2.ActionDetails, root: bool = False) -> rich.repr.Result:
+def _attempt_rich_repr(action: List[run_definition_pb2.ActionAttempt]) -> rich.repr.Result:
+    for attempt in action:
+        yield "attempt", attempt.attempt
+        yield "phase", run_definition_pb2.Phase.Name(attempt.phase)
+        yield "logs_available", attempt.logs_available
+
+
+def _action_details_rich_repr(action: run_definition_pb2.ActionDetails) -> rich.repr.Result:
     """
     Rich representation of the action details.
     """
     yield "name", action.id.run.name
     yield from _action_time_phase(action)
-    # yield "task", action.metadata.task.id.name
     yield "task", action.resolved_task_spec.task_template.id.name
     yield "task_type", action.resolved_task_spec.task_template.type
     yield "task_version", action.resolved_task_spec.task_template.id.version
-    if not root:
-        yield "group", action.metadata.group
-        yield "parent", action.metadata.parent
-    # TODO attempt info
+    yield "attempts", action.attempts
+    yield "error_info", action.error_info if action.HasField("error_info") else "NA"
+    yield "phase", run_definition_pb2.Phase.Name(action.status.phase)
+    yield "group", action.metadata.group
+    yield "parent", action.metadata.parent
 
 
 def _action_done_check(phase: run_definition_pb2.Phase) -> bool:
@@ -270,7 +279,7 @@ class Run:
         """
         Rich representation of the Run object.
         """
-        yield from _action_rich_repr(self.pb2.action, root=True)
+        yield from _action_rich_repr(self.pb2.action)
 
     def __repr__(self) -> str:
         """
@@ -375,7 +384,7 @@ class RunDetails:
         """
         Rich representation of the Run object.
         """
-        yield from _action_details_rich_repr(self.pb2.action, root=True)
+        yield from _action_details_rich_repr(self.pb2.action)
 
     def __repr__(self) -> str:
         """
@@ -657,7 +666,7 @@ class Action:
         """
         Rich representation of the Action object.
         """
-        yield from _action_rich_repr(self.pb2, root=True)
+        yield from _action_rich_repr(self.pb2)
 
     def __repr__(self) -> str:
         """
@@ -911,7 +920,7 @@ class ActionDetails:
         """
         Rich representation of the Action object.
         """
-        yield from _action_details_rich_repr(self.pb2, root=True)
+        yield from _action_details_rich_repr(self.pb2)
 
     def __repr__(self) -> str:
         """

flyte/syncify/__init__.py

@@ -1,3 +1,54 @@
+"""
+# Syncify Module
+This module provides the `syncify` decorator and the `Syncify` class.
+The decorator can be used to convert asynchronous functions or methods into synchronous ones.
+This is useful for integrating async code into synchronous contexts.
+
+Every asynchronous function or method wrapped with `syncify` can be called synchronously using the
+parenthesis `()` operator, or asynchronously using the `.aio()` method.
+
+Example::
+
+```python
+from flyte.syncify import syncify
+
+@syncify
+async def async_function(x: str) -> str:
+    return f"Hello, Async World {x}!"
+
+
+# now you can call it synchronously
+result = async_function("Async World")   # Note: no .aio() needed for sync calls
+print(result)
+# Output: Hello, Async World Async World!
+
+# or call it asynchronously
+async def main():
+    result = await async_function.aio("World")    # Note the use of .aio() for async calls
+    print(result)
+```
+
+## Creating a Syncify Instance
+```python
+from flyte.syncify. import Syncify
+
+syncer = Syncify("my_syncer")
+
+# Now you can use `syncer` to decorate your async functions or methods
+
+```
+
+## How does it work?
+The Syncify class wraps asynchronous functions, classmethods, instance methods, and static methods to
+ provide a synchronous interface. The wrapped methods are always executed in the context of a background loop,
+ whether they are called synchronously or asynchronously. This allows for seamless integration of async code, as
+ certain async libraries capture the event loop. An example is grpc.aio, which captures the event loop.
+ In such a case, the Syncify class ensures that the async function is executed in the context of the background loop.
+
+To use it correctly with grpc.aio, you should wrap every grpc.aio channel creation, and client invocation
+with the same `Syncify` instance. This ensures that the async code runs in the correct event loop context.
+"""
+
 from flyte.syncify._api import Syncify
 
 syncify = Syncify()

flyte/syncify/_api.py

@@ -54,14 +54,13 @@ class _BackgroundLoop:
     functions or methods synchronously.
     """
 
-    def __init__(self):
-        self.loop = None
-        self.thread = threading.Thread(name="syncify_bg", target=self._run, daemon=True)
+    def __init__(self, name: str):
+        self.loop = asyncio.new_event_loop()
+        self.thread = threading.Thread(name=name, target=self._run, daemon=True)
         self.thread.start()
         atexit.register(self.stop)
 
     def _run(self):
-        self.loop = asyncio.new_event_loop()
         asyncio.set_event_loop(self.loop)
         self.loop.run_forever()
 
@@ -242,8 +241,8 @@ class Syncify:
 
     """
 
-    def __init__(self):
-        self._bg_loop = _BackgroundLoop()
+    def __init__(self, name: str = "flyte_syncify"):
+        self._bg_loop = _BackgroundLoop(name=name)
 
     @overload
     def __call__(self, func: Callable[P, Awaitable[R_co]]) -> Any: ...

flyte-0.2.0b8.dist-info/METADATA

@@ -0,0 +1,180 @@
+Metadata-Version: 2.4
+Name: flyte
+Version: 0.2.0b8
+Summary: Add your description here
+Author-email: Ketan Umare <kumare3@users.noreply.github.com>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: aiofiles>=24.1.0
+Requires-Dist: click>=8.2.1
+Requires-Dist: flyteidl==1.15.4b0
+Requires-Dist: cloudpickle>=3.1.1
+Requires-Dist: fsspec>=2025.3.0
+Requires-Dist: grpcio>=1.71.0
+Requires-Dist: obstore>=0.6.0
+Requires-Dist: protobuf>=6.30.1
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: rich-click>=1.8.9
+Requires-Dist: httpx>=0.28.1
+Requires-Dist: keyring>=25.6.0
+Requires-Dist: msgpack>=1.1.0
+Requires-Dist: toml>=0.10.2
+Requires-Dist: async-lru>=2.0.5
+Requires-Dist: mashumaro
+Requires-Dist: dataclasses_json
+
+# Flyte 2 SDK
+
+The next-generation SDK for Flyte.
+
+## Quickstart
+1. Clone this repo and set `unionv2` to working directory.
+2. Run `uv venv`, and `source .venv/bin/activate` to create a new virtualenv
+3. Install the latest version of the SDK by running the following:
+
+```
+uv pip install --no-cache --prerelease=allow --upgrade flyte
+```
+4. Create the config and point it to the Dogfood GCP cluster by running the following:
+
+```
+flyte create config --endpoint dns:///dogfood-gcp.cloud-staging.union.ai --org dogfood-gcp --project andrew --domain development 
+```
+This will create a `config.yaml` file in the current directory which will be referenced ahead of any other `config.yaml`s found in your system.
+
+5. Now you can run stuff with the CLI:
+
+```
+flyte run --follow examples/basics/devbox_one.py say_hello_nested
+```
+Note that the `--follow` command is optional. Use this to stream updates to the terminal!
+
+
+
+## Hello World Example
+
+1. Only async tasks are supported right now. Style recommended, `import flyte` and then `flyte.TaskEnvironment` etc
+2. You have to create environment for even a single task
+3. look at examples/... for various examples. 
+4. For a single script recommend using uv run scripts with metadata headers.
+
+
+```python
+import flyte
+
+env = flyte.TaskEnvironment(name="hello_world")
+
+
+@env.task
+async def say_hello(data: str) -> str:
+    return f"Hello {data}"
+
+
+@env.task
+async def say_hello_nested(data: str) -> str:
+    return await say_hello.override(resources=flyte.Resources(gpu="A100 80G:4")).execute(data)
+
+
+if __name__ == "__main__":
+    import asyncio
+    
+    # to run pure python - the SDK is not invoked at all
+    asyncio.run(say_hello_nested("test"))
+    
+    # To run locally, but run through type system etc
+    flyte.init()
+    flyte.run(say_hello_nested, "World")
+    
+    # To run remote
+    flyte.init(endpoint="dns:///localhost:8090", insecure=True)
+    flyte.run(say_hello_nested, "World")
+    # It is possible to switch local and remote, but keeping init to have and endpoint, but , changing context during run
+    flyte.with_runcontext(mode="local").run(...)  # this will run locally only
+
+    # To run remote with a config
+    flyte.init_auto_from_config("config.yaml")
+```
+
+# CLI
+All commands can be run from any root directory.
+For examples, it is not needed to have `__init__.py` in the directory. If you run from a directory, the 
+code will automatically package and upload all modules that are imported. You can change the behaviour by using --copy-style flag.
+
+```bash
+flyte run examples/basics/devbox_one.py say_hello --data "World"
+```
+
+To Follow the logs for the a0 action, you can use the `--follow` flag:
+
+```bash
+flyte run --follow examples/basics/devbox_one.py say_hello --data "World"
+```
+Note follow has to be used with `run` command
+
+Change copy style
+```bash
+flyte run --copy-style examples/basics/devbox_one.py say_hello_nested --data "World"
+```
+
+# Building Images
+```python
+
+import flyte
+
+env = flyte.TaskEnvironment(
+    name="hello_world",
+    image=flyte.Image.auto().with_apt_packages(...).with_pip_packages(...),
+)
+
+```
+
+### Deploy
+```bash
+flyte deploy examples/basics/devbox_one.py say_hello_nested
+```
+
+CLI shortcuts
+
+Get runs
+```bash
+flyte get run
+```
+Get specific run
+```bash
+flyte get run "run-name"
+```
+
+Get run actions
+```bash
+flyte get actions "run-name"
+```
+
+Get specific action
+```bash
+flyte get action "run-name" "action-name"
+```
+
+Get action logs
+```bash
+flyte get logs "run-name" ["action-name"]
+```
+defaults to root action if no action name is provided
+
+
+you can run any python script directly within the script module using __main__:
+
+```python
+if __name__ == "__main__":
+    import flyte
+    flyte.init()
+    flyte.run(say_hello_nested, "World")
+```
+
+You can also run from cli
+
+you can Also run a uv script with metadata headers:
+
+```bash
+ uv run scripts / hello_world.py
+```