flyte 0.2.0b4__py3-none-any.whl → 0.2.0b7__py3-none-any.whl

flyte/__init__.py

@@ -38,6 +38,7 @@ __all__ = [
     "deploy",
     "group",
     "init",
+    "init_auto_from_config",
     "run",
     "trace",
     "with_runcontext",
@@ -49,7 +50,7 @@ from ._deploy import deploy
 from ._environment import Environment
 from ._group import group
 from ._image import Image
-from ._initialize import init
+from ._initialize import init, init_auto_from_config
 from ._resources import GPU, TPU, Device, Resources
 from ._retry import RetryStrategy
 from ._reusable_environment import ReusePolicy

flyte/_build.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
-from ._api_commons import syncer
+from flyte.syncify import syncify
+
 from ._image import Image
 
 
-@syncer.wrap
+@syncify
 async def build(image: Image) -> str:
     """
     Build an image. The existing async context will be used.

flyte/_code_bundle/_utils.py

@@ -14,7 +14,6 @@ import tempfile
 import typing
 from datetime import datetime, timezone
 from functools import lru_cache
-from pathlib import Path
 from types import ModuleType
 from typing import List, Literal, Optional, Tuple, Union
 
@@ -322,18 +321,3 @@ def hash_file(file_path: typing.Union[os.PathLike, str]) -> Tuple[bytes, str, in
             size += len(chunk)
 
     return h.digest(), h.hexdigest(), size
-
-
-def _find_project_root(source_path) -> str:
-    """
-    Find the root of the project.
-    The root of the project is considered to be the first ancestor from source_path that does
-    not contain a __init__.py file.
-
-    N.B.: This assumption only holds for regular packages (as opposed to namespace packages)
-    """
-    # Start from the directory right above source_path
-    path = Path(source_path).parent.resolve()
-    while os.path.exists(os.path.join(path, "__init__.py")):
-        path = path.parent
-    return str(path)

flyte/_code_bundle/bundle.py

@@ -146,7 +146,7 @@ async def build_code_bundle(
         logger.info(f"Code bundle created at {bundle_path}, size: {tar_size} MB, archive size: {archive_size} MB")
         if not dryrun:
             hash_digest, remote_path = await upload_file(bundle_path)
-            logger.info(f"Code bundle uploaded to {remote_path}")
+            logger.debug(f"Code bundle uploaded to {remote_path}")
         else:
             remote_path = "na"
             if copy_bundle_to:

flyte/_deploy.py

@@ -7,11 +7,11 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
 import rich.repr
 
 from flyte.models import SerializationContext
+from flyte.syncify import syncify
 
-from ._api_commons import syncer
 from ._environment import Environment
 from ._image import Image
-from ._initialize import get_client, get_common_config, requires_client, requires_initialization
+from ._initialize import ensure_client, get_client, get_common_config, requires_initialization
 from ._logging import logger
 from ._task import TaskTemplate
 from ._task_environment import TaskEnvironment
@@ -47,13 +47,13 @@ class Deployment:
         return f"Deployment(envs=[{env_names}], tasks=[{task_names_versions}])"
 
 
-@requires_client
 async def _deploy_task(
     task: TaskTemplate, serialization_context: SerializationContext, dryrun: bool = False
 ) -> task_definition_pb2.TaskSpec:
     """
     Deploy the given task.
     """
+    ensure_client()
     from ._internal.runtime.task_serde import translate_task_to_wire
     from ._protos.workflow import task_definition_pb2, task_service_pb2
 
@@ -177,7 +177,7 @@ def plan_deploy(*envs: Environment, version: Optional[str] = None) -> Deployment
     return DeploymentPlan(planned_envs, version=version)
 
 
-@syncer.wrap
+@syncify
 async def deploy(
     *envs: Environment,
     dryrun: bool = False,

flyte/_initialize.py

@@ -8,9 +8,9 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Callable, List, Literal, Optional, TypeVar
 
 from flyte.errors import InitializationError
+from flyte.syncify import syncify
 
-from ._api_commons import syncer
-from ._logging import initialize_logger
+from ._logging import initialize_logger, logger
 from ._tools import ipython_check
 
 if TYPE_CHECKING:
@@ -108,7 +108,7 @@ async def _initialize_client(
     )
 
 
-@syncer.wrap
+@syncify
 async def init(
     org: str | None = None,
     project: str | None = None,
@@ -130,7 +130,6 @@ async def init(
     rpc_retries: int = 3,
     http_proxy_url: str | None = None,
     storage: Storage | None = None,
-    config: Config | None = None,
 ) -> None:
     """
     Initialize the Flyte system with the given configuration. This method should be called before any other Flyte
@@ -179,42 +178,72 @@ async def init(
     global _init_config  # noqa: PLW0603
 
     with _init_lock:
-        if config is None:
-            import flyte.config as _f_cfg
-
-            config = _f_cfg.Config()
-        platform_cfg = config.platform
-        task_cfg = config.task
         client = None
-        if endpoint or platform_cfg.endpoint or api_key:
+        if endpoint or api_key:
             client = await _initialize_client(
                 api_key=api_key,
-                auth_type=auth_type or platform_cfg.auth_mode,
-                endpoint=endpoint or platform_cfg.endpoint,
+                auth_type=auth_type,
+                endpoint=endpoint,
                 headless=headless,
-                insecure=insecure or platform_cfg.insecure,
-                insecure_skip_verify=insecure_skip_verify or platform_cfg.insecure_skip_verify,
-                ca_cert_file_path=ca_cert_file_path or platform_cfg.ca_cert_file_path,
-                command=command or platform_cfg.command,
-                proxy_command=proxy_command or platform_cfg.proxy_command,
-                client_id=client_id or platform_cfg.client_id,
-                client_credentials_secret=client_credentials_secret or platform_cfg.client_credentials_secret,
+                insecure=insecure,
+                insecure_skip_verify=insecure_skip_verify,
+                ca_cert_file_path=ca_cert_file_path,
+                command=command,
+                proxy_command=proxy_command,
+                client_id=client_id,
+                client_credentials_secret=client_credentials_secret,
                 client_config=auth_client_config,
-                rpc_retries=rpc_retries or platform_cfg.rpc_retries,
-                http_proxy_url=http_proxy_url or platform_cfg.http_proxy_url,
+                rpc_retries=rpc_retries,
+                http_proxy_url=http_proxy_url,
             )
 
         root_dir = root_dir or get_cwd_editable_install() or Path.cwd()
         _init_config = _InitConfig(
             root_dir=root_dir,
-            project=project or task_cfg.project,
-            domain=domain or task_cfg.domain,
+            project=project,
+            domain=domain,
             client=client,
             storage=storage,
-            org=org or task_cfg.org,
+            org=org,
         )
 
 
+@syncify
+async def init_auto_from_config(path_or_config: str | Config | None = None) -> None:
+    """
+    Initialize the Flyte system using a configuration file or Config object. This method should be called before any
+    other Flyte remote API methods are called. Thread-safe implementation.
+
+    :param path_or_config: Path to the configuration file or Config object
+    :return: None
+    """
+    import flyte.config as config
+
+    cfg: config.Config
+    if path_or_config is None or isinstance(path_or_config, str):
+        # If a string is passed, treat it as a path to the config file
+        cfg = config.auto(path_or_config)
+    else:
+        # If a Config object is passed, use it directly
+        cfg = path_or_config
+
+    logger.debug(f"Flyte config initialized as {cfg}")
+    await init.aio(
+        org=cfg.task.org,
+        project=cfg.task.project,
+        domain=cfg.task.domain,
+        endpoint=cfg.platform.endpoint,
+        insecure=cfg.platform.insecure,
+        insecure_skip_verify=cfg.platform.insecure_skip_verify,
+        ca_cert_file_path=cfg.platform.ca_cert_file_path,
+        auth_type=cfg.platform.auth_mode,
+        command=cfg.platform.command,
+        proxy_command=cfg.platform.proxy_command,
+        client_id=cfg.platform.client_id,
+        client_credentials_secret=cfg.platform.client_credentials_secret,
+    )
+
+
 def _get_init_config() -> Optional[_InitConfig]:
     """
     Get the current initialization configuration. Thread-safe implementation.
@@ -298,6 +327,20 @@ def initialize_in_cluster() -> None:
 T = TypeVar("T", bound=Callable)
 
 
+def ensure_client():
+    """
+    Ensure that the client is initialized. If not, raise an InitializationError.
+    This function is used to check if the client is initialized before executing any Flyte remote API methods.
+    """
+    if _get_init_config() is None or _get_init_config().client is None:
+        raise InitializationError(
+            "ClientNotInitializedError",
+            "user",
+            "Client has not been initialized. Call flyte.init() with a valid endpoint"
+            " or api-key before using this function.",
+        )
+
+
 def requires_client(func: T) -> T:
     """
     Decorator that checks if the client has been initialized before executing the function.
@@ -308,7 +351,7 @@ def requires_client(func: T) -> T:
     """
 
     @functools.wraps(func)
-    def wrapper(*args, **kwargs) -> T:
+    async def wrapper(*args, **kwargs) -> T:
         init_config = _get_init_config()
         if init_config is None or init_config.client is None:
             raise InitializationError(

flyte/_internal/controllers/remote/_core.py

@@ -32,7 +32,7 @@ class Controller:
         max_system_retries: int = 5,
         resource_log_interval_sec: float = 10.0,
         min_backoff_on_err_sec: float = 0.1,
-        thread_wait_timeout_sec: float = 0.5,
+        thread_wait_timeout_sec: float = 5.0,
         enqueue_timeout_sec: float = 5.0,
     ):
         """

flyte/_protos/workflow/common_pb2.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: workflow/common.proto
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from flyteidl.core import interface_pb2 as flyteidl_dot_core_dot_interface__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15workflow/common.proto\x12\x11\x63loudidl.workflow\x1a\x1d\x66lyteidl/core/interface.proto\"\\\n\x0eNamedParameter\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x36\n\tparameter\x18\x02 \x01(\x0b\x32\x18.flyteidl.core.ParameterR\tparameterB\xb8\x01\n\x15\x63om.cloudidl.workflowB\x0b\x43ommonProtoH\x02P\x01Z+github.com/unionai/cloud/gen/pb-go/workflow\xa2\x02\x03\x43WX\xaa\x02\x11\x43loudidl.Workflow\xca\x02\x11\x43loudidl\\Workflow\xe2\x02\x1d\x43loudidl\\Workflow\\GPBMetadata\xea\x02\x12\x43loudidl::Workflowb\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'workflow.common_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  DESCRIPTOR._options = None
+  DESCRIPTOR._serialized_options = b'\n\025com.cloudidl.workflowB\013CommonProtoH\002P\001Z+github.com/unionai/cloud/gen/pb-go/workflow\242\002\003CWX\252\002\021Cloudidl.Workflow\312\002\021Cloudidl\\Workflow\342\002\035Cloudidl\\Workflow\\GPBMetadata\352\002\022Cloudidl::Workflow'
+  _globals['_NAMEDPARAMETER']._serialized_start=75
+  _globals['_NAMEDPARAMETER']._serialized_end=167
+# @@protoc_insertion_point(module_scope)

flyte/_protos/workflow/common_pb2.pyi

@@ -0,0 +1,14 @@
+from flyteidl.core import interface_pb2 as _interface_pb2
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class NamedParameter(_message.Message):
+    __slots__ = ["name", "parameter"]
+    NAME_FIELD_NUMBER: _ClassVar[int]
+    PARAMETER_FIELD_NUMBER: _ClassVar[int]
+    name: str
+    parameter: _interface_pb2.Parameter
+    def __init__(self, name: _Optional[str] = ..., parameter: _Optional[_Union[_interface_pb2.Parameter, _Mapping]] = ...) -> None: ...

flyte/_protos/workflow/common_pb2_grpc.py

@@ -0,0 +1,4 @@
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+"""Client and server classes corresponding to protobuf-defined services."""
+import grpc
+

flyte/_protos/workflow/run_definition_pb2.py

@@ -21,7 +21,7 @@ from flyte._protos.validate.validate import validate_pb2 as validate_dot_validat
 from flyte._protos.workflow import task_definition_pb2 as workflow_dot_task__definition__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dworkflow/run_definition.proto\x12\x11\x63loudidl.workflow\x1a\x17\x63ommon/identifier.proto\x1a\x15\x63ommon/identity.proto\x1a\x1d\x66lyteidl/core/execution.proto\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17validate/validate.proto\x1a\x1eworkflow/task_definition.proto\"\x8d\x01\n\rRunIdentifier\x12\x19\n\x03org\x18\x01 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x03org\x12!\n\x07project\x18\x02 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x07project\x12\x1f\n\x06\x64omain\x18\x03 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x06\x64omain\x12\x1d\n\x04name\x18\x04 \x01(\tB\t\xfa\x42\x06r\x04\x10\x01\x18\x1eR\x04name\"\x82\x01\n\x06Labels\x12=\n\x06values\x18\x01 \x03(\x0b\x32%.cloudidl.workflow.Labels.ValuesEntryR\x06values\x1a\x39\n\x0bValuesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\x8c\x01\n\x0b\x41nnotations\x12\x42\n\x06values\x18\x01 \x03(\x0b\x32*.cloudidl.workflow.Annotations.ValuesEntryR\x06values\x1a\x39\n\x0bValuesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\";\n\x04\x45nvs\x12\x33\n\x06values\x18\x01 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairR\x06values\"\x96\x02\n\x07RunSpec\x12\x31\n\x06labels\x18\x01 \x01(\x0b\x32\x19.cloudidl.workflow.LabelsR\x06labels\x12@\n\x0b\x61nnotations\x18\x02 \x01(\x0b\x32\x1e.cloudidl.workflow.AnnotationsR\x0b\x61nnotations\x12+\n\x04\x65nvs\x18\x03 \x01(\x0b\x32\x17.cloudidl.workflow.EnvsR\x04\x65nvs\x12@\n\rinterruptible\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValueR\rinterruptible\x12\'\n\x0foverwrite_cache\x18\x05 \x01(\x08R\x0eoverwriteCache\"8\n\x03Run\x12\x31\n\x06\x61\x63tion\x18\x01 \x01(\x0b\x32\x19.cloudidl.workflow.ActionR\x06\x61\x63tion\"}\n\nRunDetails\x12\x35\n\x08run_spec\x18\x01 \x01(\x0b\x32\x1a.cloudidl.workflow.RunSpecR\x07runSpec\x12\x38\n\x06\x61\x63tion\x18\x02 \x01(\x0b\x32 .cloudidl.workflow.ActionDetailsR\x06\x61\x63tion\"o\n\x10\x41\x63tionIdentifier\x12<\n\x03run\x18\x01 \x01(\x0b\x32 .cloudidl.workflow.RunIdentifierB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x03run\x12\x1d\n\x04name\x18\x02 \x01(\tB\t\xfa\x42\x06r\x04\x10\x01\x18\x1eR\x04name\"d\n\x12TaskActionMetadata\x12\x31\n\x02id\x18\x01 \x01(\x0b\x32!.cloudidl.workflow.TaskIdentifierR\x02id\x12\x1b\n\ttask_type\x18\x02 \x01(\tR\x08taskType\")\n\x13TraceActionMetadata\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\"\x9f\x01\n\x17\x43onditionActionMetadata\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x06run_id\x18\x02 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01H\x00R\x05runId\x12&\n\taction_id\x18\x03 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01H\x00R\x08\x61\x63tionId\x12\x18\n\x06global\x18\x04 \x01(\x08H\x00R\x06globalB\x0c\n\x05scope\x12\x03\xf8\x42\x01\"\xf1\x02\n\x0e\x41\x63tionMetadata\x12\x16\n\x06parent\x18\x03 \x01(\tR\x06parent\x12\x14\n\x05group\x18\x05 \x01(\tR\x05group\x12\x42\n\x0b\x65xecuted_by\x18\x06 \x01(\x0b\x32!.cloudidl.common.EnrichedIdentityR\nexecutedBy\x12\x45\n\x04task\x18\x07 \x01(\x0b\x32%.cloudidl.workflow.TaskActionMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01H\x00R\x04task\x12H\n\x05trace\x18\x08 \x01(\x0b\x32&.cloudidl.workflow.TraceActionMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01H\x00R\x05trace\x12T\n\tcondition\x18\t \x01(\x0b\x32*.cloudidl.workflow.ConditionActionMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01H\x00R\tconditionB\x06\n\x04spec\"\xe7\x01\n\x0c\x41\x63tionStatus\x12.\n\x05phase\x18\x01 \x01(\x0e\x32\x18.cloudidl.workflow.PhaseR\x05phase\x12\x39\n\nstart_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12:\n\x08\x65nd_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x07\x65ndTime\x88\x01\x01\x12#\n\x08\x61ttempts\x18\x04 \x01(\rB\x07\xfa\x42\x04*\x02 \x00R\x08\x61ttemptsB\x0b\n\t_end_time\"\xb5\x01\n\x06\x41\x63tion\x12\x33\n\x02id\x18\x01 \x01(\x0b\x32#.cloudidl.workflow.ActionIdentifierR\x02id\x12=\n\x08metadata\x18\x02 \x01(\x0b\x32!.cloudidl.workflow.ActionMetadataR\x08metadata\x12\x37\n\x06status\x18\x03 \x01(\x0b\x32\x1f.cloudidl.workflow.ActionStatusR\x06status\"\x9e\x02\n\x0e\x45nrichedAction\x12\x31\n\x06\x61\x63tion\x18\x01 \x01(\x0b\x32\x19.cloudidl.workflow.ActionR\x06\x61\x63tion\x12!\n\x0cmeets_filter\x18\x02 \x01(\x08R\x0bmeetsFilter\x12n\n\x15\x63hildren_phase_counts\x18\x03 \x03(\x0b\x32:.cloudidl.workflow.EnrichedAction.ChildrenPhaseCountsEntryR\x13\x63hildrenPhaseCounts\x1a\x46\n\x18\x43hildrenPhaseCountsEntry\x12\x10\n\x03key\x18\x01 \x01(\x05R\x03key\x12\x14\n\x05value\x18\x02 \x01(\x05R\x05value:\x02\x38\x01\"\x9a\x01\n\tErrorInfo\x12\x18\n\x07message\x18\x01 \x01(\tR\x07message\x12\x35\n\x04kind\x18\x02 \x01(\x0e\x32!.cloudidl.workflow.ErrorInfo.KindR\x04kind\"<\n\x04Kind\x12\x14\n\x10KIND_UNSPECIFIED\x10\x00\x12\r\n\tKIND_USER\x10\x01\x12\x0f\n\x0bKIND_SYSTEM\x10\x02\"e\n\tAbortInfo\x12\x16\n\x06reason\x18\x01 \x01(\tR\x06reason\x12@\n\naborted_by\x18\x02 \x01(\x0b\x32!.cloudidl.common.EnrichedIdentityR\tabortedBy\"\xcd\x03\n\rActionDetails\x12\x33\n\x02id\x18\x01 \x01(\x0b\x32#.cloudidl.workflow.ActionIdentifierR\x02id\x12=\n\x08metadata\x18\x02 \x01(\x0b\x32!.cloudidl.workflow.ActionMetadataR\x08metadata\x12\x37\n\x06status\x18\x03 \x01(\x0b\x32\x1f.cloudidl.workflow.ActionStatusR\x06status\x12=\n\nerror_info\x18\x04 \x01(\x0b\x32\x1c.cloudidl.workflow.ErrorInfoH\x00R\terrorInfo\x12=\n\nabort_info\x18\x05 \x01(\x0b\x32\x1c.cloudidl.workflow.AbortInfoH\x00R\tabortInfo\x12I\n\x12resolved_task_spec\x18\x06 \x01(\x0b\x32\x1b.cloudidl.workflow.TaskSpecR\x10resolvedTaskSpec\x12<\n\x08\x61ttempts\x18\x07 \x03(\x0b\x32 .cloudidl.workflow.ActionAttemptR\x08\x61ttemptsB\x08\n\x06result\"\xa9\x03\n\rActionAttempt\x12.\n\x05phase\x18\x01 \x01(\x0e\x32\x18.cloudidl.workflow.PhaseR\x05phase\x12\x39\n\nstart_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12:\n\x08\x65nd_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x07\x65ndTime\x88\x01\x01\x12@\n\nerror_info\x18\x04 \x01(\x0b\x32\x1c.cloudidl.workflow.ErrorInfoH\x01R\terrorInfo\x88\x01\x01\x12!\n\x07\x61ttempt\x18\x05 \x01(\rB\x07\xfa\x42\x04*\x02 \x00R\x07\x61ttempt\x12\x31\n\x08log_info\x18\x06 \x03(\x0b\x32\x16.flyteidl.core.TaskLogR\x07logInfo\x12=\n\x07outputs\x18\x07 \x01(\x0b\x32#.cloudidl.workflow.OutputReferencesR\x07outputsB\x0b\n\t_end_timeB\r\n\x0b_error_info\"\x95\x05\n\x0b\x41\x63tionEvent\x12=\n\x02id\x18\x01 \x01(\x0b\x32#.cloudidl.workflow.ActionIdentifierB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x02id\x12!\n\x07\x61ttempt\x18\x02 \x01(\rB\x07\xfa\x42\x04*\x02 \x00R\x07\x61ttempt\x12.\n\x05phase\x18\x03 \x01(\x0e\x32\x18.cloudidl.workflow.PhaseR\x05phase\x12\x18\n\x07version\x18\x04 \x01(\rR\x07version\x12\x39\n\nstart_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12=\n\x0cupdated_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0bupdatedTime\x12:\n\x08\x65nd_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x07\x65ndTime\x88\x01\x01\x12@\n\nerror_info\x18\x08 \x01(\x0b\x32\x1c.cloudidl.workflow.ErrorInfoH\x01R\terrorInfo\x88\x01\x01\x12\x31\n\x08log_info\x18\t \x03(\x0b\x32\x16.flyteidl.core.TaskLogR\x07logInfo\x12:\n\x0blog_context\x18\n \x01(\x0b\x32\x19.flyteidl.core.LogContextR\nlogContext\x12\x18\n\x07\x63luster\x18\x0b \x01(\tR\x07\x63luster\x12=\n\x07outputs\x18\x0c \x01(\x0b\x32#.cloudidl.workflow.OutputReferencesR\x07outputsB\x0b\n\t_end_timeB\r\n\x0b_error_info\"P\n\x0cNamedLiteral\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x16.flyteidl.core.LiteralR\x05value\"1\n\x10OutputReferences\x12\x1d\n\noutput_uri\x18\x01 \x01(\tR\toutputUri\"E\n\x06Inputs\x12;\n\x08literals\x18\x01 \x03(\x0b\x32\x1f.cloudidl.workflow.NamedLiteralR\x08literals\"F\n\x07Outputs\x12;\n\x08literals\x18\x01 \x03(\x0b\x32\x1f.cloudidl.workflow.NamedLiteralR\x08literals*\xcb\x01\n\x05Phase\x12\x15\n\x11PHASE_UNSPECIFIED\x10\x00\x12\x10\n\x0cPHASE_QUEUED\x10\x01\x12\x1f\n\x1bPHASE_WAITING_FOR_RESOURCES\x10\x02\x12\x16\n\x12PHASE_INITIALIZING\x10\x03\x12\x11\n\rPHASE_RUNNING\x10\x04\x12\x13\n\x0fPHASE_SUCCEEDED\x10\x05\x12\x10\n\x0cPHASE_FAILED\x10\x06\x12\x11\n\rPHASE_ABORTED\x10\x07\x12\x13\n\x0fPHASE_TIMED_OUT\x10\x08\x42\xbf\x01\n\x15\x63om.cloudidl.workflowB\x12RunDefinitionProtoH\x02P\x01Z+github.com/unionai/cloud/gen/pb-go/workflow\xa2\x02\x03\x43WX\xaa\x02\x11\x43loudidl.Workflow\xca\x02\x11\x43loudidl\\Workflow\xe2\x02\x1d\x43loudidl\\Workflow\\GPBMetadata\xea\x02\x12\x43loudidl::Workflowb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dworkflow/run_definition.proto\x12\x11\x63loudidl.workflow\x1a\x17\x63ommon/identifier.proto\x1a\x15\x63ommon/identity.proto\x1a\x1d\x66lyteidl/core/execution.proto\x1a\x1c\x66lyteidl/core/literals.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17validate/validate.proto\x1a\x1eworkflow/task_definition.proto\"\x8d\x01\n\rRunIdentifier\x12\x19\n\x03org\x18\x01 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x03org\x12!\n\x07project\x18\x02 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x07project\x12\x1f\n\x06\x64omain\x18\x03 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x06\x64omain\x12\x1d\n\x04name\x18\x04 \x01(\tB\t\xfa\x42\x06r\x04\x10\x01\x18\x1eR\x04name\"\x82\x01\n\x06Labels\x12=\n\x06values\x18\x01 \x03(\x0b\x32%.cloudidl.workflow.Labels.ValuesEntryR\x06values\x1a\x39\n\x0bValuesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\x8c\x01\n\x0b\x41nnotations\x12\x42\n\x06values\x18\x01 \x03(\x0b\x32*.cloudidl.workflow.Annotations.ValuesEntryR\x06values\x1a\x39\n\x0bValuesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\";\n\x04\x45nvs\x12\x33\n\x06values\x18\x01 \x03(\x0b\x32\x1b.flyteidl.core.KeyValuePairR\x06values\"\x96\x02\n\x07RunSpec\x12\x31\n\x06labels\x18\x01 \x01(\x0b\x32\x19.cloudidl.workflow.LabelsR\x06labels\x12@\n\x0b\x61nnotations\x18\x02 \x01(\x0b\x32\x1e.cloudidl.workflow.AnnotationsR\x0b\x61nnotations\x12+\n\x04\x65nvs\x18\x03 \x01(\x0b\x32\x17.cloudidl.workflow.EnvsR\x04\x65nvs\x12@\n\rinterruptible\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.BoolValueR\rinterruptible\x12\'\n\x0foverwrite_cache\x18\x05 \x01(\x08R\x0eoverwriteCache\"8\n\x03Run\x12\x31\n\x06\x61\x63tion\x18\x01 \x01(\x0b\x32\x19.cloudidl.workflow.ActionR\x06\x61\x63tion\"}\n\nRunDetails\x12\x35\n\x08run_spec\x18\x01 \x01(\x0b\x32\x1a.cloudidl.workflow.RunSpecR\x07runSpec\x12\x38\n\x06\x61\x63tion\x18\x02 \x01(\x0b\x32 .cloudidl.workflow.ActionDetailsR\x06\x61\x63tion\"o\n\x10\x41\x63tionIdentifier\x12<\n\x03run\x18\x01 \x01(\x0b\x32 .cloudidl.workflow.RunIdentifierB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x03run\x12\x1d\n\x04name\x18\x02 \x01(\tB\t\xfa\x42\x06r\x04\x10\x01\x18\x1eR\x04name\"d\n\x12TaskActionMetadata\x12\x31\n\x02id\x18\x01 \x01(\x0b\x32!.cloudidl.workflow.TaskIdentifierR\x02id\x12\x1b\n\ttask_type\x18\x02 \x01(\tR\x08taskType\")\n\x13TraceActionMetadata\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\"\x9f\x01\n\x17\x43onditionActionMetadata\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12 \n\x06run_id\x18\x02 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01H\x00R\x05runId\x12&\n\taction_id\x18\x03 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01H\x00R\x08\x61\x63tionId\x12\x18\n\x06global\x18\x04 \x01(\x08H\x00R\x06globalB\x0c\n\x05scope\x12\x03\xf8\x42\x01\"\xf1\x02\n\x0e\x41\x63tionMetadata\x12\x16\n\x06parent\x18\x03 \x01(\tR\x06parent\x12\x14\n\x05group\x18\x05 \x01(\tR\x05group\x12\x42\n\x0b\x65xecuted_by\x18\x06 \x01(\x0b\x32!.cloudidl.common.EnrichedIdentityR\nexecutedBy\x12\x45\n\x04task\x18\x07 \x01(\x0b\x32%.cloudidl.workflow.TaskActionMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01H\x00R\x04task\x12H\n\x05trace\x18\x08 \x01(\x0b\x32&.cloudidl.workflow.TraceActionMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01H\x00R\x05trace\x12T\n\tcondition\x18\t \x01(\x0b\x32*.cloudidl.workflow.ConditionActionMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01H\x00R\tconditionB\x06\n\x04spec\"\xe7\x01\n\x0c\x41\x63tionStatus\x12.\n\x05phase\x18\x01 \x01(\x0e\x32\x18.cloudidl.workflow.PhaseR\x05phase\x12\x39\n\nstart_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12:\n\x08\x65nd_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x07\x65ndTime\x88\x01\x01\x12#\n\x08\x61ttempts\x18\x04 \x01(\rB\x07\xfa\x42\x04*\x02 \x00R\x08\x61ttemptsB\x0b\n\t_end_time\"\xb5\x01\n\x06\x41\x63tion\x12\x33\n\x02id\x18\x01 \x01(\x0b\x32#.cloudidl.workflow.ActionIdentifierR\x02id\x12=\n\x08metadata\x18\x02 \x01(\x0b\x32!.cloudidl.workflow.ActionMetadataR\x08metadata\x12\x37\n\x06status\x18\x03 \x01(\x0b\x32\x1f.cloudidl.workflow.ActionStatusR\x06status\"\x9e\x02\n\x0e\x45nrichedAction\x12\x31\n\x06\x61\x63tion\x18\x01 \x01(\x0b\x32\x19.cloudidl.workflow.ActionR\x06\x61\x63tion\x12!\n\x0cmeets_filter\x18\x02 \x01(\x08R\x0bmeetsFilter\x12n\n\x15\x63hildren_phase_counts\x18\x03 \x03(\x0b\x32:.cloudidl.workflow.EnrichedAction.ChildrenPhaseCountsEntryR\x13\x63hildrenPhaseCounts\x1a\x46\n\x18\x43hildrenPhaseCountsEntry\x12\x10\n\x03key\x18\x01 \x01(\x05R\x03key\x12\x14\n\x05value\x18\x02 \x01(\x05R\x05value:\x02\x38\x01\"\x9a\x01\n\tErrorInfo\x12\x18\n\x07message\x18\x01 \x01(\tR\x07message\x12\x35\n\x04kind\x18\x02 \x01(\x0e\x32!.cloudidl.workflow.ErrorInfo.KindR\x04kind\"<\n\x04Kind\x12\x14\n\x10KIND_UNSPECIFIED\x10\x00\x12\r\n\tKIND_USER\x10\x01\x12\x0f\n\x0bKIND_SYSTEM\x10\x02\"e\n\tAbortInfo\x12\x16\n\x06reason\x18\x01 \x01(\tR\x06reason\x12@\n\naborted_by\x18\x02 \x01(\x0b\x32!.cloudidl.common.EnrichedIdentityR\tabortedBy\"\xcd\x03\n\rActionDetails\x12\x33\n\x02id\x18\x01 \x01(\x0b\x32#.cloudidl.workflow.ActionIdentifierR\x02id\x12=\n\x08metadata\x18\x02 \x01(\x0b\x32!.cloudidl.workflow.ActionMetadataR\x08metadata\x12\x37\n\x06status\x18\x03 \x01(\x0b\x32\x1f.cloudidl.workflow.ActionStatusR\x06status\x12=\n\nerror_info\x18\x04 \x01(\x0b\x32\x1c.cloudidl.workflow.ErrorInfoH\x00R\terrorInfo\x12=\n\nabort_info\x18\x05 \x01(\x0b\x32\x1c.cloudidl.workflow.AbortInfoH\x00R\tabortInfo\x12I\n\x12resolved_task_spec\x18\x06 \x01(\x0b\x32\x1b.cloudidl.workflow.TaskSpecR\x10resolvedTaskSpec\x12<\n\x08\x61ttempts\x18\x07 \x03(\x0b\x32 .cloudidl.workflow.ActionAttemptR\x08\x61ttemptsB\x08\n\x06result\"\xd0\x03\n\rActionAttempt\x12.\n\x05phase\x18\x01 \x01(\x0e\x32\x18.cloudidl.workflow.PhaseR\x05phase\x12\x39\n\nstart_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12:\n\x08\x65nd_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x07\x65ndTime\x88\x01\x01\x12@\n\nerror_info\x18\x04 \x01(\x0b\x32\x1c.cloudidl.workflow.ErrorInfoH\x01R\terrorInfo\x88\x01\x01\x12!\n\x07\x61ttempt\x18\x05 \x01(\rB\x07\xfa\x42\x04*\x02 \x00R\x07\x61ttempt\x12\x31\n\x08log_info\x18\x06 \x03(\x0b\x32\x16.flyteidl.core.TaskLogR\x07logInfo\x12=\n\x07outputs\x18\x07 \x01(\x0b\x32#.cloudidl.workflow.OutputReferencesR\x07outputs\x12%\n\x0elogs_available\x18\x08 \x01(\x08R\rlogsAvailableB\x0b\n\t_end_timeB\r\n\x0b_error_info\"\x95\x05\n\x0b\x41\x63tionEvent\x12=\n\x02id\x18\x01 \x01(\x0b\x32#.cloudidl.workflow.ActionIdentifierB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x02id\x12!\n\x07\x61ttempt\x18\x02 \x01(\rB\x07\xfa\x42\x04*\x02 \x00R\x07\x61ttempt\x12.\n\x05phase\x18\x03 \x01(\x0e\x32\x18.cloudidl.workflow.PhaseR\x05phase\x12\x18\n\x07version\x18\x04 \x01(\rR\x07version\x12\x39\n\nstart_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\tstartTime\x12=\n\x0cupdated_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\x0bupdatedTime\x12:\n\x08\x65nd_time\x18\x07 \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00R\x07\x65ndTime\x88\x01\x01\x12@\n\nerror_info\x18\x08 \x01(\x0b\x32\x1c.cloudidl.workflow.ErrorInfoH\x01R\terrorInfo\x88\x01\x01\x12\x31\n\x08log_info\x18\t \x03(\x0b\x32\x16.flyteidl.core.TaskLogR\x07logInfo\x12:\n\x0blog_context\x18\n \x01(\x0b\x32\x19.flyteidl.core.LogContextR\nlogContext\x12\x18\n\x07\x63luster\x18\x0b \x01(\tR\x07\x63luster\x12=\n\x07outputs\x18\x0c \x01(\x0b\x32#.cloudidl.workflow.OutputReferencesR\x07outputsB\x0b\n\t_end_timeB\r\n\x0b_error_info\"P\n\x0cNamedLiteral\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x16.flyteidl.core.LiteralR\x05value\"1\n\x10OutputReferences\x12\x1d\n\noutput_uri\x18\x01 \x01(\tR\toutputUri\"E\n\x06Inputs\x12;\n\x08literals\x18\x01 \x03(\x0b\x32\x1f.cloudidl.workflow.NamedLiteralR\x08literals\"F\n\x07Outputs\x12;\n\x08literals\x18\x01 \x03(\x0b\x32\x1f.cloudidl.workflow.NamedLiteralR\x08literals*\xcb\x01\n\x05Phase\x12\x15\n\x11PHASE_UNSPECIFIED\x10\x00\x12\x10\n\x0cPHASE_QUEUED\x10\x01\x12\x1f\n\x1bPHASE_WAITING_FOR_RESOURCES\x10\x02\x12\x16\n\x12PHASE_INITIALIZING\x10\x03\x12\x11\n\rPHASE_RUNNING\x10\x04\x12\x13\n\x0fPHASE_SUCCEEDED\x10\x05\x12\x10\n\x0cPHASE_FAILED\x10\x06\x12\x11\n\rPHASE_ABORTED\x10\x07\x12\x13\n\x0fPHASE_TIMED_OUT\x10\x08\x42\xbf\x01\n\x15\x63om.cloudidl.workflowB\x12RunDefinitionProtoH\x02P\x01Z+github.com/unionai/cloud/gen/pb-go/workflow\xa2\x02\x03\x43WX\xaa\x02\x11\x43loudidl.Workflow\xca\x02\x11\x43loudidl\\Workflow\xe2\x02\x1d\x43loudidl\\Workflow\\GPBMetadata\xea\x02\x12\x43loudidl::Workflowb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -67,8 +67,8 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _ACTIONEVENT.fields_by_name['id']._serialized_options = b'\372B\005\212\001\002\020\001'
   _ACTIONEVENT.fields_by_name['attempt']._options = None
   _ACTIONEVENT.fields_by_name['attempt']._serialized_options = b'\372B\004*\002 \000'
-  _globals['_PHASE']._serialized_start=4822
-  _globals['_PHASE']._serialized_end=5025
+  _globals['_PHASE']._serialized_start=4861
+  _globals['_PHASE']._serialized_end=5064
   _globals['_RUNIDENTIFIER']._serialized_start=284
   _globals['_RUNIDENTIFIER']._serialized_end=425
   _globals['_LABELS']._serialized_start=428
@@ -114,15 +114,15 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _globals['_ACTIONDETAILS']._serialized_start=2990
   _globals['_ACTIONDETAILS']._serialized_end=3451
   _globals['_ACTIONATTEMPT']._serialized_start=3454
-  _globals['_ACTIONATTEMPT']._serialized_end=3879
-  _globals['_ACTIONEVENT']._serialized_start=3882
-  _globals['_ACTIONEVENT']._serialized_end=4543
-  _globals['_NAMEDLITERAL']._serialized_start=4545
-  _globals['_NAMEDLITERAL']._serialized_end=4625
-  _globals['_OUTPUTREFERENCES']._serialized_start=4627
-  _globals['_OUTPUTREFERENCES']._serialized_end=4676
-  _globals['_INPUTS']._serialized_start=4678
-  _globals['_INPUTS']._serialized_end=4747
-  _globals['_OUTPUTS']._serialized_start=4749
-  _globals['_OUTPUTS']._serialized_end=4819
+  _globals['_ACTIONATTEMPT']._serialized_end=3918
+  _globals['_ACTIONEVENT']._serialized_start=3921
+  _globals['_ACTIONEVENT']._serialized_end=4582
+  _globals['_NAMEDLITERAL']._serialized_start=4584
+  _globals['_NAMEDLITERAL']._serialized_end=4664
+  _globals['_OUTPUTREFERENCES']._serialized_start=4666
+  _globals['_OUTPUTREFERENCES']._serialized_end=4715
+  _globals['_INPUTS']._serialized_start=4717
+  _globals['_INPUTS']._serialized_end=4786
+  _globals['_OUTPUTS']._serialized_start=4788
+  _globals['_OUTPUTS']._serialized_end=4858
 # @@protoc_insertion_point(module_scope)

flyte/_protos/workflow/run_definition_pb2.pyi

@@ -238,7 +238,7 @@ class ActionDetails(_message.Message):
     def __init__(self, id: _Optional[_Union[ActionIdentifier, _Mapping]] = ..., metadata: _Optional[_Union[ActionMetadata, _Mapping]] = ..., status: _Optional[_Union[ActionStatus, _Mapping]] = ..., error_info: _Optional[_Union[ErrorInfo, _Mapping]] = ..., abort_info: _Optional[_Union[AbortInfo, _Mapping]] = ..., resolved_task_spec: _Optional[_Union[_task_definition_pb2.TaskSpec, _Mapping]] = ..., attempts: _Optional[_Iterable[_Union[ActionAttempt, _Mapping]]] = ...) -> None: ...
 
 class ActionAttempt(_message.Message):
-    __slots__ = ["phase", "start_time", "end_time", "error_info", "attempt", "log_info", "outputs"]
+    __slots__ = ["phase", "start_time", "end_time", "error_info", "attempt", "log_info", "outputs", "logs_available"]
     PHASE_FIELD_NUMBER: _ClassVar[int]
     START_TIME_FIELD_NUMBER: _ClassVar[int]
     END_TIME_FIELD_NUMBER: _ClassVar[int]
@@ -246,6 +246,7 @@ class ActionAttempt(_message.Message):
     ATTEMPT_FIELD_NUMBER: _ClassVar[int]
     LOG_INFO_FIELD_NUMBER: _ClassVar[int]
     OUTPUTS_FIELD_NUMBER: _ClassVar[int]
+    LOGS_AVAILABLE_FIELD_NUMBER: _ClassVar[int]
     phase: Phase
     start_time: _timestamp_pb2.Timestamp
     end_time: _timestamp_pb2.Timestamp
@@ -253,7 +254,8 @@ class ActionAttempt(_message.Message):
     attempt: int
     log_info: _containers.RepeatedCompositeFieldContainer[_execution_pb2.TaskLog]
     outputs: OutputReferences
-    def __init__(self, phase: _Optional[_Union[Phase, str]] = ..., start_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., error_info: _Optional[_Union[ErrorInfo, _Mapping]] = ..., attempt: _Optional[int] = ..., log_info: _Optional[_Iterable[_Union[_execution_pb2.TaskLog, _Mapping]]] = ..., outputs: _Optional[_Union[OutputReferences, _Mapping]] = ...) -> None: ...
+    logs_available: bool
+    def __init__(self, phase: _Optional[_Union[Phase, str]] = ..., start_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., error_info: _Optional[_Union[ErrorInfo, _Mapping]] = ..., attempt: _Optional[int] = ..., log_info: _Optional[_Iterable[_Union[_execution_pb2.TaskLog, _Mapping]]] = ..., outputs: _Optional[_Union[OutputReferences, _Mapping]] = ..., logs_available: bool = ...) -> None: ...
 
 class ActionEvent(_message.Message):
     __slots__ = ["id", "attempt", "phase", "version", "start_time", "updated_time", "end_time", "error_info", "log_info", "log_context", "cluster", "outputs"]

flyte/_protos/workflow/task_definition_pb2.py

@@ -15,9 +15,10 @@ from flyte._protos.common import identifier_pb2 as common_dot_identifier__pb2
 from flyte._protos.common import identity_pb2 as common_dot_identity__pb2
 from flyteidl.core import tasks_pb2 as flyteidl_dot_core_dot_tasks__pb2
 from flyte._protos.validate.validate import validate_pb2 as validate_dot_validate__pb2
+from flyte._protos.workflow import common_pb2 as workflow_dot_common__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1eworkflow/task_definition.proto\x12\x11\x63loudidl.workflow\x1a\x17\x63ommon/identifier.proto\x1a\x15\x63ommon/identity.proto\x1a\x19\x66lyteidl/core/tasks.proto\x1a\x17validate/validate.proto\"\x88\x01\n\x08TaskName\x12\x19\n\x03org\x18\x01 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x03org\x12!\n\x07project\x18\x02 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x07project\x12\x1f\n\x06\x64omain\x18\x03 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x06\x64omain\x12\x1d\n\x04name\x18\x04 \x01(\tB\t\xfa\x42\x06r\x04\x10\x01\x18\x1eR\x04name\"\xb1\x01\n\x0eTaskIdentifier\x12\x19\n\x03org\x18\x01 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x03org\x12!\n\x07project\x18\x02 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x07project\x12\x1f\n\x06\x64omain\x18\x03 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x06\x64omain\x12\x1d\n\x04name\x18\x04 \x01(\tB\t\xfa\x42\x06r\x04\x10\x01\x18\x1eR\x04name\x12!\n\x07version\x18\x05 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x07version\"\\\n\x0cTaskMetadata\x12L\n\x0b\x64\x65ployed_by\x18\x01 \x01(\x0b\x32!.cloudidl.common.EnrichedIdentityB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\ndeployedBy\"\x93\x01\n\x04Task\x12\x44\n\x07task_id\x18\x01 \x01(\x0b\x32!.cloudidl.workflow.TaskIdentifierB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x06taskId\x12\x45\n\x08metadata\x18\x02 \x01(\x0b\x32\x1f.cloudidl.workflow.TaskMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x08metadata\"V\n\x08TaskSpec\x12J\n\rtask_template\x18\x01 \x01(\x0b\x32\x1b.flyteidl.core.TaskTemplateB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x0ctaskTemplate\"\xd5\x01\n\x0bTaskDetails\x12\x44\n\x07task_id\x18\x01 \x01(\x0b\x32!.cloudidl.workflow.TaskIdentifierB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x06taskId\x12\x45\n\x08metadata\x18\x02 \x01(\x0b\x32\x1f.cloudidl.workflow.TaskMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x08metadata\x12\x39\n\x04spec\x18\x03 \x01(\x0b\x32\x1b.cloudidl.workflow.TaskSpecB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x04specB\xc0\x01\n\x15\x63om.cloudidl.workflowB\x13TaskDefinitionProtoH\x02P\x01Z+github.com/unionai/cloud/gen/pb-go/workflow\xa2\x02\x03\x43WX\xaa\x02\x11\x43loudidl.Workflow\xca\x02\x11\x43loudidl\\Workflow\xe2\x02\x1d\x43loudidl\\Workflow\\GPBMetadata\xea\x02\x12\x43loudidl::Workflowb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1eworkflow/task_definition.proto\x12\x11\x63loudidl.workflow\x1a\x17\x63ommon/identifier.proto\x1a\x15\x63ommon/identity.proto\x1a\x19\x66lyteidl/core/tasks.proto\x1a\x17validate/validate.proto\x1a\x15workflow/common.proto\"\x88\x01\n\x08TaskName\x12\x19\n\x03org\x18\x01 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x03org\x12!\n\x07project\x18\x02 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x07project\x12\x1f\n\x06\x64omain\x18\x03 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x06\x64omain\x12\x1d\n\x04name\x18\x04 \x01(\tB\t\xfa\x42\x06r\x04\x10\x01\x18\x1eR\x04name\"\xb1\x01\n\x0eTaskIdentifier\x12\x19\n\x03org\x18\x01 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x03org\x12!\n\x07project\x18\x02 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x07project\x12\x1f\n\x06\x64omain\x18\x03 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x06\x64omain\x12\x1d\n\x04name\x18\x04 \x01(\tB\t\xfa\x42\x06r\x04\x10\x01\x18\x1eR\x04name\x12!\n\x07version\x18\x05 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x07version\"\\\n\x0cTaskMetadata\x12L\n\x0b\x64\x65ployed_by\x18\x01 \x01(\x0b\x32!.cloudidl.common.EnrichedIdentityB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\ndeployedBy\"\x93\x01\n\x04Task\x12\x44\n\x07task_id\x18\x01 \x01(\x0b\x32!.cloudidl.workflow.TaskIdentifierB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x06taskId\x12\x45\n\x08metadata\x18\x02 \x01(\x0b\x32\x1f.cloudidl.workflow.TaskMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x08metadata\"\xa0\x01\n\x08TaskSpec\x12J\n\rtask_template\x18\x01 \x01(\x0b\x32\x1b.flyteidl.core.TaskTemplateB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x0ctaskTemplate\x12H\n\x0e\x64\x65\x66\x61ult_inputs\x18\x02 \x03(\x0b\x32!.cloudidl.workflow.NamedParameterR\rdefaultInputs\"\xd5\x01\n\x0bTaskDetails\x12\x44\n\x07task_id\x18\x01 \x01(\x0b\x32!.cloudidl.workflow.TaskIdentifierB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x06taskId\x12\x45\n\x08metadata\x18\x02 \x01(\x0b\x32\x1f.cloudidl.workflow.TaskMetadataB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x08metadata\x12\x39\n\x04spec\x18\x03 \x01(\x0b\x32\x1b.cloudidl.workflow.TaskSpecB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x04specB\xc0\x01\n\x15\x63om.cloudidl.workflowB\x13TaskDefinitionProtoH\x02P\x01Z+github.com/unionai/cloud/gen/pb-go/workflow\xa2\x02\x03\x43WX\xaa\x02\x11\x43loudidl.Workflow\xca\x02\x11\x43loudidl\\Workflow\xe2\x02\x1d\x43loudidl\\Workflow\\GPBMetadata\xea\x02\x12\x43loudidl::Workflowb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -57,16 +58,16 @@ if _descriptor._USE_C_DESCRIPTORS == False:
   _TASKDETAILS.fields_by_name['metadata']._serialized_options = b'\372B\005\212\001\002\020\001'
   _TASKDETAILS.fields_by_name['spec']._options = None
   _TASKDETAILS.fields_by_name['spec']._serialized_options = b'\372B\005\212\001\002\020\001'
-  _globals['_TASKNAME']._serialized_start=154
-  _globals['_TASKNAME']._serialized_end=290
-  _globals['_TASKIDENTIFIER']._serialized_start=293
-  _globals['_TASKIDENTIFIER']._serialized_end=470
-  _globals['_TASKMETADATA']._serialized_start=472
-  _globals['_TASKMETADATA']._serialized_end=564
-  _globals['_TASK']._serialized_start=567
-  _globals['_TASK']._serialized_end=714
-  _globals['_TASKSPEC']._serialized_start=716
-  _globals['_TASKSPEC']._serialized_end=802
-  _globals['_TASKDETAILS']._serialized_start=805
-  _globals['_TASKDETAILS']._serialized_end=1018
+  _globals['_TASKNAME']._serialized_start=177
+  _globals['_TASKNAME']._serialized_end=313
+  _globals['_TASKIDENTIFIER']._serialized_start=316
+  _globals['_TASKIDENTIFIER']._serialized_end=493
+  _globals['_TASKMETADATA']._serialized_start=495
+  _globals['_TASKMETADATA']._serialized_end=587
+  _globals['_TASK']._serialized_start=590
+  _globals['_TASK']._serialized_end=737
+  _globals['_TASKSPEC']._serialized_start=740
+  _globals['_TASKSPEC']._serialized_end=900
+  _globals['_TASKDETAILS']._serialized_start=903
+  _globals['_TASKDETAILS']._serialized_end=1116
 # @@protoc_insertion_point(module_scope)

flyte/_protos/workflow/task_definition_pb2.pyi

@@ -2,9 +2,11 @@ from flyte._protos.common import identifier_pb2 as _identifier_pb2
 from flyte._protos.common import identity_pb2 as _identity_pb2
 from flyteidl.core import tasks_pb2 as _tasks_pb2
 from flyte._protos.validate.validate import validate_pb2 as _validate_pb2
+from flyte._protos.workflow import common_pb2 as _common_pb2
+from google.protobuf.internal import containers as _containers
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
-from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union
+from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union
 
 DESCRIPTOR: _descriptor.FileDescriptor
 
@@ -49,10 +51,12 @@ class Task(_message.Message):
     def __init__(self, task_id: _Optional[_Union[TaskIdentifier, _Mapping]] = ..., metadata: _Optional[_Union[TaskMetadata, _Mapping]] = ...) -> None: ...
 
 class TaskSpec(_message.Message):
-    __slots__ = ["task_template"]
+    __slots__ = ["task_template", "default_inputs"]
     TASK_TEMPLATE_FIELD_NUMBER: _ClassVar[int]
+    DEFAULT_INPUTS_FIELD_NUMBER: _ClassVar[int]
     task_template: _tasks_pb2.TaskTemplate
-    def __init__(self, task_template: _Optional[_Union[_tasks_pb2.TaskTemplate, _Mapping]] = ...) -> None: ...
+    default_inputs: _containers.RepeatedCompositeFieldContainer[_common_pb2.NamedParameter]
+    def __init__(self, task_template: _Optional[_Union[_tasks_pb2.TaskTemplate, _Mapping]] = ..., default_inputs: _Optional[_Iterable[_Union[_common_pb2.NamedParameter, _Mapping]]] = ...) -> None: ...
 
 class TaskDetails(_message.Message):
     __slots__ = ["task_id", "metadata", "spec"]

flyte/_run.py

@@ -5,8 +5,9 @@ import uuid
 from typing import TYPE_CHECKING, Any, Literal, Optional, Tuple, Union, cast
 
 from flyte.errors import InitializationError
+from flyte.models import ActionID, Checkpoints, CodeBundle, RawDataPath, SerializationContext, TaskContext
+from flyte.syncify import syncify
 
-from ._api_commons import syncer
 from ._context import contextual_run, internal_ctx
 from ._environment import Environment
 from ._initialize import (
@@ -20,7 +21,6 @@ from ._initialize import (
 from ._logging import logger
 from ._task import P, R, TaskTemplate
 from ._tools import ipython_check
-from .models import ActionID, Checkpoints, CodeBundle, RawDataPath, SerializationContext, TaskContext
 
 if TYPE_CHECKING:
     from flyte.remote import Run
@@ -38,7 +38,7 @@ async def _get_code_bundle_for_run(name: str) -> CodeBundle | None:
     from flyte._internal.runtime.task_serde import extract_code_bundle
     from flyte.remote import Run
 
-    run = await Run.get.aio(Run, name=name)
+    run = await Run.get.aio(name=name)
     if run:
         run_details = await run.details()
         spec = run_details.action_details.pb2.resolved_task_spec
@@ -46,7 +46,6 @@ async def _get_code_bundle_for_run(name: str) -> CodeBundle | None:
     return None
 
 
-@syncer.wrap
 class _Runner:
     def __init__(
         self,
@@ -323,6 +322,7 @@ class _Runner:
                 return cast(R, await convert_outputs_to_native(obj.native_interface, out))
         return cast(R, None)
 
+    @syncify
     async def run(self, task: TaskTemplate[P, Union[R, Run]], *args: P.args, **kwargs: P.kwargs) -> Union[R, Run]:
         """
         Run an async `@env.task` or `TaskTemplate` instance. The existing async context will be used.
@@ -398,6 +398,8 @@ def with_runcontext(
          interactive mode, while scripts are not. This is used to determine how the code bundle is created.
     :param raw_data_path: Use this path to store the raw data for the run. Currently only supported for local runs,
       and can be used to store raw data in specific locations. TODO coming soon for remote runs as well.
+    :param run_base_dir: Optional The base directory to use for the run. This is used to store the metadata for the run,
+     that is passed between tasks.
     :return: runner
     """
     if mode == "hybrid" and not name and not run_base_dir:
@@ -416,7 +418,7 @@ def with_runcontext(
     )
 
 
-@syncer.wrap
+@syncify
 async def run(task: TaskTemplate[P, R], *args: P.args, **kwargs: P.kwargs) -> Union[R, Run]:
     """
     Run a task with the given parameters

flyte/_trace.py

@@ -87,17 +87,12 @@ def trace(func: Callable[..., T]) -> Callable[..., T]:
                 items = []
                 result = func(*args, **kwargs)
                 # TODO ideally we should use streaming into the type-engine so that it stream uploads large blocks
-                if inspect.isasyncgen(result):
+                if inspect.isasyncgen(result) or is_async_iterable(result):
                     # If it's directly an async generator
                     async_iter = result
                     async for item in async_iter:
                         items.append(item)
                         yield item
-                elif is_async_iterable(result):
-                    # If it's an async iterable (has __aiter__)
-                    async for item in result:
-                        items.append(item)
-                        yield item
                 duration = time.time() - start_time
                 info.add_outputs(items, timedelta(seconds=duration))
                 await controller.record_trace(info)

flyte/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.2.0b4'
-__version_tuple__ = version_tuple = (0, 2, 0, 'b4')
+__version__ = version = '0.2.0b7'
+__version_tuple__ = version_tuple = (0, 2, 0, 'b7')