flyte 2.0.0b9__py3-none-any.whl → 2.0.0b14__py3-none-any.whl

flyte/_protos/workflow/run_definition_pb2.pyi

@@ -69,18 +69,20 @@ class Envs(_message.Message):
     def __init__(self, values: _Optional[_Iterable[_Union[_literals_pb2.KeyValuePair, _Mapping]]] = ...) -> None: ...
 
 class RunSpec(_message.Message):
-    __slots__ = ["labels", "annotations", "envs", "interruptible", "overwrite_cache"]
+    __slots__ = ["labels", "annotations", "envs", "interruptible", "overwrite_cache", "cluster"]
     LABELS_FIELD_NUMBER: _ClassVar[int]
     ANNOTATIONS_FIELD_NUMBER: _ClassVar[int]
     ENVS_FIELD_NUMBER: _ClassVar[int]
     INTERRUPTIBLE_FIELD_NUMBER: _ClassVar[int]
     OVERWRITE_CACHE_FIELD_NUMBER: _ClassVar[int]
+    CLUSTER_FIELD_NUMBER: _ClassVar[int]
     labels: Labels
     annotations: Annotations
     envs: Envs
     interruptible: _wrappers_pb2.BoolValue
     overwrite_cache: bool
-    def __init__(self, labels: _Optional[_Union[Labels, _Mapping]] = ..., annotations: _Optional[_Union[Annotations, _Mapping]] = ..., envs: _Optional[_Union[Envs, _Mapping]] = ..., interruptible: _Optional[_Union[_wrappers_pb2.BoolValue, _Mapping]] = ..., overwrite_cache: bool = ...) -> None: ...
+    cluster: str
+    def __init__(self, labels: _Optional[_Union[Labels, _Mapping]] = ..., annotations: _Optional[_Union[Annotations, _Mapping]] = ..., envs: _Optional[_Union[Envs, _Mapping]] = ..., interruptible: _Optional[_Union[_wrappers_pb2.BoolValue, _Mapping]] = ..., overwrite_cache: bool = ..., cluster: _Optional[str] = ...) -> None: ...
 
 class Run(_message.Message):
     __slots__ = ["action"]
@@ -140,16 +142,18 @@ class ActionMetadata(_message.Message):
     def __init__(self, parent: _Optional[str] = ..., group: _Optional[str] = ..., executed_by: _Optional[_Union[_identity_pb2.EnrichedIdentity, _Mapping]] = ..., task: _Optional[_Union[TaskActionMetadata, _Mapping]] = ..., trace: _Optional[_Union[TraceActionMetadata, _Mapping]] = ..., condition: _Optional[_Union[ConditionActionMetadata, _Mapping]] = ...) -> None: ...
 
 class ActionStatus(_message.Message):
-    __slots__ = ["phase", "start_time", "end_time", "attempts"]
+    __slots__ = ["phase", "start_time", "end_time", "attempts", "cache_status"]
     PHASE_FIELD_NUMBER: _ClassVar[int]
     START_TIME_FIELD_NUMBER: _ClassVar[int]
     END_TIME_FIELD_NUMBER: _ClassVar[int]
     ATTEMPTS_FIELD_NUMBER: _ClassVar[int]
+    CACHE_STATUS_FIELD_NUMBER: _ClassVar[int]
     phase: Phase
     start_time: _timestamp_pb2.Timestamp
     end_time: _timestamp_pb2.Timestamp
     attempts: int
-    def __init__(self, phase: _Optional[_Union[Phase, str]] = ..., start_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., attempts: _Optional[int] = ...) -> None: ...
+    cache_status: _catalog_pb2.CatalogCacheStatus
+    def __init__(self, phase: _Optional[_Union[Phase, str]] = ..., start_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., attempts: _Optional[int] = ..., cache_status: _Optional[_Union[_catalog_pb2.CatalogCacheStatus, str]] = ...) -> None: ...
 
 class Action(_message.Message):
     __slots__ = ["id", "metadata", "status"]
@@ -221,7 +225,7 @@ class ActionDetails(_message.Message):
     def __init__(self, id: _Optional[_Union[_identifier_pb2.ActionIdentifier, _Mapping]] = ..., metadata: _Optional[_Union[ActionMetadata, _Mapping]] = ..., status: _Optional[_Union[ActionStatus, _Mapping]] = ..., error_info: _Optional[_Union[ErrorInfo, _Mapping]] = ..., abort_info: _Optional[_Union[AbortInfo, _Mapping]] = ..., resolved_task_spec: _Optional[_Union[_task_definition_pb2.TaskSpec, _Mapping]] = ..., attempts: _Optional[_Iterable[_Union[ActionAttempt, _Mapping]]] = ...) -> None: ...
 
 class ActionAttempt(_message.Message):
-    __slots__ = ["phase", "start_time", "end_time", "error_info", "attempt", "log_info", "outputs", "logs_available", "cache_status", "cluster_events", "phase_transitions", "cluster"]
+    __slots__ = ["phase", "start_time", "end_time", "error_info", "attempt", "log_info", "outputs", "logs_available", "cache_status", "cluster_events", "phase_transitions", "cluster", "log_context"]
     PHASE_FIELD_NUMBER: _ClassVar[int]
     START_TIME_FIELD_NUMBER: _ClassVar[int]
     END_TIME_FIELD_NUMBER: _ClassVar[int]
@@ -234,6 +238,7 @@ class ActionAttempt(_message.Message):
     CLUSTER_EVENTS_FIELD_NUMBER: _ClassVar[int]
     PHASE_TRANSITIONS_FIELD_NUMBER: _ClassVar[int]
     CLUSTER_FIELD_NUMBER: _ClassVar[int]
+    LOG_CONTEXT_FIELD_NUMBER: _ClassVar[int]
     phase: Phase
     start_time: _timestamp_pb2.Timestamp
     end_time: _timestamp_pb2.Timestamp
@@ -246,7 +251,8 @@ class ActionAttempt(_message.Message):
     cluster_events: _containers.RepeatedCompositeFieldContainer[ClusterEvent]
     phase_transitions: _containers.RepeatedCompositeFieldContainer[PhaseTransition]
     cluster: str
-    def __init__(self, phase: _Optional[_Union[Phase, str]] = ..., start_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., error_info: _Optional[_Union[ErrorInfo, _Mapping]] = ..., attempt: _Optional[int] = ..., log_info: _Optional[_Iterable[_Union[_execution_pb2.TaskLog, _Mapping]]] = ..., outputs: _Optional[_Union[OutputReferences, _Mapping]] = ..., logs_available: bool = ..., cache_status: _Optional[_Union[_catalog_pb2.CatalogCacheStatus, str]] = ..., cluster_events: _Optional[_Iterable[_Union[ClusterEvent, _Mapping]]] = ..., phase_transitions: _Optional[_Iterable[_Union[PhaseTransition, _Mapping]]] = ..., cluster: _Optional[str] = ...) -> None: ...
+    log_context: _execution_pb2.LogContext
+    def __init__(self, phase: _Optional[_Union[Phase, str]] = ..., start_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., error_info: _Optional[_Union[ErrorInfo, _Mapping]] = ..., attempt: _Optional[int] = ..., log_info: _Optional[_Iterable[_Union[_execution_pb2.TaskLog, _Mapping]]] = ..., outputs: _Optional[_Union[OutputReferences, _Mapping]] = ..., logs_available: bool = ..., cache_status: _Optional[_Union[_catalog_pb2.CatalogCacheStatus, str]] = ..., cluster_events: _Optional[_Iterable[_Union[ClusterEvent, _Mapping]]] = ..., phase_transitions: _Optional[_Iterable[_Union[PhaseTransition, _Mapping]]] = ..., cluster: _Optional[str] = ..., log_context: _Optional[_Union[_execution_pb2.LogContext, _Mapping]] = ...) -> None: ...
 
 class ClusterEvent(_message.Message):
     __slots__ = ["occurred_at", "message"]
@@ -267,7 +273,7 @@ class PhaseTransition(_message.Message):
     def __init__(self, phase: _Optional[_Union[Phase, str]] = ..., start_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
 
 class ActionEvent(_message.Message):
-    __slots__ = ["id", "attempt", "phase", "version", "start_time", "updated_time", "end_time", "error_info", "log_info", "log_context", "cluster", "outputs", "cache_status", "cluster_events"]
+    __slots__ = ["id", "attempt", "phase", "version", "start_time", "updated_time", "end_time", "error_info", "log_info", "log_context", "cluster", "outputs", "cache_status", "cluster_events", "reported_time"]
     ID_FIELD_NUMBER: _ClassVar[int]
     ATTEMPT_FIELD_NUMBER: _ClassVar[int]
     PHASE_FIELD_NUMBER: _ClassVar[int]
@@ -282,6 +288,7 @@ class ActionEvent(_message.Message):
     OUTPUTS_FIELD_NUMBER: _ClassVar[int]
     CACHE_STATUS_FIELD_NUMBER: _ClassVar[int]
     CLUSTER_EVENTS_FIELD_NUMBER: _ClassVar[int]
+    REPORTED_TIME_FIELD_NUMBER: _ClassVar[int]
     id: _identifier_pb2.ActionIdentifier
     attempt: int
     phase: Phase
@@ -296,7 +303,8 @@ class ActionEvent(_message.Message):
     outputs: OutputReferences
     cache_status: _catalog_pb2.CatalogCacheStatus
     cluster_events: _containers.RepeatedCompositeFieldContainer[ClusterEvent]
-    def __init__(self, id: _Optional[_Union[_identifier_pb2.ActionIdentifier, _Mapping]] = ..., attempt: _Optional[int] = ..., phase: _Optional[_Union[Phase, str]] = ..., version: _Optional[int] = ..., start_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., updated_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., error_info: _Optional[_Union[ErrorInfo, _Mapping]] = ..., log_info: _Optional[_Iterable[_Union[_execution_pb2.TaskLog, _Mapping]]] = ..., log_context: _Optional[_Union[_execution_pb2.LogContext, _Mapping]] = ..., cluster: _Optional[str] = ..., outputs: _Optional[_Union[OutputReferences, _Mapping]] = ..., cache_status: _Optional[_Union[_catalog_pb2.CatalogCacheStatus, str]] = ..., cluster_events: _Optional[_Iterable[_Union[ClusterEvent, _Mapping]]] = ...) -> None: ...
+    reported_time: _timestamp_pb2.Timestamp
+    def __init__(self, id: _Optional[_Union[_identifier_pb2.ActionIdentifier, _Mapping]] = ..., attempt: _Optional[int] = ..., phase: _Optional[_Union[Phase, str]] = ..., version: _Optional[int] = ..., start_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., updated_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., end_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., error_info: _Optional[_Union[ErrorInfo, _Mapping]] = ..., log_info: _Optional[_Iterable[_Union[_execution_pb2.TaskLog, _Mapping]]] = ..., log_context: _Optional[_Union[_execution_pb2.LogContext, _Mapping]] = ..., cluster: _Optional[str] = ..., outputs: _Optional[_Union[OutputReferences, _Mapping]] = ..., cache_status: _Optional[_Union[_catalog_pb2.CatalogCacheStatus, str]] = ..., cluster_events: _Optional[_Iterable[_Union[ClusterEvent, _Mapping]]] = ..., reported_time: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ...
 
 class NamedLiteral(_message.Message):
     __slots__ = ["name", "value"]
@@ -315,10 +323,12 @@ class OutputReferences(_message.Message):
     def __init__(self, output_uri: _Optional[str] = ..., report_uri: _Optional[str] = ...) -> None: ...
 
 class Inputs(_message.Message):
-    __slots__ = ["literals"]
+    __slots__ = ["literals", "context"]
     LITERALS_FIELD_NUMBER: _ClassVar[int]
+    CONTEXT_FIELD_NUMBER: _ClassVar[int]
     literals: _containers.RepeatedCompositeFieldContainer[NamedLiteral]
-    def __init__(self, literals: _Optional[_Iterable[_Union[NamedLiteral, _Mapping]]] = ...) -> None: ...
+    context: _containers.RepeatedCompositeFieldContainer[_literals_pb2.KeyValuePair]
+    def __init__(self, literals: _Optional[_Iterable[_Union[NamedLiteral, _Mapping]]] = ..., context: _Optional[_Iterable[_Union[_literals_pb2.KeyValuePair, _Mapping]]] = ...) -> None: ...
 
 class Outputs(_message.Message):
     __slots__ = ["literals"]

flyte/_reusable_environment.py

@@ -1,6 +1,6 @@
 from dataclasses import dataclass
 from datetime import timedelta
-from typing import Optional, Tuple, Union
+from typing import Tuple, Union
 
 from flyte._logging import logger
 
@@ -17,25 +17,22 @@ class ReusePolicy:
 
     :param replicas: Either a single int representing number of replicas or a tuple of two ints representing
      the min and max.
-    :param idle_ttl: The maximum idle duration for an environment replica, specified as either seconds (int) or a
-        timedelta. If not set, the environment's global default will be used.
+    :param idle_ttl: The maximum idle duration for an environment, specified as either seconds (int) or a
+        timedelta, after which all replicas in the environment are shutdown.
+        If not set, the default is configured in the backend (can be as low as 90s).
         When a replica remains idle — meaning no tasks are running — for this duration, it will be automatically
-        terminated.
+        terminated, also referred to as environment idle timeout.
     :param concurrency: The maximum number of tasks that can run concurrently in one instance of the environment.
-          Concurrency of greater than 1 is only supported only for `async` tasks.
-    :param reuse_salt: Optional string used to control environment reuse.
-        If set, the environment will be reused even if the code bundle changes.
-        To force a new environment, either set this to `None` or change its value.
-
-        Example:
-            reuse_salt = "v1"  # Environment is reused
-            reuse_salt = "v2"  # Forces environment recreation
+          Concurrency of greater than 1 is only supported for `async` tasks.
+    :param scaledown_ttl: The minimum time to wait before scaling down each replica, specified as either seconds (int)
+        or a timedelta. This is useful to prevent rapid scaling down of replicas when tasks are running
+        frequently. If not set, the default is configured in the backend.
     """
 
     replicas: Union[int, Tuple[int, int]] = 2
-    idle_ttl: Optional[Union[int, timedelta]] = None
-    reuse_salt: str | None = None
+    idle_ttl: Union[int, timedelta] = 30  # seconds
     concurrency: int = 1
+    scaledown_ttl: Union[int, timedelta] = 30  # seconds
 
     def __post_init__(self):
         if self.replicas is None:
@@ -47,11 +44,12 @@ class ReusePolicy:
         elif len(self.replicas) != 2:
             raise ValueError("replicas must be an int or a tuple of two ints")
 
-        if self.idle_ttl:
-            if isinstance(self.idle_ttl, int):
-                self.idle_ttl = timedelta(seconds=int(self.idle_ttl))
-            elif not isinstance(self.idle_ttl, timedelta):
-                raise ValueError("idle_ttl must be an int (seconds) or a timedelta")
+        if isinstance(self.idle_ttl, int):
+            self.idle_ttl = timedelta(seconds=int(self.idle_ttl))
+        elif not isinstance(self.idle_ttl, timedelta):
+            raise ValueError("idle_ttl must be an int (seconds) or a timedelta")
+        if self.idle_ttl.total_seconds() < 30:
+            raise ValueError("idle_ttl must be at least 30 seconds")
 
         if self.replicas[1] == 1 and self.concurrency == 1:
             logger.warning(
@@ -61,6 +59,13 @@ class ReusePolicy:
                 "that runs child tasks."
             )
 
+        if isinstance(self.scaledown_ttl, int):
+            self.scaledown_ttl = timedelta(seconds=int(self.scaledown_ttl))
+        elif not isinstance(self.scaledown_ttl, timedelta):
+            raise ValueError("scaledown_ttl must be an int (seconds) or a timedelta")
+        if self.scaledown_ttl.total_seconds() < 30:
+            raise ValueError("scaledown_ttl must be at least 30 seconds")
+
     @property
     def ttl(self) -> timedelta | None:
         """
@@ -72,6 +77,23 @@ class ReusePolicy:
             return self.idle_ttl
         return timedelta(seconds=self.idle_ttl)
 
+    @property
+    def min_replicas(self) -> int:
+        """
+        Returns the minimum number of replicas.
+        """
+        return self.replicas[0] if isinstance(self.replicas, tuple) else self.replicas
+
+    def get_scaledown_ttl(self) -> timedelta | None:
+        """
+        Returns the scaledown TTL as a timedelta. If scaledown_ttl is not set, returns None.
+        """
+        if self.scaledown_ttl is None:
+            return None
+        if isinstance(self.scaledown_ttl, timedelta):
+            return self.scaledown_ttl
+        return timedelta(seconds=int(self.scaledown_ttl))
+
     @property
     def max_replicas(self) -> int:
         """

flyte/_run.py

@@ -19,8 +19,6 @@ from flyte._initialize import (
 )
 from flyte._logging import logger
 from flyte._task import P, R, TaskTemplate
-from flyte._tools import ipython_check
-from flyte.errors import InitializationError
 from flyte.models import (
     ActionID,
     Checkpoints,
@@ -89,13 +87,15 @@ class _Runner:
         overwrite_cache: bool = False,
         project: str | None = None,
         domain: str | None = None,
-        env: Dict[str, str] | None = None,
+        env_vars: Dict[str, str] | None = None,
         labels: Dict[str, str] | None = None,
         annotations: Dict[str, str] | None = None,
         interruptible: bool = False,
         log_level: int | None = None,
         disable_run_cache: bool = False,
     ):
+        from flyte._tools import ipython_check
+
         init_config = _get_init_config()
         client = init_config.client if init_config else None
         if not force_mode and client is not None:
@@ -116,7 +116,7 @@ class _Runner:
         self._overwrite_cache = overwrite_cache
         self._project = project
         self._domain = domain
-        self._env = env
+        self._env_vars = env_vars
         self._labels = labels
         self._annotations = annotations
         self._interruptible = interruptible
@@ -198,7 +198,7 @@ class _Runner:
             task_spec = translate_task_to_wire(obj, s_ctx)
             inputs = await convert_from_native_to_inputs(obj.native_interface, *args, **kwargs)
 
-        env = self._env or {}
+        env = self._env_vars or {}
         if self._log_level:
             env["LOG_LEVEL"] = str(self._log_level)
         else:
@@ -207,7 +207,7 @@ class _Runner:
         if not self._dry_run:
             if get_client() is None:
                 # This can only happen, if the user forces flyte.run(mode="remote") without initializing the client
-                raise InitializationError(
+                raise flyte.errors.InitializationError(
                     "ClientNotInitializedError",
                     "user",
                     "flyte.run requires client to be initialized. "
@@ -542,7 +542,7 @@ def with_runcontext(
     overwrite_cache: bool = False,
     project: str | None = None,
     domain: str | None = None,
-    env: Dict[str, str] | None = None,
+    env_vars: Dict[str, str] | None = None,
     labels: Dict[str, str] | None = None,
     annotations: Dict[str, str] | None = None,
     interruptible: bool = False,
@@ -582,7 +582,7 @@ def with_runcontext(
     :param overwrite_cache: Optional If true, the cache will be overwritten for the run
     :param project: Optional The project to use for the run
     :param domain: Optional The domain to use for the run
-    :param env: Optional Environment variables to set for the run
+    :param env_vars: Optional Environment variables to set for the run
     :param labels: Optional Labels to set for the run
     :param annotations: Optional Annotations to set for the run
     :param interruptible: Optional If true, the run can be interrupted by the user.
@@ -606,7 +606,7 @@ def with_runcontext(
         raw_data_path=raw_data_path,
         run_base_dir=run_base_dir,
         overwrite_cache=overwrite_cache,
-        env=env,
+        env_vars=env_vars,
         labels=labels,
         annotations=annotations,
         interruptible=interruptible,

flyte/_secret.py

@@ -17,14 +17,14 @@ class Secret:
 
     Example:
     ```python
-    @task(secrets="MY_SECRET")
+    @task(secrets="my-secret")
     async def my_task():
-        os.environ["MY_SECRET"]  # This will be set to the value of the secret
+        # This will be set to the value of the secret. Note: The env var is always uppercase, and - is replaced with _.
+        os.environ["MY_SECRET"]
 
-    @task(secrets=Secret("MY_SECRET", mount="/path/to/secret"))
+    @task(secrets=Secret("my-openai-api-key", as_env_var="OPENAI_API_KEY"))
     async def my_task2():
-        async with open("/path/to/secret") as f:
-            secret_value = f.read()
+        os.environ["OPENAI_API_KEY"]
     ```
 
     TODO: Add support for secret versioning (some stores) and secret groups (some stores) and mounting as files.
@@ -32,6 +32,7 @@ class Secret:
     :param key: The name of the secret in the secret store.
     :param group: The group of the secret in the secret store.
     :param mount: Use this to specify the path where the secret should be mounted.
+    TODO: support arbitrary mount paths. Today only "/etc/flyte/secrets" is supported
     :param as_env_var: The name of the environment variable that the secret should be mounted as.
     """
 
@@ -41,6 +42,9 @@ class Secret:
     as_env_var: Optional[str] = None
 
     def __post_init__(self):
+        if not self.mount and not self.as_env_var:
+            self.as_env_var = f"{self.group}_{self.key}" if self.group else self.key
+            self.as_env_var = self.as_env_var.replace("-", "_").upper()
         if self.as_env_var is not None:
             pattern = r"^[A-Z_][A-Z0-9_]*$"
             if not re.match(pattern, self.as_env_var):

flyte/_task.py

@@ -76,7 +76,7 @@ class TaskTemplate(Generic[P, R]):
     :param reusable: Optional The reusability policy for the task, defaults to None, which means the task environment
     will not be reused across task invocations.
     :param docs: Optional The documentation for the task, if not provided the function docstring will be used.
-    :param env: Optional The environment variables to set for the task.
+    :param env_vars: Optional The environment variables to set for the task.
     :param secrets: Optional The secrets that will be injected into the task at runtime.
     :param timeout: Optional The timeout for the task.
     :param max_inline_io_bytes: Maximum allowed size (in bytes) for all inputs and outputs passed directly to the task
@@ -85,7 +85,7 @@ class TaskTemplate(Generic[P, R]):
 
     name: str
     interface: NativeInterface
-    friendly_name: str = ""
+    short_name: str = ""
     task_type: str = "python"
     task_type_version: int = 0
     image: Union[str, Image, Literal["auto"]] = "auto"
@@ -95,7 +95,7 @@ class TaskTemplate(Generic[P, R]):
     retries: Union[int, RetryStrategy] = 0
     reusable: Union[ReusePolicy, None] = None
     docs: Optional[Documentation] = None
-    env: Optional[Dict[str, str]] = None
+    env_vars: Optional[Dict[str, str]] = None
     secrets: Optional[SecretRequest] = None
     timeout: Optional[TimeoutType] = None
     pod_template: Optional[Union[str, PodTemplate]] = None
@@ -129,9 +129,9 @@ class TaskTemplate(Generic[P, R]):
         if isinstance(self.retries, int):
             self.retries = RetryStrategy(count=self.retries)
 
-        if self.friendly_name == "":
-            # If friendly_name is not set, use the name of the task
-            self.friendly_name = self.name
+        if self.short_name == "":
+            # If short_name is not set, use the name of the task
+            self.short_name = self.name
 
     def __getstate__(self):
         """
@@ -314,14 +314,16 @@ class TaskTemplate(Generic[P, R]):
     def override(
         self,
         *,
+        short_name: Optional[str] = None,
         resources: Optional[Resources] = None,
-        cache: CacheRequest = "auto",
+        cache: Optional[CacheRequest] = None,
         retries: Union[int, RetryStrategy] = 0,
         timeout: Optional[TimeoutType] = None,
         reusable: Union[ReusePolicy, Literal["off"], None] = None,
-        env: Optional[Dict[str, str]] = None,
+        env_vars: Optional[Dict[str, str]] = None,
         secrets: Optional[SecretRequest] = None,
         max_inline_io_bytes: int | None = None,
+        pod_template: Optional[Union[str, PodTemplate]] = None,
         **kwargs: Any,
     ) -> TaskTemplate:
         """
@@ -344,7 +346,7 @@ class TaskTemplate(Generic[P, R]):
                     " Reusable tasks will use the parent env's resources. You can disable reusability and"
                     " override resources if needed. (set reusable='off')"
                 )
-            if env is not None:
+            if env_vars is not None:
                 raise ValueError(
                     "Cannot override env when reusable is set."
                     " Reusable tasks will use the parent env's env. You can disable reusability and "
@@ -358,7 +360,7 @@ class TaskTemplate(Generic[P, R]):
                 )
 
         resources = resources or self.resources
-        env = env or self.env
+        env_vars = env_vars or self.env_vars
         secrets = secrets or self.secrets
 
         for k, v in kwargs.items():
@@ -373,14 +375,17 @@ class TaskTemplate(Generic[P, R]):
 
         return replace(
             self,
+            short_name=short_name or self.short_name,
             resources=resources,
             cache=cache,
             retries=retries,
             timeout=timeout,
             reusable=cast(Optional[ReusePolicy], reusable),
-            env=env,
+            env_vars=env_vars,
             secrets=secrets,
             max_inline_io_bytes=max_inline_io_bytes,
+            pod_template=pod_template,
+            **kwargs,
         )
 
 

flyte/_task_environment.py

@@ -53,7 +53,7 @@ class TaskEnvironment(Environment):
     :param name: Name of the environment
     :param image: Docker image to use for the environment. If set to "auto", will use the default image.
     :param resources: Resources to allocate for the environment.
-    :param env: Environment variables to set for the environment.
+    :param env_vars: Environment variables to set for the environment.
     :param secrets: Secrets to inject into the environment.
     :param depends_on: Environment dependencies to hint, so when you deploy the environment,
         the dependencies are also deployed. This is useful when you have a set of environments
@@ -80,7 +80,7 @@ class TaskEnvironment(Environment):
         name: str,
         image: Optional[Union[str, Image, Literal["auto"]]] = None,
         resources: Optional[Resources] = None,
-        env: Optional[Dict[str, str]] = None,
+        env_vars: Optional[Dict[str, str]] = None,
         secrets: Optional[SecretRequest] = None,
         depends_on: Optional[List[Environment]] = None,
         **kwargs: Any,
@@ -93,7 +93,7 @@ class TaskEnvironment(Environment):
         :param name: The name of the environment.
         :param image: The image to use for the environment.
         :param resources: The resources to allocate for the environment.
-        :param env: The environment variables to set for the environment.
+        :param env_vars: The environment variables to set for the environment.
         :param secrets: The secrets to inject into the environment.
         :param depends_on: The environment dependencies to hint, so when you deploy the environment,
             the dependencies are also deployed. This is useful when you have a set of environments
@@ -119,8 +119,8 @@ class TaskEnvironment(Environment):
             kwargs["resources"] = resources
         if cache is not None:
             kwargs["cache"] = cache
-        if env is not None:
-            kwargs["env"] = env
+        if env_vars is not None:
+            kwargs["env_vars"] = env_vars
         if reusable_set:
             kwargs["reusable"] = reusable
         if secrets is not None:
@@ -133,12 +133,11 @@ class TaskEnvironment(Environment):
         self,
         _func=None,
         *,
-        name: Optional[str] = None,
+        short_name: Optional[str] = None,
         cache: CacheRequest | None = None,
         retries: Union[int, RetryStrategy] = 0,
         timeout: Union[timedelta, int] = 0,
         docs: Optional[Documentation] = None,
-        secrets: Optional[SecretRequest] = None,
         pod_template: Optional[Union[str, "V1PodTemplate"]] = None,
         report: bool = False,
         max_inline_io_bytes: int = MAX_INLINE_IO_BYTES,
@@ -148,12 +147,11 @@ class TaskEnvironment(Environment):
 
         :param _func: Optional The function to decorate. If not provided, the decorator will return a callable that
         accepts a function to be decorated.
-        :param name: Optional A friendly name for the task (defaults to the function name)
+        :param short_name: Optional A friendly name for the task (defaults to the function name)
         :param cache: Optional The cache policy for the task, defaults to auto, which will cache the results of the
         task.
         :param retries: Optional The number of retries for the task, defaults to 0, which means no retries.
         :param docs: Optional The documentation for the task, if not provided the function docstring will be used.
-        :param secrets: Optional The secrets that will be injected into the task at runtime.
         :param timeout: Optional The timeout for the task.
         :param pod_template: Optional The pod template for the task, if not provided the default pod template will be
         used.
@@ -168,7 +166,7 @@ class TaskEnvironment(Environment):
                 raise ValueError("Cannot set pod_template when environment is reusable.")
 
         def decorator(func: FunctionTypes) -> AsyncFunctionTaskTemplate[P, R]:
-            friendly_name = name or func.__name__
+            short = short_name or func.__name__
             task_name = self.name + "." + func.__name__
 
             if not inspect.iscoroutinefunction(func) and self.reusable is not None:
@@ -203,13 +201,13 @@ class TaskEnvironment(Environment):
                 timeout=timeout,
                 reusable=self.reusable,
                 docs=docs,
-                env=self.env,
-                secrets=secrets or self.secrets,
+                env_vars=self.env_vars,
+                secrets=self.secrets,
                 pod_template=pod_template or self.pod_template,
                 parent_env=weakref.ref(self),
                 interface=NativeInterface.from_callable(func),
                 report=report,
-                friendly_name=friendly_name,
+                short_name=short,
                 plugin_config=self.plugin_config,
                 max_inline_io_bytes=max_inline_io_bytes,
             )

flyte/_tools.py

@@ -1,6 +1,3 @@
-import os
-
-
 def ipython_check() -> bool:
     """
     Check if interface is launching from iPython (not colab)
@@ -17,16 +14,6 @@ def ipython_check() -> bool:
     return is_ipython
 
 
-def is_in_cluster() -> bool:
-    """
-    Check if the task is running in a cluster
-    :return is_in_cluster (bool): True or False
-    """
-    if os.getenv("_UN_CLS"):
-        return True
-    return False
-
-
 def ipywidgets_check() -> bool:
     """
     Check if the interface is running in IPython with ipywidgets support.

flyte/_version.py

@@ -1,7 +1,14 @@
 # file generated by setuptools-scm
 # don't change, don't track in version control
 
-__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"]
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -9,13 +16,19 @@ if TYPE_CHECKING:
     from typing import Union
 
     VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
 else:
     VERSION_TUPLE = object
+    COMMIT_ID = object
 
 version: str
 __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
 
-__version__ = version = '2.0.0b9'
-__version_tuple__ = version_tuple = (2, 0, 0, 'b9')
+__version__ = version = '2.0.0b14'
+__version_tuple__ = version_tuple = (2, 0, 0, 'b14')
+
+__commit_id__ = commit_id = 'g5efc235bd'

flyte/cli/_build.py

@@ -3,8 +3,7 @@ from pathlib import Path
 from types import ModuleType
 from typing import Any, Dict, List, cast
 
-import click
-from click import Context
+import rich_click as click
 
 import flyte
 
@@ -44,7 +43,7 @@ class BuildEnvCommand(click.Command):
         self.build_args = build_args
         super().__init__(*args, **kwargs)
 
-    def invoke(self, ctx: Context):
+    def invoke(self, ctx: click.Context):
         from rich.console import Console
 
         console = Console()

flyte/cli/_common.py

@@ -316,11 +316,23 @@ class FileGroup(GroupBase):
     def files(self):
         if self._files is None:
             directory = self._dir or Path(".").absolute()
-            self._files = [os.fspath(p) for p in directory.glob("*.py") if p.name != "__init__.py"]
-            if not self._files:
-                self._files = [os.fspath(".")] + [
-                    os.fspath(p.name) for p in directory.iterdir() if not p.name.startswith(("_", ".")) and p.is_dir()
+            # add python files
+            _files = [os.fspath(p) for p in directory.glob("*.py") if p.name != "__init__.py"]
+
+            # add directories
+            _files.extend(
+                [
+                    os.fspath(directory / p.name)
+                    for p in directory.iterdir()
+                    if not p.name.startswith(("_", ".")) and p.is_dir()
                 ]
+            )
+
+            # files that are in the current directory or subdirectories of the
+            # current directory should be displayed as relative paths
+            self._files = [
+                str(Path(f).relative_to(Path.cwd())) if Path(f).is_relative_to(Path.cwd()) else f for f in _files
+            ]
         return self._files
 
     def list_commands(self, ctx):
@@ -351,7 +363,6 @@ def format(title: str, vals: Iterable[Any], of: OutputFormat = "table") -> Table
     """
     Get a table from a list of values.
     """
-
     match of:
         case "table-simple":
             return _table_format(Table(title, box=None), vals)