flyte 2.0.0b23__py3-none-any.whl → 2.0.0b25__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of flyte might be problematic. Click here for more details.

Files changed (162) hide show
  1. flyte/__init__.py +11 -2
  2. flyte/_cache/local_cache.py +4 -3
  3. flyte/_code_bundle/_utils.py +3 -3
  4. flyte/_code_bundle/bundle.py +12 -5
  5. flyte/_context.py +4 -1
  6. flyte/_custom_context.py +73 -0
  7. flyte/_deploy.py +31 -7
  8. flyte/_image.py +48 -16
  9. flyte/_initialize.py +69 -26
  10. flyte/_internal/controllers/_local_controller.py +1 -0
  11. flyte/_internal/controllers/_trace.py +1 -1
  12. flyte/_internal/controllers/remote/_action.py +9 -10
  13. flyte/_internal/controllers/remote/_client.py +1 -1
  14. flyte/_internal/controllers/remote/_controller.py +4 -2
  15. flyte/_internal/controllers/remote/_core.py +10 -13
  16. flyte/_internal/controllers/remote/_informer.py +3 -3
  17. flyte/_internal/controllers/remote/_service_protocol.py +7 -7
  18. flyte/_internal/imagebuild/docker_builder.py +45 -59
  19. flyte/_internal/imagebuild/remote_builder.py +51 -11
  20. flyte/_internal/imagebuild/utils.py +51 -3
  21. flyte/_internal/runtime/convert.py +39 -18
  22. flyte/_internal/runtime/io.py +8 -7
  23. flyte/_internal/runtime/resources_serde.py +20 -6
  24. flyte/_internal/runtime/reuse.py +1 -1
  25. flyte/_internal/runtime/task_serde.py +7 -10
  26. flyte/_internal/runtime/taskrunner.py +10 -1
  27. flyte/_internal/runtime/trigger_serde.py +13 -13
  28. flyte/_internal/runtime/types_serde.py +1 -1
  29. flyte/_keyring/file.py +2 -2
  30. flyte/_map.py +65 -13
  31. flyte/_pod.py +2 -2
  32. flyte/_resources.py +175 -31
  33. flyte/_run.py +37 -21
  34. flyte/_task.py +27 -6
  35. flyte/_task_environment.py +37 -10
  36. flyte/_utils/module_loader.py +2 -2
  37. flyte/_version.py +3 -3
  38. flyte/cli/_common.py +47 -5
  39. flyte/cli/_create.py +4 -0
  40. flyte/cli/_deploy.py +8 -0
  41. flyte/cli/_get.py +4 -0
  42. flyte/cli/_params.py +4 -4
  43. flyte/cli/_run.py +50 -7
  44. flyte/cli/_update.py +4 -3
  45. flyte/config/_config.py +2 -0
  46. flyte/config/_internal.py +1 -0
  47. flyte/config/_reader.py +3 -3
  48. flyte/errors.py +1 -1
  49. flyte/extend.py +4 -0
  50. flyte/extras/_container.py +6 -1
  51. flyte/git/_config.py +11 -9
  52. flyte/io/_dataframe/basic_dfs.py +1 -1
  53. flyte/io/_dataframe/dataframe.py +12 -8
  54. flyte/io/_dir.py +48 -15
  55. flyte/io/_file.py +48 -11
  56. flyte/models.py +12 -8
  57. flyte/remote/_action.py +18 -16
  58. flyte/remote/_client/_protocols.py +4 -3
  59. flyte/remote/_client/auth/_channel.py +1 -1
  60. flyte/remote/_client/controlplane.py +4 -8
  61. flyte/remote/_data.py +4 -3
  62. flyte/remote/_logs.py +3 -3
  63. flyte/remote/_run.py +5 -5
  64. flyte/remote/_secret.py +20 -13
  65. flyte/remote/_task.py +7 -8
  66. flyte/remote/_trigger.py +25 -27
  67. flyte/storage/_parallel_reader.py +274 -0
  68. flyte/storage/_storage.py +66 -2
  69. flyte/types/_interface.py +2 -2
  70. flyte/types/_pickle.py +1 -1
  71. flyte/types/_string_literals.py +8 -9
  72. flyte/types/_type_engine.py +25 -17
  73. flyte/types/_utils.py +1 -1
  74. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/METADATA +2 -1
  75. flyte-2.0.0b25.dist-info/RECORD +184 -0
  76. flyte/_protos/__init__.py +0 -0
  77. flyte/_protos/common/authorization_pb2.py +0 -66
  78. flyte/_protos/common/authorization_pb2.pyi +0 -108
  79. flyte/_protos/common/authorization_pb2_grpc.py +0 -4
  80. flyte/_protos/common/identifier_pb2.py +0 -117
  81. flyte/_protos/common/identifier_pb2.pyi +0 -142
  82. flyte/_protos/common/identifier_pb2_grpc.py +0 -4
  83. flyte/_protos/common/identity_pb2.py +0 -48
  84. flyte/_protos/common/identity_pb2.pyi +0 -72
  85. flyte/_protos/common/identity_pb2_grpc.py +0 -4
  86. flyte/_protos/common/list_pb2.py +0 -36
  87. flyte/_protos/common/list_pb2.pyi +0 -71
  88. flyte/_protos/common/list_pb2_grpc.py +0 -4
  89. flyte/_protos/common/policy_pb2.py +0 -37
  90. flyte/_protos/common/policy_pb2.pyi +0 -27
  91. flyte/_protos/common/policy_pb2_grpc.py +0 -4
  92. flyte/_protos/common/role_pb2.py +0 -37
  93. flyte/_protos/common/role_pb2.pyi +0 -53
  94. flyte/_protos/common/role_pb2_grpc.py +0 -4
  95. flyte/_protos/common/runtime_version_pb2.py +0 -28
  96. flyte/_protos/common/runtime_version_pb2.pyi +0 -24
  97. flyte/_protos/common/runtime_version_pb2_grpc.py +0 -4
  98. flyte/_protos/imagebuilder/definition_pb2.py +0 -60
  99. flyte/_protos/imagebuilder/definition_pb2.pyi +0 -153
  100. flyte/_protos/imagebuilder/definition_pb2_grpc.py +0 -4
  101. flyte/_protos/imagebuilder/payload_pb2.py +0 -32
  102. flyte/_protos/imagebuilder/payload_pb2.pyi +0 -21
  103. flyte/_protos/imagebuilder/payload_pb2_grpc.py +0 -4
  104. flyte/_protos/imagebuilder/service_pb2.py +0 -29
  105. flyte/_protos/imagebuilder/service_pb2.pyi +0 -5
  106. flyte/_protos/imagebuilder/service_pb2_grpc.py +0 -66
  107. flyte/_protos/logs/dataplane/payload_pb2.py +0 -100
  108. flyte/_protos/logs/dataplane/payload_pb2.pyi +0 -177
  109. flyte/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
  110. flyte/_protos/secret/definition_pb2.py +0 -49
  111. flyte/_protos/secret/definition_pb2.pyi +0 -93
  112. flyte/_protos/secret/definition_pb2_grpc.py +0 -4
  113. flyte/_protos/secret/payload_pb2.py +0 -62
  114. flyte/_protos/secret/payload_pb2.pyi +0 -94
  115. flyte/_protos/secret/payload_pb2_grpc.py +0 -4
  116. flyte/_protos/secret/secret_pb2.py +0 -38
  117. flyte/_protos/secret/secret_pb2.pyi +0 -6
  118. flyte/_protos/secret/secret_pb2_grpc.py +0 -198
  119. flyte/_protos/validate/validate/validate_pb2.py +0 -76
  120. flyte/_protos/workflow/common_pb2.py +0 -38
  121. flyte/_protos/workflow/common_pb2.pyi +0 -63
  122. flyte/_protos/workflow/common_pb2_grpc.py +0 -4
  123. flyte/_protos/workflow/environment_pb2.py +0 -29
  124. flyte/_protos/workflow/environment_pb2.pyi +0 -12
  125. flyte/_protos/workflow/environment_pb2_grpc.py +0 -4
  126. flyte/_protos/workflow/node_execution_service_pb2.py +0 -26
  127. flyte/_protos/workflow/node_execution_service_pb2.pyi +0 -4
  128. flyte/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
  129. flyte/_protos/workflow/queue_service_pb2.py +0 -117
  130. flyte/_protos/workflow/queue_service_pb2.pyi +0 -182
  131. flyte/_protos/workflow/queue_service_pb2_grpc.py +0 -206
  132. flyte/_protos/workflow/run_definition_pb2.py +0 -123
  133. flyte/_protos/workflow/run_definition_pb2.pyi +0 -354
  134. flyte/_protos/workflow/run_definition_pb2_grpc.py +0 -4
  135. flyte/_protos/workflow/run_logs_service_pb2.py +0 -41
  136. flyte/_protos/workflow/run_logs_service_pb2.pyi +0 -28
  137. flyte/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
  138. flyte/_protos/workflow/run_service_pb2.py +0 -147
  139. flyte/_protos/workflow/run_service_pb2.pyi +0 -203
  140. flyte/_protos/workflow/run_service_pb2_grpc.py +0 -480
  141. flyte/_protos/workflow/state_service_pb2.py +0 -67
  142. flyte/_protos/workflow/state_service_pb2.pyi +0 -76
  143. flyte/_protos/workflow/state_service_pb2_grpc.py +0 -138
  144. flyte/_protos/workflow/task_definition_pb2.py +0 -86
  145. flyte/_protos/workflow/task_definition_pb2.pyi +0 -105
  146. flyte/_protos/workflow/task_definition_pb2_grpc.py +0 -4
  147. flyte/_protos/workflow/task_service_pb2.py +0 -61
  148. flyte/_protos/workflow/task_service_pb2.pyi +0 -62
  149. flyte/_protos/workflow/task_service_pb2_grpc.py +0 -138
  150. flyte/_protos/workflow/trigger_definition_pb2.py +0 -66
  151. flyte/_protos/workflow/trigger_definition_pb2.pyi +0 -117
  152. flyte/_protos/workflow/trigger_definition_pb2_grpc.py +0 -4
  153. flyte/_protos/workflow/trigger_service_pb2.py +0 -96
  154. flyte/_protos/workflow/trigger_service_pb2.pyi +0 -110
  155. flyte/_protos/workflow/trigger_service_pb2_grpc.py +0 -281
  156. flyte-2.0.0b23.dist-info/RECORD +0 -262
  157. {flyte-2.0.0b23.data → flyte-2.0.0b25.data}/scripts/debug.py +0 -0
  158. {flyte-2.0.0b23.data → flyte-2.0.0b25.data}/scripts/runtime.py +0 -0
  159. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/WHEEL +0 -0
  160. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/entry_points.txt +0 -0
  161. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/licenses/LICENSE +0 -0
  162. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/top_level.txt +0 -0
flyte/_task.py CHANGED
@@ -20,6 +20,7 @@ from typing import (
20
20
  TypeVar,
21
21
  Union,
22
22
  cast,
23
+ overload,
23
24
  )
24
25
 
25
26
  from flyte._pod import PodTemplate
@@ -37,7 +38,7 @@ from ._timeout import TimeoutType
37
38
  from .models import MAX_INLINE_IO_BYTES, NativeInterface, SerializationContext
38
39
 
39
40
  if TYPE_CHECKING:
40
- from flyteidl.core.tasks_pb2 import DataLoadingConfig
41
+ from flyteidl2.core.tasks_pb2 import DataLoadingConfig
41
42
 
42
43
  from flyte.trigger import Trigger
43
44
 
@@ -48,11 +49,12 @@ R = TypeVar("R") # return type
48
49
 
49
50
  AsyncFunctionType: TypeAlias = Callable[P, Coroutine[Any, Any, R]]
50
51
  SyncFunctionType: TypeAlias = Callable[P, R]
51
- FunctionTypes: TypeAlias = Union[AsyncFunctionType, SyncFunctionType]
52
+ FunctionTypes: TypeAlias = AsyncFunctionType | SyncFunctionType
53
+ F = TypeVar("F", bound=FunctionTypes)
52
54
 
53
55
 
54
56
  @dataclass(kw_only=True)
55
- class TaskTemplate(Generic[P, R]):
57
+ class TaskTemplate(Generic[P, R, F]):
56
58
  """
57
59
  Task template is a template for a task that can be executed. It defines various parameters for the task, which
58
60
  can be defined statically at the time of task definition or dynamically at the time of task invocation using
@@ -87,6 +89,7 @@ class TaskTemplate(Generic[P, R]):
87
89
  :param pod_template: Optional The pod template to use for the task.
88
90
  :param report: Optional Whether to report the task execution to the Flyte console, defaults to False.
89
91
  :param queue: Optional The queue to use for the task. If not provided, the default queue will be used.
92
+ :param debuggable: Optional Whether the task supports debugging capabilities, defaults to False.
90
93
  """
91
94
 
92
95
  name: str
@@ -107,8 +110,10 @@ class TaskTemplate(Generic[P, R]):
107
110
  pod_template: Optional[Union[str, PodTemplate]] = None
108
111
  report: bool = False
109
112
  queue: Optional[str] = None
113
+ debuggable: bool = False
110
114
 
111
115
  parent_env: Optional[weakref.ReferenceType[TaskEnvironment]] = None
116
+ parent_env_name: Optional[str] = None
112
117
  ref: bool = field(default=False, init=False, repr=False, compare=False)
113
118
  max_inline_io_bytes: int = MAX_INLINE_IO_BYTES
114
119
  triggers: Tuple[Trigger, ...] = field(default_factory=tuple)
@@ -225,6 +230,14 @@ class TaskTemplate(Generic[P, R]):
225
230
  def native_interface(self) -> NativeInterface:
226
231
  return self.interface
227
232
 
233
+ @overload
234
+ async def aio(self: TaskTemplate[P, R, SyncFunctionType], *args: P.args, **kwargs: P.kwargs) -> R: ...
235
+
236
+ @overload
237
+ async def aio(
238
+ self: TaskTemplate[P, R, AsyncFunctionType], *args: P.args, **kwargs: P.kwargs
239
+ ) -> Coroutine[Any, Any, R]: ...
240
+
228
241
  async def aio(self, *args: P.args, **kwargs: P.kwargs) -> Coroutine[Any, Any, R] | R:
229
242
  """
230
243
  The aio function allows executing "sync" tasks, in an async context. This helps with migrating v1 defined sync
@@ -248,7 +261,6 @@ class TaskTemplate(Generic[P, R]):
248
261
  :param kwargs:
249
262
  :return:
250
263
  """
251
-
252
264
  ctx = internal_ctx()
253
265
  if ctx.is_task_context():
254
266
  from ._internal.controllers import get_controller
@@ -273,6 +285,14 @@ class TaskTemplate(Generic[P, R]):
273
285
  # even for synchronous tasks. This is to support migration.
274
286
  return self.forward(*args, **kwargs)
275
287
 
288
+ @overload
289
+ def __call__(self: TaskTemplate[P, R, SyncFunctionType], *args: P.args, **kwargs: P.kwargs) -> R: ...
290
+
291
+ @overload
292
+ def __call__(
293
+ self: TaskTemplate[P, R, AsyncFunctionType], *args: P.args, **kwargs: P.kwargs
294
+ ) -> Coroutine[Any, Any, R]: ...
295
+
276
296
  def __call__(self, *args: P.args, **kwargs: P.kwargs) -> Coroutine[Any, Any, R] | R:
277
297
  """
278
298
  This is the entrypoint for an async function task at runtime. It will be called during an execution.
@@ -424,14 +444,15 @@ class TaskTemplate(Generic[P, R]):
424
444
 
425
445
 
426
446
  @dataclass(kw_only=True)
427
- class AsyncFunctionTaskTemplate(TaskTemplate[P, R]):
447
+ class AsyncFunctionTaskTemplate(TaskTemplate[P, R, F]):
428
448
  """
429
449
  A task template that wraps an asynchronous functions. This is automatically created when an asynchronous function
430
450
  is decorated with the task decorator.
431
451
  """
432
452
 
433
- func: FunctionTypes
453
+ func: F
434
454
  plugin_config: Optional[Any] = None # This is used to pass plugin specific configuration
455
+ debuggable: bool = True
435
456
 
436
457
  def __post_init__(self):
437
458
  super().__post_init__()
@@ -15,6 +15,7 @@ from typing import (
15
15
  Tuple,
16
16
  Union,
17
17
  cast,
18
+ overload,
18
19
  )
19
20
 
20
21
  import rich.repr
@@ -33,7 +34,7 @@ from ._trigger import Trigger
33
34
  from .models import MAX_INLINE_IO_BYTES, NativeInterface
34
35
 
35
36
  if TYPE_CHECKING:
36
- from ._task import FunctionTypes, P, R
37
+ from ._task import F, P, R
37
38
 
38
39
 
39
40
  @rich.repr.auto
@@ -150,9 +151,9 @@ class TaskEnvironment(Environment):
150
151
  kwargs["interruptible"] = interruptible
151
152
  return replace(self, **kwargs)
152
153
 
154
+ @overload
153
155
  def task(
154
156
  self,
155
- _func: Callable[P, R] | None = None,
156
157
  *,
157
158
  short_name: Optional[str] = None,
158
159
  cache: CacheRequest | None = None,
@@ -165,7 +166,31 @@ class TaskEnvironment(Environment):
165
166
  max_inline_io_bytes: int = MAX_INLINE_IO_BYTES,
166
167
  queue: Optional[str] = None,
167
168
  triggers: Tuple[Trigger, ...] | Trigger = (),
168
- ) -> Union[AsyncFunctionTaskTemplate[P, R], Callable[P, R]]:
169
+ ) -> Callable[[Callable[P, R]], AsyncFunctionTaskTemplate[P, R, Callable[P, R]]]: ...
170
+
171
+ @overload
172
+ def task(
173
+ self,
174
+ _func: Callable[P, R],
175
+ /,
176
+ ) -> AsyncFunctionTaskTemplate[P, R, Callable[P, R]]: ...
177
+
178
+ def task(
179
+ self,
180
+ _func: F | None = None,
181
+ *,
182
+ short_name: Optional[str] = None,
183
+ cache: CacheRequest | None = None,
184
+ retries: Union[int, RetryStrategy] = 0,
185
+ timeout: Union[timedelta, int] = 0,
186
+ docs: Optional[Documentation] = None,
187
+ pod_template: Optional[Union[str, PodTemplate]] = None,
188
+ report: bool = False,
189
+ interruptible: bool | None = None,
190
+ max_inline_io_bytes: int = MAX_INLINE_IO_BYTES,
191
+ queue: Optional[str] = None,
192
+ triggers: Tuple[Trigger, ...] | Trigger = (),
193
+ ) -> Callable[[F], AsyncFunctionTaskTemplate[P, R, F]] | AsyncFunctionTaskTemplate[P, R, F]:
169
194
  """
170
195
  Decorate a function to be a task.
171
196
 
@@ -189,13 +214,13 @@ class TaskEnvironment(Environment):
189
214
 
190
215
  :return: A TaskTemplate that can be used to deploy the task.
191
216
  """
192
- from ._task import P, R
217
+ from ._task import F, P, R
193
218
 
194
219
  if self.reusable is not None:
195
220
  if pod_template is not None:
196
221
  raise ValueError("Cannot set pod_template when environment is reusable.")
197
222
 
198
- def decorator(func: FunctionTypes) -> AsyncFunctionTaskTemplate[P, R]:
223
+ def decorator(func: F) -> AsyncFunctionTaskTemplate[P, R, F]:
199
224
  short = short_name or func.__name__
200
225
  task_name = self.name + "." + func.__name__
201
226
 
@@ -209,7 +234,7 @@ class TaskEnvironment(Environment):
209
234
  if self.plugin_config is not None:
210
235
  from flyte.extend import TaskPluginRegistry
211
236
 
212
- task_template_class: type[AsyncFunctionTaskTemplate[P, R]] | None = TaskPluginRegistry.find(
237
+ task_template_class: type[AsyncFunctionTaskTemplate[P, R, F]] | None = TaskPluginRegistry.find(
213
238
  config_type=type(self.plugin_config)
214
239
  )
215
240
  if task_template_class is None:
@@ -218,9 +243,9 @@ class TaskEnvironment(Environment):
218
243
  f"Please register a plugin using flyte.extend.TaskPluginRegistry.register() api."
219
244
  )
220
245
  else:
221
- task_template_class = AsyncFunctionTaskTemplate[P, R]
246
+ task_template_class = AsyncFunctionTaskTemplate[P, R, F]
222
247
 
223
- task_template_class = cast(type[AsyncFunctionTaskTemplate[P, R]], task_template_class)
248
+ task_template_class = cast(type[AsyncFunctionTaskTemplate[P, R, F]], task_template_class)
224
249
  tmpl = task_template_class(
225
250
  func=func,
226
251
  name=task_name,
@@ -235,6 +260,7 @@ class TaskEnvironment(Environment):
235
260
  secrets=self.secrets,
236
261
  pod_template=pod_template or self.pod_template,
237
262
  parent_env=weakref.ref(self),
263
+ parent_env_name=self.name,
238
264
  interface=NativeInterface.from_callable(func),
239
265
  report=report,
240
266
  short_name=short,
@@ -248,8 +274,8 @@ class TaskEnvironment(Environment):
248
274
  return tmpl
249
275
 
250
276
  if _func is None:
251
- return cast(AsyncFunctionTaskTemplate, decorator)
252
- return cast(AsyncFunctionTaskTemplate, decorator(_func))
277
+ return cast(Callable[[F], AsyncFunctionTaskTemplate[P, R, F]], decorator)
278
+ return cast(AsyncFunctionTaskTemplate[P, R, F], decorator(_func))
253
279
 
254
280
  @property
255
281
  def tasks(self) -> Dict[str, TaskTemplate]:
@@ -286,4 +312,5 @@ class TaskEnvironment(Environment):
286
312
  for t in tasks:
287
313
  env._tasks[t.name] = t
288
314
  t.parent_env = weakref.ref(env)
315
+ t.parent_env_name = name
289
316
  return env
@@ -5,8 +5,6 @@ import sys
5
5
  from pathlib import Path
6
6
  from typing import List, Tuple
7
7
 
8
- from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn, TimeRemainingColumn
9
-
10
8
  import flyte.errors
11
9
 
12
10
 
@@ -18,6 +16,8 @@ def load_python_modules(path: Path, recursive: bool = False) -> Tuple[List[str],
18
16
  :param recursive: If True, load modules recursively from subdirectories
19
17
  :return: List of loaded module names, and list of file paths that failed to load
20
18
  """
19
+ from rich.progress import BarColumn, Progress, TextColumn, TimeElapsedColumn, TimeRemainingColumn
20
+
21
21
  loaded_modules = []
22
22
  failed_paths = []
23
23
 
flyte/_version.py CHANGED
@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
28
28
  commit_id: COMMIT_ID
29
29
  __commit_id__: COMMIT_ID
30
30
 
31
- __version__ = version = '2.0.0b23'
32
- __version_tuple__ = version_tuple = (2, 0, 0, 'b23')
31
+ __version__ = version = '2.0.0b25'
32
+ __version_tuple__ = version_tuple = (2, 0, 0, 'b25')
33
33
 
34
- __commit_id__ = commit_id = 'g52dc01de2'
34
+ __commit_id__ = commit_id = 'gf704e4b75'
flyte/cli/_common.py CHANGED
@@ -7,6 +7,7 @@ import os
7
7
  import sys
8
8
  from abc import abstractmethod
9
9
  from dataclasses import dataclass, replace
10
+ from functools import lru_cache
10
11
  from pathlib import Path
11
12
  from types import MappingProxyType, ModuleType
12
13
  from typing import Any, Dict, Iterable, List, Literal, Optional
@@ -20,6 +21,7 @@ from rich.pretty import pretty_repr
20
21
  from rich.table import Table
21
22
  from rich.traceback import Traceback
22
23
 
24
+ import flyte.config
23
25
  import flyte.errors
24
26
  from flyte.config import Config
25
27
 
@@ -112,13 +114,19 @@ class CLIConfig:
112
114
  """
113
115
  return replace(self, **kwargs)
114
116
 
115
- def init(self, project: str | None = None, domain: str | None = None, root_dir: str | None = None):
117
+ def init(
118
+ self,
119
+ project: str | None = None,
120
+ domain: str | None = None,
121
+ root_dir: str | None = None,
122
+ images: tuple[str, ...] | None = None,
123
+ ):
116
124
  from flyte.config._config import TaskConfig
117
125
 
118
126
  task_cfg = TaskConfig(
119
127
  org=self.org or self.config.task.org,
120
- project=project or self.config.task.project,
121
- domain=domain or self.config.task.domain,
128
+ project=project if project is not None else self.config.task.project,
129
+ domain=domain if domain is not None else self.config.task.domain,
122
130
  )
123
131
 
124
132
  kwargs: Dict[str, Any] = {}
@@ -132,7 +140,7 @@ class CLIConfig:
132
140
 
133
141
  updated_config = self.config.with_params(platform_cfg, task_cfg)
134
142
 
135
- flyte.init_from_config(updated_config, log_level=self.log_level, root_dir=root_dir)
143
+ flyte.init_from_config(updated_config, log_level=self.log_level, root_dir=root_dir, images=images)
136
144
 
137
145
 
138
146
  class InvokeBaseMixin:
@@ -355,7 +363,7 @@ def _table_format(table: Table, vals: Iterable[Any]) -> Table:
355
363
  if headers is None:
356
364
  headers = [k for k, _ in o]
357
365
  for h in headers:
358
- table.add_column(h.capitalize())
366
+ table.add_column(h.capitalize(), no_wrap=True if "name" in h.casefold() else False)
359
367
  table.add_row(*[str(v) for _, v in o])
360
368
  return table
361
369
 
@@ -375,6 +383,7 @@ def format(title: str, vals: Iterable[Any], of: OutputFormat = "table") -> Table
375
383
  header_style=HEADER_STYLE,
376
384
  show_header=True,
377
385
  border_style=PREFERRED_BORDER_COLOR,
386
+ expand=True,
378
387
  ),
379
388
  vals,
380
389
  )
@@ -408,3 +417,36 @@ def get_console() -> Console:
408
417
  Get a console that is configured to use colors if the terminal supports it.
409
418
  """
410
419
  return Console(color_system="auto", force_terminal=True, width=120)
420
+
421
+
422
+ def parse_images(cfg: Config, values: tuple[str, ...] | None) -> None:
423
+ """
424
+ Parse image values and update the config.
425
+
426
+ Args:
427
+ cfg: The Config object to write images to
428
+ values: List of image strings in format "imagename=imageuri" or just "imageuri"
429
+ """
430
+ if values is None:
431
+ return
432
+ for value in values:
433
+ if "=" in value:
434
+ image_name, image_uri = value.split("=", 1)
435
+ cfg.image.image_refs[image_name] = image_uri
436
+ else:
437
+ # If no name specified, use "default" as the name
438
+ cfg.image.image_refs["default"] = value
439
+
440
+
441
+ @lru_cache()
442
+ def initialize_config(
443
+ ctx: click.Context, project: str, domain: str, root_dir: str | None = None, images: tuple[str, ...] | None = None
444
+ ):
445
+ obj: CLIConfig | None = ctx.obj
446
+ if obj is None:
447
+ import flyte.config
448
+
449
+ obj = CLIConfig(flyte.config.auto(), ctx)
450
+
451
+ obj.init(project, domain, root_dir, images)
452
+ return obj
flyte/cli/_create.py CHANGED
@@ -79,6 +79,10 @@ def secret(
79
79
  """
80
80
  from flyte.remote import Secret
81
81
 
82
+ # todo: remove this hack when secrets creation more easily distinguishes between org and project/domain level
83
+ # (and domain level) secrets
84
+ project = "" if project is None else project
85
+ domain = "" if domain is None else domain
82
86
  cfg.init(project, domain)
83
87
  if from_file:
84
88
  with open(from_file, "rb") as f:
flyte/cli/_deploy.py CHANGED
@@ -189,6 +189,14 @@ class EnvPerFileGroup(common.ObjectsPerFileGroup):
189
189
  def _filter_objects(self, module: ModuleType) -> Dict[str, Any]:
190
190
  return {k: v for k, v in module.__dict__.items() if isinstance(v, flyte.Environment)}
191
191
 
192
+ def list_commands(self, ctx):
193
+ common.initialize_config(ctx, self.deploy_args.project, self.deploy_args.domain, self.deploy_args.root_dir)
194
+ return super().list_commands(ctx)
195
+
196
+ def get_command(self, ctx, obj_name):
197
+ common.initialize_config(ctx, self.deploy_args.project, self.deploy_args.domain, self.deploy_args.root_dir)
198
+ return super().get_command(ctx, obj_name)
199
+
192
200
  def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command:
193
201
  obj = cast(flyte.Environment, obj)
194
202
  return DeployEnvCommand(
flyte/cli/_get.py CHANGED
@@ -246,6 +246,10 @@ def secret(
246
246
  """
247
247
  Get a list of all secrets, or details of a specific secret by name.
248
248
  """
249
+ if project is None:
250
+ project = ""
251
+ if domain is None:
252
+ domain = ""
249
253
  cfg.init(project=project, domain=domain)
250
254
 
251
255
  console = common.get_console()
flyte/cli/_params.py CHANGED
@@ -15,9 +15,9 @@ from typing import get_args
15
15
  import rich_click as click
16
16
  import yaml
17
17
  from click import Parameter
18
- from flyteidl.core.interface_pb2 import Variable
19
- from flyteidl.core.literals_pb2 import Literal
20
- from flyteidl.core.types_pb2 import BlobType, LiteralType, SimpleType
18
+ from flyteidl2.core.interface_pb2 import Variable
19
+ from flyteidl2.core.literals_pb2 import Literal
20
+ from flyteidl2.core.types_pb2 import BlobType, LiteralType, SimpleType
21
21
  from google.protobuf.json_format import MessageToDict
22
22
  from mashumaro.codecs.json import JSONEncoder
23
23
 
@@ -505,7 +505,7 @@ def to_click_option(
505
505
  This handles converting workflow input types to supported click parameters with callbacks to initialize
506
506
  the input values to their expected types.
507
507
  """
508
- from flyteidl.core.types_pb2 import SimpleType
508
+ from flyteidl2.core.types_pb2 import SimpleType
509
509
 
510
510
  if input_name != input_name.lower():
511
511
  # Click does not support uppercase option names: https://github.com/pallets/click/issues/837
flyte/cli/_run.py CHANGED
@@ -15,7 +15,7 @@ from .._code_bundle._utils import CopyFiles
15
15
  from .._task import TaskTemplate
16
16
  from ..remote import Run
17
17
  from . import _common as common
18
- from ._common import CLIConfig
18
+ from ._common import CLIConfig, initialize_config
19
19
  from ._params import to_click_option
20
20
 
21
21
  RUN_REMOTE_CMD = "deployed-task"
@@ -43,7 +43,7 @@ def _list_tasks(
43
43
  ) -> list[str]:
44
44
  import flyte.remote
45
45
 
46
- _initialize_config(ctx, project, domain)
46
+ common.initialize_config(ctx, project, domain)
47
47
  return [task.name for task in flyte.remote.Task.listall(by_task_name=by_task_name, by_task_env=by_task_env)]
48
48
 
49
49
 
@@ -108,6 +108,17 @@ class RunArguments:
108
108
  )
109
109
  },
110
110
  )
111
+ image: List[str] = field(
112
+ default_factory=list,
113
+ metadata={
114
+ "click.option": click.Option(
115
+ ["--image"],
116
+ type=str,
117
+ multiple=True,
118
+ help="Image to be used in the run. Format: imagename=imageuri. Can be specified multiple times.",
119
+ )
120
+ },
121
+ )
111
122
 
112
123
  @classmethod
113
124
  def from_dict(cls, d: Dict[str, Any]) -> RunArguments:
@@ -130,7 +141,9 @@ class RunTaskCommand(click.RichCommand):
130
141
  super().__init__(obj_name, *args, **kwargs)
131
142
 
132
143
  def invoke(self, ctx: click.Context):
133
- obj: CLIConfig = _initialize_config(ctx, self.run_args.project, self.run_args.domain, self.run_args.root_dir)
144
+ obj: CLIConfig = initialize_config(
145
+ ctx, self.run_args.project, self.run_args.domain, self.run_args.root_dir, tuple(self.run_args.image) or None
146
+ )
134
147
 
135
148
  async def _run():
136
149
  import flyte
@@ -205,6 +218,14 @@ class TaskPerFileGroup(common.ObjectsPerFileGroup):
205
218
  def _filter_objects(self, module: ModuleType) -> Dict[str, Any]:
206
219
  return {k: v for k, v in module.__dict__.items() if isinstance(v, TaskTemplate)}
207
220
 
221
+ def list_commands(self, ctx):
222
+ common.initialize_config(ctx, self.run_args.project, self.run_args.domain, self.run_args.root_dir)
223
+ return super().list_commands(ctx)
224
+
225
+ def get_command(self, ctx, obj_name):
226
+ common.initialize_config(ctx, self.run_args.project, self.run_args.domain, self.run_args.root_dir)
227
+ return super().get_command(ctx, obj_name)
228
+
208
229
  def _get_command_for_obj(self, ctx: click.Context, obj_name: str, obj: Any) -> click.Command:
209
230
  obj = cast(TaskTemplate, obj)
210
231
  return RunTaskCommand(
@@ -224,10 +245,11 @@ class RunReferenceTaskCommand(click.RichCommand):
224
245
  super().__init__(*args, **kwargs)
225
246
 
226
247
  def invoke(self, ctx: click.Context):
227
- obj: CLIConfig = _initialize_config(ctx, self.run_args.project, self.run_args.domain)
248
+ obj: CLIConfig = common.initialize_config(
249
+ ctx, self.run_args.project, self.run_args.domain, self.run_args.root_dir, tuple(self.run_args.image) or None
250
+ )
228
251
 
229
252
  async def _run():
230
- import flyte
231
253
  import flyte.remote
232
254
 
233
255
  task = flyte.remote.Task.get(self.task_name, version=self.version, auto_version="latest")
@@ -262,7 +284,7 @@ class RunReferenceTaskCommand(click.RichCommand):
262
284
  import flyte.remote
263
285
  from flyte._internal.runtime.types_serde import transform_native_to_typed_interface
264
286
 
265
- _initialize_config(ctx, self.run_args.project, self.run_args.domain)
287
+ common.initialize_config(ctx, self.run_args.project, self.run_args.domain)
266
288
 
267
289
  task = flyte.remote.Task.get(self.task_name, auto_version="latest")
268
290
  task_details = task.fetch()
@@ -408,7 +430,6 @@ class TaskFiles(common.FileGroup):
408
430
 
409
431
  def get_command(self, ctx, cmd_name):
410
432
  run_args = RunArguments.from_dict(ctx.params)
411
-
412
433
  if cmd_name == RUN_REMOTE_CMD:
413
434
  return ReferenceTaskGroup(
414
435
  name=cmd_name,
@@ -455,6 +476,28 @@ Flyte environment:
455
476
  flyte run --local hello.py my_task --arg1 value1 --arg2 value2
456
477
  ```
457
478
 
479
+ You can provide image mappings with `--image` flag. This allows you to specify
480
+ the image URI for the task environment during CLI execution without changing
481
+ the code. Any images defined with `Image.from_ref_name("name")` will resolve to the
482
+ corresponding URIs you specify here.
483
+
484
+ ```bash
485
+ flyte run hello.py my_task --image my_image=ghcr.io/myorg/my-image:v1.0
486
+ ```
487
+
488
+ If the image name is not provided, it is regarded as a default image and will
489
+ be used when no image is specified in TaskEnvironment:
490
+
491
+ ```bash
492
+ flyte run hello.py my_task --image ghcr.io/myorg/default-image:latest
493
+ ```
494
+
495
+ You can specify multiple image arguments:
496
+
497
+ ```bash
498
+ flyte run hello.py my_task --image ghcr.io/org/default:latest --image gpu=ghcr.io/org/gpu:v2.0
499
+ ```
500
+
458
501
  To run tasks that you've already deployed to Flyte, use the {RUN_REMOTE_CMD} command:
459
502
 
460
503
  ```bash
flyte/cli/_update.py CHANGED
@@ -15,7 +15,7 @@ def update():
15
15
  @update.command("trigger", cls=common.CommandBase)
16
16
  @click.argument("name", type=str)
17
17
  @click.argument("task_name", type=str)
18
- @click.option("--activate/--deactivate", default=None, required=True, help="Activate or deactivate the trigger.")
18
+ @click.option("--activate/--deactivate", required=True, help="Activate or deactivate the trigger.")
19
19
  @click.pass_obj
20
20
  def trigger(cfg: common.CLIConfig, name: str, task_name: str, activate: bool, project: str | None, domain: str | None):
21
21
  """
@@ -31,6 +31,7 @@ def trigger(cfg: common.CLIConfig, name: str, task_name: str, activate: bool, pr
31
31
  """
32
32
  cfg.init(project, domain)
33
33
  console = common.get_console()
34
- with console.status(f"Updating trigger {name} for task {task_name} to {'active' if activate else 'deactive'}..."):
34
+ to_state = "active" if activate else "deactivate"
35
+ with console.status(f"Updating trigger {name} for task {task_name} to {to_state}..."):
35
36
  remote.Trigger.update(name, task_name, activate)
36
- console.print("Trigger updated.")
37
+ console.print(f"Trigger updated and is set to [fuchsia]{to_state}[/fuchsia]")
flyte/config/_config.py CHANGED
@@ -148,6 +148,7 @@ class ImageConfig(object):
148
148
  """
149
149
 
150
150
  builder: str | None = None
151
+ image_refs: typing.Dict[str, str] = field(default_factory=dict)
151
152
 
152
153
  @classmethod
153
154
  def auto(cls, config_file: typing.Optional[typing.Union[str, ConfigFile]] = None) -> "ImageConfig":
@@ -159,6 +160,7 @@ class ImageConfig(object):
159
160
  config_file = get_config_file(config_file)
160
161
  kwargs: typing.Dict[str, typing.Any] = {}
161
162
  kwargs = set_if_exists(kwargs, "builder", _internal.Image.BUILDER.read(config_file))
163
+ kwargs = set_if_exists(kwargs, "image_refs", _internal.Image.IMAGE_REFS.read(config_file))
162
164
  return ImageConfig(**kwargs)
163
165
 
164
166
 
flyte/config/_internal.py CHANGED
@@ -70,3 +70,4 @@ class Image(object):
70
70
  """
71
71
 
72
72
  BUILDER = ConfigEntry(YamlConfigEntry("image.builder"))
73
+ IMAGE_REFS = ConfigEntry(YamlConfigEntry("image.image_refs"))
flyte/config/_reader.py CHANGED
@@ -138,10 +138,10 @@ class ConfigFile(object):
138
138
  def _config_path_from_git_root() -> pathlib.Path | None:
139
139
  from flyte.git import config_from_root
140
140
 
141
- try:
142
- return config_from_root().source
143
- except RuntimeError:
141
+ config = config_from_root()
142
+ if config is None:
144
143
  return None
144
+ return config.source
145
145
 
146
146
 
147
147
  def resolve_config_path() -> pathlib.Path | None:
flyte/errors.py CHANGED
@@ -174,7 +174,7 @@ class RuntimeDataValidationError(RuntimeUserError):
174
174
 
175
175
  def __init__(self, var: str, e: Exception | str, task_name: str = ""):
176
176
  super().__init__(
177
- "DataValiationError", f"In task {task_name} variable {var}, failed to serialize/deserialize because of {e}"
177
+ "DataValidationError", f"In task {task_name} variable {var}, failed to serialize/deserialize because of {e}"
178
178
  )
179
179
 
180
180
 
flyte/extend.py CHANGED
@@ -1,4 +1,6 @@
1
1
  from ._initialize import is_initialized
2
+ from ._internal.runtime.entrypoints import download_code_bundle
3
+ from ._internal.runtime.resources_serde import get_proto_resources
2
4
  from ._resources import PRIMARY_CONTAINER_DEFAULT_NAME, pod_spec_from_resources
3
5
  from ._task import AsyncFunctionTaskTemplate
4
6
  from ._task_plugins import TaskPluginRegistry
@@ -7,6 +9,8 @@ __all__ = [
7
9
  "PRIMARY_CONTAINER_DEFAULT_NAME",
8
10
  "AsyncFunctionTaskTemplate",
9
11
  "TaskPluginRegistry",
12
+ "download_code_bundle",
13
+ "get_proto_resources",
10
14
  "is_initialized",
11
15
  "pod_spec_from_resources",
12
16
  ]
@@ -2,7 +2,7 @@ import os
2
2
  import pathlib
3
3
  from typing import Any, Dict, List, Literal, Optional, Tuple, Type, Union
4
4
 
5
- from flyteidl.core import tasks_pb2
5
+ from flyteidl2.core import tasks_pb2
6
6
 
7
7
  from flyte import Image, storage
8
8
  from flyte._logging import logger
@@ -83,6 +83,11 @@ class ContainerTask(TaskTemplate):
83
83
  self._image = Image.from_debian_base()
84
84
  else:
85
85
  self._image = Image.from_base(image)
86
+
87
+ if command and any(not isinstance(c, str) for c in command):
88
+ raise ValueError("All elements in the command list must be strings.")
89
+ if arguments and any(not isinstance(a, str) for a in arguments):
90
+ raise ValueError("All elements in the arguments list must be strings.")
86
91
  self._cmd = command
87
92
  self._args = arguments
88
93
  self._input_data_dir = input_data_dir
flyte/git/_config.py CHANGED
@@ -4,16 +4,18 @@ import subprocess
4
4
  import flyte.config
5
5
 
6
6
 
7
- def config_from_root(path: pathlib.Path | str = ".flyte/config.yaml") -> flyte.config.Config:
7
+ def config_from_root(path: pathlib.Path | str = ".flyte/config.yaml") -> flyte.config.Config | None:
8
8
  """Get the config file from the git root directory.
9
9
 
10
10
  By default, the config file is expected to be in `.flyte/config.yaml` in the git root directory.
11
11
  """
12
-
13
- result = subprocess.run(["git", "rev-parse", "--show-toplevel"], check=False, capture_output=True, text=True)
14
- if result.returncode != 0:
15
- raise RuntimeError(f"Failed to get git root directory: {result.stderr}")
16
- root = pathlib.Path(result.stdout.strip())
17
- if not (root / path).exists():
18
- raise RuntimeError(f"Config file {root / path} does not exist")
19
- return flyte.config.auto(root / path)
12
+ try:
13
+ result = subprocess.run(["git", "rev-parse", "--show-toplevel"], check=False, capture_output=True, text=True)
14
+ if result.returncode != 0:
15
+ return None
16
+ root = pathlib.Path(result.stdout.strip())
17
+ if not (root / path).exists():
18
+ return None
19
+ return flyte.config.auto(root / path)
20
+ except Exception:
21
+ return None