flyte 2.0.0b23__py3-none-any.whl → 2.0.0b25__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of flyte might be problematic. Click here for more details.

Files changed (162) hide show
  1. flyte/__init__.py +11 -2
  2. flyte/_cache/local_cache.py +4 -3
  3. flyte/_code_bundle/_utils.py +3 -3
  4. flyte/_code_bundle/bundle.py +12 -5
  5. flyte/_context.py +4 -1
  6. flyte/_custom_context.py +73 -0
  7. flyte/_deploy.py +31 -7
  8. flyte/_image.py +48 -16
  9. flyte/_initialize.py +69 -26
  10. flyte/_internal/controllers/_local_controller.py +1 -0
  11. flyte/_internal/controllers/_trace.py +1 -1
  12. flyte/_internal/controllers/remote/_action.py +9 -10
  13. flyte/_internal/controllers/remote/_client.py +1 -1
  14. flyte/_internal/controllers/remote/_controller.py +4 -2
  15. flyte/_internal/controllers/remote/_core.py +10 -13
  16. flyte/_internal/controllers/remote/_informer.py +3 -3
  17. flyte/_internal/controllers/remote/_service_protocol.py +7 -7
  18. flyte/_internal/imagebuild/docker_builder.py +45 -59
  19. flyte/_internal/imagebuild/remote_builder.py +51 -11
  20. flyte/_internal/imagebuild/utils.py +51 -3
  21. flyte/_internal/runtime/convert.py +39 -18
  22. flyte/_internal/runtime/io.py +8 -7
  23. flyte/_internal/runtime/resources_serde.py +20 -6
  24. flyte/_internal/runtime/reuse.py +1 -1
  25. flyte/_internal/runtime/task_serde.py +7 -10
  26. flyte/_internal/runtime/taskrunner.py +10 -1
  27. flyte/_internal/runtime/trigger_serde.py +13 -13
  28. flyte/_internal/runtime/types_serde.py +1 -1
  29. flyte/_keyring/file.py +2 -2
  30. flyte/_map.py +65 -13
  31. flyte/_pod.py +2 -2
  32. flyte/_resources.py +175 -31
  33. flyte/_run.py +37 -21
  34. flyte/_task.py +27 -6
  35. flyte/_task_environment.py +37 -10
  36. flyte/_utils/module_loader.py +2 -2
  37. flyte/_version.py +3 -3
  38. flyte/cli/_common.py +47 -5
  39. flyte/cli/_create.py +4 -0
  40. flyte/cli/_deploy.py +8 -0
  41. flyte/cli/_get.py +4 -0
  42. flyte/cli/_params.py +4 -4
  43. flyte/cli/_run.py +50 -7
  44. flyte/cli/_update.py +4 -3
  45. flyte/config/_config.py +2 -0
  46. flyte/config/_internal.py +1 -0
  47. flyte/config/_reader.py +3 -3
  48. flyte/errors.py +1 -1
  49. flyte/extend.py +4 -0
  50. flyte/extras/_container.py +6 -1
  51. flyte/git/_config.py +11 -9
  52. flyte/io/_dataframe/basic_dfs.py +1 -1
  53. flyte/io/_dataframe/dataframe.py +12 -8
  54. flyte/io/_dir.py +48 -15
  55. flyte/io/_file.py +48 -11
  56. flyte/models.py +12 -8
  57. flyte/remote/_action.py +18 -16
  58. flyte/remote/_client/_protocols.py +4 -3
  59. flyte/remote/_client/auth/_channel.py +1 -1
  60. flyte/remote/_client/controlplane.py +4 -8
  61. flyte/remote/_data.py +4 -3
  62. flyte/remote/_logs.py +3 -3
  63. flyte/remote/_run.py +5 -5
  64. flyte/remote/_secret.py +20 -13
  65. flyte/remote/_task.py +7 -8
  66. flyte/remote/_trigger.py +25 -27
  67. flyte/storage/_parallel_reader.py +274 -0
  68. flyte/storage/_storage.py +66 -2
  69. flyte/types/_interface.py +2 -2
  70. flyte/types/_pickle.py +1 -1
  71. flyte/types/_string_literals.py +8 -9
  72. flyte/types/_type_engine.py +25 -17
  73. flyte/types/_utils.py +1 -1
  74. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/METADATA +2 -1
  75. flyte-2.0.0b25.dist-info/RECORD +184 -0
  76. flyte/_protos/__init__.py +0 -0
  77. flyte/_protos/common/authorization_pb2.py +0 -66
  78. flyte/_protos/common/authorization_pb2.pyi +0 -108
  79. flyte/_protos/common/authorization_pb2_grpc.py +0 -4
  80. flyte/_protos/common/identifier_pb2.py +0 -117
  81. flyte/_protos/common/identifier_pb2.pyi +0 -142
  82. flyte/_protos/common/identifier_pb2_grpc.py +0 -4
  83. flyte/_protos/common/identity_pb2.py +0 -48
  84. flyte/_protos/common/identity_pb2.pyi +0 -72
  85. flyte/_protos/common/identity_pb2_grpc.py +0 -4
  86. flyte/_protos/common/list_pb2.py +0 -36
  87. flyte/_protos/common/list_pb2.pyi +0 -71
  88. flyte/_protos/common/list_pb2_grpc.py +0 -4
  89. flyte/_protos/common/policy_pb2.py +0 -37
  90. flyte/_protos/common/policy_pb2.pyi +0 -27
  91. flyte/_protos/common/policy_pb2_grpc.py +0 -4
  92. flyte/_protos/common/role_pb2.py +0 -37
  93. flyte/_protos/common/role_pb2.pyi +0 -53
  94. flyte/_protos/common/role_pb2_grpc.py +0 -4
  95. flyte/_protos/common/runtime_version_pb2.py +0 -28
  96. flyte/_protos/common/runtime_version_pb2.pyi +0 -24
  97. flyte/_protos/common/runtime_version_pb2_grpc.py +0 -4
  98. flyte/_protos/imagebuilder/definition_pb2.py +0 -60
  99. flyte/_protos/imagebuilder/definition_pb2.pyi +0 -153
  100. flyte/_protos/imagebuilder/definition_pb2_grpc.py +0 -4
  101. flyte/_protos/imagebuilder/payload_pb2.py +0 -32
  102. flyte/_protos/imagebuilder/payload_pb2.pyi +0 -21
  103. flyte/_protos/imagebuilder/payload_pb2_grpc.py +0 -4
  104. flyte/_protos/imagebuilder/service_pb2.py +0 -29
  105. flyte/_protos/imagebuilder/service_pb2.pyi +0 -5
  106. flyte/_protos/imagebuilder/service_pb2_grpc.py +0 -66
  107. flyte/_protos/logs/dataplane/payload_pb2.py +0 -100
  108. flyte/_protos/logs/dataplane/payload_pb2.pyi +0 -177
  109. flyte/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
  110. flyte/_protos/secret/definition_pb2.py +0 -49
  111. flyte/_protos/secret/definition_pb2.pyi +0 -93
  112. flyte/_protos/secret/definition_pb2_grpc.py +0 -4
  113. flyte/_protos/secret/payload_pb2.py +0 -62
  114. flyte/_protos/secret/payload_pb2.pyi +0 -94
  115. flyte/_protos/secret/payload_pb2_grpc.py +0 -4
  116. flyte/_protos/secret/secret_pb2.py +0 -38
  117. flyte/_protos/secret/secret_pb2.pyi +0 -6
  118. flyte/_protos/secret/secret_pb2_grpc.py +0 -198
  119. flyte/_protos/validate/validate/validate_pb2.py +0 -76
  120. flyte/_protos/workflow/common_pb2.py +0 -38
  121. flyte/_protos/workflow/common_pb2.pyi +0 -63
  122. flyte/_protos/workflow/common_pb2_grpc.py +0 -4
  123. flyte/_protos/workflow/environment_pb2.py +0 -29
  124. flyte/_protos/workflow/environment_pb2.pyi +0 -12
  125. flyte/_protos/workflow/environment_pb2_grpc.py +0 -4
  126. flyte/_protos/workflow/node_execution_service_pb2.py +0 -26
  127. flyte/_protos/workflow/node_execution_service_pb2.pyi +0 -4
  128. flyte/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
  129. flyte/_protos/workflow/queue_service_pb2.py +0 -117
  130. flyte/_protos/workflow/queue_service_pb2.pyi +0 -182
  131. flyte/_protos/workflow/queue_service_pb2_grpc.py +0 -206
  132. flyte/_protos/workflow/run_definition_pb2.py +0 -123
  133. flyte/_protos/workflow/run_definition_pb2.pyi +0 -354
  134. flyte/_protos/workflow/run_definition_pb2_grpc.py +0 -4
  135. flyte/_protos/workflow/run_logs_service_pb2.py +0 -41
  136. flyte/_protos/workflow/run_logs_service_pb2.pyi +0 -28
  137. flyte/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
  138. flyte/_protos/workflow/run_service_pb2.py +0 -147
  139. flyte/_protos/workflow/run_service_pb2.pyi +0 -203
  140. flyte/_protos/workflow/run_service_pb2_grpc.py +0 -480
  141. flyte/_protos/workflow/state_service_pb2.py +0 -67
  142. flyte/_protos/workflow/state_service_pb2.pyi +0 -76
  143. flyte/_protos/workflow/state_service_pb2_grpc.py +0 -138
  144. flyte/_protos/workflow/task_definition_pb2.py +0 -86
  145. flyte/_protos/workflow/task_definition_pb2.pyi +0 -105
  146. flyte/_protos/workflow/task_definition_pb2_grpc.py +0 -4
  147. flyte/_protos/workflow/task_service_pb2.py +0 -61
  148. flyte/_protos/workflow/task_service_pb2.pyi +0 -62
  149. flyte/_protos/workflow/task_service_pb2_grpc.py +0 -138
  150. flyte/_protos/workflow/trigger_definition_pb2.py +0 -66
  151. flyte/_protos/workflow/trigger_definition_pb2.pyi +0 -117
  152. flyte/_protos/workflow/trigger_definition_pb2_grpc.py +0 -4
  153. flyte/_protos/workflow/trigger_service_pb2.py +0 -96
  154. flyte/_protos/workflow/trigger_service_pb2.pyi +0 -110
  155. flyte/_protos/workflow/trigger_service_pb2_grpc.py +0 -281
  156. flyte-2.0.0b23.dist-info/RECORD +0 -262
  157. {flyte-2.0.0b23.data → flyte-2.0.0b25.data}/scripts/debug.py +0 -0
  158. {flyte-2.0.0b23.data → flyte-2.0.0b25.data}/scripts/runtime.py +0 -0
  159. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/WHEEL +0 -0
  160. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/entry_points.txt +0 -0
  161. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/licenses/LICENSE +0 -0
  162. {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/top_level.txt +0 -0
flyte/_initialize.py CHANGED
@@ -3,7 +3,7 @@ from __future__ import annotations
3
3
  import functools
4
4
  import threading
5
5
  import typing
6
- from dataclasses import dataclass, replace
6
+ from dataclasses import dataclass, field, replace
7
7
  from pathlib import Path
8
8
  from typing import TYPE_CHECKING, Callable, List, Literal, Optional, TypeVar
9
9
 
@@ -41,6 +41,7 @@ class _InitConfig(CommonInit):
41
41
  client: Optional[ClientSet] = None
42
42
  storage: Optional[Storage] = None
43
43
  image_builder: "ImageBuildEngine.ImageBuilderType" = "local"
44
+ images: typing.Dict[str, str] = field(default_factory=dict)
44
45
 
45
46
  def replace(self, **kwargs) -> _InitConfig:
46
47
  return replace(self, **kwargs)
@@ -141,6 +142,7 @@ async def init(
141
142
  storage: Storage | None = None,
142
143
  batch_size: int = 1000,
143
144
  image_builder: ImageBuildEngine.ImageBuilderType = "local",
145
+ images: typing.Dict[str, str] | None = None,
144
146
  source_config_path: Optional[Path] = None,
145
147
  ) -> None:
146
148
  """
@@ -170,40 +172,18 @@ async def init(
170
172
  :param ca_cert_file_path: [optional] str Root Cert to be loaded and used to verify admin
171
173
  :param http_proxy_url: [optional] HTTP Proxy to be used for OAuth requests
172
174
  :param rpc_retries: [optional] int Number of times to retry the platform calls
173
- :param audience: oauth2 audience for the token request. This is used to validate the token
174
175
  :param insecure: insecure flag for the client
175
176
  :param storage: Optional blob store (S3, GCS, Azure) configuration if needed to access (i.e. using Minio)
176
177
  :param org: Optional organization override for the client. Should be set by auth instead.
177
178
  :param batch_size: Optional batch size for operations that use listings, defaults to 1000, so limit larger than
178
179
  batch_size will be split into multiple requests.
179
180
  :param image_builder: Optional image builder configuration, if not provided, the default image builder will be used.
181
+ :param images: Optional dict of images that can be used by referencing the image name.
180
182
  :param source_config_path: Optional path to the source configuration file (This is only used for documentation)
181
183
  :return: None
182
184
  """
183
185
  from flyte._utils import get_cwd_editable_install, org_from_endpoint, sanitize_endpoint
184
186
 
185
- if endpoint or api_key:
186
- if project is None:
187
- raise ValueError(
188
- "Project must be provided to initialize the client. "
189
- "Please set 'project' in the 'task' section of your config file, "
190
- "or pass it directly to flyte.init(project='your-project-name')."
191
- )
192
-
193
- if domain is None:
194
- raise ValueError(
195
- "Domain must be provided to initialize the client. "
196
- "Please set 'domain' in the 'task' section of your config file, "
197
- "or pass it directly to flyte.init(domain='your-domain-name')."
198
- )
199
-
200
- if org is None and org_from_endpoint(endpoint) is None:
201
- raise ValueError(
202
- "Organization must be provided to initialize the client. "
203
- "Please set 'org' in the 'task' section of your config file, "
204
- "or pass it directly to flyte.init(org='your-org-name')."
205
- )
206
-
207
187
  _initialize_logger(log_level=log_level)
208
188
 
209
189
  global _init_config # noqa: PLW0603
@@ -238,7 +218,7 @@ async def init(
238
218
  else:
239
219
  logger.info("No editable install found, using current working directory as root directory.")
240
220
  root_dir = Path.cwd()
241
- root_dir = root_dir or get_cwd_editable_install() or Path.cwd()
221
+
242
222
  _init_config = _InitConfig(
243
223
  root_dir=root_dir,
244
224
  project=project,
@@ -248,6 +228,7 @@ async def init(
248
228
  org=org or org_from_endpoint(endpoint),
249
229
  batch_size=batch_size,
250
230
  image_builder=image_builder,
231
+ images=images or {},
251
232
  source_config_path=source_config_path,
252
233
  )
253
234
 
@@ -258,6 +239,7 @@ async def init_from_config(
258
239
  root_dir: Path | None = None,
259
240
  log_level: int | None = None,
260
241
  storage: Storage | None = None,
242
+ images: tuple[str, ...] | None = None,
261
243
  ) -> None:
262
244
  """
263
245
  Initialize the Flyte system using a configuration file or Config object. This method should be called before any
@@ -276,6 +258,7 @@ async def init_from_config(
276
258
  from rich.highlighter import ReprHighlighter
277
259
 
278
260
  import flyte.config as config
261
+ from flyte.cli._common import parse_images
279
262
 
280
263
  cfg: config.Config
281
264
  cfg_path: Optional[Path] = None
@@ -301,6 +284,9 @@ async def init_from_config(
301
284
 
302
285
  logger.info(f"Flyte config initialized as {cfg}", extra={"highlighter": ReprHighlighter()})
303
286
 
287
+ # parse image, this will overwrite the image_refs set in the config file
288
+ parse_images(cfg, images)
289
+
304
290
  await init.aio(
305
291
  org=cfg.task.org,
306
292
  project=cfg.task.project,
@@ -317,6 +303,7 @@ async def init_from_config(
317
303
  root_dir=root_dir,
318
304
  log_level=log_level,
319
305
  image_builder=cfg.image.builder,
306
+ images=cfg.image.image_refs,
320
307
  storage=storage,
321
308
  source_config_path=cfg_path,
322
309
  )
@@ -332,7 +319,7 @@ def _get_init_config() -> Optional[_InitConfig]:
332
319
  return _init_config
333
320
 
334
321
 
335
- def get_common_config() -> CommonInit:
322
+ def get_init_config() -> _InitConfig:
336
323
  """
337
324
  Get the current initialization configuration. Thread-safe implementation.
338
325
 
@@ -490,6 +477,34 @@ def requires_initialization(func: T) -> T:
490
477
  return typing.cast(T, wrapper)
491
478
 
492
479
 
480
+ def require_project_and_domain(func):
481
+ """
482
+ Decorator that ensures the current Flyte configuration defines
483
+ both 'project' and 'domain'. Raises a clear error if not found.
484
+ """
485
+
486
+ @functools.wraps(func)
487
+ def wrapper(*args, **kwargs):
488
+ cfg = get_init_config()
489
+ if cfg.project is None:
490
+ raise ValueError(
491
+ "Project must be provided to initialize the client. "
492
+ "Please set 'project' in the 'task' section of your config file, "
493
+ "or pass it directly to flyte.init(project='your-project-name')."
494
+ )
495
+
496
+ if cfg.domain is None:
497
+ raise ValueError(
498
+ "Domain must be provided to initialize the client. "
499
+ "Please set 'domain' in the 'task' section of your config file, "
500
+ "or pass it directly to flyte.init(domain='your-domain-name')."
501
+ )
502
+
503
+ return func(*args, **kwargs)
504
+
505
+ return wrapper
506
+
507
+
493
508
  async def _init_for_testing(
494
509
  project: str | None = None,
495
510
  domain: str | None = None,
@@ -519,3 +534,31 @@ def replace_client(client):
519
534
 
520
535
  with _init_lock:
521
536
  _init_config = _init_config.replace(client=client)
537
+
538
+
539
+ def current_domain() -> str:
540
+ """
541
+ Returns the current domain from Runtime environment (on the cluster) or from the initialized configuration.
542
+ This is safe to be used during `deploy`, `run` and within `task` code.
543
+
544
+ NOTE: This will not work if you deploy a task to a domain and then run it in another domain.
545
+
546
+ Raises InitializationError if the configuration is not initialized or domain is not set.
547
+ :return: The current domain
548
+ """
549
+ from ._context import ctx
550
+
551
+ tctx = ctx()
552
+ if tctx is not None:
553
+ domain = tctx.action.domain
554
+ if domain is not None:
555
+ return domain
556
+
557
+ cfg = _get_init_config()
558
+ if cfg is None or cfg.domain is None:
559
+ raise InitializationError(
560
+ "DomainNotInitializedError",
561
+ "user",
562
+ "Domain has not been initialized. Call flyte.init() with a valid domain before using this function.",
563
+ )
564
+ return cfg.domain
@@ -186,6 +186,7 @@ class LocalController:
186
186
  tctx = ctx.data.task_context
187
187
  if not tctx:
188
188
  raise flyte.errors.NotInTaskContextError("BadContext", "Task context not initialized")
189
+
189
190
  converted_inputs = convert.Inputs.empty()
190
191
  if _interface.inputs:
191
192
  converted_inputs = await convert.convert_from_native_to_inputs(_interface, *args, **kwargs)
@@ -1,7 +1,7 @@
1
1
  from dataclasses import dataclass, field
2
2
  from typing import Any, Optional
3
3
 
4
- from flyteidl.core import interface_pb2
4
+ from flyteidl2.core import interface_pb2
5
5
 
6
6
  from flyte.models import ActionID, NativeInterface
7
7
 
@@ -3,16 +3,15 @@ from __future__ import annotations
3
3
  from dataclasses import dataclass
4
4
  from typing import Literal, Optional
5
5
 
6
- from flyteidl.core import execution_pb2, interface_pb2
7
- from google.protobuf import timestamp_pb2
8
-
9
- from flyte._protos.common import identifier_pb2
10
- from flyte._protos.workflow import (
11
- queue_service_pb2,
6
+ from flyteidl2.common import identifier_pb2
7
+ from flyteidl2.core import execution_pb2, interface_pb2
8
+ from flyteidl2.task import common_pb2, task_definition_pb2
9
+ from flyteidl2.workflow import (
12
10
  run_definition_pb2,
13
11
  state_service_pb2,
14
- task_definition_pb2,
15
12
  )
13
+ from google.protobuf import timestamp_pb2
14
+
16
15
  from flyte.models import GroupData
17
16
 
18
17
  ActionType = Literal["task", "trace"]
@@ -31,7 +30,7 @@ class Action:
31
30
  friendly_name: str | None = None
32
31
  group: GroupData | None = None
33
32
  task: task_definition_pb2.TaskSpec | None = None
34
- trace: queue_service_pb2.TraceAction | None = None
33
+ trace: run_definition_pb2.TraceAction | None = None
35
34
  inputs_uri: str | None = None
36
35
  run_output_base: str | None = None
37
36
  realized_outputs_uri: str | None = None
@@ -198,12 +197,12 @@ class Action:
198
197
  realized_outputs_uri=outputs_uri,
199
198
  phase=run_definition_pb2.Phase.PHASE_SUCCEEDED,
200
199
  run_output_base=run_output_base,
201
- trace=queue_service_pb2.TraceAction(
200
+ trace=run_definition_pb2.TraceAction(
202
201
  name=friendly_name,
203
202
  phase=run_definition_pb2.Phase.PHASE_SUCCEEDED,
204
203
  start_time=st,
205
204
  end_time=et,
206
- outputs=run_definition_pb2.OutputReferences(
205
+ outputs=common_pb2.OutputReferences(
207
206
  output_uri=outputs_uri,
208
207
  report_uri=report_uri,
209
208
  ),
@@ -1,8 +1,8 @@
1
1
  from __future__ import annotations
2
2
 
3
3
  import grpc.aio
4
+ from flyteidl2.workflow import queue_service_pb2_grpc, state_service_pb2_grpc
4
5
 
5
- from flyte._protos.workflow import queue_service_pb2_grpc, state_service_pb2_grpc
6
6
  from flyte.remote import create_channel
7
7
 
8
8
  from ._service_protocol import QueueService, StateService
@@ -9,6 +9,9 @@ from collections.abc import Callable
9
9
  from pathlib import Path
10
10
  from typing import Any, Awaitable, DefaultDict, Tuple, TypeVar
11
11
 
12
+ from flyteidl2.common import identifier_pb2
13
+ from flyteidl2.workflow import run_definition_pb2
14
+
12
15
  import flyte
13
16
  import flyte.errors
14
17
  import flyte.storage as storage
@@ -22,8 +25,6 @@ from flyte._internal.runtime import convert, io
22
25
  from flyte._internal.runtime.task_serde import translate_task_to_wire
23
26
  from flyte._internal.runtime.types_serde import transform_native_to_typed_interface
24
27
  from flyte._logging import logger
25
- from flyte._protos.common import identifier_pb2
26
- from flyte._protos.workflow import run_definition_pb2
27
28
  from flyte._task import TaskTemplate
28
29
  from flyte._utils.helpers import _selector_policy
29
30
  from flyte.models import MAX_INLINE_IO_BYTES, ActionID, NativeInterface, SerializationContext
@@ -376,6 +377,7 @@ class RemoteController(Controller):
376
377
 
377
378
  func_name = _func.__name__
378
379
  invoke_seq_num = self.generate_task_call_sequence(_func, current_action_id)
380
+
379
381
  inputs = await convert.convert_from_native_to_inputs(_interface, *args, **kwargs)
380
382
  serialized_inputs = inputs.proto_inputs.SerializeToString(deterministic=True)
381
383
  inputs_hash = convert.generate_inputs_hash_from_proto(inputs.proto_inputs)
@@ -9,15 +9,13 @@ from typing import Awaitable, Coroutine, Optional
9
9
 
10
10
  import grpc.aio
11
11
  from aiolimiter import AsyncLimiter
12
+ from flyteidl2.common import identifier_pb2
13
+ from flyteidl2.task import task_definition_pb2
14
+ from flyteidl2.workflow import queue_service_pb2, run_definition_pb2
12
15
  from google.protobuf.wrappers_pb2 import StringValue
13
16
 
14
17
  import flyte.errors
15
18
  from flyte._logging import log, logger
16
- from flyte._protos.common import identifier_pb2
17
- from flyte._protos.workflow import (
18
- queue_service_pb2,
19
- task_definition_pb2,
20
- )
21
19
 
22
20
  from ._action import Action
23
21
  from ._informer import InformerCache
@@ -303,11 +301,10 @@ class Controller:
303
301
  async with self._rate_limiter:
304
302
  logger.info(f"Cancelling action: {action.name}")
305
303
  try:
306
- # TODO add support when the queue service supports aborting actions
307
- # await self._queue_service.AbortQueuedAction(
308
- # queue_service_pb2.AbortQueuedActionRequest(action_id=action.action_id),
309
- # wait_for_ready=True,
310
- # )
304
+ await self._queue_service.AbortQueuedAction(
305
+ queue_service_pb2.AbortQueuedActionRequest(action_id=action.action_id),
306
+ wait_for_ready=True,
307
+ )
311
308
  logger.info(f"Successfully cancelled action: {action.name}")
312
309
  except grpc.aio.AioRpcError as e:
313
310
  if e.code() in [
@@ -330,8 +327,8 @@ class Controller:
330
327
  """
331
328
  if not action.is_started():
332
329
  async with self._rate_limiter:
333
- task: queue_service_pb2.TaskAction | None = None
334
- trace: queue_service_pb2.TraceAction | None = None
330
+ task: run_definition_pb2.TaskAction | None = None
331
+ trace: run_definition_pb2.TraceAction | None = None
335
332
  if action.type == "task":
336
333
  if action.task is None:
337
334
  raise flyte.errors.RuntimeSystemError(
@@ -342,7 +339,7 @@ class Controller:
342
339
  if action.cache_key:
343
340
  cache_key = StringValue(value=action.cache_key)
344
341
 
345
- task = queue_service_pb2.TaskAction(
342
+ task = run_definition_pb2.TaskAction(
346
343
  id=task_definition_pb2.TaskIdentifier(
347
344
  version=action.task.task_template.id.version,
348
345
  org=action.task.task_template.id.org,
@@ -5,10 +5,10 @@ from asyncio import Queue
5
5
  from typing import AsyncIterator, Callable, Dict, Optional, Tuple, cast
6
6
 
7
7
  import grpc.aio
8
+ from flyteidl2.common import identifier_pb2
9
+ from flyteidl2.workflow import run_definition_pb2, state_service_pb2
8
10
 
9
11
  from flyte._logging import log, logger
10
- from flyte._protos.common import identifier_pb2
11
- from flyte._protos.workflow import run_definition_pb2, state_service_pb2
12
12
 
13
13
  from ._action import Action
14
14
  from ._service_protocol import StateService
@@ -373,7 +373,7 @@ class InformerCache:
373
373
  """Stop all informers and remove them from the cache"""
374
374
  async with self._lock:
375
375
  while self._cache:
376
- name, informer = self._cache.popitem()
376
+ _name, informer = self._cache.popitem()
377
377
  try:
378
378
  await informer.stop()
379
379
  except asyncio.CancelledError:
@@ -2,7 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  from typing import AsyncIterator, Protocol
4
4
 
5
- from flyte._protos.workflow import queue_service_pb2, state_service_pb2
5
+ from flyteidl2.workflow import queue_service_pb2, state_service_pb2
6
6
 
7
7
 
8
8
  class StateService(Protocol):
@@ -28,12 +28,12 @@ class QueueService(Protocol):
28
28
  ) -> queue_service_pb2.EnqueueActionResponse:
29
29
  """Enqueue a task"""
30
30
 
31
- # async def AbortQueuedAction(
32
- # self,
33
- # req: queue_service_pb2.AbortQueuedActionRequest,
34
- # **kwargs,
35
- # ) -> queue_service_pb2.AbortQueuedActionResponse:
36
- # """Dequeue a task"""
31
+ async def AbortQueuedAction(
32
+ self,
33
+ req: queue_service_pb2.AbortQueuedActionRequest,
34
+ **kwargs,
35
+ ) -> queue_service_pb2.AbortQueuedActionResponse:
36
+ """Cancel an enqueued task"""
37
37
 
38
38
 
39
39
  class ClientSet(Protocol):
@@ -6,7 +6,7 @@ import tempfile
6
6
  import typing
7
7
  from pathlib import Path
8
8
  from string import Template
9
- from typing import ClassVar, List, Optional, Protocol, cast
9
+ from typing import ClassVar, Optional, Protocol, cast
10
10
 
11
11
  import aiofiles
12
12
  import click
@@ -38,7 +38,7 @@ from flyte._internal.imagebuild.image_builder import (
38
38
  LocalDockerCommandImageChecker,
39
39
  LocalPodmanCommandImageChecker,
40
40
  )
41
- from flyte._internal.imagebuild.utils import copy_files_to_context
41
+ from flyte._internal.imagebuild.utils import copy_files_to_context, get_and_list_dockerignore
42
42
  from flyte._logging import logger
43
43
 
44
44
  _F_IMG_ID = "_F_IMG_ID"
@@ -263,24 +263,35 @@ class AptPackagesHandler:
263
263
 
264
264
  class UVProjectHandler:
265
265
  @staticmethod
266
- async def handle(layer: UVProject, context_path: Path, dockerfile: str) -> str:
266
+ async def handle(
267
+ layer: UVProject, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
268
+ ) -> str:
267
269
  secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
268
- if layer.extra_index_urls and "--no-install-project" in layer.extra_index_urls:
270
+ if layer.project_install_mode == "dependencies_only":
271
+ pip_install_args = " ".join(layer.get_pip_install_args())
272
+ if "--no-install-project" not in pip_install_args:
273
+ pip_install_args += " --no-install-project"
269
274
  # Only Copy pyproject.yaml and uv.lock.
270
275
  pyproject_dst = copy_files_to_context(layer.pyproject, context_path)
271
276
  uvlock_dst = copy_files_to_context(layer.uvlock, context_path)
272
277
  delta = UV_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE.substitute(
273
278
  UV_LOCK_PATH=uvlock_dst.relative_to(context_path),
274
279
  PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
275
- PIP_INSTALL_ARGS=" ".join(layer.get_pip_install_args()),
280
+ PIP_INSTALL_ARGS=pip_install_args,
276
281
  SECRET_MOUNT=secret_mounts,
277
282
  )
278
283
  else:
279
284
  # Copy the entire project.
280
- pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path)
281
- if layer.uvlock:
282
- # Sometimes the uv.lock file is in a different folder, if it's specified, let's copy it there explicitly
285
+ pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path, docker_ignore_patterns)
286
+
287
+ # Make sure pyproject.toml and uv.lock files are not removed by docker ignore
288
+ uv_lock_context_path = pyproject_dst / "uv.lock"
289
+ pyproject_context_path = pyproject_dst / "pyproject.toml"
290
+ if not uv_lock_context_path.exists():
283
291
  shutil.copy(layer.uvlock, pyproject_dst)
292
+ if not pyproject_context_path.exists():
293
+ shutil.copy(layer.pyproject, pyproject_dst)
294
+
284
295
  delta = UV_LOCK_INSTALL_TEMPLATE.substitute(
285
296
  PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
286
297
  PIP_INSTALL_ARGS=" ".join(layer.get_pip_install_args()),
@@ -293,7 +304,9 @@ class UVProjectHandler:
293
304
 
294
305
  class PoetryProjectHandler:
295
306
  @staticmethod
296
- async def handel(layer: PoetryProject, context_path: Path, dockerfile: str) -> str:
307
+ async def handel(
308
+ layer: PoetryProject, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
309
+ ) -> str:
297
310
  secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
298
311
  if layer.extra_args and "--no-root" in layer.extra_args:
299
312
  # Only Copy pyproject.yaml and poetry.lock.
@@ -307,7 +320,16 @@ class PoetryProjectHandler:
307
320
  )
308
321
  else:
309
322
  # Copy the entire project.
310
- pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path)
323
+ pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path, docker_ignore_patterns)
324
+
325
+ # Make sure pyproject.toml and poetry.lock files are not removed by docker ignore
326
+ poetry_lock_context_path = pyproject_dst / "poetry.lock"
327
+ pyproject_context_path = pyproject_dst / "pyproject.toml"
328
+ if not poetry_lock_context_path.exists():
329
+ shutil.copy(layer.poetry_lock, pyproject_dst)
330
+ if not pyproject_context_path.exists():
331
+ shutil.copy(layer.pyproject, pyproject_dst)
332
+
311
333
  delta = POETRY_LOCK_INSTALL_TEMPLATE.substitute(
312
334
  PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
313
335
  POETRY_INSTALL_ARGS=layer.extra_args or "",
@@ -324,37 +346,9 @@ class DockerIgnoreHandler:
324
346
 
325
347
 
326
348
  class CopyConfigHandler:
327
- @staticmethod
328
- def list_dockerignore(root_path: Optional[Path], docker_ignore_file_path: Optional[Path]) -> List[str]:
329
- patterns: List[str] = []
330
- dockerignore_path: Optional[Path] = None
331
- if root_path:
332
- dockerignore_path = root_path / ".dockerignore"
333
- # DockerIgnore layer should be first priority
334
- if docker_ignore_file_path:
335
- dockerignore_path = docker_ignore_file_path
336
-
337
- # Return empty list if no .dockerignore file found
338
- if not dockerignore_path or not dockerignore_path.exists() or not dockerignore_path.is_file():
339
- logger.info(f".dockerignore file not found at path: {dockerignore_path}")
340
- return patterns
341
-
342
- try:
343
- with open(dockerignore_path, "r", encoding="utf-8") as f:
344
- for line in f:
345
- stripped_line = line.strip()
346
- # Skip empty lines, whitespace-only lines, and comments
347
- if not stripped_line or stripped_line.startswith("#"):
348
- continue
349
- patterns.append(stripped_line)
350
- except Exception as e:
351
- logger.error(f"Failed to read .dockerignore file at {dockerignore_path}: {e}")
352
- return []
353
- return patterns
354
-
355
349
  @staticmethod
356
350
  async def handle(
357
- layer: CopyConfig, context_path: Path, dockerfile: str, docker_ignore_file_path: Optional[Path]
351
+ layer: CopyConfig, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
358
352
  ) -> str:
359
353
  # Copy the source config file or directory to the context path
360
354
  if layer.src.is_absolute() or ".." in str(layer.src):
@@ -370,11 +364,6 @@ class CopyConfigHandler:
370
364
  shutil.copy(abs_path, dst_path)
371
365
  elif layer.src.is_dir():
372
366
  # Copy the entire directory
373
- from flyte._initialize import _get_init_config
374
-
375
- init_config = _get_init_config()
376
- root_path = init_config.root_dir if init_config else None
377
- docker_ignore_patterns = CopyConfigHandler.list_dockerignore(root_path, docker_ignore_file_path)
378
367
  shutil.copytree(
379
368
  abs_path, dst_path, dirs_exist_ok=True, ignore=shutil.ignore_patterns(*docker_ignore_patterns)
380
369
  )
@@ -450,7 +439,7 @@ def _get_secret_mounts_layer(secrets: typing.Tuple[str | Secret, ...] | None) ->
450
439
 
451
440
 
452
441
  async def _process_layer(
453
- layer: Layer, context_path: Path, dockerfile: str, docker_ignore_file_path: Optional[Path]
442
+ layer: Layer, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
454
443
  ) -> str:
455
444
  match layer:
456
445
  case PythonWheels():
@@ -483,15 +472,19 @@ async def _process_layer(
483
472
 
484
473
  case UVProject():
485
474
  # Handle UV project
486
- dockerfile = await UVProjectHandler.handle(layer, context_path, dockerfile)
475
+ dockerfile = await UVProjectHandler.handle(layer, context_path, dockerfile, docker_ignore_patterns)
487
476
 
488
477
  case PoetryProject():
489
478
  # Handle Poetry project
490
- dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile)
479
+ dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile, docker_ignore_patterns)
480
+
481
+ case PoetryProject():
482
+ # Handle Poetry project
483
+ dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile, docker_ignore_patterns)
491
484
 
492
485
  case CopyConfig():
493
486
  # Handle local files and folders
494
- dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile, docker_ignore_file_path)
487
+ dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile, docker_ignore_patterns)
495
488
 
496
489
  case Commands():
497
490
  # Handle commands
@@ -618,15 +611,6 @@ class DockerImageBuilder(ImageBuilder):
618
611
  else:
619
612
  logger.info("Buildx builder already exists.")
620
613
 
621
- def get_docker_ignore(self, image: Image) -> Optional[Path]:
622
- """Get the .dockerignore file path from the image layers."""
623
- # Look for DockerIgnore layer in the image layers
624
- for layer in image._layers:
625
- if isinstance(layer, DockerIgnore) and layer.path.strip():
626
- return Path(layer.path)
627
-
628
- return None
629
-
630
614
  async def _build_image(self, image: Image, *, push: bool = True, dry_run: bool = False) -> str:
631
615
  """
632
616
  if default image (only base image and locked), raise an error, don't have a dockerfile
@@ -657,9 +641,11 @@ class DockerImageBuilder(ImageBuilder):
657
641
  PYTHON_VERSION=f"{image.python_version[0]}.{image.python_version[1]}",
658
642
  )
659
643
 
660
- docker_ignore_file_path = self.get_docker_ignore(image)
644
+ # Get .dockerignore file patterns first
645
+ docker_ignore_patterns = get_and_list_dockerignore(image)
646
+
661
647
  for layer in image._layers:
662
- dockerfile = await _process_layer(layer, tmp_path, dockerfile, docker_ignore_file_path)
648
+ dockerfile = await _process_layer(layer, tmp_path, dockerfile, docker_ignore_patterns)
663
649
 
664
650
  dockerfile += DOCKER_FILE_BASE_FOOTER.substitute(F_IMG_ID=image.uri)
665
651