flyte 2.0.0b13__py3-none-any.whl → 2.0.0b30__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (211) hide show
  1. flyte/__init__.py +18 -2
  2. flyte/_bin/debug.py +38 -0
  3. flyte/_bin/runtime.py +62 -8
  4. flyte/_cache/cache.py +4 -2
  5. flyte/_cache/local_cache.py +216 -0
  6. flyte/_code_bundle/_ignore.py +12 -4
  7. flyte/_code_bundle/_packaging.py +13 -9
  8. flyte/_code_bundle/_utils.py +18 -10
  9. flyte/_code_bundle/bundle.py +17 -9
  10. flyte/_constants.py +1 -0
  11. flyte/_context.py +4 -1
  12. flyte/_custom_context.py +73 -0
  13. flyte/_debug/constants.py +38 -0
  14. flyte/_debug/utils.py +17 -0
  15. flyte/_debug/vscode.py +307 -0
  16. flyte/_deploy.py +235 -61
  17. flyte/_environment.py +20 -6
  18. flyte/_excepthook.py +1 -1
  19. flyte/_hash.py +1 -16
  20. flyte/_image.py +178 -81
  21. flyte/_initialize.py +132 -51
  22. flyte/_interface.py +39 -2
  23. flyte/_internal/controllers/__init__.py +4 -5
  24. flyte/_internal/controllers/_local_controller.py +70 -29
  25. flyte/_internal/controllers/_trace.py +1 -1
  26. flyte/_internal/controllers/remote/__init__.py +0 -2
  27. flyte/_internal/controllers/remote/_action.py +14 -16
  28. flyte/_internal/controllers/remote/_client.py +1 -1
  29. flyte/_internal/controllers/remote/_controller.py +68 -70
  30. flyte/_internal/controllers/remote/_core.py +127 -99
  31. flyte/_internal/controllers/remote/_informer.py +19 -10
  32. flyte/_internal/controllers/remote/_service_protocol.py +7 -7
  33. flyte/_internal/imagebuild/docker_builder.py +181 -69
  34. flyte/_internal/imagebuild/image_builder.py +0 -5
  35. flyte/_internal/imagebuild/remote_builder.py +155 -64
  36. flyte/_internal/imagebuild/utils.py +51 -2
  37. flyte/_internal/resolvers/_task_module.py +5 -38
  38. flyte/_internal/resolvers/default.py +2 -2
  39. flyte/_internal/runtime/convert.py +110 -21
  40. flyte/_internal/runtime/entrypoints.py +27 -1
  41. flyte/_internal/runtime/io.py +21 -8
  42. flyte/_internal/runtime/resources_serde.py +20 -6
  43. flyte/_internal/runtime/reuse.py +1 -1
  44. flyte/_internal/runtime/rusty.py +20 -5
  45. flyte/_internal/runtime/task_serde.py +34 -19
  46. flyte/_internal/runtime/taskrunner.py +22 -4
  47. flyte/_internal/runtime/trigger_serde.py +160 -0
  48. flyte/_internal/runtime/types_serde.py +1 -1
  49. flyte/_keyring/__init__.py +0 -0
  50. flyte/_keyring/file.py +115 -0
  51. flyte/_logging.py +201 -39
  52. flyte/_map.py +111 -14
  53. flyte/_module.py +70 -0
  54. flyte/_pod.py +4 -3
  55. flyte/_resources.py +213 -31
  56. flyte/_run.py +110 -39
  57. flyte/_task.py +75 -16
  58. flyte/_task_environment.py +105 -29
  59. flyte/_task_plugins.py +4 -2
  60. flyte/_trace.py +5 -0
  61. flyte/_trigger.py +1000 -0
  62. flyte/_utils/__init__.py +2 -1
  63. flyte/_utils/asyn.py +3 -1
  64. flyte/_utils/coro_management.py +2 -1
  65. flyte/_utils/docker_credentials.py +173 -0
  66. flyte/_utils/module_loader.py +17 -2
  67. flyte/_version.py +3 -3
  68. flyte/cli/_abort.py +3 -3
  69. flyte/cli/_build.py +3 -6
  70. flyte/cli/_common.py +78 -7
  71. flyte/cli/_create.py +182 -4
  72. flyte/cli/_delete.py +23 -1
  73. flyte/cli/_deploy.py +63 -16
  74. flyte/cli/_get.py +79 -34
  75. flyte/cli/_params.py +26 -10
  76. flyte/cli/_plugins.py +209 -0
  77. flyte/cli/_run.py +151 -26
  78. flyte/cli/_serve.py +64 -0
  79. flyte/cli/_update.py +37 -0
  80. flyte/cli/_user.py +17 -0
  81. flyte/cli/main.py +30 -4
  82. flyte/config/_config.py +10 -6
  83. flyte/config/_internal.py +1 -0
  84. flyte/config/_reader.py +29 -8
  85. flyte/connectors/__init__.py +11 -0
  86. flyte/connectors/_connector.py +270 -0
  87. flyte/connectors/_server.py +197 -0
  88. flyte/connectors/utils.py +135 -0
  89. flyte/errors.py +22 -2
  90. flyte/extend.py +8 -1
  91. flyte/extras/_container.py +6 -1
  92. flyte/git/__init__.py +3 -0
  93. flyte/git/_config.py +21 -0
  94. flyte/io/__init__.py +2 -0
  95. flyte/io/_dataframe/__init__.py +2 -0
  96. flyte/io/_dataframe/basic_dfs.py +17 -8
  97. flyte/io/_dataframe/dataframe.py +98 -132
  98. flyte/io/_dir.py +575 -113
  99. flyte/io/_file.py +582 -139
  100. flyte/io/_hashing_io.py +342 -0
  101. flyte/models.py +74 -15
  102. flyte/remote/__init__.py +6 -1
  103. flyte/remote/_action.py +34 -26
  104. flyte/remote/_client/_protocols.py +39 -4
  105. flyte/remote/_client/auth/_authenticators/device_code.py +4 -5
  106. flyte/remote/_client/auth/_authenticators/pkce.py +1 -1
  107. flyte/remote/_client/auth/_channel.py +10 -6
  108. flyte/remote/_client/controlplane.py +17 -5
  109. flyte/remote/_console.py +3 -2
  110. flyte/remote/_data.py +6 -6
  111. flyte/remote/_logs.py +3 -3
  112. flyte/remote/_run.py +64 -8
  113. flyte/remote/_secret.py +26 -17
  114. flyte/remote/_task.py +75 -33
  115. flyte/remote/_trigger.py +306 -0
  116. flyte/remote/_user.py +33 -0
  117. flyte/report/_report.py +1 -1
  118. flyte/storage/__init__.py +6 -1
  119. flyte/storage/_config.py +5 -1
  120. flyte/storage/_parallel_reader.py +274 -0
  121. flyte/storage/_storage.py +200 -103
  122. flyte/types/__init__.py +16 -0
  123. flyte/types/_interface.py +2 -2
  124. flyte/types/_pickle.py +35 -8
  125. flyte/types/_string_literals.py +8 -9
  126. flyte/types/_type_engine.py +40 -70
  127. flyte/types/_utils.py +1 -1
  128. flyte-2.0.0b30.data/scripts/debug.py +38 -0
  129. {flyte-2.0.0b13.data → flyte-2.0.0b30.data}/scripts/runtime.py +62 -8
  130. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/METADATA +11 -3
  131. flyte-2.0.0b30.dist-info/RECORD +192 -0
  132. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/entry_points.txt +3 -0
  133. flyte/_protos/common/authorization_pb2.py +0 -66
  134. flyte/_protos/common/authorization_pb2.pyi +0 -108
  135. flyte/_protos/common/authorization_pb2_grpc.py +0 -4
  136. flyte/_protos/common/identifier_pb2.py +0 -93
  137. flyte/_protos/common/identifier_pb2.pyi +0 -110
  138. flyte/_protos/common/identifier_pb2_grpc.py +0 -4
  139. flyte/_protos/common/identity_pb2.py +0 -48
  140. flyte/_protos/common/identity_pb2.pyi +0 -72
  141. flyte/_protos/common/identity_pb2_grpc.py +0 -4
  142. flyte/_protos/common/list_pb2.py +0 -36
  143. flyte/_protos/common/list_pb2.pyi +0 -71
  144. flyte/_protos/common/list_pb2_grpc.py +0 -4
  145. flyte/_protos/common/policy_pb2.py +0 -37
  146. flyte/_protos/common/policy_pb2.pyi +0 -27
  147. flyte/_protos/common/policy_pb2_grpc.py +0 -4
  148. flyte/_protos/common/role_pb2.py +0 -37
  149. flyte/_protos/common/role_pb2.pyi +0 -53
  150. flyte/_protos/common/role_pb2_grpc.py +0 -4
  151. flyte/_protos/common/runtime_version_pb2.py +0 -28
  152. flyte/_protos/common/runtime_version_pb2.pyi +0 -24
  153. flyte/_protos/common/runtime_version_pb2_grpc.py +0 -4
  154. flyte/_protos/imagebuilder/definition_pb2.py +0 -59
  155. flyte/_protos/imagebuilder/definition_pb2.pyi +0 -140
  156. flyte/_protos/imagebuilder/definition_pb2_grpc.py +0 -4
  157. flyte/_protos/imagebuilder/payload_pb2.py +0 -32
  158. flyte/_protos/imagebuilder/payload_pb2.pyi +0 -21
  159. flyte/_protos/imagebuilder/payload_pb2_grpc.py +0 -4
  160. flyte/_protos/imagebuilder/service_pb2.py +0 -29
  161. flyte/_protos/imagebuilder/service_pb2.pyi +0 -5
  162. flyte/_protos/imagebuilder/service_pb2_grpc.py +0 -66
  163. flyte/_protos/logs/dataplane/payload_pb2.py +0 -100
  164. flyte/_protos/logs/dataplane/payload_pb2.pyi +0 -177
  165. flyte/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
  166. flyte/_protos/secret/definition_pb2.py +0 -49
  167. flyte/_protos/secret/definition_pb2.pyi +0 -93
  168. flyte/_protos/secret/definition_pb2_grpc.py +0 -4
  169. flyte/_protos/secret/payload_pb2.py +0 -62
  170. flyte/_protos/secret/payload_pb2.pyi +0 -94
  171. flyte/_protos/secret/payload_pb2_grpc.py +0 -4
  172. flyte/_protos/secret/secret_pb2.py +0 -38
  173. flyte/_protos/secret/secret_pb2.pyi +0 -6
  174. flyte/_protos/secret/secret_pb2_grpc.py +0 -198
  175. flyte/_protos/secret/secret_pb2_grpc_grpc.py +0 -198
  176. flyte/_protos/validate/validate/validate_pb2.py +0 -76
  177. flyte/_protos/workflow/common_pb2.py +0 -27
  178. flyte/_protos/workflow/common_pb2.pyi +0 -14
  179. flyte/_protos/workflow/common_pb2_grpc.py +0 -4
  180. flyte/_protos/workflow/environment_pb2.py +0 -29
  181. flyte/_protos/workflow/environment_pb2.pyi +0 -12
  182. flyte/_protos/workflow/environment_pb2_grpc.py +0 -4
  183. flyte/_protos/workflow/node_execution_service_pb2.py +0 -26
  184. flyte/_protos/workflow/node_execution_service_pb2.pyi +0 -4
  185. flyte/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
  186. flyte/_protos/workflow/queue_service_pb2.py +0 -109
  187. flyte/_protos/workflow/queue_service_pb2.pyi +0 -166
  188. flyte/_protos/workflow/queue_service_pb2_grpc.py +0 -172
  189. flyte/_protos/workflow/run_definition_pb2.py +0 -121
  190. flyte/_protos/workflow/run_definition_pb2.pyi +0 -327
  191. flyte/_protos/workflow/run_definition_pb2_grpc.py +0 -4
  192. flyte/_protos/workflow/run_logs_service_pb2.py +0 -41
  193. flyte/_protos/workflow/run_logs_service_pb2.pyi +0 -28
  194. flyte/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
  195. flyte/_protos/workflow/run_service_pb2.py +0 -137
  196. flyte/_protos/workflow/run_service_pb2.pyi +0 -185
  197. flyte/_protos/workflow/run_service_pb2_grpc.py +0 -446
  198. flyte/_protos/workflow/state_service_pb2.py +0 -67
  199. flyte/_protos/workflow/state_service_pb2.pyi +0 -76
  200. flyte/_protos/workflow/state_service_pb2_grpc.py +0 -138
  201. flyte/_protos/workflow/task_definition_pb2.py +0 -79
  202. flyte/_protos/workflow/task_definition_pb2.pyi +0 -81
  203. flyte/_protos/workflow/task_definition_pb2_grpc.py +0 -4
  204. flyte/_protos/workflow/task_service_pb2.py +0 -60
  205. flyte/_protos/workflow/task_service_pb2.pyi +0 -59
  206. flyte/_protos/workflow/task_service_pb2_grpc.py +0 -138
  207. flyte-2.0.0b13.dist-info/RECORD +0 -239
  208. /flyte/{_protos → _debug}/__init__.py +0 -0
  209. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/WHEEL +0 -0
  210. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/licenses/LICENSE +0 -0
  211. {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/top_level.txt +0 -0
@@ -5,10 +5,10 @@ from asyncio import Queue
5
5
  from typing import AsyncIterator, Callable, Dict, Optional, Tuple, cast
6
6
 
7
7
  import grpc.aio
8
+ from flyteidl2.common import identifier_pb2
9
+ from flyteidl2.workflow import run_definition_pb2, state_service_pb2
8
10
 
9
11
  from flyte._logging import log, logger
10
- from flyte._protos.common import identifier_pb2
11
- from flyte._protos.workflow import run_definition_pb2, state_service_pb2
12
12
 
13
13
  from ._action import Action
14
14
  from ._service_protocol import StateService
@@ -132,8 +132,10 @@ class Informer:
132
132
  parent_action_name: str,
133
133
  shared_queue: Queue,
134
134
  client: Optional[StateService] = None,
135
- watch_backoff_interval_sec: float = 1.0,
135
+ min_watch_backoff: float = 1.0,
136
+ max_watch_backoff: float = 30.0,
136
137
  watch_conn_timeout_sec: float = 5.0,
138
+ max_watch_retries: int = 10,
137
139
  ):
138
140
  self.name = self.mkname(run_name=run_id.name, parent_action_name=parent_action_name)
139
141
  self.parent_action_name = parent_action_name
@@ -144,8 +146,10 @@ class Informer:
144
146
  self._running = False
145
147
  self._watch_task: asyncio.Task | None = None
146
148
  self._ready = asyncio.Event()
147
- self._watch_backoff_interval_sec = watch_backoff_interval_sec
149
+ self._min_watch_backoff = min_watch_backoff
150
+ self._max_watch_backoff = max_watch_backoff
148
151
  self._watch_conn_timeout_sec = watch_conn_timeout_sec
152
+ self._max_watch_retries = max_watch_retries
149
153
 
150
154
  @classmethod
151
155
  def mkname(cls, *, run_name: str, parent_action_name: str) -> str:
@@ -211,13 +215,16 @@ class Informer:
211
215
  """
212
216
  # sentinel = False
213
217
  retries = 0
214
- max_retries = 5
215
218
  last_exc = None
216
219
  while self._running:
217
- if retries >= max_retries:
218
- logger.error(f"Informer watch failure retries crossed threshold {retries}/{max_retries}, exiting!")
220
+ if retries >= self._max_watch_retries:
221
+ logger.error(
222
+ f"Informer watch failure retries crossed threshold {retries}/{self._max_watch_retries}, exiting!"
223
+ )
219
224
  raise last_exc
220
225
  try:
226
+ if retries >= 1:
227
+ logger.warning(f"Informer watch retrying, attempt {retries}/{self._max_watch_retries}")
221
228
  watcher = self._client.Watch(
222
229
  state_service_pb2.WatchRequest(
223
230
  parent_action_id=identifier_pb2.ActionIdentifier(
@@ -252,7 +259,9 @@ class Informer:
252
259
  logger.exception(f"Watch error: {self.name}", exc_info=e)
253
260
  last_exc = e
254
261
  retries += 1
255
- await asyncio.sleep(self._watch_backoff_interval_sec)
262
+ backoff = min(self._min_watch_backoff * (2**retries), self._max_watch_backoff)
263
+ logger.warning(f"Watch for {self.name} failed, retrying in {backoff} seconds...")
264
+ await asyncio.sleep(backoff)
256
265
 
257
266
  @log
258
267
  async def start(self, timeout: Optional[float] = None) -> asyncio.Task:
@@ -261,7 +270,7 @@ class Informer:
261
270
  logger.warning("Informer already running")
262
271
  return cast(asyncio.Task, self._watch_task)
263
272
  self._running = True
264
- self._watch_task = asyncio.create_task(self.watch())
273
+ self._watch_task = asyncio.create_task(self.watch(), name=f"InformerWatch-{self.parent_action_name}")
265
274
  await self.wait_for_cache_sync(timeout=timeout)
266
275
  return self._watch_task
267
276
 
@@ -364,7 +373,7 @@ class InformerCache:
364
373
  """Stop all informers and remove them from the cache"""
365
374
  async with self._lock:
366
375
  while self._cache:
367
- name, informer = self._cache.popitem()
376
+ _name, informer = self._cache.popitem()
368
377
  try:
369
378
  await informer.stop()
370
379
  except asyncio.CancelledError:
@@ -2,7 +2,7 @@ from __future__ import annotations
2
2
 
3
3
  from typing import AsyncIterator, Protocol
4
4
 
5
- from flyte._protos.workflow import queue_service_pb2, state_service_pb2
5
+ from flyteidl2.workflow import queue_service_pb2, state_service_pb2
6
6
 
7
7
 
8
8
  class StateService(Protocol):
@@ -28,12 +28,12 @@ class QueueService(Protocol):
28
28
  ) -> queue_service_pb2.EnqueueActionResponse:
29
29
  """Enqueue a task"""
30
30
 
31
- # async def AbortQueuedAction(
32
- # self,
33
- # req: queue_service_pb2.AbortQueuedActionRequest,
34
- # **kwargs,
35
- # ) -> queue_service_pb2.AbortQueuedActionResponse:
36
- # """Dequeue a task"""
31
+ async def AbortQueuedAction(
32
+ self,
33
+ req: queue_service_pb2.AbortQueuedActionRequest,
34
+ **kwargs,
35
+ ) -> queue_service_pb2.AbortQueuedActionResponse:
36
+ """Cancel an enqueued task"""
37
37
 
38
38
 
39
39
  class ClientSet(Protocol):
@@ -22,6 +22,7 @@ from flyte._image import (
22
22
  Layer,
23
23
  PipOption,
24
24
  PipPackages,
25
+ PoetryProject,
25
26
  PythonWheels,
26
27
  Requirements,
27
28
  UVProject,
@@ -37,80 +38,112 @@ from flyte._internal.imagebuild.image_builder import (
37
38
  LocalDockerCommandImageChecker,
38
39
  LocalPodmanCommandImageChecker,
39
40
  )
40
- from flyte._internal.imagebuild.utils import copy_files_to_context
41
+ from flyte._internal.imagebuild.utils import copy_files_to_context, get_and_list_dockerignore
41
42
  from flyte._logging import logger
42
43
 
43
44
  _F_IMG_ID = "_F_IMG_ID"
44
45
  FLYTE_DOCKER_BUILDER_CACHE_FROM = "FLYTE_DOCKER_BUILDER_CACHE_FROM"
45
46
  FLYTE_DOCKER_BUILDER_CACHE_TO = "FLYTE_DOCKER_BUILDER_CACHE_TO"
46
47
 
48
+ UV_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE = Template("""\
49
+ RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
50
+ --mount=type=bind,target=uv.lock,src=$UV_LOCK_PATH,rw \
51
+ --mount=type=bind,target=pyproject.toml,src=$PYPROJECT_PATH \
52
+ $SECRET_MOUNT \
53
+ uv sync --active --inexact $PIP_INSTALL_ARGS
54
+ """)
55
+
47
56
  UV_LOCK_INSTALL_TEMPLATE = Template("""\
48
- WORKDIR /root
49
57
  RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
50
- --mount=type=bind,target=uv.lock,src=uv.lock \
51
- --mount=type=bind,target=pyproject.toml,src=pyproject.toml \
52
- $SECRET_MOUNT \
53
- uv sync $PIP_INSTALL_ARGS
54
- WORKDIR /
55
-
56
- # Update PATH and UV_PYTHON to point to the venv created by uv sync
57
- ENV PATH="/root/.venv/bin:$$PATH" \
58
- VIRTUALENV=/root/.venv \
59
- UV_PYTHON=/root/.venv/bin/python
58
+ --mount=type=bind,target=/root/.flyte/$PYPROJECT_PATH,src=$PYPROJECT_PATH,rw \
59
+ $SECRET_MOUNT \
60
+ uv sync --active --inexact --no-editable $PIP_INSTALL_ARGS --project /root/.flyte/$PYPROJECT_PATH
61
+ """)
62
+
63
+ POETRY_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE = Template("""\
64
+ RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
65
+ uv pip install poetry
66
+
67
+ ENV POETRY_CACHE_DIR=/tmp/poetry_cache \
68
+ POETRY_VIRTUALENVS_IN_PROJECT=true
69
+
70
+ RUN --mount=type=cache,sharing=locked,mode=0777,target=/tmp/poetry_cache,id=poetry \
71
+ --mount=type=bind,target=poetry.lock,src=$POETRY_LOCK_PATH \
72
+ --mount=type=bind,target=pyproject.toml,src=$PYPROJECT_PATH \
73
+ $SECRET_MOUNT \
74
+ poetry install $POETRY_INSTALL_ARGS
75
+ """)
76
+
77
+ POETRY_LOCK_INSTALL_TEMPLATE = Template("""\
78
+ RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
79
+ uv pip install poetry
80
+
81
+ ENV POETRY_CACHE_DIR=/tmp/poetry_cache \
82
+ POETRY_VIRTUALENVS_IN_PROJECT=true
83
+
84
+ RUN --mount=type=cache,sharing=locked,mode=0777,target=/tmp/poetry_cache,id=poetry \
85
+ --mount=type=bind,target=/root/.flyte/$PYPROJECT_PATH,src=$PYPROJECT_PATH,rw \
86
+ $SECRET_MOUNT \
87
+ poetry install $POETRY_INSTALL_ARGS -C /root/.flyte/$PYPROJECT_PATH
60
88
  """)
61
89
 
62
90
  UV_PACKAGE_INSTALL_COMMAND_TEMPLATE = Template("""\
63
91
  RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
64
- $REQUIREMENTS_MOUNT \
65
- $SECRET_MOUNT \
66
- uv pip install --python $$UV_PYTHON $PIP_INSTALL_ARGS
92
+ $REQUIREMENTS_MOUNT \
93
+ $SECRET_MOUNT \
94
+ uv pip install --python $$UV_PYTHON $PIP_INSTALL_ARGS
67
95
  """)
68
96
 
69
97
  UV_WHEEL_INSTALL_COMMAND_TEMPLATE = Template("""\
70
98
  RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=wheel \
71
- --mount=source=/dist,target=/dist,type=bind \
72
- $SECRET_MOUNT \
73
- uv pip install --python $$UV_PYTHON $PIP_INSTALL_ARGS
99
+ --mount=source=/dist,target=/dist,type=bind \
100
+ $SECRET_MOUNT \
101
+ uv pip install --python $$UV_PYTHON $PIP_INSTALL_ARGS
74
102
  """)
75
103
 
76
104
  APT_INSTALL_COMMAND_TEMPLATE = Template("""\
77
105
  RUN --mount=type=cache,sharing=locked,mode=0777,target=/var/cache/apt,id=apt \
78
- $SECRET_MOUNT \
79
- apt-get update && apt-get install -y --no-install-recommends \
80
- $APT_PACKAGES
106
+ $SECRET_MOUNT \
107
+ apt-get update && apt-get install -y --no-install-recommends \
108
+ $APT_PACKAGES
81
109
  """)
82
110
 
83
111
  UV_PYTHON_INSTALL_COMMAND = Template("""\
84
112
  RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
85
- $SECRET_MOUNT \
86
- uv pip install $PIP_INSTALL_ARGS
113
+ $SECRET_MOUNT \
114
+ uv pip install $PIP_INSTALL_ARGS
87
115
  """)
88
116
 
89
117
  # uv pip install --python /root/env/bin/python
90
118
  # new template
91
119
  DOCKER_FILE_UV_BASE_TEMPLATE = Template("""\
92
120
  # syntax=docker/dockerfile:1.10
93
- FROM ghcr.io/astral-sh/uv:0.6.12 AS uv
121
+ FROM ghcr.io/astral-sh/uv:0.8.13 AS uv
94
122
  FROM $BASE_IMAGE
95
123
 
124
+
96
125
  USER root
97
126
 
127
+
98
128
  # Copy in uv so that later commands don't have to mount it in
99
129
  COPY --from=uv /uv /usr/bin/uv
100
130
 
131
+
101
132
  # Configure default envs
102
133
  ENV UV_COMPILE_BYTECODE=1 \
103
- UV_LINK_MODE=copy \
104
- VIRTUALENV=/opt/venv \
105
- UV_PYTHON=/opt/venv/bin/python \
106
- PATH="/opt/venv/bin:$$PATH"
134
+ UV_LINK_MODE=copy \
135
+ VIRTUALENV=/opt/venv \
136
+ UV_PYTHON=/opt/venv/bin/python \
137
+ PATH="/opt/venv/bin:$$PATH"
138
+
107
139
 
108
140
  # Create a virtualenv with the user specified python version
109
141
  RUN uv venv $$VIRTUALENV --python=$PYTHON_VERSION
110
142
 
143
+
111
144
  # Adds nvidia just in case it exists
112
145
  ENV PATH="$$PATH:/usr/local/nvidia/bin:/usr/local/cuda/bin" \
113
- LD_LIBRARY_PATH="/usr/local/nvidia/lib64"
146
+ LD_LIBRARY_PATH="/usr/local/nvidia/lib64"
114
147
  """)
115
148
 
116
149
  # This gets added on to the end of the dockerfile
@@ -177,6 +210,7 @@ class PythonWheelHandler:
177
210
  "/dist",
178
211
  "--no-deps",
179
212
  "--no-index",
213
+ "--reinstall",
180
214
  layer.package_name,
181
215
  ],
182
216
  ]
@@ -229,21 +263,84 @@ class AptPackagesHandler:
229
263
 
230
264
  class UVProjectHandler:
231
265
  @staticmethod
232
- async def handle(layer: UVProject, context_path: Path, dockerfile: str) -> str:
233
- # copy the two files
234
- shutil.copy(layer.pyproject, context_path)
235
- shutil.copy(layer.uvlock, context_path)
236
-
237
- # --locked: Assert that the `uv.lock` will remain unchanged
238
- # --no-dev: Omit the development dependency group
239
- # --no-install-project: Do not install the current project
240
- additional_pip_install_args = ["--locked", "--no-dev", "--no-install-project"]
266
+ async def handle(
267
+ layer: UVProject, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
268
+ ) -> str:
241
269
  secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
242
- delta = UV_LOCK_INSTALL_TEMPLATE.substitute(
243
- PIP_INSTALL_ARGS=" ".join(additional_pip_install_args), SECRET_MOUNT=secret_mounts
244
- )
270
+ if layer.project_install_mode == "dependencies_only":
271
+ pip_install_args = " ".join(layer.get_pip_install_args())
272
+ if "--no-install-project" not in pip_install_args:
273
+ pip_install_args += " --no-install-project"
274
+ if "--no-sources" not in pip_install_args:
275
+ pip_install_args += " --no-sources"
276
+ # Only Copy pyproject.yaml and uv.lock.
277
+ pyproject_dst = copy_files_to_context(layer.pyproject, context_path)
278
+ uvlock_dst = copy_files_to_context(layer.uvlock, context_path)
279
+ delta = UV_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE.substitute(
280
+ UV_LOCK_PATH=uvlock_dst.relative_to(context_path),
281
+ PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
282
+ PIP_INSTALL_ARGS=pip_install_args,
283
+ SECRET_MOUNT=secret_mounts,
284
+ )
285
+ else:
286
+ # Copy the entire project.
287
+ pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path, docker_ignore_patterns)
288
+
289
+ # Make sure pyproject.toml and uv.lock files are not removed by docker ignore
290
+ uv_lock_context_path = pyproject_dst / "uv.lock"
291
+ pyproject_context_path = pyproject_dst / "pyproject.toml"
292
+ if not uv_lock_context_path.exists():
293
+ shutil.copy(layer.uvlock, pyproject_dst)
294
+ if not pyproject_context_path.exists():
295
+ shutil.copy(layer.pyproject, pyproject_dst)
296
+
297
+ delta = UV_LOCK_INSTALL_TEMPLATE.substitute(
298
+ PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
299
+ PIP_INSTALL_ARGS=" ".join(layer.get_pip_install_args()),
300
+ SECRET_MOUNT=secret_mounts,
301
+ )
302
+
245
303
  dockerfile += delta
304
+ return dockerfile
305
+
246
306
 
307
+ class PoetryProjectHandler:
308
+ @staticmethod
309
+ async def handel(
310
+ layer: PoetryProject, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
311
+ ) -> str:
312
+ secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
313
+ extra_args = layer.extra_args or ""
314
+ if layer.project_install_mode == "dependencies_only":
315
+ # Only Copy pyproject.yaml and poetry.lock.
316
+ pyproject_dst = copy_files_to_context(layer.pyproject, context_path)
317
+ poetry_lock_dst = copy_files_to_context(layer.poetry_lock, context_path)
318
+ if "--no-root" not in extra_args:
319
+ extra_args += " --no-root"
320
+ delta = POETRY_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE.substitute(
321
+ POETRY_LOCK_PATH=poetry_lock_dst.relative_to(context_path),
322
+ PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
323
+ POETRY_INSTALL_ARGS=extra_args,
324
+ SECRET_MOUNT=secret_mounts,
325
+ )
326
+ else:
327
+ # Copy the entire project.
328
+ pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path, docker_ignore_patterns)
329
+
330
+ # Make sure pyproject.toml and poetry.lock files are not removed by docker ignore
331
+ poetry_lock_context_path = pyproject_dst / "poetry.lock"
332
+ pyproject_context_path = pyproject_dst / "pyproject.toml"
333
+ if not poetry_lock_context_path.exists():
334
+ shutil.copy(layer.poetry_lock, pyproject_dst)
335
+ if not pyproject_context_path.exists():
336
+ shutil.copy(layer.pyproject, pyproject_dst)
337
+
338
+ delta = POETRY_LOCK_INSTALL_TEMPLATE.substitute(
339
+ PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
340
+ POETRY_INSTALL_ARGS=extra_args,
341
+ SECRET_MOUNT=secret_mounts,
342
+ )
343
+ dockerfile += delta
247
344
  return dockerfile
248
345
 
249
346
 
@@ -255,7 +352,9 @@ class DockerIgnoreHandler:
255
352
 
256
353
  class CopyConfigHandler:
257
354
  @staticmethod
258
- async def handle(layer: CopyConfig, context_path: Path, dockerfile: str) -> str:
355
+ async def handle(
356
+ layer: CopyConfig, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
357
+ ) -> str:
259
358
  # Copy the source config file or directory to the context path
260
359
  if layer.src.is_absolute() or ".." in str(layer.src):
261
360
  dst_path = context_path / str(layer.src.absolute()).replace("/", "./_flyte_abs_context/", 1)
@@ -264,18 +363,21 @@ class CopyConfigHandler:
264
363
 
265
364
  dst_path.parent.mkdir(parents=True, exist_ok=True)
266
365
  abs_path = layer.src.absolute()
366
+
267
367
  if layer.src.is_file():
268
368
  # Copy the file
269
369
  shutil.copy(abs_path, dst_path)
270
370
  elif layer.src.is_dir():
271
371
  # Copy the entire directory
272
- shutil.copytree(abs_path, dst_path, dirs_exist_ok=True)
372
+ shutil.copytree(
373
+ abs_path, dst_path, dirs_exist_ok=True, ignore=shutil.ignore_patterns(*docker_ignore_patterns)
374
+ )
273
375
  else:
274
- raise ValueError(f"Source path is neither file nor directory: {layer.src}")
376
+ logger.error(f"Source path not exists: {layer.src}")
377
+ return dockerfile
275
378
 
276
379
  # Add a copy command to the dockerfile
277
380
  dockerfile += f"\nCOPY {dst_path.relative_to(context_path)} {layer.dst}\n"
278
-
279
381
  return dockerfile
280
382
 
281
383
 
@@ -283,8 +385,9 @@ class CommandsHandler:
283
385
  @staticmethod
284
386
  async def handle(layer: Commands, _: Path, dockerfile: str) -> str:
285
387
  # Append raw commands to the dockerfile
388
+ secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
286
389
  for command in layer.commands:
287
- dockerfile += f"\nRUN {command}\n"
390
+ dockerfile += f"\nRUN {secret_mounts} {command}\n"
288
391
 
289
392
  return dockerfile
290
393
 
@@ -302,15 +405,12 @@ def _get_secret_commands(layers: typing.Tuple[Layer, ...]) -> typing.List[str]:
302
405
  commands = []
303
406
 
304
407
  def _get_secret_command(secret: str | Secret) -> typing.List[str]:
305
- secret_id = hash(secret)
306
408
  if isinstance(secret, str):
307
- if not os.path.exists(secret):
308
- raise FileNotFoundError(f"Secret file '{secret}' not found")
309
- return ["--secret", f"id={secret_id},src={secret}"]
409
+ secret = Secret(key=secret)
410
+ secret_id = hash(secret)
310
411
  secret_env_key = "_".join([k.upper() for k in filter(None, (secret.group, secret.key))])
311
- secret_env = os.getenv(secret_env_key)
312
- if secret_env:
313
- return ["--secret", f"id={secret_id},env={secret_env}"]
412
+ if os.getenv(secret_env_key):
413
+ return ["--secret", f"id={secret_id},env={secret_env_key}"]
314
414
  secret_file_name = "_".join(list(filter(None, (secret.group, secret.key))))
315
415
  secret_file_path = f"/etc/secrets/{secret_file_name}"
316
416
  if not os.path.exists(secret_file_path):
@@ -318,7 +418,7 @@ def _get_secret_commands(layers: typing.Tuple[Layer, ...]) -> typing.List[str]:
318
418
  return ["--secret", f"id={secret_id},src={secret_file_path}"]
319
419
 
320
420
  for layer in layers:
321
- if isinstance(layer, (PipOption, AptPackages)):
421
+ if isinstance(layer, (PipOption, AptPackages, Commands)):
322
422
  if layer.secret_mounts:
323
423
  for secret_mount in layer.secret_mounts:
324
424
  commands.extend(_get_secret_command(secret_mount))
@@ -329,23 +429,23 @@ def _get_secret_mounts_layer(secrets: typing.Tuple[str | Secret, ...] | None) ->
329
429
  if secrets is None:
330
430
  return ""
331
431
  secret_mounts_layer = ""
332
- for secret in secrets:
432
+ for s in secrets:
433
+ secret = Secret(key=s) if isinstance(s, str) else s
333
434
  secret_id = hash(secret)
334
- if isinstance(secret, str):
335
- secret_mounts_layer += f"--mount=type=secret,id={secret_id},target=/run/secrets/{os.path.basename(secret)}"
336
- elif isinstance(secret, Secret):
337
- if secret.mount:
338
- secret_mounts_layer += f"--mount=type=secret,id={secret_id},target={secret.mount}"
339
- elif secret.as_env_var:
340
- secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret.as_env_var}"
341
- else:
342
- secret_file_name = "_".join(list(filter(None, (secret.group, secret.key))))
343
- secret_mounts_layer += f"--mount=type=secret,id={secret_id},src=/run/secrets/{secret_file_name}"
435
+ if secret.mount:
436
+ secret_mounts_layer += f"--mount=type=secret,id={secret_id},target={secret.mount}"
437
+ elif secret.as_env_var:
438
+ secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret.as_env_var}"
439
+ else:
440
+ secret_default_env_key = "_".join(list(filter(None, (secret.group, secret.key))))
441
+ secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret_default_env_key}"
344
442
 
345
443
  return secret_mounts_layer
346
444
 
347
445
 
348
- async def _process_layer(layer: Layer, context_path: Path, dockerfile: str) -> str:
446
+ async def _process_layer(
447
+ layer: Layer, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
448
+ ) -> str:
349
449
  match layer:
350
450
  case PythonWheels():
351
451
  # Handle Python wheels
@@ -377,11 +477,19 @@ async def _process_layer(layer: Layer, context_path: Path, dockerfile: str) -> s
377
477
 
378
478
  case UVProject():
379
479
  # Handle UV project
380
- dockerfile = await UVProjectHandler.handle(layer, context_path, dockerfile)
480
+ dockerfile = await UVProjectHandler.handle(layer, context_path, dockerfile, docker_ignore_patterns)
481
+
482
+ case PoetryProject():
483
+ # Handle Poetry project
484
+ dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile, docker_ignore_patterns)
485
+
486
+ case PoetryProject():
487
+ # Handle Poetry project
488
+ dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile, docker_ignore_patterns)
381
489
 
382
490
  case CopyConfig():
383
491
  # Handle local files and folders
384
- dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile)
492
+ dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile, docker_ignore_patterns)
385
493
 
386
494
  case Commands():
387
495
  # Handle commands
@@ -516,6 +624,7 @@ class DockerImageBuilder(ImageBuilder):
516
624
  - start from the base image
517
625
  - use python to create a default venv and export variables
518
626
 
627
+
519
628
  Then for the layers
520
629
  - for each layer
521
630
  - find the appropriate layer handler
@@ -537,8 +646,11 @@ class DockerImageBuilder(ImageBuilder):
537
646
  PYTHON_VERSION=f"{image.python_version[0]}.{image.python_version[1]}",
538
647
  )
539
648
 
649
+ # Get .dockerignore file patterns first
650
+ docker_ignore_patterns = get_and_list_dockerignore(image)
651
+
540
652
  for layer in image._layers:
541
- dockerfile = await _process_layer(layer, tmp_path, dockerfile)
653
+ dockerfile = await _process_layer(layer, tmp_path, dockerfile, docker_ignore_patterns)
542
654
 
543
655
  dockerfile += DOCKER_FILE_BASE_FOOTER.substitute(F_IMG_ID=image.uri)
544
656
 
@@ -135,11 +135,6 @@ class ImageBuildEngine:
135
135
 
136
136
  ImageBuilderType = typing.Literal["local", "remote"]
137
137
 
138
- _SEEN_IMAGES: typing.ClassVar[typing.Dict[str, str]] = {
139
- # Set default for the auto container. See Image._identifier_override for more info.
140
- "auto": Image.from_debian_base().uri,
141
- }
142
-
143
138
  @staticmethod
144
139
  @alru_cache
145
140
  async def image_exists(image: Image) -> Optional[str]: