flyte 2.0.0b13__py3-none-any.whl → 2.0.0b30__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flyte/__init__.py +18 -2
- flyte/_bin/debug.py +38 -0
- flyte/_bin/runtime.py +62 -8
- flyte/_cache/cache.py +4 -2
- flyte/_cache/local_cache.py +216 -0
- flyte/_code_bundle/_ignore.py +12 -4
- flyte/_code_bundle/_packaging.py +13 -9
- flyte/_code_bundle/_utils.py +18 -10
- flyte/_code_bundle/bundle.py +17 -9
- flyte/_constants.py +1 -0
- flyte/_context.py +4 -1
- flyte/_custom_context.py +73 -0
- flyte/_debug/constants.py +38 -0
- flyte/_debug/utils.py +17 -0
- flyte/_debug/vscode.py +307 -0
- flyte/_deploy.py +235 -61
- flyte/_environment.py +20 -6
- flyte/_excepthook.py +1 -1
- flyte/_hash.py +1 -16
- flyte/_image.py +178 -81
- flyte/_initialize.py +132 -51
- flyte/_interface.py +39 -2
- flyte/_internal/controllers/__init__.py +4 -5
- flyte/_internal/controllers/_local_controller.py +70 -29
- flyte/_internal/controllers/_trace.py +1 -1
- flyte/_internal/controllers/remote/__init__.py +0 -2
- flyte/_internal/controllers/remote/_action.py +14 -16
- flyte/_internal/controllers/remote/_client.py +1 -1
- flyte/_internal/controllers/remote/_controller.py +68 -70
- flyte/_internal/controllers/remote/_core.py +127 -99
- flyte/_internal/controllers/remote/_informer.py +19 -10
- flyte/_internal/controllers/remote/_service_protocol.py +7 -7
- flyte/_internal/imagebuild/docker_builder.py +181 -69
- flyte/_internal/imagebuild/image_builder.py +0 -5
- flyte/_internal/imagebuild/remote_builder.py +155 -64
- flyte/_internal/imagebuild/utils.py +51 -2
- flyte/_internal/resolvers/_task_module.py +5 -38
- flyte/_internal/resolvers/default.py +2 -2
- flyte/_internal/runtime/convert.py +110 -21
- flyte/_internal/runtime/entrypoints.py +27 -1
- flyte/_internal/runtime/io.py +21 -8
- flyte/_internal/runtime/resources_serde.py +20 -6
- flyte/_internal/runtime/reuse.py +1 -1
- flyte/_internal/runtime/rusty.py +20 -5
- flyte/_internal/runtime/task_serde.py +34 -19
- flyte/_internal/runtime/taskrunner.py +22 -4
- flyte/_internal/runtime/trigger_serde.py +160 -0
- flyte/_internal/runtime/types_serde.py +1 -1
- flyte/_keyring/__init__.py +0 -0
- flyte/_keyring/file.py +115 -0
- flyte/_logging.py +201 -39
- flyte/_map.py +111 -14
- flyte/_module.py +70 -0
- flyte/_pod.py +4 -3
- flyte/_resources.py +213 -31
- flyte/_run.py +110 -39
- flyte/_task.py +75 -16
- flyte/_task_environment.py +105 -29
- flyte/_task_plugins.py +4 -2
- flyte/_trace.py +5 -0
- flyte/_trigger.py +1000 -0
- flyte/_utils/__init__.py +2 -1
- flyte/_utils/asyn.py +3 -1
- flyte/_utils/coro_management.py +2 -1
- flyte/_utils/docker_credentials.py +173 -0
- flyte/_utils/module_loader.py +17 -2
- flyte/_version.py +3 -3
- flyte/cli/_abort.py +3 -3
- flyte/cli/_build.py +3 -6
- flyte/cli/_common.py +78 -7
- flyte/cli/_create.py +182 -4
- flyte/cli/_delete.py +23 -1
- flyte/cli/_deploy.py +63 -16
- flyte/cli/_get.py +79 -34
- flyte/cli/_params.py +26 -10
- flyte/cli/_plugins.py +209 -0
- flyte/cli/_run.py +151 -26
- flyte/cli/_serve.py +64 -0
- flyte/cli/_update.py +37 -0
- flyte/cli/_user.py +17 -0
- flyte/cli/main.py +30 -4
- flyte/config/_config.py +10 -6
- flyte/config/_internal.py +1 -0
- flyte/config/_reader.py +29 -8
- flyte/connectors/__init__.py +11 -0
- flyte/connectors/_connector.py +270 -0
- flyte/connectors/_server.py +197 -0
- flyte/connectors/utils.py +135 -0
- flyte/errors.py +22 -2
- flyte/extend.py +8 -1
- flyte/extras/_container.py +6 -1
- flyte/git/__init__.py +3 -0
- flyte/git/_config.py +21 -0
- flyte/io/__init__.py +2 -0
- flyte/io/_dataframe/__init__.py +2 -0
- flyte/io/_dataframe/basic_dfs.py +17 -8
- flyte/io/_dataframe/dataframe.py +98 -132
- flyte/io/_dir.py +575 -113
- flyte/io/_file.py +582 -139
- flyte/io/_hashing_io.py +342 -0
- flyte/models.py +74 -15
- flyte/remote/__init__.py +6 -1
- flyte/remote/_action.py +34 -26
- flyte/remote/_client/_protocols.py +39 -4
- flyte/remote/_client/auth/_authenticators/device_code.py +4 -5
- flyte/remote/_client/auth/_authenticators/pkce.py +1 -1
- flyte/remote/_client/auth/_channel.py +10 -6
- flyte/remote/_client/controlplane.py +17 -5
- flyte/remote/_console.py +3 -2
- flyte/remote/_data.py +6 -6
- flyte/remote/_logs.py +3 -3
- flyte/remote/_run.py +64 -8
- flyte/remote/_secret.py +26 -17
- flyte/remote/_task.py +75 -33
- flyte/remote/_trigger.py +306 -0
- flyte/remote/_user.py +33 -0
- flyte/report/_report.py +1 -1
- flyte/storage/__init__.py +6 -1
- flyte/storage/_config.py +5 -1
- flyte/storage/_parallel_reader.py +274 -0
- flyte/storage/_storage.py +200 -103
- flyte/types/__init__.py +16 -0
- flyte/types/_interface.py +2 -2
- flyte/types/_pickle.py +35 -8
- flyte/types/_string_literals.py +8 -9
- flyte/types/_type_engine.py +40 -70
- flyte/types/_utils.py +1 -1
- flyte-2.0.0b30.data/scripts/debug.py +38 -0
- {flyte-2.0.0b13.data → flyte-2.0.0b30.data}/scripts/runtime.py +62 -8
- {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/METADATA +11 -3
- flyte-2.0.0b30.dist-info/RECORD +192 -0
- {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/entry_points.txt +3 -0
- flyte/_protos/common/authorization_pb2.py +0 -66
- flyte/_protos/common/authorization_pb2.pyi +0 -108
- flyte/_protos/common/authorization_pb2_grpc.py +0 -4
- flyte/_protos/common/identifier_pb2.py +0 -93
- flyte/_protos/common/identifier_pb2.pyi +0 -110
- flyte/_protos/common/identifier_pb2_grpc.py +0 -4
- flyte/_protos/common/identity_pb2.py +0 -48
- flyte/_protos/common/identity_pb2.pyi +0 -72
- flyte/_protos/common/identity_pb2_grpc.py +0 -4
- flyte/_protos/common/list_pb2.py +0 -36
- flyte/_protos/common/list_pb2.pyi +0 -71
- flyte/_protos/common/list_pb2_grpc.py +0 -4
- flyte/_protos/common/policy_pb2.py +0 -37
- flyte/_protos/common/policy_pb2.pyi +0 -27
- flyte/_protos/common/policy_pb2_grpc.py +0 -4
- flyte/_protos/common/role_pb2.py +0 -37
- flyte/_protos/common/role_pb2.pyi +0 -53
- flyte/_protos/common/role_pb2_grpc.py +0 -4
- flyte/_protos/common/runtime_version_pb2.py +0 -28
- flyte/_protos/common/runtime_version_pb2.pyi +0 -24
- flyte/_protos/common/runtime_version_pb2_grpc.py +0 -4
- flyte/_protos/imagebuilder/definition_pb2.py +0 -59
- flyte/_protos/imagebuilder/definition_pb2.pyi +0 -140
- flyte/_protos/imagebuilder/definition_pb2_grpc.py +0 -4
- flyte/_protos/imagebuilder/payload_pb2.py +0 -32
- flyte/_protos/imagebuilder/payload_pb2.pyi +0 -21
- flyte/_protos/imagebuilder/payload_pb2_grpc.py +0 -4
- flyte/_protos/imagebuilder/service_pb2.py +0 -29
- flyte/_protos/imagebuilder/service_pb2.pyi +0 -5
- flyte/_protos/imagebuilder/service_pb2_grpc.py +0 -66
- flyte/_protos/logs/dataplane/payload_pb2.py +0 -100
- flyte/_protos/logs/dataplane/payload_pb2.pyi +0 -177
- flyte/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
- flyte/_protos/secret/definition_pb2.py +0 -49
- flyte/_protos/secret/definition_pb2.pyi +0 -93
- flyte/_protos/secret/definition_pb2_grpc.py +0 -4
- flyte/_protos/secret/payload_pb2.py +0 -62
- flyte/_protos/secret/payload_pb2.pyi +0 -94
- flyte/_protos/secret/payload_pb2_grpc.py +0 -4
- flyte/_protos/secret/secret_pb2.py +0 -38
- flyte/_protos/secret/secret_pb2.pyi +0 -6
- flyte/_protos/secret/secret_pb2_grpc.py +0 -198
- flyte/_protos/secret/secret_pb2_grpc_grpc.py +0 -198
- flyte/_protos/validate/validate/validate_pb2.py +0 -76
- flyte/_protos/workflow/common_pb2.py +0 -27
- flyte/_protos/workflow/common_pb2.pyi +0 -14
- flyte/_protos/workflow/common_pb2_grpc.py +0 -4
- flyte/_protos/workflow/environment_pb2.py +0 -29
- flyte/_protos/workflow/environment_pb2.pyi +0 -12
- flyte/_protos/workflow/environment_pb2_grpc.py +0 -4
- flyte/_protos/workflow/node_execution_service_pb2.py +0 -26
- flyte/_protos/workflow/node_execution_service_pb2.pyi +0 -4
- flyte/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
- flyte/_protos/workflow/queue_service_pb2.py +0 -109
- flyte/_protos/workflow/queue_service_pb2.pyi +0 -166
- flyte/_protos/workflow/queue_service_pb2_grpc.py +0 -172
- flyte/_protos/workflow/run_definition_pb2.py +0 -121
- flyte/_protos/workflow/run_definition_pb2.pyi +0 -327
- flyte/_protos/workflow/run_definition_pb2_grpc.py +0 -4
- flyte/_protos/workflow/run_logs_service_pb2.py +0 -41
- flyte/_protos/workflow/run_logs_service_pb2.pyi +0 -28
- flyte/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
- flyte/_protos/workflow/run_service_pb2.py +0 -137
- flyte/_protos/workflow/run_service_pb2.pyi +0 -185
- flyte/_protos/workflow/run_service_pb2_grpc.py +0 -446
- flyte/_protos/workflow/state_service_pb2.py +0 -67
- flyte/_protos/workflow/state_service_pb2.pyi +0 -76
- flyte/_protos/workflow/state_service_pb2_grpc.py +0 -138
- flyte/_protos/workflow/task_definition_pb2.py +0 -79
- flyte/_protos/workflow/task_definition_pb2.pyi +0 -81
- flyte/_protos/workflow/task_definition_pb2_grpc.py +0 -4
- flyte/_protos/workflow/task_service_pb2.py +0 -60
- flyte/_protos/workflow/task_service_pb2.pyi +0 -59
- flyte/_protos/workflow/task_service_pb2_grpc.py +0 -138
- flyte-2.0.0b13.dist-info/RECORD +0 -239
- /flyte/{_protos → _debug}/__init__.py +0 -0
- {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/WHEEL +0 -0
- {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/licenses/LICENSE +0 -0
- {flyte-2.0.0b13.dist-info → flyte-2.0.0b30.dist-info}/top_level.txt +0 -0
|
@@ -5,10 +5,10 @@ from asyncio import Queue
|
|
|
5
5
|
from typing import AsyncIterator, Callable, Dict, Optional, Tuple, cast
|
|
6
6
|
|
|
7
7
|
import grpc.aio
|
|
8
|
+
from flyteidl2.common import identifier_pb2
|
|
9
|
+
from flyteidl2.workflow import run_definition_pb2, state_service_pb2
|
|
8
10
|
|
|
9
11
|
from flyte._logging import log, logger
|
|
10
|
-
from flyte._protos.common import identifier_pb2
|
|
11
|
-
from flyte._protos.workflow import run_definition_pb2, state_service_pb2
|
|
12
12
|
|
|
13
13
|
from ._action import Action
|
|
14
14
|
from ._service_protocol import StateService
|
|
@@ -132,8 +132,10 @@ class Informer:
|
|
|
132
132
|
parent_action_name: str,
|
|
133
133
|
shared_queue: Queue,
|
|
134
134
|
client: Optional[StateService] = None,
|
|
135
|
-
|
|
135
|
+
min_watch_backoff: float = 1.0,
|
|
136
|
+
max_watch_backoff: float = 30.0,
|
|
136
137
|
watch_conn_timeout_sec: float = 5.0,
|
|
138
|
+
max_watch_retries: int = 10,
|
|
137
139
|
):
|
|
138
140
|
self.name = self.mkname(run_name=run_id.name, parent_action_name=parent_action_name)
|
|
139
141
|
self.parent_action_name = parent_action_name
|
|
@@ -144,8 +146,10 @@ class Informer:
|
|
|
144
146
|
self._running = False
|
|
145
147
|
self._watch_task: asyncio.Task | None = None
|
|
146
148
|
self._ready = asyncio.Event()
|
|
147
|
-
self.
|
|
149
|
+
self._min_watch_backoff = min_watch_backoff
|
|
150
|
+
self._max_watch_backoff = max_watch_backoff
|
|
148
151
|
self._watch_conn_timeout_sec = watch_conn_timeout_sec
|
|
152
|
+
self._max_watch_retries = max_watch_retries
|
|
149
153
|
|
|
150
154
|
@classmethod
|
|
151
155
|
def mkname(cls, *, run_name: str, parent_action_name: str) -> str:
|
|
@@ -211,13 +215,16 @@ class Informer:
|
|
|
211
215
|
"""
|
|
212
216
|
# sentinel = False
|
|
213
217
|
retries = 0
|
|
214
|
-
max_retries = 5
|
|
215
218
|
last_exc = None
|
|
216
219
|
while self._running:
|
|
217
|
-
if retries >=
|
|
218
|
-
logger.error(
|
|
220
|
+
if retries >= self._max_watch_retries:
|
|
221
|
+
logger.error(
|
|
222
|
+
f"Informer watch failure retries crossed threshold {retries}/{self._max_watch_retries}, exiting!"
|
|
223
|
+
)
|
|
219
224
|
raise last_exc
|
|
220
225
|
try:
|
|
226
|
+
if retries >= 1:
|
|
227
|
+
logger.warning(f"Informer watch retrying, attempt {retries}/{self._max_watch_retries}")
|
|
221
228
|
watcher = self._client.Watch(
|
|
222
229
|
state_service_pb2.WatchRequest(
|
|
223
230
|
parent_action_id=identifier_pb2.ActionIdentifier(
|
|
@@ -252,7 +259,9 @@ class Informer:
|
|
|
252
259
|
logger.exception(f"Watch error: {self.name}", exc_info=e)
|
|
253
260
|
last_exc = e
|
|
254
261
|
retries += 1
|
|
255
|
-
|
|
262
|
+
backoff = min(self._min_watch_backoff * (2**retries), self._max_watch_backoff)
|
|
263
|
+
logger.warning(f"Watch for {self.name} failed, retrying in {backoff} seconds...")
|
|
264
|
+
await asyncio.sleep(backoff)
|
|
256
265
|
|
|
257
266
|
@log
|
|
258
267
|
async def start(self, timeout: Optional[float] = None) -> asyncio.Task:
|
|
@@ -261,7 +270,7 @@ class Informer:
|
|
|
261
270
|
logger.warning("Informer already running")
|
|
262
271
|
return cast(asyncio.Task, self._watch_task)
|
|
263
272
|
self._running = True
|
|
264
|
-
self._watch_task = asyncio.create_task(self.watch())
|
|
273
|
+
self._watch_task = asyncio.create_task(self.watch(), name=f"InformerWatch-{self.parent_action_name}")
|
|
265
274
|
await self.wait_for_cache_sync(timeout=timeout)
|
|
266
275
|
return self._watch_task
|
|
267
276
|
|
|
@@ -364,7 +373,7 @@ class InformerCache:
|
|
|
364
373
|
"""Stop all informers and remove them from the cache"""
|
|
365
374
|
async with self._lock:
|
|
366
375
|
while self._cache:
|
|
367
|
-
|
|
376
|
+
_name, informer = self._cache.popitem()
|
|
368
377
|
try:
|
|
369
378
|
await informer.stop()
|
|
370
379
|
except asyncio.CancelledError:
|
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from typing import AsyncIterator, Protocol
|
|
4
4
|
|
|
5
|
-
from
|
|
5
|
+
from flyteidl2.workflow import queue_service_pb2, state_service_pb2
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class StateService(Protocol):
|
|
@@ -28,12 +28,12 @@ class QueueService(Protocol):
|
|
|
28
28
|
) -> queue_service_pb2.EnqueueActionResponse:
|
|
29
29
|
"""Enqueue a task"""
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
31
|
+
async def AbortQueuedAction(
|
|
32
|
+
self,
|
|
33
|
+
req: queue_service_pb2.AbortQueuedActionRequest,
|
|
34
|
+
**kwargs,
|
|
35
|
+
) -> queue_service_pb2.AbortQueuedActionResponse:
|
|
36
|
+
"""Cancel an enqueued task"""
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
class ClientSet(Protocol):
|
|
@@ -22,6 +22,7 @@ from flyte._image import (
|
|
|
22
22
|
Layer,
|
|
23
23
|
PipOption,
|
|
24
24
|
PipPackages,
|
|
25
|
+
PoetryProject,
|
|
25
26
|
PythonWheels,
|
|
26
27
|
Requirements,
|
|
27
28
|
UVProject,
|
|
@@ -37,80 +38,112 @@ from flyte._internal.imagebuild.image_builder import (
|
|
|
37
38
|
LocalDockerCommandImageChecker,
|
|
38
39
|
LocalPodmanCommandImageChecker,
|
|
39
40
|
)
|
|
40
|
-
from flyte._internal.imagebuild.utils import copy_files_to_context
|
|
41
|
+
from flyte._internal.imagebuild.utils import copy_files_to_context, get_and_list_dockerignore
|
|
41
42
|
from flyte._logging import logger
|
|
42
43
|
|
|
43
44
|
_F_IMG_ID = "_F_IMG_ID"
|
|
44
45
|
FLYTE_DOCKER_BUILDER_CACHE_FROM = "FLYTE_DOCKER_BUILDER_CACHE_FROM"
|
|
45
46
|
FLYTE_DOCKER_BUILDER_CACHE_TO = "FLYTE_DOCKER_BUILDER_CACHE_TO"
|
|
46
47
|
|
|
48
|
+
UV_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE = Template("""\
|
|
49
|
+
RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
|
|
50
|
+
--mount=type=bind,target=uv.lock,src=$UV_LOCK_PATH,rw \
|
|
51
|
+
--mount=type=bind,target=pyproject.toml,src=$PYPROJECT_PATH \
|
|
52
|
+
$SECRET_MOUNT \
|
|
53
|
+
uv sync --active --inexact $PIP_INSTALL_ARGS
|
|
54
|
+
""")
|
|
55
|
+
|
|
47
56
|
UV_LOCK_INSTALL_TEMPLATE = Template("""\
|
|
48
|
-
WORKDIR /root
|
|
49
57
|
RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
58
|
+
--mount=type=bind,target=/root/.flyte/$PYPROJECT_PATH,src=$PYPROJECT_PATH,rw \
|
|
59
|
+
$SECRET_MOUNT \
|
|
60
|
+
uv sync --active --inexact --no-editable $PIP_INSTALL_ARGS --project /root/.flyte/$PYPROJECT_PATH
|
|
61
|
+
""")
|
|
62
|
+
|
|
63
|
+
POETRY_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE = Template("""\
|
|
64
|
+
RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
|
|
65
|
+
uv pip install poetry
|
|
66
|
+
|
|
67
|
+
ENV POETRY_CACHE_DIR=/tmp/poetry_cache \
|
|
68
|
+
POETRY_VIRTUALENVS_IN_PROJECT=true
|
|
69
|
+
|
|
70
|
+
RUN --mount=type=cache,sharing=locked,mode=0777,target=/tmp/poetry_cache,id=poetry \
|
|
71
|
+
--mount=type=bind,target=poetry.lock,src=$POETRY_LOCK_PATH \
|
|
72
|
+
--mount=type=bind,target=pyproject.toml,src=$PYPROJECT_PATH \
|
|
73
|
+
$SECRET_MOUNT \
|
|
74
|
+
poetry install $POETRY_INSTALL_ARGS
|
|
75
|
+
""")
|
|
76
|
+
|
|
77
|
+
POETRY_LOCK_INSTALL_TEMPLATE = Template("""\
|
|
78
|
+
RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
|
|
79
|
+
uv pip install poetry
|
|
80
|
+
|
|
81
|
+
ENV POETRY_CACHE_DIR=/tmp/poetry_cache \
|
|
82
|
+
POETRY_VIRTUALENVS_IN_PROJECT=true
|
|
83
|
+
|
|
84
|
+
RUN --mount=type=cache,sharing=locked,mode=0777,target=/tmp/poetry_cache,id=poetry \
|
|
85
|
+
--mount=type=bind,target=/root/.flyte/$PYPROJECT_PATH,src=$PYPROJECT_PATH,rw \
|
|
86
|
+
$SECRET_MOUNT \
|
|
87
|
+
poetry install $POETRY_INSTALL_ARGS -C /root/.flyte/$PYPROJECT_PATH
|
|
60
88
|
""")
|
|
61
89
|
|
|
62
90
|
UV_PACKAGE_INSTALL_COMMAND_TEMPLATE = Template("""\
|
|
63
91
|
RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
92
|
+
$REQUIREMENTS_MOUNT \
|
|
93
|
+
$SECRET_MOUNT \
|
|
94
|
+
uv pip install --python $$UV_PYTHON $PIP_INSTALL_ARGS
|
|
67
95
|
""")
|
|
68
96
|
|
|
69
97
|
UV_WHEEL_INSTALL_COMMAND_TEMPLATE = Template("""\
|
|
70
98
|
RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=wheel \
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
99
|
+
--mount=source=/dist,target=/dist,type=bind \
|
|
100
|
+
$SECRET_MOUNT \
|
|
101
|
+
uv pip install --python $$UV_PYTHON $PIP_INSTALL_ARGS
|
|
74
102
|
""")
|
|
75
103
|
|
|
76
104
|
APT_INSTALL_COMMAND_TEMPLATE = Template("""\
|
|
77
105
|
RUN --mount=type=cache,sharing=locked,mode=0777,target=/var/cache/apt,id=apt \
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
106
|
+
$SECRET_MOUNT \
|
|
107
|
+
apt-get update && apt-get install -y --no-install-recommends \
|
|
108
|
+
$APT_PACKAGES
|
|
81
109
|
""")
|
|
82
110
|
|
|
83
111
|
UV_PYTHON_INSTALL_COMMAND = Template("""\
|
|
84
112
|
RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
|
|
85
|
-
|
|
86
|
-
|
|
113
|
+
$SECRET_MOUNT \
|
|
114
|
+
uv pip install $PIP_INSTALL_ARGS
|
|
87
115
|
""")
|
|
88
116
|
|
|
89
117
|
# uv pip install --python /root/env/bin/python
|
|
90
118
|
# new template
|
|
91
119
|
DOCKER_FILE_UV_BASE_TEMPLATE = Template("""\
|
|
92
120
|
# syntax=docker/dockerfile:1.10
|
|
93
|
-
FROM ghcr.io/astral-sh/uv:0.
|
|
121
|
+
FROM ghcr.io/astral-sh/uv:0.8.13 AS uv
|
|
94
122
|
FROM $BASE_IMAGE
|
|
95
123
|
|
|
124
|
+
|
|
96
125
|
USER root
|
|
97
126
|
|
|
127
|
+
|
|
98
128
|
# Copy in uv so that later commands don't have to mount it in
|
|
99
129
|
COPY --from=uv /uv /usr/bin/uv
|
|
100
130
|
|
|
131
|
+
|
|
101
132
|
# Configure default envs
|
|
102
133
|
ENV UV_COMPILE_BYTECODE=1 \
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
134
|
+
UV_LINK_MODE=copy \
|
|
135
|
+
VIRTUALENV=/opt/venv \
|
|
136
|
+
UV_PYTHON=/opt/venv/bin/python \
|
|
137
|
+
PATH="/opt/venv/bin:$$PATH"
|
|
138
|
+
|
|
107
139
|
|
|
108
140
|
# Create a virtualenv with the user specified python version
|
|
109
141
|
RUN uv venv $$VIRTUALENV --python=$PYTHON_VERSION
|
|
110
142
|
|
|
143
|
+
|
|
111
144
|
# Adds nvidia just in case it exists
|
|
112
145
|
ENV PATH="$$PATH:/usr/local/nvidia/bin:/usr/local/cuda/bin" \
|
|
113
|
-
|
|
146
|
+
LD_LIBRARY_PATH="/usr/local/nvidia/lib64"
|
|
114
147
|
""")
|
|
115
148
|
|
|
116
149
|
# This gets added on to the end of the dockerfile
|
|
@@ -177,6 +210,7 @@ class PythonWheelHandler:
|
|
|
177
210
|
"/dist",
|
|
178
211
|
"--no-deps",
|
|
179
212
|
"--no-index",
|
|
213
|
+
"--reinstall",
|
|
180
214
|
layer.package_name,
|
|
181
215
|
],
|
|
182
216
|
]
|
|
@@ -229,21 +263,84 @@ class AptPackagesHandler:
|
|
|
229
263
|
|
|
230
264
|
class UVProjectHandler:
|
|
231
265
|
@staticmethod
|
|
232
|
-
async def handle(
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
shutil.copy(layer.uvlock, context_path)
|
|
236
|
-
|
|
237
|
-
# --locked: Assert that the `uv.lock` will remain unchanged
|
|
238
|
-
# --no-dev: Omit the development dependency group
|
|
239
|
-
# --no-install-project: Do not install the current project
|
|
240
|
-
additional_pip_install_args = ["--locked", "--no-dev", "--no-install-project"]
|
|
266
|
+
async def handle(
|
|
267
|
+
layer: UVProject, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
|
|
268
|
+
) -> str:
|
|
241
269
|
secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
270
|
+
if layer.project_install_mode == "dependencies_only":
|
|
271
|
+
pip_install_args = " ".join(layer.get_pip_install_args())
|
|
272
|
+
if "--no-install-project" not in pip_install_args:
|
|
273
|
+
pip_install_args += " --no-install-project"
|
|
274
|
+
if "--no-sources" not in pip_install_args:
|
|
275
|
+
pip_install_args += " --no-sources"
|
|
276
|
+
# Only Copy pyproject.yaml and uv.lock.
|
|
277
|
+
pyproject_dst = copy_files_to_context(layer.pyproject, context_path)
|
|
278
|
+
uvlock_dst = copy_files_to_context(layer.uvlock, context_path)
|
|
279
|
+
delta = UV_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE.substitute(
|
|
280
|
+
UV_LOCK_PATH=uvlock_dst.relative_to(context_path),
|
|
281
|
+
PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
|
|
282
|
+
PIP_INSTALL_ARGS=pip_install_args,
|
|
283
|
+
SECRET_MOUNT=secret_mounts,
|
|
284
|
+
)
|
|
285
|
+
else:
|
|
286
|
+
# Copy the entire project.
|
|
287
|
+
pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path, docker_ignore_patterns)
|
|
288
|
+
|
|
289
|
+
# Make sure pyproject.toml and uv.lock files are not removed by docker ignore
|
|
290
|
+
uv_lock_context_path = pyproject_dst / "uv.lock"
|
|
291
|
+
pyproject_context_path = pyproject_dst / "pyproject.toml"
|
|
292
|
+
if not uv_lock_context_path.exists():
|
|
293
|
+
shutil.copy(layer.uvlock, pyproject_dst)
|
|
294
|
+
if not pyproject_context_path.exists():
|
|
295
|
+
shutil.copy(layer.pyproject, pyproject_dst)
|
|
296
|
+
|
|
297
|
+
delta = UV_LOCK_INSTALL_TEMPLATE.substitute(
|
|
298
|
+
PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
|
|
299
|
+
PIP_INSTALL_ARGS=" ".join(layer.get_pip_install_args()),
|
|
300
|
+
SECRET_MOUNT=secret_mounts,
|
|
301
|
+
)
|
|
302
|
+
|
|
245
303
|
dockerfile += delta
|
|
304
|
+
return dockerfile
|
|
305
|
+
|
|
246
306
|
|
|
307
|
+
class PoetryProjectHandler:
|
|
308
|
+
@staticmethod
|
|
309
|
+
async def handel(
|
|
310
|
+
layer: PoetryProject, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
|
|
311
|
+
) -> str:
|
|
312
|
+
secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
|
|
313
|
+
extra_args = layer.extra_args or ""
|
|
314
|
+
if layer.project_install_mode == "dependencies_only":
|
|
315
|
+
# Only Copy pyproject.yaml and poetry.lock.
|
|
316
|
+
pyproject_dst = copy_files_to_context(layer.pyproject, context_path)
|
|
317
|
+
poetry_lock_dst = copy_files_to_context(layer.poetry_lock, context_path)
|
|
318
|
+
if "--no-root" not in extra_args:
|
|
319
|
+
extra_args += " --no-root"
|
|
320
|
+
delta = POETRY_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE.substitute(
|
|
321
|
+
POETRY_LOCK_PATH=poetry_lock_dst.relative_to(context_path),
|
|
322
|
+
PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
|
|
323
|
+
POETRY_INSTALL_ARGS=extra_args,
|
|
324
|
+
SECRET_MOUNT=secret_mounts,
|
|
325
|
+
)
|
|
326
|
+
else:
|
|
327
|
+
# Copy the entire project.
|
|
328
|
+
pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path, docker_ignore_patterns)
|
|
329
|
+
|
|
330
|
+
# Make sure pyproject.toml and poetry.lock files are not removed by docker ignore
|
|
331
|
+
poetry_lock_context_path = pyproject_dst / "poetry.lock"
|
|
332
|
+
pyproject_context_path = pyproject_dst / "pyproject.toml"
|
|
333
|
+
if not poetry_lock_context_path.exists():
|
|
334
|
+
shutil.copy(layer.poetry_lock, pyproject_dst)
|
|
335
|
+
if not pyproject_context_path.exists():
|
|
336
|
+
shutil.copy(layer.pyproject, pyproject_dst)
|
|
337
|
+
|
|
338
|
+
delta = POETRY_LOCK_INSTALL_TEMPLATE.substitute(
|
|
339
|
+
PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
|
|
340
|
+
POETRY_INSTALL_ARGS=extra_args,
|
|
341
|
+
SECRET_MOUNT=secret_mounts,
|
|
342
|
+
)
|
|
343
|
+
dockerfile += delta
|
|
247
344
|
return dockerfile
|
|
248
345
|
|
|
249
346
|
|
|
@@ -255,7 +352,9 @@ class DockerIgnoreHandler:
|
|
|
255
352
|
|
|
256
353
|
class CopyConfigHandler:
|
|
257
354
|
@staticmethod
|
|
258
|
-
async def handle(
|
|
355
|
+
async def handle(
|
|
356
|
+
layer: CopyConfig, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
|
|
357
|
+
) -> str:
|
|
259
358
|
# Copy the source config file or directory to the context path
|
|
260
359
|
if layer.src.is_absolute() or ".." in str(layer.src):
|
|
261
360
|
dst_path = context_path / str(layer.src.absolute()).replace("/", "./_flyte_abs_context/", 1)
|
|
@@ -264,18 +363,21 @@ class CopyConfigHandler:
|
|
|
264
363
|
|
|
265
364
|
dst_path.parent.mkdir(parents=True, exist_ok=True)
|
|
266
365
|
abs_path = layer.src.absolute()
|
|
366
|
+
|
|
267
367
|
if layer.src.is_file():
|
|
268
368
|
# Copy the file
|
|
269
369
|
shutil.copy(abs_path, dst_path)
|
|
270
370
|
elif layer.src.is_dir():
|
|
271
371
|
# Copy the entire directory
|
|
272
|
-
shutil.copytree(
|
|
372
|
+
shutil.copytree(
|
|
373
|
+
abs_path, dst_path, dirs_exist_ok=True, ignore=shutil.ignore_patterns(*docker_ignore_patterns)
|
|
374
|
+
)
|
|
273
375
|
else:
|
|
274
|
-
|
|
376
|
+
logger.error(f"Source path not exists: {layer.src}")
|
|
377
|
+
return dockerfile
|
|
275
378
|
|
|
276
379
|
# Add a copy command to the dockerfile
|
|
277
380
|
dockerfile += f"\nCOPY {dst_path.relative_to(context_path)} {layer.dst}\n"
|
|
278
|
-
|
|
279
381
|
return dockerfile
|
|
280
382
|
|
|
281
383
|
|
|
@@ -283,8 +385,9 @@ class CommandsHandler:
|
|
|
283
385
|
@staticmethod
|
|
284
386
|
async def handle(layer: Commands, _: Path, dockerfile: str) -> str:
|
|
285
387
|
# Append raw commands to the dockerfile
|
|
388
|
+
secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
|
|
286
389
|
for command in layer.commands:
|
|
287
|
-
dockerfile += f"\nRUN {command}\n"
|
|
390
|
+
dockerfile += f"\nRUN {secret_mounts} {command}\n"
|
|
288
391
|
|
|
289
392
|
return dockerfile
|
|
290
393
|
|
|
@@ -302,15 +405,12 @@ def _get_secret_commands(layers: typing.Tuple[Layer, ...]) -> typing.List[str]:
|
|
|
302
405
|
commands = []
|
|
303
406
|
|
|
304
407
|
def _get_secret_command(secret: str | Secret) -> typing.List[str]:
|
|
305
|
-
secret_id = hash(secret)
|
|
306
408
|
if isinstance(secret, str):
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
return ["--secret", f"id={secret_id},src={secret}"]
|
|
409
|
+
secret = Secret(key=secret)
|
|
410
|
+
secret_id = hash(secret)
|
|
310
411
|
secret_env_key = "_".join([k.upper() for k in filter(None, (secret.group, secret.key))])
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
return ["--secret", f"id={secret_id},env={secret_env}"]
|
|
412
|
+
if os.getenv(secret_env_key):
|
|
413
|
+
return ["--secret", f"id={secret_id},env={secret_env_key}"]
|
|
314
414
|
secret_file_name = "_".join(list(filter(None, (secret.group, secret.key))))
|
|
315
415
|
secret_file_path = f"/etc/secrets/{secret_file_name}"
|
|
316
416
|
if not os.path.exists(secret_file_path):
|
|
@@ -318,7 +418,7 @@ def _get_secret_commands(layers: typing.Tuple[Layer, ...]) -> typing.List[str]:
|
|
|
318
418
|
return ["--secret", f"id={secret_id},src={secret_file_path}"]
|
|
319
419
|
|
|
320
420
|
for layer in layers:
|
|
321
|
-
if isinstance(layer, (PipOption, AptPackages)):
|
|
421
|
+
if isinstance(layer, (PipOption, AptPackages, Commands)):
|
|
322
422
|
if layer.secret_mounts:
|
|
323
423
|
for secret_mount in layer.secret_mounts:
|
|
324
424
|
commands.extend(_get_secret_command(secret_mount))
|
|
@@ -329,23 +429,23 @@ def _get_secret_mounts_layer(secrets: typing.Tuple[str | Secret, ...] | None) ->
|
|
|
329
429
|
if secrets is None:
|
|
330
430
|
return ""
|
|
331
431
|
secret_mounts_layer = ""
|
|
332
|
-
for
|
|
432
|
+
for s in secrets:
|
|
433
|
+
secret = Secret(key=s) if isinstance(s, str) else s
|
|
333
434
|
secret_id = hash(secret)
|
|
334
|
-
if
|
|
335
|
-
secret_mounts_layer += f"--mount=type=secret,id={secret_id},target
|
|
336
|
-
elif
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
else:
|
|
342
|
-
secret_file_name = "_".join(list(filter(None, (secret.group, secret.key))))
|
|
343
|
-
secret_mounts_layer += f"--mount=type=secret,id={secret_id},src=/run/secrets/{secret_file_name}"
|
|
435
|
+
if secret.mount:
|
|
436
|
+
secret_mounts_layer += f"--mount=type=secret,id={secret_id},target={secret.mount}"
|
|
437
|
+
elif secret.as_env_var:
|
|
438
|
+
secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret.as_env_var}"
|
|
439
|
+
else:
|
|
440
|
+
secret_default_env_key = "_".join(list(filter(None, (secret.group, secret.key))))
|
|
441
|
+
secret_mounts_layer += f"--mount=type=secret,id={secret_id},env={secret_default_env_key}"
|
|
344
442
|
|
|
345
443
|
return secret_mounts_layer
|
|
346
444
|
|
|
347
445
|
|
|
348
|
-
async def _process_layer(
|
|
446
|
+
async def _process_layer(
|
|
447
|
+
layer: Layer, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
|
|
448
|
+
) -> str:
|
|
349
449
|
match layer:
|
|
350
450
|
case PythonWheels():
|
|
351
451
|
# Handle Python wheels
|
|
@@ -377,11 +477,19 @@ async def _process_layer(layer: Layer, context_path: Path, dockerfile: str) -> s
|
|
|
377
477
|
|
|
378
478
|
case UVProject():
|
|
379
479
|
# Handle UV project
|
|
380
|
-
dockerfile = await UVProjectHandler.handle(layer, context_path, dockerfile)
|
|
480
|
+
dockerfile = await UVProjectHandler.handle(layer, context_path, dockerfile, docker_ignore_patterns)
|
|
481
|
+
|
|
482
|
+
case PoetryProject():
|
|
483
|
+
# Handle Poetry project
|
|
484
|
+
dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile, docker_ignore_patterns)
|
|
485
|
+
|
|
486
|
+
case PoetryProject():
|
|
487
|
+
# Handle Poetry project
|
|
488
|
+
dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile, docker_ignore_patterns)
|
|
381
489
|
|
|
382
490
|
case CopyConfig():
|
|
383
491
|
# Handle local files and folders
|
|
384
|
-
dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile)
|
|
492
|
+
dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile, docker_ignore_patterns)
|
|
385
493
|
|
|
386
494
|
case Commands():
|
|
387
495
|
# Handle commands
|
|
@@ -516,6 +624,7 @@ class DockerImageBuilder(ImageBuilder):
|
|
|
516
624
|
- start from the base image
|
|
517
625
|
- use python to create a default venv and export variables
|
|
518
626
|
|
|
627
|
+
|
|
519
628
|
Then for the layers
|
|
520
629
|
- for each layer
|
|
521
630
|
- find the appropriate layer handler
|
|
@@ -537,8 +646,11 @@ class DockerImageBuilder(ImageBuilder):
|
|
|
537
646
|
PYTHON_VERSION=f"{image.python_version[0]}.{image.python_version[1]}",
|
|
538
647
|
)
|
|
539
648
|
|
|
649
|
+
# Get .dockerignore file patterns first
|
|
650
|
+
docker_ignore_patterns = get_and_list_dockerignore(image)
|
|
651
|
+
|
|
540
652
|
for layer in image._layers:
|
|
541
|
-
dockerfile = await _process_layer(layer, tmp_path, dockerfile)
|
|
653
|
+
dockerfile = await _process_layer(layer, tmp_path, dockerfile, docker_ignore_patterns)
|
|
542
654
|
|
|
543
655
|
dockerfile += DOCKER_FILE_BASE_FOOTER.substitute(F_IMG_ID=image.uri)
|
|
544
656
|
|
|
@@ -135,11 +135,6 @@ class ImageBuildEngine:
|
|
|
135
135
|
|
|
136
136
|
ImageBuilderType = typing.Literal["local", "remote"]
|
|
137
137
|
|
|
138
|
-
_SEEN_IMAGES: typing.ClassVar[typing.Dict[str, str]] = {
|
|
139
|
-
# Set default for the auto container. See Image._identifier_override for more info.
|
|
140
|
-
"auto": Image.from_debian_base().uri,
|
|
141
|
-
}
|
|
142
|
-
|
|
143
138
|
@staticmethod
|
|
144
139
|
@alru_cache
|
|
145
140
|
async def image_exists(image: Image) -> Optional[str]:
|