flyte 0.2.0b18__py3-none-any.whl → 0.2.0b19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/_code_bundle/bundle.py +2 -2
- flyte/_deploy.py +2 -3
- flyte/_image.py +100 -64
- flyte/_initialize.py +9 -1
- flyte/_internal/imagebuild/__init__.py +4 -0
- flyte/_internal/imagebuild/docker_builder.py +57 -24
- flyte/_internal/imagebuild/image_builder.py +69 -42
- flyte/_internal/imagebuild/remote_builder.py +259 -0
- flyte/_protos/imagebuilder/definition_pb2.py +59 -0
- flyte/_protos/imagebuilder/definition_pb2.pyi +140 -0
- flyte/_protos/imagebuilder/definition_pb2_grpc.py +4 -0
- flyte/_protos/imagebuilder/payload_pb2.py +32 -0
- flyte/_protos/imagebuilder/payload_pb2.pyi +21 -0
- flyte/_protos/imagebuilder/payload_pb2_grpc.py +4 -0
- flyte/_protos/imagebuilder/service_pb2.py +29 -0
- flyte/_protos/imagebuilder/service_pb2.pyi +5 -0
- flyte/_protos/imagebuilder/service_pb2_grpc.py +66 -0
- flyte/_run.py +20 -8
- flyte/_task_environment.py +1 -0
- flyte/_version.py +2 -2
- flyte/cli/__init__.py +9 -0
- flyte/cli/_create.py +15 -0
- flyte/config/_config.py +30 -2
- flyte/config/_internal.py +8 -0
- flyte/config/_reader.py +0 -3
- flyte/extras/_container.py +2 -2
- flyte/remote/_data.py +2 -0
- flyte/remote/_run.py +5 -4
- flyte/remote/_task.py +35 -7
- {flyte-0.2.0b18.dist-info → flyte-0.2.0b19.dist-info}/METADATA +1 -1
- {flyte-0.2.0b18.dist-info → flyte-0.2.0b19.dist-info}/RECORD +34 -25
- {flyte-0.2.0b18.dist-info → flyte-0.2.0b19.dist-info}/WHEEL +0 -0
- {flyte-0.2.0b18.dist-info → flyte-0.2.0b19.dist-info}/entry_points.txt +0 -0
- {flyte-0.2.0b18.dist-info → flyte-0.2.0b19.dist-info}/top_level.txt +0 -0
|
@@ -10,18 +10,26 @@ from pydantic import BaseModel
|
|
|
10
10
|
from typing_extensions import Protocol
|
|
11
11
|
|
|
12
12
|
from flyte._image import Architecture, Image
|
|
13
|
+
from flyte._initialize import _get_init_config
|
|
13
14
|
from flyte._logging import logger
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
class ImageBuilder(Protocol):
|
|
17
18
|
async def build_image(self, image: Image, dry_run: bool) -> str: ...
|
|
18
19
|
|
|
20
|
+
def get_checkers(self) -> Optional[typing.List[typing.Type[ImageChecker]]]:
|
|
21
|
+
"""
|
|
22
|
+
Returns ImageCheckers that can be used to check if the image exists in the registry.
|
|
23
|
+
If None, then use the default checkers.
|
|
24
|
+
"""
|
|
25
|
+
return None
|
|
26
|
+
|
|
19
27
|
|
|
20
28
|
class ImageChecker(Protocol):
|
|
21
29
|
@classmethod
|
|
22
30
|
async def image_exists(
|
|
23
31
|
cls, repository: str, tag: str, arch: Tuple[Architecture, ...] = ("linux/amd64",)
|
|
24
|
-
) ->
|
|
32
|
+
) -> Optional[str]: ...
|
|
25
33
|
|
|
26
34
|
|
|
27
35
|
class DockerAPIImageChecker(ImageChecker):
|
|
@@ -32,7 +40,9 @@ class DockerAPIImageChecker(ImageChecker):
|
|
|
32
40
|
"""
|
|
33
41
|
|
|
34
42
|
@classmethod
|
|
35
|
-
async def image_exists(
|
|
43
|
+
async def image_exists(
|
|
44
|
+
cls, repository: str, tag: str, arch: Tuple[Architecture, ...] = ("linux/amd64",)
|
|
45
|
+
) -> Optional[str]:
|
|
36
46
|
import httpx
|
|
37
47
|
|
|
38
48
|
if "/" not in repository:
|
|
@@ -50,6 +60,7 @@ class DockerAPIImageChecker(ImageChecker):
|
|
|
50
60
|
|
|
51
61
|
token = auth_response.json()["token"]
|
|
52
62
|
|
|
63
|
+
# ghcr.io/union-oss/flyte:latest
|
|
53
64
|
manifest_url = f"https://registry-1.docker.io/v2/{repository}/manifests/{tag}"
|
|
54
65
|
headers = {
|
|
55
66
|
"Authorization": f"Bearer {token}",
|
|
@@ -62,24 +73,26 @@ class DockerAPIImageChecker(ImageChecker):
|
|
|
62
73
|
manifest_response = await client.get(manifest_url, headers=headers)
|
|
63
74
|
if manifest_response.status_code != 200:
|
|
64
75
|
logger.warning(f"Image not found: {repository}:{tag} (HTTP {manifest_response.status_code})")
|
|
65
|
-
return
|
|
76
|
+
return None
|
|
66
77
|
|
|
67
78
|
manifest_list = manifest_response.json()["manifests"]
|
|
68
79
|
architectures = [f"{m['platform']['os']}/{m['platform']['architecture']}" for m in manifest_list]
|
|
69
80
|
|
|
70
81
|
if set(arch).issubset(set(architectures)):
|
|
71
82
|
logger.debug(f"Image {repository}:{tag} found with arch {architectures}")
|
|
72
|
-
return
|
|
83
|
+
return f"{repository}:{tag}"
|
|
73
84
|
else:
|
|
74
85
|
logger.debug(f"Image {repository}:{tag} has {architectures}, but missing {arch}")
|
|
75
|
-
return
|
|
86
|
+
return None
|
|
76
87
|
|
|
77
88
|
|
|
78
89
|
class LocalDockerCommandImageChecker(ImageChecker):
|
|
79
90
|
command_name: ClassVar[str] = "docker"
|
|
80
91
|
|
|
81
92
|
@classmethod
|
|
82
|
-
async def image_exists(
|
|
93
|
+
async def image_exists(
|
|
94
|
+
cls, repository: str, tag: str, arch: Tuple[Architecture, ...] = ("linux/amd64",)
|
|
95
|
+
) -> Optional[str]:
|
|
83
96
|
# Check if the image exists locally by running the docker inspect command
|
|
84
97
|
process = await asyncio.create_subprocess_exec(
|
|
85
98
|
cls.command_name,
|
|
@@ -90,9 +103,9 @@ class LocalDockerCommandImageChecker(ImageChecker):
|
|
|
90
103
|
stderr=asyncio.subprocess.PIPE,
|
|
91
104
|
)
|
|
92
105
|
stdout, stderr = await process.communicate()
|
|
93
|
-
if stderr and "manifest unknown" in stderr.decode():
|
|
106
|
+
if (stderr and "manifest unknown") or "no such manifest" in stderr.decode():
|
|
94
107
|
logger.debug(f"Image {repository}:{tag} not found using the docker command.")
|
|
95
|
-
return
|
|
108
|
+
return None
|
|
96
109
|
|
|
97
110
|
if process.returncode != 0:
|
|
98
111
|
raise RuntimeError(f"Failed to run docker image inspect {repository}:{tag}")
|
|
@@ -104,11 +117,11 @@ class LocalDockerCommandImageChecker(ImageChecker):
|
|
|
104
117
|
architectures = [f"{x['platform']['os']}/{x['platform']['architecture']}" for x in manifest_list]
|
|
105
118
|
if set(architectures) >= set(arch):
|
|
106
119
|
logger.debug(f"Image {repository}:{tag} found for architecture(s) {arch}, has {architectures}")
|
|
107
|
-
return
|
|
120
|
+
return f"{repository}:{tag}"
|
|
108
121
|
|
|
109
122
|
# Otherwise write a message and return false to trigger build
|
|
110
123
|
logger.debug(f"Image {repository}:{tag} not found for architecture(s) {arch}, only has {architectures}")
|
|
111
|
-
return
|
|
124
|
+
return None
|
|
112
125
|
|
|
113
126
|
|
|
114
127
|
class LocalPodmanCommandImageChecker(LocalDockerCommandImageChecker):
|
|
@@ -120,55 +133,59 @@ class ImageBuildEngine:
|
|
|
120
133
|
ImageBuildEngine contains a list of builders that can be used to build an ImageSpec.
|
|
121
134
|
"""
|
|
122
135
|
|
|
123
|
-
|
|
136
|
+
ImageBuilderType = typing.Literal["local", "remote"]
|
|
137
|
+
|
|
124
138
|
_SEEN_IMAGES: typing.ClassVar[typing.Dict[str, str]] = {
|
|
125
139
|
# Set default for the auto container. See Image._identifier_override for more info.
|
|
126
140
|
"auto": Image.from_debian_base().uri,
|
|
127
141
|
}
|
|
128
142
|
|
|
129
|
-
@classmethod
|
|
130
|
-
def register(cls, builder_type: str, image_builder: ImageBuilder, priority: int = 5):
|
|
131
|
-
cls._REGISTRY[builder_type] = (image_builder, priority)
|
|
132
|
-
|
|
133
|
-
@classmethod
|
|
134
|
-
def get_registry(cls) -> Dict[str, Tuple[ImageBuilder, int]]:
|
|
135
|
-
return cls._REGISTRY
|
|
136
|
-
|
|
137
143
|
@staticmethod
|
|
138
144
|
@alru_cache
|
|
139
|
-
async def image_exists(image: Image) ->
|
|
145
|
+
async def image_exists(image: Image) -> Optional[str]:
|
|
140
146
|
if image.base_image is not None and not image._layers:
|
|
141
147
|
logger.debug(f"Image {image} has a base image: {image.base_image} and no layers. Skip existence check.")
|
|
142
|
-
return
|
|
143
|
-
assert image.registry is not None, f"Image registry is not set for {image}"
|
|
148
|
+
return image.uri
|
|
144
149
|
assert image.name is not None, f"Image name is not set for {image}"
|
|
145
150
|
|
|
146
|
-
repository = image.registry + "/" + image.name
|
|
147
151
|
tag = image._final_tag
|
|
148
152
|
|
|
149
153
|
if tag == "latest":
|
|
150
154
|
logger.debug(f"Image {image} has tag 'latest', skip existence check, always build")
|
|
151
|
-
return
|
|
155
|
+
return image.uri
|
|
152
156
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
157
|
+
builder = None
|
|
158
|
+
cfg = _get_init_config()
|
|
159
|
+
if cfg and cfg.image_builder:
|
|
160
|
+
builder = cfg.image_builder
|
|
161
|
+
image_builder = ImageBuildEngine._get_builder(builder)
|
|
162
|
+
image_checker = image_builder.get_checkers()
|
|
163
|
+
if image_checker is None:
|
|
164
|
+
logger.info(f"No image checkers found for builder `{image_builder}`, assuming it exists")
|
|
165
|
+
return image.uri
|
|
166
|
+
for checker in image_checker:
|
|
156
167
|
try:
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
168
|
+
repository = image.registry + "/" + image.name if image.registry else image.name
|
|
169
|
+
image_uri = await checker.image_exists(repository, tag, tuple(image.platform))
|
|
170
|
+
if image_uri:
|
|
171
|
+
logger.debug(f"Image {image_uri} in registry")
|
|
172
|
+
return image_uri
|
|
160
173
|
except Exception as e:
|
|
161
174
|
logger.debug(f"Error checking image existence with {checker.__name__}: {e}")
|
|
162
175
|
continue
|
|
163
176
|
|
|
164
177
|
# If all checkers fail, then assume the image exists. This is current flytekit behavior
|
|
165
178
|
logger.info(f"All checkers failed to check existence of {image.uri}, assuming it does exists")
|
|
166
|
-
return
|
|
179
|
+
return image.uri
|
|
167
180
|
|
|
168
181
|
@classmethod
|
|
169
182
|
@alru_cache
|
|
170
183
|
async def build(
|
|
171
|
-
cls,
|
|
184
|
+
cls,
|
|
185
|
+
image: Image,
|
|
186
|
+
builder: ImageBuildEngine.ImageBuilderType | None = None,
|
|
187
|
+
dry_run: bool = False,
|
|
188
|
+
force: bool = False,
|
|
172
189
|
) -> str:
|
|
173
190
|
"""
|
|
174
191
|
Build the image. Images to be tagged with latest will always be built. Otherwise, this engine will check the
|
|
@@ -181,30 +198,40 @@ class ImageBuildEngine:
|
|
|
181
198
|
:return:
|
|
182
199
|
"""
|
|
183
200
|
# Always trigger a build if this is a dry run since builder shouldn't really do anything, or a force.
|
|
201
|
+
image_uri = (await cls.image_exists(image)) or image.uri
|
|
184
202
|
if force or dry_run or not await cls.image_exists(image):
|
|
185
|
-
logger.info(f"Image {
|
|
203
|
+
logger.info(f"Image {image_uri} does not exist in registry or force/dry-run, building...")
|
|
186
204
|
|
|
187
205
|
# Validate the image before building
|
|
188
206
|
image.validate()
|
|
189
207
|
|
|
190
|
-
# If builder is not specified, use the first registered builder
|
|
208
|
+
# If a builder is not specified, use the first registered builder
|
|
209
|
+
cfg = _get_init_config()
|
|
210
|
+
if cfg and cfg.image_builder:
|
|
211
|
+
builder = builder or cfg.image_builder
|
|
191
212
|
img_builder = ImageBuildEngine._get_builder(builder)
|
|
213
|
+
logger.debug(f"Using `{img_builder}` image builder to build image.")
|
|
192
214
|
|
|
193
215
|
result = await img_builder.build_image(image, dry_run=dry_run)
|
|
194
216
|
return result
|
|
195
217
|
else:
|
|
196
|
-
logger.info(f"Image {
|
|
197
|
-
return
|
|
218
|
+
logger.info(f"Image {image_uri} already exists in registry. Skipping build.")
|
|
219
|
+
return image_uri
|
|
198
220
|
|
|
199
221
|
@classmethod
|
|
200
|
-
def _get_builder(cls, builder:
|
|
201
|
-
if
|
|
202
|
-
|
|
222
|
+
def _get_builder(cls, builder: ImageBuildEngine.ImageBuilderType | None = "local") -> ImageBuilder:
|
|
223
|
+
if builder is None:
|
|
224
|
+
builder = "local"
|
|
225
|
+
if builder == "remote":
|
|
226
|
+
from flyte._internal.imagebuild.remote_builder import RemoteImageBuilder
|
|
227
|
+
|
|
228
|
+
return RemoteImageBuilder()
|
|
229
|
+
elif builder == "local":
|
|
230
|
+
from flyte._internal.imagebuild.docker_builder import DockerImageBuilder
|
|
203
231
|
|
|
204
232
|
return DockerImageBuilder()
|
|
205
|
-
|
|
206
|
-
raise
|
|
207
|
-
return cls._REGISTRY[builder][0]
|
|
233
|
+
else:
|
|
234
|
+
raise ValueError(f"Unknown image builder type: {builder}. Supported types are 'local' and 'remote'.")
|
|
208
235
|
|
|
209
236
|
|
|
210
237
|
class ImageCache(BaseModel):
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import shutil
|
|
3
|
+
import tempfile
|
|
4
|
+
import typing
|
|
5
|
+
from datetime import datetime, timezone
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import TYPE_CHECKING, Optional, Tuple, cast
|
|
8
|
+
from uuid import uuid4
|
|
9
|
+
|
|
10
|
+
import click
|
|
11
|
+
|
|
12
|
+
import flyte
|
|
13
|
+
from flyte import Image, remote
|
|
14
|
+
from flyte._image import (
|
|
15
|
+
AptPackages,
|
|
16
|
+
Architecture,
|
|
17
|
+
Commands,
|
|
18
|
+
CopyConfig,
|
|
19
|
+
Env,
|
|
20
|
+
PipPackages,
|
|
21
|
+
PythonWheels,
|
|
22
|
+
Requirements,
|
|
23
|
+
UVProject,
|
|
24
|
+
)
|
|
25
|
+
from flyte._internal.imagebuild.image_builder import ImageBuilder, ImageChecker
|
|
26
|
+
from flyte._logging import logger
|
|
27
|
+
from flyte.remote import ActionOutputs, Run
|
|
28
|
+
|
|
29
|
+
if TYPE_CHECKING:
|
|
30
|
+
from flyte._protos.imagebuilder import definition_pb2 as image_definition_pb2
|
|
31
|
+
|
|
32
|
+
IMAGE_TASK_NAME = "build-image"
|
|
33
|
+
IMAGE_TASK_PROJECT = "system"
|
|
34
|
+
IMAGE_TASK_DOMAIN = "production"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class RemoteImageChecker(ImageChecker):
|
|
38
|
+
_images_client = None
|
|
39
|
+
|
|
40
|
+
@classmethod
|
|
41
|
+
async def image_exists(
|
|
42
|
+
cls, repository: str, tag: str, arch: Tuple[Architecture, ...] = ("linux/amd64",)
|
|
43
|
+
) -> Optional[str]:
|
|
44
|
+
try:
|
|
45
|
+
import flyte.remote as remote
|
|
46
|
+
|
|
47
|
+
remote.Task.get(
|
|
48
|
+
name=IMAGE_TASK_NAME,
|
|
49
|
+
project=IMAGE_TASK_PROJECT,
|
|
50
|
+
domain=IMAGE_TASK_DOMAIN,
|
|
51
|
+
auto_version="latest",
|
|
52
|
+
)
|
|
53
|
+
except Exception as e:
|
|
54
|
+
msg = "remote image builder is not enabled. Please contact Union support to enable it."
|
|
55
|
+
raise click.ClickException(msg) from e
|
|
56
|
+
|
|
57
|
+
image_name = f"{repository.split('/')[-1]}:{tag}"
|
|
58
|
+
|
|
59
|
+
try:
|
|
60
|
+
from flyte._initialize import _get_init_config
|
|
61
|
+
from flyte._protos.imagebuilder import definition_pb2 as image_definition__pb2
|
|
62
|
+
from flyte._protos.imagebuilder import payload_pb2 as image_payload__pb2
|
|
63
|
+
from flyte._protos.imagebuilder import service_pb2_grpc as image_service_pb2_grpc
|
|
64
|
+
|
|
65
|
+
cfg = _get_init_config()
|
|
66
|
+
if cfg is None:
|
|
67
|
+
raise ValueError("Init config should not be None")
|
|
68
|
+
image_id = image_definition__pb2.ImageIdentifier(name=image_name)
|
|
69
|
+
req = image_payload__pb2.GetImageRequest(id=image_id, organization=cfg.org)
|
|
70
|
+
if cls._images_client is None:
|
|
71
|
+
if cfg.client is None:
|
|
72
|
+
raise ValueError("remote client should not be None")
|
|
73
|
+
cls._images_client = image_service_pb2_grpc.ImageServiceStub(cfg.client._channel)
|
|
74
|
+
resp = await cls._images_client.GetImage(req)
|
|
75
|
+
logger.warning(click.style(f"Image {resp.image.fqin} found. Skip building.", fg="blue"))
|
|
76
|
+
return resp.image.fqin
|
|
77
|
+
except Exception:
|
|
78
|
+
logger.warning(click.style(f"Image {image_name} was not found or has expired.", fg="blue"))
|
|
79
|
+
return None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class RemoteImageBuilder(ImageBuilder):
|
|
83
|
+
def get_checkers(self) -> Optional[typing.List[typing.Type[ImageChecker]]]:
|
|
84
|
+
"""Return the image checker."""
|
|
85
|
+
return [RemoteImageChecker]
|
|
86
|
+
|
|
87
|
+
async def build_image(self, image: Image, dry_run: bool = False) -> str:
|
|
88
|
+
from flyte._protos.workflow import run_definition_pb2
|
|
89
|
+
|
|
90
|
+
image_name = f"{image.name}:{image._final_tag}"
|
|
91
|
+
spec, context = await _validate_configuration(image)
|
|
92
|
+
|
|
93
|
+
start = datetime.now(timezone.utc)
|
|
94
|
+
entity = remote.Task.get(
|
|
95
|
+
name=IMAGE_TASK_NAME,
|
|
96
|
+
project=IMAGE_TASK_PROJECT,
|
|
97
|
+
domain=IMAGE_TASK_DOMAIN,
|
|
98
|
+
auto_version="latest",
|
|
99
|
+
)
|
|
100
|
+
run = cast(
|
|
101
|
+
Run,
|
|
102
|
+
await flyte.with_runcontext(project=IMAGE_TASK_PROJECT, domain=IMAGE_TASK_DOMAIN).run.aio(
|
|
103
|
+
entity, spec=spec, context=context, target_image=image_name
|
|
104
|
+
),
|
|
105
|
+
)
|
|
106
|
+
logger.warning(click.style("🐳 Submitting a new build...", fg="blue", bold=True))
|
|
107
|
+
|
|
108
|
+
logger.warning(click.style("⏳ Waiting for build to finish at: " + click.style(run.url, fg="cyan"), bold=True))
|
|
109
|
+
await run.wait.aio(quiet=True)
|
|
110
|
+
run_details = await run.details.aio()
|
|
111
|
+
|
|
112
|
+
elapsed = str(datetime.now(timezone.utc) - start).split(".")[0]
|
|
113
|
+
|
|
114
|
+
if run_details.action_details.raw_phase == run_definition_pb2.PHASE_SUCCEEDED:
|
|
115
|
+
logger.warning(click.style(f"✅ Build completed in {elapsed}!", bold=True, fg="green"))
|
|
116
|
+
else:
|
|
117
|
+
raise click.ClickException(f"❌ Build failed in {elapsed} at {click.style(run.url, fg='cyan')}")
|
|
118
|
+
|
|
119
|
+
outputs = await run_details.outputs()
|
|
120
|
+
return _get_fully_qualified_image_name(outputs)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
async def _validate_configuration(image: Image) -> Tuple[str, Optional[str]]:
|
|
124
|
+
"""Validate the configuration and prepare the spec and context files.""" # Prepare the spec file
|
|
125
|
+
tmp_path = Path(tempfile.gettempdir()) / str(uuid4())
|
|
126
|
+
os.makedirs(tmp_path, exist_ok=True)
|
|
127
|
+
|
|
128
|
+
context_path = tmp_path / "build.uc-image-builder"
|
|
129
|
+
context_path.mkdir(exist_ok=True)
|
|
130
|
+
|
|
131
|
+
image_idl = _get_layers_proto(image, context_path)
|
|
132
|
+
|
|
133
|
+
spec_path = tmp_path / "spec.pb"
|
|
134
|
+
with spec_path.open("wb") as f:
|
|
135
|
+
f.write(image_idl.SerializeToString())
|
|
136
|
+
|
|
137
|
+
_, spec_url = await remote.upload_file.aio(spec_path)
|
|
138
|
+
|
|
139
|
+
if any(context_path.iterdir()):
|
|
140
|
+
# If there are files in the context directory, upload it
|
|
141
|
+
_, context_url = await remote.upload_file.aio(
|
|
142
|
+
Path(shutil.make_archive(str(tmp_path / "context"), "xztar", context_path))
|
|
143
|
+
)
|
|
144
|
+
else:
|
|
145
|
+
context_url = ""
|
|
146
|
+
|
|
147
|
+
return spec_url, context_url
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _get_layers_proto(image: Image, context_path: Path) -> "image_definition_pb2.ImageSpec":
|
|
151
|
+
from flyte._protos.imagebuilder import definition_pb2 as image_definition_pb2
|
|
152
|
+
|
|
153
|
+
layers = []
|
|
154
|
+
for layer in image._layers:
|
|
155
|
+
if isinstance(layer, AptPackages):
|
|
156
|
+
apt_layer = image_definition_pb2.Layer(
|
|
157
|
+
apt_packages=image_definition_pb2.AptPackages(packages=layer.packages)
|
|
158
|
+
)
|
|
159
|
+
layers.append(apt_layer)
|
|
160
|
+
elif isinstance(layer, PythonWheels):
|
|
161
|
+
dst_path = _copy_files_to_context(layer.wheel_dir, context_path)
|
|
162
|
+
wheel_layer = image_definition_pb2.Layer(
|
|
163
|
+
python_wheels=image_definition_pb2.PythonWheels(
|
|
164
|
+
dir=str(dst_path.relative_to(context_path)),
|
|
165
|
+
options=image_definition_pb2.PipOptions(
|
|
166
|
+
index_url=layer.index_url,
|
|
167
|
+
extra_index_urls=layer.extra_index_urls,
|
|
168
|
+
pre=layer.pre,
|
|
169
|
+
extra_args=layer.extra_args,
|
|
170
|
+
),
|
|
171
|
+
)
|
|
172
|
+
)
|
|
173
|
+
layers.append(wheel_layer)
|
|
174
|
+
|
|
175
|
+
elif isinstance(layer, Requirements):
|
|
176
|
+
dst_path = _copy_files_to_context(layer.file, context_path)
|
|
177
|
+
requirements_layer = image_definition_pb2.Layer(
|
|
178
|
+
requirements=image_definition_pb2.Requirements(
|
|
179
|
+
file=str(dst_path.relative_to(context_path)),
|
|
180
|
+
options=image_definition_pb2.PipOptions(
|
|
181
|
+
index_url=layer.index_url,
|
|
182
|
+
extra_index_urls=layer.extra_index_urls,
|
|
183
|
+
pre=layer.pre,
|
|
184
|
+
extra_args=layer.extra_args,
|
|
185
|
+
),
|
|
186
|
+
)
|
|
187
|
+
)
|
|
188
|
+
layers.append(requirements_layer)
|
|
189
|
+
elif isinstance(layer, PipPackages):
|
|
190
|
+
pip_layer = image_definition_pb2.Layer(
|
|
191
|
+
pip_packages=image_definition_pb2.PipPackages(
|
|
192
|
+
packages=layer.packages,
|
|
193
|
+
options=image_definition_pb2.PipOptions(
|
|
194
|
+
index_url=layer.index_url,
|
|
195
|
+
extra_index_urls=layer.extra_index_urls,
|
|
196
|
+
pre=layer.pre,
|
|
197
|
+
extra_args=layer.extra_args,
|
|
198
|
+
),
|
|
199
|
+
)
|
|
200
|
+
)
|
|
201
|
+
layers.append(pip_layer)
|
|
202
|
+
elif isinstance(layer, UVProject):
|
|
203
|
+
for line in layer.pyproject.read_text().splitlines():
|
|
204
|
+
if "tool.uv.index" in line:
|
|
205
|
+
raise ValueError("External sources are not supported in pyproject.toml")
|
|
206
|
+
shutil.copy2(layer.pyproject, context_path / layer.pyproject.name)
|
|
207
|
+
|
|
208
|
+
uv_layer = image_definition_pb2.Layer(
|
|
209
|
+
uv_project=image_definition_pb2.UVProject(
|
|
210
|
+
pyproject=str(layer.pyproject.name),
|
|
211
|
+
uvlock=str(layer.uvlock.name),
|
|
212
|
+
)
|
|
213
|
+
)
|
|
214
|
+
layers.append(uv_layer)
|
|
215
|
+
elif isinstance(layer, Commands):
|
|
216
|
+
commands_layer = image_definition_pb2.Layer(
|
|
217
|
+
commands=image_definition_pb2.Commands(cmd=list(layer.commands))
|
|
218
|
+
)
|
|
219
|
+
layers.append(commands_layer)
|
|
220
|
+
elif isinstance(layer, CopyConfig):
|
|
221
|
+
dst_path = _copy_files_to_context(layer.src, context_path)
|
|
222
|
+
|
|
223
|
+
copy_layer = image_definition_pb2.Layer(
|
|
224
|
+
copy_config=image_definition_pb2.CopyConfig(
|
|
225
|
+
src=str(dst_path.relative_to(context_path)),
|
|
226
|
+
dst=str(layer.dst),
|
|
227
|
+
)
|
|
228
|
+
)
|
|
229
|
+
layers.append(copy_layer)
|
|
230
|
+
elif isinstance(layer, Env):
|
|
231
|
+
env_layer = image_definition_pb2.Layer(
|
|
232
|
+
env=image_definition_pb2.Env(
|
|
233
|
+
env_variables=dict(layer.env_vars),
|
|
234
|
+
)
|
|
235
|
+
)
|
|
236
|
+
layers.append(env_layer)
|
|
237
|
+
|
|
238
|
+
return image_definition_pb2.ImageSpec(
|
|
239
|
+
base_image=image.base_image,
|
|
240
|
+
python_version=f"{image.python_version[0]}.{image.python_version[1]}",
|
|
241
|
+
layers=layers,
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _copy_files_to_context(src: Path, context_path: Path) -> Path:
|
|
246
|
+
if src.is_absolute() or ".." in str(src):
|
|
247
|
+
dst_path = context_path / str(src.absolute()).replace("/", "./_flyte_abs_context/", 1)
|
|
248
|
+
else:
|
|
249
|
+
dst_path = context_path / src
|
|
250
|
+
dst_path.parent.mkdir(parents=True, exist_ok=True)
|
|
251
|
+
if src.is_dir():
|
|
252
|
+
shutil.copytree(src, dst_path, dirs_exist_ok=True)
|
|
253
|
+
else:
|
|
254
|
+
shutil.copy(src, dst_path)
|
|
255
|
+
return dst_path
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _get_fully_qualified_image_name(outputs: ActionOutputs) -> str:
|
|
259
|
+
return outputs.pb2.literals[0].value.scalar.primitive.string_value
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
|
+
# source: imagebuilder/definition.proto
|
|
4
|
+
"""Generated protocol buffer code."""
|
|
5
|
+
from google.protobuf import descriptor as _descriptor
|
|
6
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
7
|
+
from google.protobuf import symbol_database as _symbol_database
|
|
8
|
+
from google.protobuf.internal import builder as _builder
|
|
9
|
+
# @@protoc_insertion_point(imports)
|
|
10
|
+
|
|
11
|
+
_sym_db = _symbol_database.Default()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
from flyte._protos.validate.validate import validate_pb2 as validate_dot_validate__pb2
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1dimagebuilder/definition.proto\x12\x15\x63loudidl.imagebuilder\x1a\x17validate/validate.proto\".\n\x0fImageIdentifier\x12\x1b\n\x04name\x18\x01 \x01(\tB\x07\xfa\x42\x04r\x02\x10\x01R\x04name\"S\n\x05Image\x12\x36\n\x02id\x18\x01 \x01(\x0b\x32&.cloudidl.imagebuilder.ImageIdentifierR\x02id\x12\x12\n\x04\x66qin\x18\x02 \x01(\tR\x04\x66qin\")\n\x0b\x41ptPackages\x12\x1a\n\x08packages\x18\x01 \x03(\tR\x08packages\"\x84\x01\n\nPipOptions\x12\x1b\n\tindex_url\x18\x02 \x01(\tR\x08indexUrl\x12(\n\x10\x65xtra_index_urls\x18\x03 \x03(\tR\x0e\x65xtraIndexUrls\x12\x10\n\x03pre\x18\x04 \x01(\x08R\x03pre\x12\x1d\n\nextra_args\x18\x05 \x01(\tR\textraArgs\"f\n\x0bPipPackages\x12\x1a\n\x08packages\x18\x01 \x03(\tR\x08packages\x12;\n\x07options\x18\x02 \x01(\x0b\x32!.cloudidl.imagebuilder.PipOptionsR\x07options\"_\n\x0cRequirements\x12\x12\n\x04\x66ile\x18\x01 \x01(\tR\x04\x66ile\x12;\n\x07options\x18\x02 \x01(\x0b\x32!.cloudidl.imagebuilder.PipOptionsR\x07options\"]\n\x0cPythonWheels\x12\x10\n\x03\x64ir\x18\x01 \x01(\tR\x03\x64ir\x12;\n\x07options\x18\x02 \x01(\x0b\x32!.cloudidl.imagebuilder.PipOptionsR\x07options\"~\n\tUVProject\x12\x1c\n\tpyproject\x18\x01 \x01(\tR\tpyproject\x12\x16\n\x06uvlock\x18\x02 \x01(\tR\x06uvlock\x12;\n\x07options\x18\x03 \x01(\x0b\x32!.cloudidl.imagebuilder.PipOptionsR\x07options\"\x1c\n\x08\x43ommands\x12\x10\n\x03\x63md\x18\x02 \x03(\tR\x03\x63md\"#\n\x07WorkDir\x12\x18\n\x07workdir\x18\x01 \x01(\tR\x07workdir\"0\n\nCopyConfig\x12\x10\n\x03src\x18\x01 \x01(\tR\x03src\x12\x10\n\x03\x64st\x18\x02 \x01(\tR\x03\x64st\"\x99\x01\n\x03\x45nv\x12Q\n\renv_variables\x18\x01 \x03(\x0b\x32,.cloudidl.imagebuilder.Env.EnvVariablesEntryR\x0c\x65nvVariables\x1a?\n\x11\x45nvVariablesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\xed\x04\n\x05Layer\x12G\n\x0c\x61pt_packages\x18\x01 \x01(\x0b\x32\".cloudidl.imagebuilder.AptPackagesH\x00R\x0b\x61ptPackages\x12G\n\x0cpip_packages\x18\x02 \x01(\x0b\x32\".cloudidl.imagebuilder.PipPackagesH\x00R\x0bpipPackages\x12=\n\x08\x63ommands\x18\x03 \x01(\x0b\x32\x1f.cloudidl.imagebuilder.CommandsH\x00R\x08\x63ommands\x12I\n\x0crequirements\x18\x04 \x01(\x0b\x32#.cloudidl.imagebuilder.RequirementsH\x00R\x0crequirements\x12J\n\rpython_wheels\x18\x05 \x01(\x0b\x32#.cloudidl.imagebuilder.PythonWheelsH\x00R\x0cpythonWheels\x12:\n\x07workdir\x18\x06 \x01(\x0b\x32\x1e.cloudidl.imagebuilder.WorkDirH\x00R\x07workdir\x12\x44\n\x0b\x63opy_config\x18\x07 \x01(\x0b\x32!.cloudidl.imagebuilder.CopyConfigH\x00R\ncopyConfig\x12\x41\n\nuv_project\x18\x08 \x01(\x0b\x32 .cloudidl.imagebuilder.UVProjectH\x00R\tuvProject\x12.\n\x03\x65nv\x18\t \x01(\x0b\x32\x1a.cloudidl.imagebuilder.EnvH\x00R\x03\x65nvB\x07\n\x05layer\"\xa3\x01\n\tImageSpec\x12\x1d\n\nbase_image\x18\x01 \x01(\tR\tbaseImage\x12%\n\x0epython_version\x18\x02 \x01(\tR\rpythonVersion\x12\x34\n\x06layers\x18\x03 \x03(\x0b\x32\x1c.cloudidl.imagebuilder.LayerR\x06layers\x12\x1a\n\x08platform\x18\x04 \x03(\tR\x08platformB\xd4\x01\n\x19\x63om.cloudidl.imagebuilderB\x0f\x44\x65\x66initionProtoH\x02P\x01Z/github.com/unionai/cloud/gen/pb-go/imagebuilder\xa2\x02\x03\x43IX\xaa\x02\x15\x43loudidl.Imagebuilder\xca\x02\x15\x43loudidl\\Imagebuilder\xe2\x02!Cloudidl\\Imagebuilder\\GPBMetadata\xea\x02\x16\x43loudidl::Imagebuilderb\x06proto3')
|
|
18
|
+
|
|
19
|
+
_globals = globals()
|
|
20
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
21
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'imagebuilder.definition_pb2', _globals)
|
|
22
|
+
if _descriptor._USE_C_DESCRIPTORS == False:
|
|
23
|
+
DESCRIPTOR._options = None
|
|
24
|
+
DESCRIPTOR._serialized_options = b'\n\031com.cloudidl.imagebuilderB\017DefinitionProtoH\002P\001Z/github.com/unionai/cloud/gen/pb-go/imagebuilder\242\002\003CIX\252\002\025Cloudidl.Imagebuilder\312\002\025Cloudidl\\Imagebuilder\342\002!Cloudidl\\Imagebuilder\\GPBMetadata\352\002\026Cloudidl::Imagebuilder'
|
|
25
|
+
_IMAGEIDENTIFIER.fields_by_name['name']._options = None
|
|
26
|
+
_IMAGEIDENTIFIER.fields_by_name['name']._serialized_options = b'\372B\004r\002\020\001'
|
|
27
|
+
_ENV_ENVVARIABLESENTRY._options = None
|
|
28
|
+
_ENV_ENVVARIABLESENTRY._serialized_options = b'8\001'
|
|
29
|
+
_globals['_IMAGEIDENTIFIER']._serialized_start=81
|
|
30
|
+
_globals['_IMAGEIDENTIFIER']._serialized_end=127
|
|
31
|
+
_globals['_IMAGE']._serialized_start=129
|
|
32
|
+
_globals['_IMAGE']._serialized_end=212
|
|
33
|
+
_globals['_APTPACKAGES']._serialized_start=214
|
|
34
|
+
_globals['_APTPACKAGES']._serialized_end=255
|
|
35
|
+
_globals['_PIPOPTIONS']._serialized_start=258
|
|
36
|
+
_globals['_PIPOPTIONS']._serialized_end=390
|
|
37
|
+
_globals['_PIPPACKAGES']._serialized_start=392
|
|
38
|
+
_globals['_PIPPACKAGES']._serialized_end=494
|
|
39
|
+
_globals['_REQUIREMENTS']._serialized_start=496
|
|
40
|
+
_globals['_REQUIREMENTS']._serialized_end=591
|
|
41
|
+
_globals['_PYTHONWHEELS']._serialized_start=593
|
|
42
|
+
_globals['_PYTHONWHEELS']._serialized_end=686
|
|
43
|
+
_globals['_UVPROJECT']._serialized_start=688
|
|
44
|
+
_globals['_UVPROJECT']._serialized_end=814
|
|
45
|
+
_globals['_COMMANDS']._serialized_start=816
|
|
46
|
+
_globals['_COMMANDS']._serialized_end=844
|
|
47
|
+
_globals['_WORKDIR']._serialized_start=846
|
|
48
|
+
_globals['_WORKDIR']._serialized_end=881
|
|
49
|
+
_globals['_COPYCONFIG']._serialized_start=883
|
|
50
|
+
_globals['_COPYCONFIG']._serialized_end=931
|
|
51
|
+
_globals['_ENV']._serialized_start=934
|
|
52
|
+
_globals['_ENV']._serialized_end=1087
|
|
53
|
+
_globals['_ENV_ENVVARIABLESENTRY']._serialized_start=1024
|
|
54
|
+
_globals['_ENV_ENVVARIABLESENTRY']._serialized_end=1087
|
|
55
|
+
_globals['_LAYER']._serialized_start=1090
|
|
56
|
+
_globals['_LAYER']._serialized_end=1711
|
|
57
|
+
_globals['_IMAGESPEC']._serialized_start=1714
|
|
58
|
+
_globals['_IMAGESPEC']._serialized_end=1877
|
|
59
|
+
# @@protoc_insertion_point(module_scope)
|