flyte 0.2.0b17__py3-none-any.whl → 0.2.0b19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/_bin/runtime.py +10 -2
- flyte/_code_bundle/bundle.py +2 -2
- flyte/_deploy.py +2 -3
- flyte/_image.py +100 -64
- flyte/_initialize.py +9 -1
- flyte/_internal/imagebuild/__init__.py +4 -0
- flyte/_internal/imagebuild/docker_builder.py +57 -24
- flyte/_internal/imagebuild/image_builder.py +69 -42
- flyte/_internal/imagebuild/remote_builder.py +259 -0
- flyte/_protos/imagebuilder/definition_pb2.py +59 -0
- flyte/_protos/imagebuilder/definition_pb2.pyi +140 -0
- flyte/_protos/imagebuilder/definition_pb2_grpc.py +4 -0
- flyte/_protos/imagebuilder/payload_pb2.py +32 -0
- flyte/_protos/imagebuilder/payload_pb2.pyi +21 -0
- flyte/_protos/imagebuilder/payload_pb2_grpc.py +4 -0
- flyte/_protos/imagebuilder/service_pb2.py +29 -0
- flyte/_protos/imagebuilder/service_pb2.pyi +5 -0
- flyte/_protos/imagebuilder/service_pb2_grpc.py +66 -0
- flyte/_run.py +71 -8
- flyte/_task_environment.py +1 -0
- flyte/_version.py +2 -2
- flyte/cli/__init__.py +9 -0
- flyte/cli/_build.py +1 -1
- flyte/cli/_common.py +14 -11
- flyte/cli/_create.py +15 -0
- flyte/cli/_deploy.py +2 -2
- flyte/cli/_get.py +9 -6
- flyte/cli/_run.py +1 -0
- flyte/cli/main.py +8 -0
- flyte/config/_config.py +30 -2
- flyte/config/_internal.py +8 -0
- flyte/config/_reader.py +0 -3
- flyte/extras/_container.py +2 -2
- flyte/remote/_data.py +2 -0
- flyte/remote/_run.py +10 -4
- flyte/remote/_task.py +35 -7
- flyte/types/_type_engine.py +2 -4
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/METADATA +1 -1
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/RECORD +42 -33
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/WHEEL +0 -0
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/entry_points.txt +0 -0
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/top_level.txt +0 -0
flyte/_bin/runtime.py
CHANGED
|
@@ -26,6 +26,7 @@ DOMAIN_NAME = "FLYTE_INTERNAL_TASK_DOMAIN"
|
|
|
26
26
|
ORG_NAME = "_U_ORG_NAME"
|
|
27
27
|
ENDPOINT_OVERRIDE = "_U_EP_OVERRIDE"
|
|
28
28
|
RUN_OUTPUT_BASE_DIR = "_U_RUN_BASE"
|
|
29
|
+
ENABLE_REF_TASKS = "_REF_TASKS" # This is a temporary flag to enable reference tasks in the runtime.
|
|
29
30
|
|
|
30
31
|
# TODO: Remove this after proper auth is implemented
|
|
31
32
|
_UNION_EAGER_API_KEY_ENV_VAR = "_UNION_EAGER_API_KEY"
|
|
@@ -112,15 +113,22 @@ def main(
|
|
|
112
113
|
logger.debug(f"Using controller endpoint: {ep} with kwargs: {controller_kwargs}")
|
|
113
114
|
|
|
114
115
|
bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
|
|
116
|
+
enable_ref_tasks = os.getenv(ENABLE_REF_TASKS, "false").lower() in ("true", "1", "yes")
|
|
115
117
|
# We init regular client here so that reference tasks can work
|
|
116
118
|
# Current reference tasks will not work with remote controller, because we create 2 different
|
|
117
119
|
# channels on different threads and this is not supported by grpcio or the auth system. It ends up leading
|
|
118
120
|
# File "src/python/grpcio/grpc/_cython/_cygrpc/aio/completion_queue.pyx.pxi", line 147,
|
|
119
121
|
# in grpc._cython.cygrpc.PollerCompletionQueue._handle_events
|
|
120
122
|
# BlockingIOError: [Errno 11] Resource temporarily unavailable
|
|
121
|
-
# init(org=org, project=project, domain=domain, **controller_kwargs)
|
|
122
123
|
# TODO solution is to use a single channel for both controller and reference tasks, but this requires a refactor
|
|
123
|
-
|
|
124
|
+
if enable_ref_tasks:
|
|
125
|
+
logger.warning(
|
|
126
|
+
"Reference tasks are enabled. This will initialize client and you will see a BlockIOError. "
|
|
127
|
+
"This is harmless, but a nuisance. We are working on a fix."
|
|
128
|
+
)
|
|
129
|
+
init(org=org, project=project, domain=domain, **controller_kwargs)
|
|
130
|
+
else:
|
|
131
|
+
init()
|
|
124
132
|
# Controller is created with the same kwargs as init, so that it can be used to run tasks
|
|
125
133
|
controller = create_controller(ct="remote", **controller_kwargs)
|
|
126
134
|
|
flyte/_code_bundle/bundle.py
CHANGED
|
@@ -80,7 +80,7 @@ async def build_pkl_bundle(
|
|
|
80
80
|
logger.debug("Uploading pickled code bundle to control plane.")
|
|
81
81
|
from flyte.remote import upload_file
|
|
82
82
|
|
|
83
|
-
hash_digest, remote_path = await upload_file(dest)
|
|
83
|
+
hash_digest, remote_path = await upload_file.aio(dest)
|
|
84
84
|
return CodeBundle(pkl=remote_path, computed_version=hash_digest)
|
|
85
85
|
|
|
86
86
|
elif upload_from_dataplane_base_path:
|
|
@@ -145,7 +145,7 @@ async def build_code_bundle(
|
|
|
145
145
|
bundle_path, tar_size, archive_size = create_bundle(from_dir, pathlib.Path(tmp_dir), files, digest)
|
|
146
146
|
logger.info(f"Code bundle created at {bundle_path}, size: {tar_size} MB, archive size: {archive_size} MB")
|
|
147
147
|
if not dryrun:
|
|
148
|
-
hash_digest, remote_path = await upload_file(bundle_path)
|
|
148
|
+
hash_digest, remote_path = await upload_file.aio(bundle_path)
|
|
149
149
|
logger.debug(f"Code bundle uploaded to {remote_path}")
|
|
150
150
|
else:
|
|
151
151
|
remote_path = "na"
|
flyte/_deploy.py
CHANGED
|
@@ -133,7 +133,7 @@ async def _build_images(deployment: DeploymentPlan) -> ImageCache:
|
|
|
133
133
|
image_identifier_map = {}
|
|
134
134
|
for env_name, env in deployment.envs.items():
|
|
135
135
|
if not isinstance(env.image, str):
|
|
136
|
-
logger.
|
|
136
|
+
logger.debug(f"Building Image for environment {env_name}, image: {env.image}")
|
|
137
137
|
images.append(_build_image_bg(env_name, env.image))
|
|
138
138
|
|
|
139
139
|
elif env.image == "auto" and "auto" not in image_identifier_map:
|
|
@@ -145,7 +145,7 @@ async def _build_images(deployment: DeploymentPlan) -> ImageCache:
|
|
|
145
145
|
logger.warning(f"Built Image for environment {env_name}, image: {image_uri}")
|
|
146
146
|
env = deployment.envs[env_name]
|
|
147
147
|
if isinstance(env.image, Image):
|
|
148
|
-
image_identifier_map[env.image.identifier] =
|
|
148
|
+
image_identifier_map[env.image.identifier] = image_uri
|
|
149
149
|
|
|
150
150
|
return ImageCache(image_lookup=image_identifier_map)
|
|
151
151
|
|
|
@@ -158,7 +158,6 @@ async def apply(deployment: DeploymentPlan, copy_style: CopyFiles, dryrun: bool
|
|
|
158
158
|
image_cache = await _build_images(deployment)
|
|
159
159
|
|
|
160
160
|
version = deployment.version
|
|
161
|
-
code_bundle = None
|
|
162
161
|
if copy_style == "none" and not version:
|
|
163
162
|
raise flyte.errors.DeploymentError("Version must be set when copy_style is none")
|
|
164
163
|
else:
|
flyte/_image.py
CHANGED
|
@@ -53,7 +53,7 @@ class Layer:
|
|
|
53
53
|
|
|
54
54
|
:param hasher: The hash object to update with the layer's data.
|
|
55
55
|
"""
|
|
56
|
-
|
|
56
|
+
print("hash hash")
|
|
57
57
|
|
|
58
58
|
def validate(self):
|
|
59
59
|
"""
|
|
@@ -64,24 +64,32 @@ class Layer:
|
|
|
64
64
|
|
|
65
65
|
@rich.repr.auto
|
|
66
66
|
@dataclass(kw_only=True, frozen=True, repr=True)
|
|
67
|
-
class
|
|
68
|
-
packages: Optional[Tuple[str, ...]] = None
|
|
67
|
+
class PipOption:
|
|
69
68
|
index_url: Optional[str] = None
|
|
70
69
|
extra_index_urls: Optional[Tuple[str] | Tuple[str, ...] | List[str]] = None
|
|
71
70
|
pre: bool = False
|
|
72
71
|
extra_args: Optional[str] = None
|
|
73
72
|
|
|
74
|
-
|
|
75
|
-
|
|
73
|
+
def get_pip_install_args(self) -> List[str]:
|
|
74
|
+
pip_install_args = []
|
|
75
|
+
if self.index_url:
|
|
76
|
+
pip_install_args.append(f"--index-url {self.index_url}")
|
|
77
|
+
|
|
78
|
+
if self.extra_index_urls:
|
|
79
|
+
pip_install_args.extend([f"--extra-index-url {url}" for url in self.extra_index_urls])
|
|
80
|
+
|
|
81
|
+
if self.pre:
|
|
82
|
+
pip_install_args.append("--pre")
|
|
83
|
+
|
|
84
|
+
if self.extra_args:
|
|
85
|
+
pip_install_args.append(self.extra_args)
|
|
86
|
+
return pip_install_args
|
|
76
87
|
|
|
77
88
|
def update_hash(self, hasher: hashlib._Hash):
|
|
78
89
|
"""
|
|
79
|
-
Update the hash with the
|
|
90
|
+
Update the hash with the PipOption
|
|
80
91
|
"""
|
|
81
92
|
hash_input = ""
|
|
82
|
-
if self.packages:
|
|
83
|
-
for package in self.packages:
|
|
84
|
-
hash_input += package
|
|
85
93
|
if self.index_url:
|
|
86
94
|
hash_input += self.index_url
|
|
87
95
|
if self.extra_index_urls:
|
|
@@ -95,6 +103,44 @@ class PipPackages(Layer):
|
|
|
95
103
|
hasher.update(hash_input.encode("utf-8"))
|
|
96
104
|
|
|
97
105
|
|
|
106
|
+
@rich.repr.auto
|
|
107
|
+
@dataclass(kw_only=True, frozen=True, repr=True)
|
|
108
|
+
class PipPackages(PipOption, Layer):
|
|
109
|
+
packages: Optional[Tuple[str, ...]] = None
|
|
110
|
+
|
|
111
|
+
# todo: to be implemented
|
|
112
|
+
# secret_mounts: Optional[List[Tuple[str, str]]] = None
|
|
113
|
+
|
|
114
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
115
|
+
"""
|
|
116
|
+
Update the hash with the pip packages
|
|
117
|
+
"""
|
|
118
|
+
super().update_hash(hasher)
|
|
119
|
+
hash_input = ""
|
|
120
|
+
if self.packages:
|
|
121
|
+
for package in self.packages:
|
|
122
|
+
hash_input += package
|
|
123
|
+
|
|
124
|
+
hasher.update(hash_input.encode("utf-8"))
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@rich.repr.auto
|
|
128
|
+
@dataclass(kw_only=True, frozen=True, repr=True)
|
|
129
|
+
class PythonWheels(PipOption, Layer):
|
|
130
|
+
wheel_dir: Path
|
|
131
|
+
|
|
132
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
133
|
+
super().update_hash(hasher)
|
|
134
|
+
from ._utils import filehash_update
|
|
135
|
+
|
|
136
|
+
# Iterate through all the wheel files in the directory and update the hash
|
|
137
|
+
for wheel_file in self.wheel_dir.glob("*.whl"):
|
|
138
|
+
if not wheel_file.is_file():
|
|
139
|
+
# Skip if it's not a file (e.g., directory or symlink)
|
|
140
|
+
continue
|
|
141
|
+
filehash_update(wheel_file, hasher)
|
|
142
|
+
|
|
143
|
+
|
|
98
144
|
@rich.repr.auto
|
|
99
145
|
@dataclass(kw_only=True, frozen=True, repr=True)
|
|
100
146
|
class Requirements(PipPackages):
|
|
@@ -107,6 +153,19 @@ class Requirements(PipPackages):
|
|
|
107
153
|
filehash_update(self.file, hasher)
|
|
108
154
|
|
|
109
155
|
|
|
156
|
+
@rich.repr.auto
|
|
157
|
+
@dataclass(frozen=True, repr=True)
|
|
158
|
+
class UVProject(PipOption, Layer):
|
|
159
|
+
pyproject: Path
|
|
160
|
+
uvlock: Path
|
|
161
|
+
|
|
162
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
163
|
+
from ._utils import filehash_update
|
|
164
|
+
|
|
165
|
+
super().update_hash(hasher)
|
|
166
|
+
filehash_update(self.uvlock, hasher)
|
|
167
|
+
|
|
168
|
+
|
|
110
169
|
@rich.repr.auto
|
|
111
170
|
@dataclass(frozen=True, repr=True)
|
|
112
171
|
class AptPackages(Layer):
|
|
@@ -138,35 +197,23 @@ class WorkDir(Layer):
|
|
|
138
197
|
@dataclass(frozen=True, repr=True)
|
|
139
198
|
class CopyConfig(Layer):
|
|
140
199
|
path_type: CopyConfigType
|
|
141
|
-
|
|
142
|
-
|
|
200
|
+
src: Path
|
|
201
|
+
dst: str = "."
|
|
143
202
|
|
|
144
203
|
def validate(self):
|
|
145
|
-
if not self.
|
|
146
|
-
raise ValueError(f"Source folder {self.
|
|
147
|
-
if not self.
|
|
148
|
-
raise ValueError(f"Source folder {self.
|
|
149
|
-
if not self.
|
|
150
|
-
raise ValueError(f"Source file {self.
|
|
204
|
+
if not self.src.exists():
|
|
205
|
+
raise ValueError(f"Source folder {self.src.absolute()} does not exist")
|
|
206
|
+
if not self.src.is_dir() and self.path_type == 1:
|
|
207
|
+
raise ValueError(f"Source folder {self.src.absolute()} is not a directory")
|
|
208
|
+
if not self.src.is_file() and self.path_type == 0:
|
|
209
|
+
raise ValueError(f"Source file {self.src.absolute()} is not a file")
|
|
151
210
|
|
|
152
211
|
def update_hash(self, hasher: hashlib._Hash):
|
|
153
212
|
from ._utils import update_hasher_for_source
|
|
154
213
|
|
|
155
|
-
update_hasher_for_source(self.
|
|
156
|
-
if self.
|
|
157
|
-
hasher.update(self.
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
@rich.repr.auto
|
|
161
|
-
@dataclass(frozen=True, repr=True)
|
|
162
|
-
class UVProject(Layer):
|
|
163
|
-
pyproject: Path
|
|
164
|
-
uvlock: Path
|
|
165
|
-
|
|
166
|
-
def update_hash(self, hasher: hashlib._Hash):
|
|
167
|
-
from ._utils import filehash_update
|
|
168
|
-
|
|
169
|
-
filehash_update(self.uvlock, hasher)
|
|
214
|
+
update_hasher_for_source(self.src, hasher)
|
|
215
|
+
if self.dst:
|
|
216
|
+
hasher.update(self.dst.encode("utf-8"))
|
|
170
217
|
|
|
171
218
|
|
|
172
219
|
@rich.repr.auto
|
|
@@ -384,8 +431,6 @@ class Image:
|
|
|
384
431
|
python_version = _detect_python_version()
|
|
385
432
|
|
|
386
433
|
base_image = cls._get_default_image_for(python_version=python_version, flyte_version=flyte_version)
|
|
387
|
-
if name is not None and registry is None:
|
|
388
|
-
raise ValueError("Both name and registry must be specified to override the default image name.")
|
|
389
434
|
|
|
390
435
|
if registry and name:
|
|
391
436
|
return base_image.clone(registry=registry, name=name)
|
|
@@ -436,6 +481,8 @@ class Image:
|
|
|
436
481
|
|
|
437
482
|
:param name: name of the image
|
|
438
483
|
:param registry: registry to use for the image
|
|
484
|
+
:param python_version: Python version to use for the image, if not specified, will use the current Python
|
|
485
|
+
version
|
|
439
486
|
:param script: path to the uv script
|
|
440
487
|
:param arch: architecture to use for the image, default is linux/amd64, use tuple for multiple values
|
|
441
488
|
:param python_version: Python version for the image, if not specified, will use the current Python version
|
|
@@ -470,13 +517,17 @@ class Image:
|
|
|
470
517
|
return img
|
|
471
518
|
|
|
472
519
|
def clone(
|
|
473
|
-
self,
|
|
520
|
+
self,
|
|
521
|
+
registry: Optional[str] = None,
|
|
522
|
+
name: Optional[str] = None,
|
|
523
|
+
addl_layer: Optional[Layer] = None,
|
|
474
524
|
) -> Image:
|
|
475
525
|
"""
|
|
476
526
|
Use this method to clone the current image and change the registry and name
|
|
477
527
|
|
|
478
528
|
:param registry: Registry to use for the image
|
|
479
529
|
:param name: Name of the image
|
|
530
|
+
:param addl_layer: Additional layer to add to the image. This will be added to the end of the layers.
|
|
480
531
|
:param addl_layer: Additional layer to add to the image. This will be added on top of the existing layers.
|
|
481
532
|
|
|
482
533
|
:return:
|
|
@@ -566,7 +617,7 @@ class Image:
|
|
|
566
617
|
new_image = self.clone(addl_layer=WorkDir(workdir=workdir))
|
|
567
618
|
return new_image
|
|
568
619
|
|
|
569
|
-
def with_requirements(self, file: Path) -> Image:
|
|
620
|
+
def with_requirements(self, file: str | Path) -> Image:
|
|
570
621
|
"""
|
|
571
622
|
Use this method to create a new image with the specified requirements file layered on top of the current image
|
|
572
623
|
Cannot be used in conjunction with conda
|
|
@@ -574,10 +625,8 @@ class Image:
|
|
|
574
625
|
:param file: path to the requirements file, must be a .txt file
|
|
575
626
|
:return:
|
|
576
627
|
"""
|
|
577
|
-
if
|
|
578
|
-
|
|
579
|
-
if not file.is_file():
|
|
580
|
-
raise ValueError(f"Requirements file {file} is not a file")
|
|
628
|
+
if isinstance(file, str):
|
|
629
|
+
file = Path(file)
|
|
581
630
|
if file.suffix != ".txt":
|
|
582
631
|
raise ValueError(f"Requirements file {file} must have a .txt extension")
|
|
583
632
|
new_image = self.clone(addl_layer=Requirements(file=file))
|
|
@@ -638,30 +687,28 @@ class Image:
|
|
|
638
687
|
new_image = self.clone(addl_layer=Env.from_dict(env_vars))
|
|
639
688
|
return new_image
|
|
640
689
|
|
|
641
|
-
def with_source_folder(self,
|
|
690
|
+
def with_source_folder(self, src: Path, dst: str = ".") -> Image:
|
|
642
691
|
"""
|
|
643
692
|
Use this method to create a new image with the specified local directory layered on top of the current image.
|
|
644
693
|
If dest is not specified, it will be copied to the working directory of the image
|
|
645
694
|
|
|
646
|
-
:param
|
|
647
|
-
:param
|
|
695
|
+
:param src: root folder of the source code from the build context to be copied
|
|
696
|
+
:param dst: destination folder in the image
|
|
648
697
|
:return: Image
|
|
649
698
|
"""
|
|
650
|
-
|
|
651
|
-
new_image = self.clone(addl_layer=CopyConfig(path_type=1, context_source=context_source, image_dest=image_dest))
|
|
699
|
+
new_image = self.clone(addl_layer=CopyConfig(path_type=1, src=src, dst=dst))
|
|
652
700
|
return new_image
|
|
653
701
|
|
|
654
|
-
def with_source_file(self,
|
|
702
|
+
def with_source_file(self, src: Path, dst: str = ".") -> Image:
|
|
655
703
|
"""
|
|
656
704
|
Use this method to create a new image with the specified local file layered on top of the current image.
|
|
657
705
|
If dest is not specified, it will be copied to the working directory of the image
|
|
658
706
|
|
|
659
|
-
:param
|
|
660
|
-
:param
|
|
707
|
+
:param src: root folder of the source code from the build context to be copied
|
|
708
|
+
:param dst: destination folder in the image
|
|
661
709
|
:return: Image
|
|
662
710
|
"""
|
|
663
|
-
|
|
664
|
-
new_image = self.clone(addl_layer=CopyConfig(path_type=0, context_source=context_source, image_dest=image_dest))
|
|
711
|
+
new_image = self.clone(addl_layer=CopyConfig(path_type=0, src=src, dst=dst))
|
|
665
712
|
return new_image
|
|
666
713
|
|
|
667
714
|
def with_uv_project(self, pyproject_file: Path) -> Image:
|
|
@@ -714,22 +761,11 @@ class Image:
|
|
|
714
761
|
:return: Image
|
|
715
762
|
"""
|
|
716
763
|
dist_folder = Path(__file__).parent.parent.parent / "dist"
|
|
717
|
-
# Manually declare the
|
|
764
|
+
# Manually declare the PythonWheel so we can set the hashing
|
|
718
765
|
# used to compute the identifier. Can remove if we ever decide to expose the lambda in with_ commands
|
|
719
|
-
with_dist = self.clone(
|
|
720
|
-
addl_layer=CopyConfig(
|
|
721
|
-
path_type=1, context_source=dist_folder, image_dest=".", _compute_identifier=lambda x: "/dist"
|
|
722
|
-
)
|
|
723
|
-
)
|
|
766
|
+
with_dist = self.clone(addl_layer=PythonWheels(wheel_dir=dist_folder, _compute_identifier=lambda x: "/dist"))
|
|
724
767
|
|
|
725
|
-
return with_dist
|
|
726
|
-
[
|
|
727
|
-
"--mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv"
|
|
728
|
-
" --mount=from=uv,source=/uv,target=/usr/bin/uv"
|
|
729
|
-
" --mount=source=dist,target=/dist,type=bind"
|
|
730
|
-
" uv pip install --python $VIRTUALENV $(ls /dist/*whl)"
|
|
731
|
-
]
|
|
732
|
-
)
|
|
768
|
+
return with_dist
|
|
733
769
|
|
|
734
770
|
def __img_str__(self) -> str:
|
|
735
771
|
"""
|
flyte/_initialize.py
CHANGED
|
@@ -14,6 +14,7 @@ from ._logging import initialize_logger, logger
|
|
|
14
14
|
from ._tools import ipython_check
|
|
15
15
|
|
|
16
16
|
if TYPE_CHECKING:
|
|
17
|
+
from flyte._internal.imagebuild import ImageBuildEngine
|
|
17
18
|
from flyte.config import Config
|
|
18
19
|
from flyte.remote._client.auth import AuthType, ClientConfig
|
|
19
20
|
from flyte.remote._client.controlplane import ClientSet
|
|
@@ -39,6 +40,7 @@ class CommonInit:
|
|
|
39
40
|
class _InitConfig(CommonInit):
|
|
40
41
|
client: Optional[ClientSet] = None
|
|
41
42
|
storage: Optional[Storage] = None
|
|
43
|
+
image_builder: "ImageBuildEngine.ImageBuilderType" = "local"
|
|
42
44
|
|
|
43
45
|
def replace(self, **kwargs) -> _InitConfig:
|
|
44
46
|
return replace(self, **kwargs)
|
|
@@ -132,6 +134,7 @@ async def init(
|
|
|
132
134
|
http_proxy_url: str | None = None,
|
|
133
135
|
storage: Storage | None = None,
|
|
134
136
|
batch_size: int = 1000,
|
|
137
|
+
image_builder: ImageBuildEngine.ImageBuilderType = "local",
|
|
135
138
|
) -> None:
|
|
136
139
|
"""
|
|
137
140
|
Initialize the Flyte system with the given configuration. This method should be called before any other Flyte
|
|
@@ -166,6 +169,7 @@ async def init(
|
|
|
166
169
|
:param org: Optional organization override for the client. Should be set by auth instead.
|
|
167
170
|
:param batch_size: Optional batch size for operations that use listings, defaults to 1000, so limit larger than
|
|
168
171
|
batch_size will be split into multiple requests.
|
|
172
|
+
:param image_builder: Optional image builder configuration, if not provided, the default image builder will be used.
|
|
169
173
|
|
|
170
174
|
:return: None
|
|
171
175
|
"""
|
|
@@ -210,12 +214,15 @@ async def init(
|
|
|
210
214
|
storage=storage,
|
|
211
215
|
org=org or org_from_endpoint(endpoint),
|
|
212
216
|
batch_size=batch_size,
|
|
217
|
+
image_builder=image_builder,
|
|
213
218
|
)
|
|
214
219
|
|
|
215
220
|
|
|
216
221
|
@syncify
|
|
217
222
|
async def init_from_config(
|
|
218
|
-
path_or_config: str | Config | None = None,
|
|
223
|
+
path_or_config: str | Config | None = None,
|
|
224
|
+
root_dir: Path | None = None,
|
|
225
|
+
log_level: int | None = None,
|
|
219
226
|
) -> None:
|
|
220
227
|
"""
|
|
221
228
|
Initialize the Flyte system using a configuration file or Config object. This method should be called before any
|
|
@@ -266,6 +273,7 @@ async def init_from_config(
|
|
|
266
273
|
client_credentials_secret=cfg.platform.client_credentials_secret,
|
|
267
274
|
root_dir=root_dir,
|
|
268
275
|
log_level=log_level,
|
|
276
|
+
image_builder=cfg.image.builder,
|
|
269
277
|
)
|
|
270
278
|
|
|
271
279
|
|
|
@@ -3,6 +3,10 @@ from typing import List
|
|
|
3
3
|
|
|
4
4
|
from flyte._image import Image
|
|
5
5
|
from flyte._internal.imagebuild.docker_builder import DockerImageBuilder
|
|
6
|
+
from flyte._internal.imagebuild.image_builder import ImageBuildEngine
|
|
7
|
+
from flyte._internal.imagebuild.remote_builder import RemoteImageBuilder
|
|
8
|
+
|
|
9
|
+
__all__ = ["DockerImageBuilder", "ImageBuildEngine", "RemoteImageBuilder"]
|
|
6
10
|
|
|
7
11
|
|
|
8
12
|
async def build(images: List[Image]) -> List[str]:
|
|
@@ -3,9 +3,10 @@ import os
|
|
|
3
3
|
import shutil
|
|
4
4
|
import subprocess
|
|
5
5
|
import tempfile
|
|
6
|
+
import typing
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
from string import Template
|
|
8
|
-
from typing import ClassVar, Protocol, cast
|
|
9
|
+
from typing import ClassVar, Optional, Protocol, cast
|
|
9
10
|
|
|
10
11
|
import aiofiles
|
|
11
12
|
import click
|
|
@@ -18,11 +19,19 @@ from flyte._image import (
|
|
|
18
19
|
Image,
|
|
19
20
|
Layer,
|
|
20
21
|
PipPackages,
|
|
22
|
+
PythonWheels,
|
|
21
23
|
Requirements,
|
|
22
24
|
UVProject,
|
|
23
25
|
WorkDir,
|
|
24
26
|
_DockerLines,
|
|
25
27
|
)
|
|
28
|
+
from flyte._internal.imagebuild.image_builder import (
|
|
29
|
+
DockerAPIImageChecker,
|
|
30
|
+
ImageBuilder,
|
|
31
|
+
ImageChecker,
|
|
32
|
+
LocalDockerCommandImageChecker,
|
|
33
|
+
LocalPodmanCommandImageChecker,
|
|
34
|
+
)
|
|
26
35
|
from flyte._logging import logger
|
|
27
36
|
|
|
28
37
|
_F_IMG_ID = "_F_IMG_ID"
|
|
@@ -49,6 +58,12 @@ RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv \
|
|
|
49
58
|
uv pip install --prerelease=allow --python $$UV_PYTHON $PIP_INSTALL_ARGS
|
|
50
59
|
""")
|
|
51
60
|
|
|
61
|
+
UV_WHEEL_INSTALL_COMMAND_TEMPLATE = Template("""\
|
|
62
|
+
RUN --mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=wheel \
|
|
63
|
+
--mount=source=/dist,target=/dist,type=bind \
|
|
64
|
+
uv pip install --prerelease=allow --python $$UV_PYTHON $PIP_INSTALL_ARGS
|
|
65
|
+
""")
|
|
66
|
+
|
|
52
67
|
APT_INSTALL_COMMAND_TEMPLATE = Template("""\
|
|
53
68
|
RUN --mount=type=cache,sharing=locked,mode=0777,target=/var/cache/apt,id=apt \
|
|
54
69
|
apt-get update && apt-get install -y --no-install-recommends \
|
|
@@ -104,10 +119,15 @@ class PipAndRequirementsHandler:
|
|
|
104
119
|
@staticmethod
|
|
105
120
|
async def handle(layer: PipPackages, context_path: Path, dockerfile: str) -> str:
|
|
106
121
|
if isinstance(layer, Requirements):
|
|
122
|
+
if not layer.file.exists():
|
|
123
|
+
raise FileNotFoundError(f"Requirements file {layer.file} does not exist")
|
|
124
|
+
if not layer.file.is_file():
|
|
125
|
+
raise ValueError(f"Requirements file {layer.file} is not a file")
|
|
126
|
+
|
|
107
127
|
async with aiofiles.open(layer.file) as f:
|
|
108
128
|
requirements = []
|
|
109
129
|
async for line in f:
|
|
110
|
-
requirement =
|
|
130
|
+
requirement = line
|
|
111
131
|
requirements.append(requirement.strip())
|
|
112
132
|
else:
|
|
113
133
|
requirements = list(layer.packages) if layer.packages else []
|
|
@@ -116,22 +136,23 @@ class PipAndRequirementsHandler:
|
|
|
116
136
|
reqs = "\n".join(requirements)
|
|
117
137
|
await f.write(reqs)
|
|
118
138
|
|
|
119
|
-
pip_install_args =
|
|
120
|
-
|
|
121
|
-
pip_install_args.append(f"--index-url {layer.index_url}")
|
|
139
|
+
pip_install_args = layer.get_pip_install_args()
|
|
140
|
+
pip_install_args.extend(["--requirement", "requirements_uv.txt"])
|
|
122
141
|
|
|
123
|
-
|
|
124
|
-
|
|
142
|
+
delta = UV_PACKAGE_INSTALL_COMMAND_TEMPLATE.substitute(PIP_INSTALL_ARGS=" ".join(pip_install_args))
|
|
143
|
+
dockerfile += delta
|
|
125
144
|
|
|
126
|
-
|
|
127
|
-
pip_install_args.append("--pre")
|
|
145
|
+
return dockerfile
|
|
128
146
|
|
|
129
|
-
if layer.extra_args:
|
|
130
|
-
pip_install_args.append(layer.extra_args)
|
|
131
147
|
|
|
132
|
-
|
|
148
|
+
class PythonWheelHandler:
|
|
149
|
+
@staticmethod
|
|
150
|
+
async def handle(layer: PythonWheels, context_path: Path, dockerfile: str) -> str:
|
|
151
|
+
shutil.copytree(layer.wheel_dir, context_path / "dist", dirs_exist_ok=True)
|
|
152
|
+
pip_install_args = layer.get_pip_install_args()
|
|
153
|
+
pip_install_args.extend(["/dist/*.whl"])
|
|
133
154
|
|
|
134
|
-
delta =
|
|
155
|
+
delta = UV_WHEEL_INSTALL_COMMAND_TEMPLATE.substitute(PIP_INSTALL_ARGS=" ".join(pip_install_args))
|
|
135
156
|
dockerfile += delta
|
|
136
157
|
|
|
137
158
|
return dockerfile
|
|
@@ -188,27 +209,31 @@ class CopyConfigHandler:
|
|
|
188
209
|
@staticmethod
|
|
189
210
|
async def handle(layer: CopyConfig, context_path: Path, dockerfile: str) -> str:
|
|
190
211
|
# Copy the source config file or directory to the context path
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
212
|
+
if layer.src.is_absolute() or ".." in str(layer.src):
|
|
213
|
+
dst_path = context_path / str(layer.src.absolute()).replace("/", "./_flyte_abs_context/")
|
|
214
|
+
else:
|
|
215
|
+
dst_path = context_path / layer.src
|
|
216
|
+
|
|
217
|
+
dst_path.parent.mkdir(parents=True, exist_ok=True)
|
|
218
|
+
abs_path = layer.src.absolute()
|
|
219
|
+
if layer.src.is_file():
|
|
195
220
|
# Copy the file
|
|
196
|
-
shutil.copy(abs_path,
|
|
197
|
-
elif layer.
|
|
221
|
+
shutil.copy(abs_path, dst_path)
|
|
222
|
+
elif layer.src.is_dir():
|
|
198
223
|
# Copy the entire directory
|
|
199
|
-
shutil.copytree(abs_path,
|
|
224
|
+
shutil.copytree(abs_path, dst_path, dirs_exist_ok=True)
|
|
200
225
|
else:
|
|
201
|
-
raise ValueError(f"Source path is neither file nor directory: {layer.
|
|
226
|
+
raise ValueError(f"Source path is neither file nor directory: {layer.src}")
|
|
202
227
|
|
|
203
228
|
# Add a copy command to the dockerfile
|
|
204
|
-
dockerfile += f"\nCOPY {
|
|
229
|
+
dockerfile += f"\nCOPY {dst_path.relative_to(context_path)} {layer.dst}\n"
|
|
205
230
|
|
|
206
231
|
return dockerfile
|
|
207
232
|
|
|
208
233
|
|
|
209
234
|
class CommandsHandler:
|
|
210
235
|
@staticmethod
|
|
211
|
-
async def handle(layer: Commands,
|
|
236
|
+
async def handle(layer: Commands, _: Path, dockerfile: str) -> str:
|
|
212
237
|
# Append raw commands to the dockerfile
|
|
213
238
|
for command in layer.commands:
|
|
214
239
|
dockerfile += f"\nRUN {command}\n"
|
|
@@ -227,6 +252,10 @@ class WorkDirHandler:
|
|
|
227
252
|
|
|
228
253
|
async def _process_layer(layer: Layer, context_path: Path, dockerfile: str) -> str:
|
|
229
254
|
match layer:
|
|
255
|
+
case PythonWheels():
|
|
256
|
+
# Handle Python wheels
|
|
257
|
+
dockerfile = await PythonWheelHandler.handle(layer, context_path, dockerfile)
|
|
258
|
+
|
|
230
259
|
case Requirements() | PipPackages():
|
|
231
260
|
# Handle pip packages and requirements
|
|
232
261
|
dockerfile = await PipAndRequirementsHandler.handle(layer, context_path, dockerfile)
|
|
@@ -265,12 +294,16 @@ async def _process_layer(layer: Layer, context_path: Path, dockerfile: str) -> s
|
|
|
265
294
|
return dockerfile
|
|
266
295
|
|
|
267
296
|
|
|
268
|
-
class DockerImageBuilder:
|
|
297
|
+
class DockerImageBuilder(ImageBuilder):
|
|
269
298
|
"""Image builder using Docker and buildkit."""
|
|
270
299
|
|
|
271
300
|
builder_type: ClassVar = "docker"
|
|
272
301
|
_builder_name: ClassVar = "flytex"
|
|
273
302
|
|
|
303
|
+
def get_checkers(self) -> Optional[typing.List[typing.Type[ImageChecker]]]:
|
|
304
|
+
# Can get a public token for docker.io but ghcr requires a pat, so harder to get the manifest anonymously
|
|
305
|
+
return [LocalDockerCommandImageChecker, LocalPodmanCommandImageChecker, DockerAPIImageChecker]
|
|
306
|
+
|
|
274
307
|
async def build_image(self, image: Image, dry_run: bool = False) -> str:
|
|
275
308
|
if len(image._layers) == 0:
|
|
276
309
|
logger.warning("No layers to build, returning the image URI as is.")
|