flyte 0.1.0__py3-none-any.whl → 0.2.0a0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of flyte might be problematic. Click here for more details.

Files changed (219) hide show
  1. flyte/__init__.py +78 -2
  2. flyte/_bin/__init__.py +0 -0
  3. flyte/_bin/runtime.py +152 -0
  4. flyte/_build.py +26 -0
  5. flyte/_cache/__init__.py +12 -0
  6. flyte/_cache/cache.py +145 -0
  7. flyte/_cache/defaults.py +9 -0
  8. flyte/_cache/policy_function_body.py +42 -0
  9. flyte/_code_bundle/__init__.py +8 -0
  10. flyte/_code_bundle/_ignore.py +113 -0
  11. flyte/_code_bundle/_packaging.py +187 -0
  12. flyte/_code_bundle/_utils.py +323 -0
  13. flyte/_code_bundle/bundle.py +209 -0
  14. flyte/_context.py +152 -0
  15. flyte/_deploy.py +243 -0
  16. flyte/_doc.py +29 -0
  17. flyte/_docstring.py +32 -0
  18. flyte/_environment.py +84 -0
  19. flyte/_excepthook.py +37 -0
  20. flyte/_group.py +32 -0
  21. flyte/_hash.py +23 -0
  22. flyte/_image.py +762 -0
  23. flyte/_initialize.py +492 -0
  24. flyte/_interface.py +84 -0
  25. flyte/_internal/__init__.py +3 -0
  26. flyte/_internal/controllers/__init__.py +128 -0
  27. flyte/_internal/controllers/_local_controller.py +193 -0
  28. flyte/_internal/controllers/_trace.py +41 -0
  29. flyte/_internal/controllers/remote/__init__.py +60 -0
  30. flyte/_internal/controllers/remote/_action.py +146 -0
  31. flyte/_internal/controllers/remote/_client.py +47 -0
  32. flyte/_internal/controllers/remote/_controller.py +494 -0
  33. flyte/_internal/controllers/remote/_core.py +410 -0
  34. flyte/_internal/controllers/remote/_informer.py +361 -0
  35. flyte/_internal/controllers/remote/_service_protocol.py +50 -0
  36. flyte/_internal/imagebuild/__init__.py +11 -0
  37. flyte/_internal/imagebuild/docker_builder.py +427 -0
  38. flyte/_internal/imagebuild/image_builder.py +246 -0
  39. flyte/_internal/imagebuild/remote_builder.py +0 -0
  40. flyte/_internal/resolvers/__init__.py +0 -0
  41. flyte/_internal/resolvers/_task_module.py +54 -0
  42. flyte/_internal/resolvers/common.py +31 -0
  43. flyte/_internal/resolvers/default.py +28 -0
  44. flyte/_internal/runtime/__init__.py +0 -0
  45. flyte/_internal/runtime/convert.py +342 -0
  46. flyte/_internal/runtime/entrypoints.py +135 -0
  47. flyte/_internal/runtime/io.py +136 -0
  48. flyte/_internal/runtime/resources_serde.py +138 -0
  49. flyte/_internal/runtime/task_serde.py +330 -0
  50. flyte/_internal/runtime/taskrunner.py +191 -0
  51. flyte/_internal/runtime/types_serde.py +54 -0
  52. flyte/_logging.py +135 -0
  53. flyte/_map.py +215 -0
  54. flyte/_pod.py +19 -0
  55. flyte/_protos/__init__.py +0 -0
  56. flyte/_protos/common/authorization_pb2.py +66 -0
  57. flyte/_protos/common/authorization_pb2.pyi +108 -0
  58. flyte/_protos/common/authorization_pb2_grpc.py +4 -0
  59. flyte/_protos/common/identifier_pb2.py +71 -0
  60. flyte/_protos/common/identifier_pb2.pyi +82 -0
  61. flyte/_protos/common/identifier_pb2_grpc.py +4 -0
  62. flyte/_protos/common/identity_pb2.py +48 -0
  63. flyte/_protos/common/identity_pb2.pyi +72 -0
  64. flyte/_protos/common/identity_pb2_grpc.py +4 -0
  65. flyte/_protos/common/list_pb2.py +36 -0
  66. flyte/_protos/common/list_pb2.pyi +71 -0
  67. flyte/_protos/common/list_pb2_grpc.py +4 -0
  68. flyte/_protos/common/policy_pb2.py +37 -0
  69. flyte/_protos/common/policy_pb2.pyi +27 -0
  70. flyte/_protos/common/policy_pb2_grpc.py +4 -0
  71. flyte/_protos/common/role_pb2.py +37 -0
  72. flyte/_protos/common/role_pb2.pyi +53 -0
  73. flyte/_protos/common/role_pb2_grpc.py +4 -0
  74. flyte/_protos/common/runtime_version_pb2.py +28 -0
  75. flyte/_protos/common/runtime_version_pb2.pyi +24 -0
  76. flyte/_protos/common/runtime_version_pb2_grpc.py +4 -0
  77. flyte/_protos/logs/dataplane/payload_pb2.py +100 -0
  78. flyte/_protos/logs/dataplane/payload_pb2.pyi +177 -0
  79. flyte/_protos/logs/dataplane/payload_pb2_grpc.py +4 -0
  80. flyte/_protos/secret/definition_pb2.py +49 -0
  81. flyte/_protos/secret/definition_pb2.pyi +93 -0
  82. flyte/_protos/secret/definition_pb2_grpc.py +4 -0
  83. flyte/_protos/secret/payload_pb2.py +62 -0
  84. flyte/_protos/secret/payload_pb2.pyi +94 -0
  85. flyte/_protos/secret/payload_pb2_grpc.py +4 -0
  86. flyte/_protos/secret/secret_pb2.py +38 -0
  87. flyte/_protos/secret/secret_pb2.pyi +6 -0
  88. flyte/_protos/secret/secret_pb2_grpc.py +198 -0
  89. flyte/_protos/secret/secret_pb2_grpc_grpc.py +198 -0
  90. flyte/_protos/validate/validate/validate_pb2.py +76 -0
  91. flyte/_protos/workflow/common_pb2.py +27 -0
  92. flyte/_protos/workflow/common_pb2.pyi +14 -0
  93. flyte/_protos/workflow/common_pb2_grpc.py +4 -0
  94. flyte/_protos/workflow/environment_pb2.py +29 -0
  95. flyte/_protos/workflow/environment_pb2.pyi +12 -0
  96. flyte/_protos/workflow/environment_pb2_grpc.py +4 -0
  97. flyte/_protos/workflow/node_execution_service_pb2.py +26 -0
  98. flyte/_protos/workflow/node_execution_service_pb2.pyi +4 -0
  99. flyte/_protos/workflow/node_execution_service_pb2_grpc.py +32 -0
  100. flyte/_protos/workflow/queue_service_pb2.py +105 -0
  101. flyte/_protos/workflow/queue_service_pb2.pyi +146 -0
  102. flyte/_protos/workflow/queue_service_pb2_grpc.py +172 -0
  103. flyte/_protos/workflow/run_definition_pb2.py +128 -0
  104. flyte/_protos/workflow/run_definition_pb2.pyi +314 -0
  105. flyte/_protos/workflow/run_definition_pb2_grpc.py +4 -0
  106. flyte/_protos/workflow/run_logs_service_pb2.py +41 -0
  107. flyte/_protos/workflow/run_logs_service_pb2.pyi +28 -0
  108. flyte/_protos/workflow/run_logs_service_pb2_grpc.py +69 -0
  109. flyte/_protos/workflow/run_service_pb2.py +129 -0
  110. flyte/_protos/workflow/run_service_pb2.pyi +171 -0
  111. flyte/_protos/workflow/run_service_pb2_grpc.py +412 -0
  112. flyte/_protos/workflow/state_service_pb2.py +66 -0
  113. flyte/_protos/workflow/state_service_pb2.pyi +75 -0
  114. flyte/_protos/workflow/state_service_pb2_grpc.py +138 -0
  115. flyte/_protos/workflow/task_definition_pb2.py +79 -0
  116. flyte/_protos/workflow/task_definition_pb2.pyi +81 -0
  117. flyte/_protos/workflow/task_definition_pb2_grpc.py +4 -0
  118. flyte/_protos/workflow/task_service_pb2.py +60 -0
  119. flyte/_protos/workflow/task_service_pb2.pyi +59 -0
  120. flyte/_protos/workflow/task_service_pb2_grpc.py +138 -0
  121. flyte/_resources.py +226 -0
  122. flyte/_retry.py +32 -0
  123. flyte/_reusable_environment.py +25 -0
  124. flyte/_run.py +482 -0
  125. flyte/_secret.py +61 -0
  126. flyte/_task.py +449 -0
  127. flyte/_task_environment.py +183 -0
  128. flyte/_timeout.py +47 -0
  129. flyte/_tools.py +27 -0
  130. flyte/_trace.py +120 -0
  131. flyte/_utils/__init__.py +26 -0
  132. flyte/_utils/asyn.py +119 -0
  133. flyte/_utils/async_cache.py +139 -0
  134. flyte/_utils/coro_management.py +23 -0
  135. flyte/_utils/file_handling.py +72 -0
  136. flyte/_utils/helpers.py +134 -0
  137. flyte/_utils/lazy_module.py +54 -0
  138. flyte/_utils/org_discovery.py +57 -0
  139. flyte/_utils/uv_script_parser.py +49 -0
  140. flyte/_version.py +21 -0
  141. flyte/cli/__init__.py +3 -0
  142. flyte/cli/_abort.py +28 -0
  143. flyte/cli/_common.py +337 -0
  144. flyte/cli/_create.py +145 -0
  145. flyte/cli/_delete.py +23 -0
  146. flyte/cli/_deploy.py +152 -0
  147. flyte/cli/_gen.py +163 -0
  148. flyte/cli/_get.py +310 -0
  149. flyte/cli/_params.py +538 -0
  150. flyte/cli/_run.py +231 -0
  151. flyte/cli/main.py +166 -0
  152. flyte/config/__init__.py +3 -0
  153. flyte/config/_config.py +216 -0
  154. flyte/config/_internal.py +64 -0
  155. flyte/config/_reader.py +207 -0
  156. flyte/connectors/__init__.py +0 -0
  157. flyte/errors.py +172 -0
  158. flyte/extras/__init__.py +5 -0
  159. flyte/extras/_container.py +263 -0
  160. flyte/io/__init__.py +27 -0
  161. flyte/io/_dir.py +448 -0
  162. flyte/io/_file.py +467 -0
  163. flyte/io/_structured_dataset/__init__.py +129 -0
  164. flyte/io/_structured_dataset/basic_dfs.py +219 -0
  165. flyte/io/_structured_dataset/structured_dataset.py +1061 -0
  166. flyte/models.py +391 -0
  167. flyte/remote/__init__.py +26 -0
  168. flyte/remote/_client/__init__.py +0 -0
  169. flyte/remote/_client/_protocols.py +133 -0
  170. flyte/remote/_client/auth/__init__.py +12 -0
  171. flyte/remote/_client/auth/_auth_utils.py +14 -0
  172. flyte/remote/_client/auth/_authenticators/__init__.py +0 -0
  173. flyte/remote/_client/auth/_authenticators/base.py +397 -0
  174. flyte/remote/_client/auth/_authenticators/client_credentials.py +73 -0
  175. flyte/remote/_client/auth/_authenticators/device_code.py +118 -0
  176. flyte/remote/_client/auth/_authenticators/external_command.py +79 -0
  177. flyte/remote/_client/auth/_authenticators/factory.py +200 -0
  178. flyte/remote/_client/auth/_authenticators/pkce.py +516 -0
  179. flyte/remote/_client/auth/_channel.py +215 -0
  180. flyte/remote/_client/auth/_client_config.py +83 -0
  181. flyte/remote/_client/auth/_default_html.py +32 -0
  182. flyte/remote/_client/auth/_grpc_utils/__init__.py +0 -0
  183. flyte/remote/_client/auth/_grpc_utils/auth_interceptor.py +288 -0
  184. flyte/remote/_client/auth/_grpc_utils/default_metadata_interceptor.py +151 -0
  185. flyte/remote/_client/auth/_keyring.py +143 -0
  186. flyte/remote/_client/auth/_token_client.py +260 -0
  187. flyte/remote/_client/auth/errors.py +16 -0
  188. flyte/remote/_client/controlplane.py +95 -0
  189. flyte/remote/_console.py +18 -0
  190. flyte/remote/_data.py +159 -0
  191. flyte/remote/_logs.py +176 -0
  192. flyte/remote/_project.py +85 -0
  193. flyte/remote/_run.py +970 -0
  194. flyte/remote/_secret.py +132 -0
  195. flyte/remote/_task.py +391 -0
  196. flyte/report/__init__.py +3 -0
  197. flyte/report/_report.py +178 -0
  198. flyte/report/_template.html +124 -0
  199. flyte/storage/__init__.py +29 -0
  200. flyte/storage/_config.py +233 -0
  201. flyte/storage/_remote_fs.py +34 -0
  202. flyte/storage/_storage.py +271 -0
  203. flyte/storage/_utils.py +5 -0
  204. flyte/syncify/__init__.py +56 -0
  205. flyte/syncify/_api.py +371 -0
  206. flyte/types/__init__.py +36 -0
  207. flyte/types/_interface.py +40 -0
  208. flyte/types/_pickle.py +118 -0
  209. flyte/types/_renderer.py +162 -0
  210. flyte/types/_string_literals.py +120 -0
  211. flyte/types/_type_engine.py +2287 -0
  212. flyte/types/_utils.py +80 -0
  213. flyte-0.2.0a0.dist-info/METADATA +249 -0
  214. flyte-0.2.0a0.dist-info/RECORD +218 -0
  215. {flyte-0.1.0.dist-info → flyte-0.2.0a0.dist-info}/WHEEL +2 -1
  216. flyte-0.2.0a0.dist-info/entry_points.txt +3 -0
  217. flyte-0.2.0a0.dist-info/top_level.txt +1 -0
  218. flyte-0.1.0.dist-info/METADATA +0 -6
  219. flyte-0.1.0.dist-info/RECORD +0 -5
flyte/_image.py ADDED
@@ -0,0 +1,762 @@
1
+ from __future__ import annotations
2
+
3
+ import base64
4
+ import hashlib
5
+ import sys
6
+ from abc import abstractmethod
7
+ from dataclasses import asdict, dataclass, field
8
+ from functools import cached_property
9
+ from pathlib import Path
10
+ from typing import Callable, ClassVar, Dict, List, Literal, Optional, Tuple, TypeVar, Union
11
+
12
+ import rich.repr
13
+
14
+ # Supported Python versions
15
+ PYTHON_3_10 = (3, 10)
16
+ PYTHON_3_11 = (3, 11)
17
+ PYTHON_3_12 = (3, 12)
18
+ PYTHON_3_13 = (3, 13)
19
+
20
+ # 0 is a file, 1 is a directory
21
+ CopyConfigType = Literal[0, 1]
22
+
23
+ T = TypeVar("T")
24
+
25
+
26
+ def _ensure_tuple(val: Union[T, List[T], Tuple[T, ...]]) -> Tuple[T] | Tuple[T, ...]:
27
+ """
28
+ Ensure that the input is a tuple. If it is a string, convert it to a tuple with one element.
29
+ If it is a list, convert it to a tuple.
30
+ """
31
+ if isinstance(val, list):
32
+ return tuple(val)
33
+ elif isinstance(val, tuple):
34
+ return val
35
+ else:
36
+ return (val,)
37
+
38
+
39
+ @rich.repr.auto
40
+ @dataclass(frozen=True, repr=True, kw_only=True)
41
+ class Layer:
42
+ """
43
+ This is an abstract representation of Container Image Layers, which can be used to create
44
+ layered images programmatically.
45
+ """
46
+
47
+ _compute_identifier: Callable[[Layer], str] = field(default=lambda x: x.__str__(), init=True)
48
+
49
+ @abstractmethod
50
+ def update_hash(self, hasher: hashlib._Hash):
51
+ """
52
+ This method should be implemented by subclasses to provide a hash representation of the layer.
53
+
54
+ :param hasher: The hash object to update with the layer's data.
55
+ """
56
+ ...
57
+
58
+ def validate(self):
59
+ """
60
+ Raise any validation errors for the layer
61
+ :return:
62
+ """
63
+
64
+
65
+ @rich.repr.auto
66
+ @dataclass(kw_only=True, frozen=True, repr=True)
67
+ class PipPackages(Layer):
68
+ packages: Optional[Tuple[str, ...]] = None
69
+ index_url: Optional[str] = None
70
+ extra_index_urls: Optional[Tuple[str] | Tuple[str, ...] | List[str]] = None
71
+ pre: bool = False
72
+ extra_args: Optional[str] = None
73
+
74
+ # todo: to be implemented
75
+ # secret_mounts: Optional[List[Tuple[str, str]]] = None
76
+
77
+ def update_hash(self, hasher: hashlib._Hash):
78
+ """
79
+ Update the hash with the pip packages
80
+ """
81
+ hash_input = ""
82
+ if self.packages:
83
+ for package in self.packages:
84
+ hash_input += package
85
+ if self.index_url:
86
+ hash_input += self.index_url
87
+ if self.extra_index_urls:
88
+ for url in self.extra_index_urls:
89
+ hash_input += url
90
+ if self.pre:
91
+ hash_input += str(self.pre)
92
+ if self.extra_args:
93
+ hash_input += self.extra_args
94
+
95
+ hasher.update(hash_input.encode("utf-8"))
96
+
97
+
98
+ @rich.repr.auto
99
+ @dataclass(kw_only=True, frozen=True, repr=True)
100
+ class Requirements(PipPackages):
101
+ file: Path
102
+
103
+ def update_hash(self, hasher: hashlib._Hash):
104
+ from ._utils import filehash_update
105
+
106
+ super().update_hash(hasher)
107
+ filehash_update(self.file, hasher)
108
+
109
+
110
+ @rich.repr.auto
111
+ @dataclass(frozen=True, repr=True)
112
+ class AptPackages(Layer):
113
+ packages: Tuple[str, ...]
114
+
115
+ def update_hash(self, hasher: hashlib._Hash):
116
+ hasher.update("".join(self.packages).encode("utf-8"))
117
+
118
+
119
+ @rich.repr.auto
120
+ @dataclass(frozen=True, repr=True)
121
+ class Commands(Layer):
122
+ commands: Tuple[str, ...]
123
+
124
+ def update_hash(self, hasher: hashlib._Hash):
125
+ hasher.update("".join(self.commands).encode("utf-8"))
126
+
127
+
128
+ @rich.repr.auto
129
+ @dataclass(frozen=True, repr=True)
130
+ class WorkDir(Layer):
131
+ workdir: str
132
+
133
+ def update_hash(self, hasher: hashlib._Hash):
134
+ hasher.update(self.workdir.encode("utf-8"))
135
+
136
+
137
+ @rich.repr.auto
138
+ @dataclass(frozen=True, repr=True)
139
+ class CopyConfig(Layer):
140
+ path_type: CopyConfigType
141
+ context_source: Path
142
+ image_dest: str = "."
143
+
144
+ def validate(self):
145
+ if not self.context_source.exists():
146
+ raise ValueError(f"Source folder {self.context_source.absolute()} does not exist")
147
+ if not self.context_source.is_dir() and self.path_type == 1:
148
+ raise ValueError(f"Source folder {self.context_source.absolute()} is not a directory")
149
+ if not self.context_source.is_file() and self.path_type == 0:
150
+ raise ValueError(f"Source file {self.context_source.absolute()} is not a file")
151
+
152
+ def update_hash(self, hasher: hashlib._Hash):
153
+ from ._utils import update_hasher_for_source
154
+
155
+ update_hasher_for_source(self.context_source, hasher)
156
+ if self.image_dest:
157
+ hasher.update(self.image_dest.encode("utf-8"))
158
+
159
+
160
+ @rich.repr.auto
161
+ @dataclass(frozen=True, repr=True)
162
+ class UVProject(Layer):
163
+ pyproject: Path
164
+ uvlock: Path
165
+
166
+ def update_hash(self, hasher: hashlib._Hash):
167
+ from ._utils import filehash_update
168
+
169
+ filehash_update(self.uvlock, hasher)
170
+
171
+
172
+ @rich.repr.auto
173
+ @dataclass(frozen=True, repr=True)
174
+ class _DockerLines(Layer):
175
+ """
176
+ This is an internal class and should only be used by the default images. It is not supported by most
177
+ builders so please don't use it.
178
+ """
179
+
180
+ lines: Tuple[str, ...]
181
+
182
+ def update_hash(self, hasher: hashlib._Hash):
183
+ hasher.update("".join(self.lines).encode("utf-8"))
184
+
185
+
186
+ @rich.repr.auto
187
+ @dataclass(frozen=True, repr=True)
188
+ class Env(Layer):
189
+ """
190
+ This is an internal class and should only be used by the default images. It is not supported by most
191
+ builders so please don't use it.
192
+ """
193
+
194
+ env_vars: Tuple[Tuple[str, str], ...] = field(default_factory=tuple)
195
+
196
+ def update_hash(self, hasher: hashlib._Hash):
197
+ txt = [f"{k}={v}" for k, v in self.env_vars]
198
+ hasher.update(" ".join(txt).encode("utf-8"))
199
+
200
+ @classmethod
201
+ def from_dict(cls, envs: Dict[str, str]) -> Env:
202
+ return cls(env_vars=tuple((k, v) for k, v in envs.items()))
203
+
204
+
205
+ Architecture = Literal["linux/amd64", "linux/arm64"]
206
+
207
+ _BASE_REGISTRY = "ghcr.io/unionai-oss"
208
+ _DEFAULT_IMAGE_NAME = "flyte"
209
+
210
+
211
+ def _detect_python_version() -> Tuple[int, int]:
212
+ """
213
+ Detect the current Python version.
214
+ :return: Tuple of major and minor version
215
+ """
216
+ return sys.version_info.major, sys.version_info.minor
217
+
218
+
219
+ @dataclass(frozen=True, repr=True, eq=True)
220
+ class Image:
221
+ """
222
+ This is a representation of Container Images, which can be used to create layered images programmatically.
223
+
224
+ Use by first calling one of the base constructor methods. These all begin with `from` or `default_`
225
+ The image can then be amended with additional layers using the various `with_*` methods.
226
+
227
+ Invariant for this class: The construction of Image objects must be doable everywhere. That is, if a
228
+ user has a custom image that is not accessible, calling .with_source_file on a file that doesn't exist, the
229
+ instantiation of the object itself must still go through. Further, the .identifier property of the image must
230
+ also still go through. This is because it may have been already built somewhere else.
231
+ Use validate() functions to check each layer for actual errors. These are invoked at actual
232
+ build time. See self.id for more information
233
+ """
234
+
235
+ # These are base properties of an image
236
+ base_image: Optional[str] = field(default=None)
237
+ dockerfile: Optional[Path] = field(default=None)
238
+ registry: Optional[str] = field(default=None)
239
+ name: Optional[str] = field(default=None)
240
+ platform: Tuple[Architecture, ...] = field(default=("linux/amd64",))
241
+ tag: Optional[str] = field(default=None)
242
+ python_version: Tuple[int, int] = field(default_factory=_detect_python_version)
243
+
244
+ # For .auto() images. Don't compute an actual identifier.
245
+ _identifier_override: Optional[str] = field(default=None, init=False)
246
+ # This is set on default images. These images are built from the base Dockerfile in this library and shouldn't be
247
+ # modified with additional layers.
248
+ is_final: bool = field(default=False)
249
+
250
+ # Layers to be added to the image. In init, because frozen, but users shouldn't access, so underscore.
251
+ _layers: Tuple[Layer, ...] = field(default_factory=tuple)
252
+
253
+ _DEFAULT_IMAGE_PREFIXES: ClassVar = {
254
+ PYTHON_3_10: "py3.10-",
255
+ PYTHON_3_11: "py3.11-",
256
+ PYTHON_3_12: "py3.12-",
257
+ PYTHON_3_13: "py3.13-",
258
+ }
259
+
260
+ @cached_property
261
+ def identifier(self) -> str:
262
+ """
263
+ This identifier is a hash of the layers and properties of the image. It is used to look up previously built
264
+ images. Why is this useful? For example, if a user has Image.from_uv_base().with_source_file("a/local/file"),
265
+ it's not necessarily the case that that file exists within the image (further commands may have removed/changed
266
+ it), and certainly not the case that the path to the file, inside the image (which is used as part of the layer
267
+ hash computation), is the same. That is, inside the image when a task runs, as we come across the same Image
268
+ declaration, we need a way of identifying the image and its uri, without hashing all the layers again. This
269
+ is what this identifier is for. See the ImageCache object for additional information.
270
+
271
+ :return: A unique identifier of the Image
272
+ """
273
+ if self._identifier_override:
274
+ return self._identifier_override
275
+
276
+ # Only get the non-None values in the ImageSpec to ensure the hash is consistent
277
+ # across different SDK versions.
278
+ # Can potentially add a second hashing function to the Layer protocol, but relying on just asdict/str
279
+ # representation for now.
280
+ image_dict = asdict(self, dict_factory=lambda x: {k: v for (k, v) in x if v is not None and k != "_layers"})
281
+ layers_str_repr = "".join([layer._compute_identifier(layer) for layer in self._layers])
282
+ image_dict["layers"] = layers_str_repr
283
+ spec_bytes = image_dict.__str__().encode("utf-8")
284
+ return base64.urlsafe_b64encode(hashlib.md5(spec_bytes).digest()).decode("ascii").rstrip("=")
285
+
286
+ def validate(self):
287
+ for layer in self._layers:
288
+ layer.validate()
289
+
290
+ @classmethod
291
+ def _get_default_image_for(cls, python_version: Tuple[int, int], flyte_version: Optional[str] = None) -> Image:
292
+ # Would love a way to move this outside of this class (but still needs to be accessible via Image.auto())
293
+ # this default image definition may need to be updated once there is a released pypi version
294
+ from flyte._version import __version__
295
+
296
+ dev_mode = cls._is_editable_install() or (__version__ and "dev" in __version__)
297
+ if flyte_version is None:
298
+ flyte_version = __version__.replace("+", "-")
299
+ preset_tag = flyte_version if flyte_version.startswith("v") else f"v{flyte_version}"
300
+ preset_tag = f"py{python_version[0]}.{python_version[1]}-{preset_tag}"
301
+ image = Image(
302
+ base_image=f"python:{python_version[0]}.{python_version[1]}-slim-bookworm",
303
+ registry=_BASE_REGISTRY,
304
+ name=_DEFAULT_IMAGE_NAME,
305
+ tag=preset_tag,
306
+ platform=("linux/amd64", "linux/arm64"),
307
+ )
308
+ labels_and_user = _DockerLines(
309
+ (
310
+ "LABEL org.opencontainers.image.authors='Union.AI <sales@union.ai>'",
311
+ "LABEL org.opencontainers.image.source=https://github.com/unionai/unionv2",
312
+ "RUN useradd --create-home --shell /bin/bash flytekit &&"
313
+ " chown -R flytekit /root && chown -R flytekit /home",
314
+ "WORKDIR /root",
315
+ )
316
+ )
317
+ image = image.clone(addl_layer=labels_and_user)
318
+ image = image.with_env_vars(
319
+ {
320
+ "VIRTUAL_ENV": "/opt/venv",
321
+ "PATH": "/opt/venv/bin:$PATH",
322
+ "PYTHONPATH": "/root",
323
+ "UV_LINK_MODE": "copy",
324
+ }
325
+ )
326
+ image = image.with_apt_packages(["build-essential", "ca-certificates"])
327
+
328
+ base_packages = ["kubernetes", "msgpack", "mashumaro"]
329
+
330
+ # Add in flyte library
331
+ if dev_mode:
332
+ image = image.with_pip_packages(base_packages)
333
+ image = image.with_local_v2()
334
+ else:
335
+ base_packages.append(f"flyte=={flyte_version}")
336
+ image = image.with_pip_packages(base_packages)
337
+
338
+ return image
339
+
340
+ @staticmethod
341
+ def _is_editable_install():
342
+ """Internal hacky function to see if the current install is editable or not."""
343
+ curr = Path(__file__)
344
+ pyproject = curr.parent.parent.parent / "pyproject.toml"
345
+ return pyproject.exists()
346
+
347
+ @classmethod
348
+ def from_uv_debian(
349
+ cls,
350
+ registry: str,
351
+ name: str,
352
+ tag: Optional[str] = None,
353
+ python_version: Optional[Tuple[int, int]] = None,
354
+ arch: Union[Architecture, Tuple[Architecture, ...]] = "linux/amd64",
355
+ ) -> Image:
356
+ """
357
+ This creates a new debian-based base image.
358
+ If using the Union or docker builders, image will have uv available and a virtualenv created at /opt/venv.
359
+
360
+ :param registry: Registry to use for the image
361
+ :param name: Name of the image
362
+ :param tag: Tag to use for the image
363
+ :param python_version: Python version to use for the image
364
+ :param arch: Architecture to use for the image, default is linux/amd64
365
+ :return: Image
366
+ """
367
+ base_image = "debian:bookworm-slim"
368
+ plat = arch if isinstance(arch, tuple) else (arch,)
369
+ if python_version is None:
370
+ python_version = _detect_python_version()
371
+ img = cls(
372
+ base_image=base_image, name=name, registry=registry, tag=tag, platform=plat, python_version=python_version
373
+ )
374
+ return img
375
+
376
+ @classmethod
377
+ def auto(
378
+ cls,
379
+ python_version: Optional[Tuple[int, int]] = None,
380
+ flyte_version: Optional[str] = None,
381
+ registry: Optional[str] = None,
382
+ name: Optional[str] = None,
383
+ ) -> Image:
384
+ """
385
+ Use this method to start using the default base image, built from this library's base Dockerfile
386
+ Default images are multi-arch amd/arm64
387
+
388
+ :param python_version: If not specified, will use the current Python version
389
+ :param flyte_version: Union version to use
390
+ :param registry: Registry to use for the image
391
+ :param name: Name of the image if you want to override the default name
392
+
393
+ :return: Image
394
+ """
395
+ if python_version is None:
396
+ python_version = _detect_python_version()
397
+
398
+ base_image = cls._get_default_image_for(python_version=python_version, flyte_version=flyte_version)
399
+ if name is not None and registry is None:
400
+ raise ValueError("Both name and registry must be specified to override the default image name.")
401
+
402
+ if registry and name:
403
+ return base_image.clone(registry=registry, name=name)
404
+
405
+ # Set this to auto for all auto images because the meaning of "auto" can change (based on logic inside
406
+ # _get_default_image_for, acts differently in a running task container) so let's make sure it stays auto.
407
+ object.__setattr__(base_image, "_identifier_override", "auto")
408
+ return base_image
409
+
410
+ @classmethod
411
+ def from_prebuilt(cls, image_uri: str) -> Image:
412
+ """
413
+ Use this method to start with a pre-built base image. This image must already exist in the registry of course.
414
+
415
+ :param image_uri: The full URI of the image, in the format <registry>/<name>:<tag>
416
+ :return:
417
+ """
418
+ img = cls(base_image=image_uri)
419
+ return img
420
+
421
+ @classmethod
422
+ def from_uv_script(
423
+ cls,
424
+ script: Path | str,
425
+ *,
426
+ name: str,
427
+ registry: str | None = None,
428
+ python_version: Optional[Tuple[int, int]] = None,
429
+ arch: Union[Architecture, Tuple[Architecture, ...]] = "linux/amd64",
430
+ ) -> Image:
431
+ """
432
+ Use this method to create a new image with the specified uv script.
433
+ It uses the header of the script to determine the python version, dependencies to install.
434
+ The script must be a valid uv script, otherwise an error will be raised.
435
+
436
+ Usually the header of the script will look like this:
437
+ Example:
438
+ ```python
439
+ #!/usr/bin/env -S uv run --script
440
+ # /// script
441
+ # requires-python = ">=3.12"
442
+ # dependencies = ["httpx"]
443
+ # ///
444
+ ```
445
+
446
+ For more information on the uv script format, see the documentation:
447
+ [UV: Declaring script dependencies](https://docs.astral.sh/uv/guides/scripts/#declaring-script-dependencies)
448
+
449
+ :param name: name of the image
450
+ :param registry: registry to use for the image
451
+ :param script: path to the uv script
452
+ :param arch: architecture to use for the image, default is linux/amd64, use tuple for multiple values
453
+
454
+ :return: Image
455
+ """
456
+ from ._utils import parse_uv_script_file
457
+
458
+ if isinstance(script, str):
459
+ script = Path(script)
460
+ if not script.exists():
461
+ raise FileNotFoundError(f"UV script {script} does not exist")
462
+ if not script.is_file():
463
+ raise ValueError(f"UV script {script} is not a file")
464
+ if not script.suffix == ".py":
465
+ raise ValueError(f"UV script {script} must have a .py extension")
466
+ header = parse_uv_script_file(script)
467
+ if registry is None:
468
+ raise ValueError("registry must be specified")
469
+
470
+ img = cls.from_uv_debian(registry=registry, name=name, arch=arch, python_version=python_version)
471
+
472
+ # add ca-certificates to the image by default
473
+ img = img.with_apt_packages(["ca-certificates"])
474
+
475
+ if header.dependencies:
476
+ return img.with_pip_packages(header.dependencies)
477
+
478
+ # todo: override the _identifier_override to be the script name or a hash of the script contents
479
+ # This is needed because inside the image, the identifier will be computed to be something different.
480
+ return img
481
+
482
+ def clone(
483
+ self, registry: Optional[str] = None, name: Optional[str] = None, addl_layer: Optional[Layer] = None
484
+ ) -> Image:
485
+ """
486
+ Use this method to clone the current image and change the registry and name
487
+
488
+ :param registry: Registry to use for the image
489
+ :param name: Name of the image
490
+
491
+ :return:
492
+ """
493
+ registry = registry if registry else self.registry
494
+ name = name if name else self.name
495
+ new_layers = (*self._layers, addl_layer) if addl_layer else self._layers
496
+ img = Image(
497
+ base_image=self.base_image,
498
+ dockerfile=self.dockerfile,
499
+ registry=registry,
500
+ name=name,
501
+ tag=self.tag,
502
+ platform=self.platform,
503
+ python_version=self.python_version,
504
+ is_final=self.is_final,
505
+ _layers=new_layers,
506
+ )
507
+
508
+ return img
509
+
510
+ @classmethod
511
+ def from_dockerfile(cls, file: Path, registry: str, name: str, tag: Optional[str] = None) -> Image:
512
+ """
513
+ Use this method to create a new image with the specified dockerfile
514
+
515
+ :param file: path to the dockerfile
516
+ :param name: name of the image
517
+ :param registry: registry to use for the image
518
+ :param tag: tag to use for the image
519
+
520
+ :return:
521
+ """
522
+ tag = tag or "latest"
523
+ img = cls(dockerfile=file, registry=registry, name=name, tag=tag)
524
+
525
+ return img
526
+
527
+ def _get_hash_digest(self) -> str:
528
+ """
529
+ Returns the hash digest of the image, which is a combination of all the layers and properties of the image
530
+ """
531
+ import hashlib
532
+
533
+ from ._utils import filehash_update
534
+
535
+ hasher = hashlib.md5()
536
+ if self.dockerfile:
537
+ # Note the location of the dockerfile shouldn't matter, only the contents
538
+ filehash_update(self.dockerfile, hasher)
539
+ if self._layers:
540
+ for layer in self._layers:
541
+ layer.update_hash(hasher)
542
+ return hasher.hexdigest()
543
+
544
+ @property
545
+ def _final_tag(self) -> str:
546
+ t = self.tag if self.tag else self._get_hash_digest()
547
+ return t or "latest"
548
+
549
+ @cached_property
550
+ def uri(self) -> str:
551
+ """
552
+ Returns the URI of the image in the format <registry>/<name>:<tag>
553
+ """
554
+ if self.registry and self.name:
555
+ tag = self._final_tag
556
+ return f"{self.registry}/{self.name}:{tag}"
557
+ elif self.name:
558
+ return f"{self.name}:{self._final_tag}"
559
+ elif self.base_image:
560
+ return self.base_image
561
+
562
+ raise ValueError("Image is not fully defined. Please set registry, name and tag.")
563
+
564
+ def with_workdir(self, workdir: str) -> Image:
565
+ """
566
+ Use this method to create a new image with the specified working directory
567
+ This will override any existing working directory
568
+
569
+ :param workdir: working directory to use
570
+ :return:
571
+ """
572
+ new_image = self.clone(addl_layer=WorkDir(workdir=workdir))
573
+ return new_image
574
+
575
+ def with_requirements(self, file: Path) -> Image:
576
+ """
577
+ Use this method to create a new image with the specified requirements file layered on top of the current image
578
+ Cannot be used in conjunction with conda
579
+
580
+ :param file: path to the requirements file, must be a .txt file
581
+ :return:
582
+ """
583
+ if not file.exists():
584
+ raise FileNotFoundError(f"Requirements file {file} does not exist")
585
+ if not file.is_file():
586
+ raise ValueError(f"Requirements file {file} is not a file")
587
+ if file.suffix != ".txt":
588
+ raise ValueError(f"Requirements file {file} must have a .txt extension")
589
+ new_image = self.clone(addl_layer=Requirements(file=file))
590
+ return new_image
591
+
592
+ def with_pip_packages(
593
+ self,
594
+ packages: Union[str, List[str], Tuple[str, ...]],
595
+ index_url: Optional[str] = None,
596
+ extra_index_urls: Union[str, List[str], Tuple[str, ...], None] = None,
597
+ pre: bool = False,
598
+ extra_args: Optional[str] = None,
599
+ ) -> Image:
600
+ """
601
+ Use this method to create a new image with the specified pip packages layered on top of the current image
602
+ Cannot be used in conjunction with conda
603
+
604
+ Example:
605
+ ```python
606
+ @flyte.task(image=(flyte.Image
607
+ .ubuntu_python()
608
+ .with_pip_packages(["requests", "numpy"])))
609
+ def my_task(x: int) -> int:
610
+ import numpy as np
611
+ return np.sum([x, 1])
612
+ ```
613
+
614
+ :param packages: list of pip packages to install, follows pip install syntax
615
+ :param index_url: index url to use for pip install, default is None
616
+ :param extra_index_urls: extra index urls to use for pip install, default is None
617
+ :param pre: whether to allow pre-release versions, default is False
618
+ :param extra_args: extra arguments to pass to pip install, default is None
619
+ # :param secret_mounts: todo
620
+ :param extra_args: extra arguments to pass to pip install, default is None
621
+ :return: Image
622
+ """
623
+
624
+ new_packages: Optional[Tuple] = _ensure_tuple(packages) if packages else None
625
+ new_extra_index_urls: Optional[Tuple] = _ensure_tuple(extra_index_urls) if extra_index_urls else None
626
+
627
+ ll = PipPackages(
628
+ packages=new_packages,
629
+ index_url=index_url,
630
+ extra_index_urls=new_extra_index_urls,
631
+ pre=pre,
632
+ extra_args=extra_args,
633
+ )
634
+ new_image = self.clone(addl_layer=ll)
635
+ return new_image
636
+
637
+ def with_env_vars(self, env_vars: Dict[str, str]) -> Image:
638
+ """
639
+ Use this method to create a new image with the specified environment variables layered on top of
640
+ the current image. Cannot be used in conjunction with conda
641
+
642
+ :param env_vars: dictionary of environment variables to set
643
+ :return: Image
644
+ """
645
+ new_image = self.clone(addl_layer=Env.from_dict(env_vars))
646
+ return new_image
647
+
648
+ def with_source_folder(self, context_source: Path, image_dest: Optional[str] = None) -> Image:
649
+ """
650
+ Use this method to create a new image with the specified local directory layered on top of the current image.
651
+ If dest is not specified, it will be copied to the working directory of the image
652
+
653
+ :param context_source: root folder of the source code from the build context to be copied
654
+ :param image_dest: destination folder in the image
655
+ :return: Image
656
+ """
657
+ image_dest = image_dest if image_dest else "."
658
+ new_image = self.clone(addl_layer=CopyConfig(path_type=1, context_source=context_source, image_dest=image_dest))
659
+ return new_image
660
+
661
+ def with_source_file(self, context_source: Path, image_dest: Optional[str] = None) -> Image:
662
+ """
663
+ Use this method to create a new image with the specified local file layered on top of the current image.
664
+ If dest is not specified, it will be copied to the working directory of the image
665
+
666
+ :param context_source: root folder of the source code from the build context to be copied
667
+ :param image_dest: destination folder in the image
668
+ :return: Image
669
+ """
670
+ image_dest = image_dest if image_dest else "."
671
+ new_image = self.clone(addl_layer=CopyConfig(path_type=0, context_source=context_source, image_dest=image_dest))
672
+ return new_image
673
+
674
+ def with_uv_project(self, pyproject_file: Path) -> Image:
675
+ """
676
+ Use this method to create a new image with the specified uv.lock file layered on top of the current image
677
+ Must have a corresponding pyproject.toml file in the same directory
678
+ Cannot be used in conjunction with conda
679
+ In the Union builders, using this will change the virtual env to /root/.venv
680
+
681
+ :param pyproject_file: path to the pyproject.toml file, needs to have a corresponding uv.lock file
682
+ :return:
683
+ """
684
+ if not pyproject_file.exists():
685
+ raise FileNotFoundError(f"UVLock file {pyproject_file} does not exist")
686
+ if not pyproject_file.is_file():
687
+ raise ValueError(f"UVLock file {pyproject_file} is not a file")
688
+ lock = pyproject_file.parent / "uv.lock"
689
+ if not lock.exists():
690
+ raise ValueError(f"UVLock file {lock} does not exist")
691
+ new_image = self.clone(addl_layer=UVProject(pyproject=pyproject_file, uvlock=lock))
692
+ return new_image
693
+
694
+ def with_apt_packages(self, packages: Union[str, List[str], Tuple[str, ...]]) -> Image:
695
+ """
696
+ Use this method to create a new image with the specified apt packages layered on top of the current image
697
+
698
+ :param packages: list of apt packages to install
699
+ :return: Image
700
+ """
701
+ pkgs = _ensure_tuple(packages)
702
+ new_image = self.clone(addl_layer=AptPackages(packages=pkgs))
703
+ return new_image
704
+
705
+ def with_commands(self, commands: List[str]) -> Image:
706
+ """
707
+ Use this method to create a new image with the specified commands layered on top of the current image
708
+ Be sure not to use RUN in your command.
709
+
710
+ :param commands: list of commands to run
711
+ :return: Image
712
+ """
713
+ new_commands: Tuple = _ensure_tuple(commands)
714
+ new_image = self.clone(addl_layer=Commands(commands=new_commands))
715
+ return new_image
716
+
717
+ def with_local_v2(self) -> Image:
718
+ """
719
+ Use this method to create a new image with the local v2 builder
720
+ This will override any existing builder
721
+
722
+ :return: Image
723
+ """
724
+ dist_folder = Path(__file__).parent.parent.parent / "dist"
725
+ # Manually declare the CopyConfig instead of using with_source_folder so we can set the hashing
726
+ # used to compute the identifier. Can remove if we ever decide to expose the lambda in with_ commands
727
+ with_dist = self.clone(
728
+ addl_layer=CopyConfig(
729
+ path_type=1, context_source=dist_folder, image_dest=".", _compute_identifier=lambda x: "/dist"
730
+ )
731
+ )
732
+
733
+ return with_dist.with_commands(
734
+ [
735
+ "--mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv"
736
+ " --mount=from=uv,source=/uv,target=/usr/bin/uv"
737
+ " --mount=source=dist,target=/dist,type=bind"
738
+ " uv pip install --python $VIRTUALENV $(ls /dist/*whl)"
739
+ ]
740
+ )
741
+
742
+ def __img_str__(self) -> str:
743
+ """
744
+ For the current image only, print all the details if they are not None
745
+ """
746
+ details = []
747
+ if self.base_image:
748
+ details.append(f"Base Image: {self.base_image}")
749
+ elif self.dockerfile:
750
+ details.append(f"Dockerfile: {self.dockerfile}")
751
+ if self.registry:
752
+ details.append(f"Registry: {self.registry}")
753
+ if self.name:
754
+ details.append(f"Name: {self.name}")
755
+ if self.platform:
756
+ details.append(f"Platform: {self.platform}")
757
+
758
+ if self.__getattribute__("_layers"):
759
+ for layer in self._layers:
760
+ details.append(f"Layer: {layer}")
761
+
762
+ return "\n".join(details)