flyte 0.0.1b0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/__init__.py +62 -0
- flyte/_api_commons.py +3 -0
- flyte/_bin/__init__.py +0 -0
- flyte/_bin/runtime.py +126 -0
- flyte/_build.py +25 -0
- flyte/_cache/__init__.py +12 -0
- flyte/_cache/cache.py +146 -0
- flyte/_cache/defaults.py +9 -0
- flyte/_cache/policy_function_body.py +42 -0
- flyte/_cli/__init__.py +0 -0
- flyte/_cli/_common.py +287 -0
- flyte/_cli/_create.py +42 -0
- flyte/_cli/_delete.py +23 -0
- flyte/_cli/_deploy.py +140 -0
- flyte/_cli/_get.py +235 -0
- flyte/_cli/_run.py +152 -0
- flyte/_cli/main.py +72 -0
- flyte/_code_bundle/__init__.py +8 -0
- flyte/_code_bundle/_ignore.py +113 -0
- flyte/_code_bundle/_packaging.py +187 -0
- flyte/_code_bundle/_utils.py +339 -0
- flyte/_code_bundle/bundle.py +178 -0
- flyte/_context.py +146 -0
- flyte/_datastructures.py +342 -0
- flyte/_deploy.py +202 -0
- flyte/_doc.py +29 -0
- flyte/_docstring.py +32 -0
- flyte/_environment.py +43 -0
- flyte/_group.py +31 -0
- flyte/_hash.py +23 -0
- flyte/_image.py +760 -0
- flyte/_initialize.py +634 -0
- flyte/_interface.py +84 -0
- flyte/_internal/__init__.py +3 -0
- flyte/_internal/controllers/__init__.py +115 -0
- flyte/_internal/controllers/_local_controller.py +118 -0
- flyte/_internal/controllers/_trace.py +40 -0
- flyte/_internal/controllers/pbhash.py +39 -0
- flyte/_internal/controllers/remote/__init__.py +40 -0
- flyte/_internal/controllers/remote/_action.py +141 -0
- flyte/_internal/controllers/remote/_client.py +43 -0
- flyte/_internal/controllers/remote/_controller.py +361 -0
- flyte/_internal/controllers/remote/_core.py +402 -0
- flyte/_internal/controllers/remote/_informer.py +361 -0
- flyte/_internal/controllers/remote/_service_protocol.py +50 -0
- flyte/_internal/imagebuild/__init__.py +11 -0
- flyte/_internal/imagebuild/docker_builder.py +416 -0
- flyte/_internal/imagebuild/image_builder.py +241 -0
- flyte/_internal/imagebuild/remote_builder.py +0 -0
- flyte/_internal/resolvers/__init__.py +0 -0
- flyte/_internal/resolvers/_task_module.py +54 -0
- flyte/_internal/resolvers/common.py +31 -0
- flyte/_internal/resolvers/default.py +28 -0
- flyte/_internal/runtime/__init__.py +0 -0
- flyte/_internal/runtime/convert.py +199 -0
- flyte/_internal/runtime/entrypoints.py +135 -0
- flyte/_internal/runtime/io.py +136 -0
- flyte/_internal/runtime/resources_serde.py +138 -0
- flyte/_internal/runtime/task_serde.py +210 -0
- flyte/_internal/runtime/taskrunner.py +190 -0
- flyte/_internal/runtime/types_serde.py +54 -0
- flyte/_logging.py +124 -0
- flyte/_protos/__init__.py +0 -0
- flyte/_protos/common/authorization_pb2.py +66 -0
- flyte/_protos/common/authorization_pb2.pyi +108 -0
- flyte/_protos/common/authorization_pb2_grpc.py +4 -0
- flyte/_protos/common/identifier_pb2.py +71 -0
- flyte/_protos/common/identifier_pb2.pyi +82 -0
- flyte/_protos/common/identifier_pb2_grpc.py +4 -0
- flyte/_protos/common/identity_pb2.py +48 -0
- flyte/_protos/common/identity_pb2.pyi +72 -0
- flyte/_protos/common/identity_pb2_grpc.py +4 -0
- flyte/_protos/common/list_pb2.py +36 -0
- flyte/_protos/common/list_pb2.pyi +69 -0
- flyte/_protos/common/list_pb2_grpc.py +4 -0
- flyte/_protos/common/policy_pb2.py +37 -0
- flyte/_protos/common/policy_pb2.pyi +27 -0
- flyte/_protos/common/policy_pb2_grpc.py +4 -0
- flyte/_protos/common/role_pb2.py +37 -0
- flyte/_protos/common/role_pb2.pyi +53 -0
- flyte/_protos/common/role_pb2_grpc.py +4 -0
- flyte/_protos/common/runtime_version_pb2.py +28 -0
- flyte/_protos/common/runtime_version_pb2.pyi +24 -0
- flyte/_protos/common/runtime_version_pb2_grpc.py +4 -0
- flyte/_protos/logs/dataplane/payload_pb2.py +96 -0
- flyte/_protos/logs/dataplane/payload_pb2.pyi +168 -0
- flyte/_protos/logs/dataplane/payload_pb2_grpc.py +4 -0
- flyte/_protos/secret/definition_pb2.py +49 -0
- flyte/_protos/secret/definition_pb2.pyi +93 -0
- flyte/_protos/secret/definition_pb2_grpc.py +4 -0
- flyte/_protos/secret/payload_pb2.py +62 -0
- flyte/_protos/secret/payload_pb2.pyi +94 -0
- flyte/_protos/secret/payload_pb2_grpc.py +4 -0
- flyte/_protos/secret/secret_pb2.py +38 -0
- flyte/_protos/secret/secret_pb2.pyi +6 -0
- flyte/_protos/secret/secret_pb2_grpc.py +198 -0
- flyte/_protos/secret/secret_pb2_grpc_grpc.py +198 -0
- flyte/_protos/validate/validate/validate_pb2.py +76 -0
- flyte/_protos/workflow/node_execution_service_pb2.py +26 -0
- flyte/_protos/workflow/node_execution_service_pb2.pyi +4 -0
- flyte/_protos/workflow/node_execution_service_pb2_grpc.py +32 -0
- flyte/_protos/workflow/queue_service_pb2.py +106 -0
- flyte/_protos/workflow/queue_service_pb2.pyi +141 -0
- flyte/_protos/workflow/queue_service_pb2_grpc.py +172 -0
- flyte/_protos/workflow/run_definition_pb2.py +128 -0
- flyte/_protos/workflow/run_definition_pb2.pyi +310 -0
- flyte/_protos/workflow/run_definition_pb2_grpc.py +4 -0
- flyte/_protos/workflow/run_logs_service_pb2.py +41 -0
- flyte/_protos/workflow/run_logs_service_pb2.pyi +28 -0
- flyte/_protos/workflow/run_logs_service_pb2_grpc.py +69 -0
- flyte/_protos/workflow/run_service_pb2.py +133 -0
- flyte/_protos/workflow/run_service_pb2.pyi +175 -0
- flyte/_protos/workflow/run_service_pb2_grpc.py +412 -0
- flyte/_protos/workflow/state_service_pb2.py +58 -0
- flyte/_protos/workflow/state_service_pb2.pyi +71 -0
- flyte/_protos/workflow/state_service_pb2_grpc.py +138 -0
- flyte/_protos/workflow/task_definition_pb2.py +72 -0
- flyte/_protos/workflow/task_definition_pb2.pyi +65 -0
- flyte/_protos/workflow/task_definition_pb2_grpc.py +4 -0
- flyte/_protos/workflow/task_service_pb2.py +44 -0
- flyte/_protos/workflow/task_service_pb2.pyi +31 -0
- flyte/_protos/workflow/task_service_pb2_grpc.py +104 -0
- flyte/_resources.py +226 -0
- flyte/_retry.py +32 -0
- flyte/_reusable_environment.py +25 -0
- flyte/_run.py +411 -0
- flyte/_secret.py +61 -0
- flyte/_task.py +367 -0
- flyte/_task_environment.py +200 -0
- flyte/_timeout.py +47 -0
- flyte/_tools.py +27 -0
- flyte/_trace.py +128 -0
- flyte/_utils/__init__.py +20 -0
- flyte/_utils/asyn.py +119 -0
- flyte/_utils/coro_management.py +25 -0
- flyte/_utils/file_handling.py +72 -0
- flyte/_utils/helpers.py +108 -0
- flyte/_utils/lazy_module.py +54 -0
- flyte/_utils/uv_script_parser.py +49 -0
- flyte/_version.py +21 -0
- flyte/connectors/__init__.py +0 -0
- flyte/errors.py +143 -0
- flyte/extras/__init__.py +5 -0
- flyte/extras/_container.py +273 -0
- flyte/io/__init__.py +11 -0
- flyte/io/_dataframe.py +0 -0
- flyte/io/_dir.py +448 -0
- flyte/io/_file.py +468 -0
- flyte/io/pickle/__init__.py +0 -0
- flyte/io/pickle/transformer.py +117 -0
- flyte/io/structured_dataset/__init__.py +129 -0
- flyte/io/structured_dataset/basic_dfs.py +219 -0
- flyte/io/structured_dataset/structured_dataset.py +1061 -0
- flyte/py.typed +0 -0
- flyte/remote/__init__.py +25 -0
- flyte/remote/_client/__init__.py +0 -0
- flyte/remote/_client/_protocols.py +131 -0
- flyte/remote/_client/auth/__init__.py +12 -0
- flyte/remote/_client/auth/_authenticators/__init__.py +0 -0
- flyte/remote/_client/auth/_authenticators/base.py +397 -0
- flyte/remote/_client/auth/_authenticators/client_credentials.py +73 -0
- flyte/remote/_client/auth/_authenticators/device_code.py +118 -0
- flyte/remote/_client/auth/_authenticators/external_command.py +79 -0
- flyte/remote/_client/auth/_authenticators/factory.py +200 -0
- flyte/remote/_client/auth/_authenticators/pkce.py +516 -0
- flyte/remote/_client/auth/_channel.py +184 -0
- flyte/remote/_client/auth/_client_config.py +83 -0
- flyte/remote/_client/auth/_default_html.py +32 -0
- flyte/remote/_client/auth/_grpc_utils/__init__.py +0 -0
- flyte/remote/_client/auth/_grpc_utils/auth_interceptor.py +288 -0
- flyte/remote/_client/auth/_grpc_utils/default_metadata_interceptor.py +151 -0
- flyte/remote/_client/auth/_keyring.py +143 -0
- flyte/remote/_client/auth/_token_client.py +260 -0
- flyte/remote/_client/auth/errors.py +16 -0
- flyte/remote/_client/controlplane.py +95 -0
- flyte/remote/_console.py +18 -0
- flyte/remote/_data.py +155 -0
- flyte/remote/_logs.py +116 -0
- flyte/remote/_project.py +86 -0
- flyte/remote/_run.py +873 -0
- flyte/remote/_secret.py +132 -0
- flyte/remote/_task.py +227 -0
- flyte/report/__init__.py +3 -0
- flyte/report/_report.py +178 -0
- flyte/report/_template.html +124 -0
- flyte/storage/__init__.py +24 -0
- flyte/storage/_remote_fs.py +34 -0
- flyte/storage/_storage.py +251 -0
- flyte/storage/_utils.py +5 -0
- flyte/types/__init__.py +13 -0
- flyte/types/_interface.py +25 -0
- flyte/types/_renderer.py +162 -0
- flyte/types/_string_literals.py +120 -0
- flyte/types/_type_engine.py +2210 -0
- flyte/types/_utils.py +80 -0
- flyte-0.0.1b0.dist-info/METADATA +179 -0
- flyte-0.0.1b0.dist-info/RECORD +390 -0
- flyte-0.0.1b0.dist-info/WHEEL +5 -0
- flyte-0.0.1b0.dist-info/entry_points.txt +3 -0
- flyte-0.0.1b0.dist-info/top_level.txt +1 -0
- union/__init__.py +54 -0
- union/_api_commons.py +3 -0
- union/_bin/__init__.py +0 -0
- union/_bin/runtime.py +113 -0
- union/_build.py +25 -0
- union/_cache/__init__.py +12 -0
- union/_cache/cache.py +141 -0
- union/_cache/defaults.py +9 -0
- union/_cache/policy_function_body.py +42 -0
- union/_cli/__init__.py +0 -0
- union/_cli/_common.py +263 -0
- union/_cli/_create.py +40 -0
- union/_cli/_delete.py +23 -0
- union/_cli/_deploy.py +120 -0
- union/_cli/_get.py +162 -0
- union/_cli/_params.py +579 -0
- union/_cli/_run.py +150 -0
- union/_cli/main.py +72 -0
- union/_code_bundle/__init__.py +8 -0
- union/_code_bundle/_ignore.py +113 -0
- union/_code_bundle/_packaging.py +187 -0
- union/_code_bundle/_utils.py +342 -0
- union/_code_bundle/bundle.py +176 -0
- union/_context.py +146 -0
- union/_datastructures.py +295 -0
- union/_deploy.py +185 -0
- union/_doc.py +29 -0
- union/_docstring.py +26 -0
- union/_environment.py +43 -0
- union/_group.py +31 -0
- union/_hash.py +23 -0
- union/_image.py +760 -0
- union/_initialize.py +585 -0
- union/_interface.py +84 -0
- union/_internal/__init__.py +3 -0
- union/_internal/controllers/__init__.py +77 -0
- union/_internal/controllers/_local_controller.py +77 -0
- union/_internal/controllers/pbhash.py +39 -0
- union/_internal/controllers/remote/__init__.py +40 -0
- union/_internal/controllers/remote/_action.py +131 -0
- union/_internal/controllers/remote/_client.py +43 -0
- union/_internal/controllers/remote/_controller.py +169 -0
- union/_internal/controllers/remote/_core.py +341 -0
- union/_internal/controllers/remote/_informer.py +260 -0
- union/_internal/controllers/remote/_service_protocol.py +44 -0
- union/_internal/imagebuild/__init__.py +11 -0
- union/_internal/imagebuild/docker_builder.py +416 -0
- union/_internal/imagebuild/image_builder.py +243 -0
- union/_internal/imagebuild/remote_builder.py +0 -0
- union/_internal/resolvers/__init__.py +0 -0
- union/_internal/resolvers/_task_module.py +31 -0
- union/_internal/resolvers/common.py +24 -0
- union/_internal/resolvers/default.py +27 -0
- union/_internal/runtime/__init__.py +0 -0
- union/_internal/runtime/convert.py +163 -0
- union/_internal/runtime/entrypoints.py +121 -0
- union/_internal/runtime/io.py +136 -0
- union/_internal/runtime/resources_serde.py +134 -0
- union/_internal/runtime/task_serde.py +202 -0
- union/_internal/runtime/taskrunner.py +179 -0
- union/_internal/runtime/types_serde.py +53 -0
- union/_logging.py +124 -0
- union/_protos/__init__.py +0 -0
- union/_protos/common/authorization_pb2.py +66 -0
- union/_protos/common/authorization_pb2.pyi +106 -0
- union/_protos/common/authorization_pb2_grpc.py +4 -0
- union/_protos/common/identifier_pb2.py +71 -0
- union/_protos/common/identifier_pb2.pyi +82 -0
- union/_protos/common/identifier_pb2_grpc.py +4 -0
- union/_protos/common/identity_pb2.py +48 -0
- union/_protos/common/identity_pb2.pyi +72 -0
- union/_protos/common/identity_pb2_grpc.py +4 -0
- union/_protos/common/list_pb2.py +36 -0
- union/_protos/common/list_pb2.pyi +69 -0
- union/_protos/common/list_pb2_grpc.py +4 -0
- union/_protos/common/policy_pb2.py +37 -0
- union/_protos/common/policy_pb2.pyi +27 -0
- union/_protos/common/policy_pb2_grpc.py +4 -0
- union/_protos/common/role_pb2.py +37 -0
- union/_protos/common/role_pb2.pyi +51 -0
- union/_protos/common/role_pb2_grpc.py +4 -0
- union/_protos/common/runtime_version_pb2.py +28 -0
- union/_protos/common/runtime_version_pb2.pyi +24 -0
- union/_protos/common/runtime_version_pb2_grpc.py +4 -0
- union/_protos/logs/dataplane/payload_pb2.py +96 -0
- union/_protos/logs/dataplane/payload_pb2.pyi +168 -0
- union/_protos/logs/dataplane/payload_pb2_grpc.py +4 -0
- union/_protos/secret/definition_pb2.py +49 -0
- union/_protos/secret/definition_pb2.pyi +93 -0
- union/_protos/secret/definition_pb2_grpc.py +4 -0
- union/_protos/secret/payload_pb2.py +62 -0
- union/_protos/secret/payload_pb2.pyi +94 -0
- union/_protos/secret/payload_pb2_grpc.py +4 -0
- union/_protos/secret/secret_pb2.py +38 -0
- union/_protos/secret/secret_pb2.pyi +6 -0
- union/_protos/secret/secret_pb2_grpc.py +198 -0
- union/_protos/validate/validate/validate_pb2.py +76 -0
- union/_protos/workflow/node_execution_service_pb2.py +26 -0
- union/_protos/workflow/node_execution_service_pb2.pyi +4 -0
- union/_protos/workflow/node_execution_service_pb2_grpc.py +32 -0
- union/_protos/workflow/queue_service_pb2.py +75 -0
- union/_protos/workflow/queue_service_pb2.pyi +103 -0
- union/_protos/workflow/queue_service_pb2_grpc.py +172 -0
- union/_protos/workflow/run_definition_pb2.py +100 -0
- union/_protos/workflow/run_definition_pb2.pyi +256 -0
- union/_protos/workflow/run_definition_pb2_grpc.py +4 -0
- union/_protos/workflow/run_logs_service_pb2.py +41 -0
- union/_protos/workflow/run_logs_service_pb2.pyi +28 -0
- union/_protos/workflow/run_logs_service_pb2_grpc.py +69 -0
- union/_protos/workflow/run_service_pb2.py +133 -0
- union/_protos/workflow/run_service_pb2.pyi +173 -0
- union/_protos/workflow/run_service_pb2_grpc.py +412 -0
- union/_protos/workflow/state_service_pb2.py +58 -0
- union/_protos/workflow/state_service_pb2.pyi +69 -0
- union/_protos/workflow/state_service_pb2_grpc.py +138 -0
- union/_protos/workflow/task_definition_pb2.py +72 -0
- union/_protos/workflow/task_definition_pb2.pyi +65 -0
- union/_protos/workflow/task_definition_pb2_grpc.py +4 -0
- union/_protos/workflow/task_service_pb2.py +44 -0
- union/_protos/workflow/task_service_pb2.pyi +31 -0
- union/_protos/workflow/task_service_pb2_grpc.py +104 -0
- union/_resources.py +226 -0
- union/_retry.py +32 -0
- union/_reusable_environment.py +25 -0
- union/_run.py +374 -0
- union/_secret.py +61 -0
- union/_task.py +354 -0
- union/_task_environment.py +186 -0
- union/_timeout.py +47 -0
- union/_tools.py +27 -0
- union/_utils/__init__.py +11 -0
- union/_utils/asyn.py +119 -0
- union/_utils/file_handling.py +71 -0
- union/_utils/helpers.py +46 -0
- union/_utils/lazy_module.py +54 -0
- union/_utils/uv_script_parser.py +49 -0
- union/_version.py +21 -0
- union/connectors/__init__.py +0 -0
- union/errors.py +128 -0
- union/extras/__init__.py +5 -0
- union/extras/_container.py +263 -0
- union/io/__init__.py +11 -0
- union/io/_dataframe.py +0 -0
- union/io/_dir.py +425 -0
- union/io/_file.py +418 -0
- union/io/pickle/__init__.py +0 -0
- union/io/pickle/transformer.py +117 -0
- union/io/structured_dataset/__init__.py +122 -0
- union/io/structured_dataset/basic_dfs.py +219 -0
- union/io/structured_dataset/structured_dataset.py +1057 -0
- union/py.typed +0 -0
- union/remote/__init__.py +23 -0
- union/remote/_client/__init__.py +0 -0
- union/remote/_client/_protocols.py +129 -0
- union/remote/_client/auth/__init__.py +12 -0
- union/remote/_client/auth/_authenticators/__init__.py +0 -0
- union/remote/_client/auth/_authenticators/base.py +391 -0
- union/remote/_client/auth/_authenticators/client_credentials.py +73 -0
- union/remote/_client/auth/_authenticators/device_code.py +120 -0
- union/remote/_client/auth/_authenticators/external_command.py +77 -0
- union/remote/_client/auth/_authenticators/factory.py +200 -0
- union/remote/_client/auth/_authenticators/pkce.py +515 -0
- union/remote/_client/auth/_channel.py +184 -0
- union/remote/_client/auth/_client_config.py +83 -0
- union/remote/_client/auth/_default_html.py +32 -0
- union/remote/_client/auth/_grpc_utils/__init__.py +0 -0
- union/remote/_client/auth/_grpc_utils/auth_interceptor.py +204 -0
- union/remote/_client/auth/_grpc_utils/default_metadata_interceptor.py +144 -0
- union/remote/_client/auth/_keyring.py +154 -0
- union/remote/_client/auth/_token_client.py +258 -0
- union/remote/_client/auth/errors.py +16 -0
- union/remote/_client/controlplane.py +86 -0
- union/remote/_data.py +149 -0
- union/remote/_logs.py +74 -0
- union/remote/_project.py +86 -0
- union/remote/_run.py +820 -0
- union/remote/_secret.py +132 -0
- union/remote/_task.py +193 -0
- union/report/__init__.py +3 -0
- union/report/_report.py +178 -0
- union/report/_template.html +124 -0
- union/storage/__init__.py +24 -0
- union/storage/_remote_fs.py +34 -0
- union/storage/_storage.py +247 -0
- union/storage/_utils.py +5 -0
- union/types/__init__.py +11 -0
- union/types/_renderer.py +162 -0
- union/types/_string_literals.py +120 -0
- union/types/_type_engine.py +2131 -0
- union/types/_utils.py +80 -0
flyte/_image.py
ADDED
|
@@ -0,0 +1,760 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import hashlib
|
|
5
|
+
import sys
|
|
6
|
+
from abc import abstractmethod
|
|
7
|
+
from dataclasses import asdict, dataclass, field
|
|
8
|
+
from functools import cached_property
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Callable, ClassVar, Dict, List, Literal, Optional, Tuple, TypeVar, Union
|
|
11
|
+
|
|
12
|
+
import rich.repr
|
|
13
|
+
|
|
14
|
+
# Supported Python versions
|
|
15
|
+
PYTHON_3_10 = (3, 10)
|
|
16
|
+
PYTHON_3_11 = (3, 11)
|
|
17
|
+
PYTHON_3_12 = (3, 12)
|
|
18
|
+
PYTHON_3_13 = (3, 13)
|
|
19
|
+
|
|
20
|
+
# 0 is a file, 1 is a directory
|
|
21
|
+
CopyConfigType = Literal[0, 1]
|
|
22
|
+
|
|
23
|
+
T = TypeVar("T")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _ensure_tuple(val: Union[T, List[T], Tuple[T, ...]]) -> Tuple[T] | Tuple[T, ...]:
|
|
27
|
+
"""
|
|
28
|
+
Ensure that the input is a tuple. If it is a string, convert it to a tuple with one element.
|
|
29
|
+
If it is a list, convert it to a tuple.
|
|
30
|
+
"""
|
|
31
|
+
if isinstance(val, list):
|
|
32
|
+
return tuple(val)
|
|
33
|
+
elif isinstance(val, tuple):
|
|
34
|
+
return val
|
|
35
|
+
else:
|
|
36
|
+
return (val,)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@rich.repr.auto
|
|
40
|
+
@dataclass(frozen=True, repr=True, kw_only=True)
|
|
41
|
+
class Layer:
|
|
42
|
+
"""
|
|
43
|
+
This is an abstract representation of Container Image Layers, which can be used to create
|
|
44
|
+
layered images programmatically.
|
|
45
|
+
"""
|
|
46
|
+
|
|
47
|
+
_compute_identifier: Callable[[Layer], str] = field(default=lambda x: x.__str__(), init=True)
|
|
48
|
+
|
|
49
|
+
@abstractmethod
|
|
50
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
51
|
+
"""
|
|
52
|
+
This method should be implemented by subclasses to provide a hash representation of the layer.
|
|
53
|
+
|
|
54
|
+
:param hasher: The hash object to update with the layer's data.
|
|
55
|
+
"""
|
|
56
|
+
...
|
|
57
|
+
|
|
58
|
+
def validate(self):
|
|
59
|
+
"""
|
|
60
|
+
Raise any validation errors for the layer
|
|
61
|
+
:return:
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@rich.repr.auto
|
|
66
|
+
@dataclass(kw_only=True, frozen=True, repr=True)
|
|
67
|
+
class PipPackages(Layer):
|
|
68
|
+
packages: Optional[Tuple[str, ...]] = None
|
|
69
|
+
index_url: Optional[str] = None
|
|
70
|
+
extra_index_urls: Optional[Tuple[str] | Tuple[str, ...] | List[str]] = None
|
|
71
|
+
pre: bool = False
|
|
72
|
+
extra_args: Optional[str] = None
|
|
73
|
+
|
|
74
|
+
# todo: to be implemented
|
|
75
|
+
# secret_mounts: Optional[List[Tuple[str, str]]] = None
|
|
76
|
+
|
|
77
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
78
|
+
"""
|
|
79
|
+
Update the hash with the pip packages
|
|
80
|
+
"""
|
|
81
|
+
hash_input = ""
|
|
82
|
+
if self.packages:
|
|
83
|
+
for package in self.packages:
|
|
84
|
+
hash_input += package
|
|
85
|
+
if self.index_url:
|
|
86
|
+
hash_input += self.index_url
|
|
87
|
+
if self.extra_index_urls:
|
|
88
|
+
for url in self.extra_index_urls:
|
|
89
|
+
hash_input += url
|
|
90
|
+
if self.pre:
|
|
91
|
+
hash_input += str(self.pre)
|
|
92
|
+
if self.extra_args:
|
|
93
|
+
hash_input += self.extra_args
|
|
94
|
+
|
|
95
|
+
hasher.update(hash_input.encode("utf-8"))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@rich.repr.auto
|
|
99
|
+
@dataclass(kw_only=True, frozen=True, repr=True)
|
|
100
|
+
class Requirements(PipPackages):
|
|
101
|
+
file: Path
|
|
102
|
+
|
|
103
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
104
|
+
from ._utils import filehash_update
|
|
105
|
+
|
|
106
|
+
super().update_hash(hasher)
|
|
107
|
+
filehash_update(self.file, hasher)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
@rich.repr.auto
|
|
111
|
+
@dataclass(frozen=True, repr=True)
|
|
112
|
+
class AptPackages(Layer):
|
|
113
|
+
packages: Tuple[str, ...]
|
|
114
|
+
|
|
115
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
116
|
+
hasher.update("".join(self.packages).encode("utf-8"))
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@rich.repr.auto
|
|
120
|
+
@dataclass(frozen=True, repr=True)
|
|
121
|
+
class Commands(Layer):
|
|
122
|
+
commands: Tuple[str, ...]
|
|
123
|
+
|
|
124
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
125
|
+
hasher.update("".join(self.commands).encode("utf-8"))
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@rich.repr.auto
|
|
129
|
+
@dataclass(frozen=True, repr=True)
|
|
130
|
+
class WorkDir(Layer):
|
|
131
|
+
workdir: str
|
|
132
|
+
|
|
133
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
134
|
+
hasher.update(self.workdir.encode("utf-8"))
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@rich.repr.auto
|
|
138
|
+
@dataclass(frozen=True, repr=True)
|
|
139
|
+
class CopyConfig(Layer):
|
|
140
|
+
path_type: CopyConfigType
|
|
141
|
+
context_source: Path
|
|
142
|
+
image_dest: str = "."
|
|
143
|
+
|
|
144
|
+
def validate(self):
|
|
145
|
+
if not self.context_source.exists():
|
|
146
|
+
raise ValueError(f"Source folder {self.context_source.absolute()} does not exist")
|
|
147
|
+
if not self.context_source.is_dir() and self.path_type == 1:
|
|
148
|
+
raise ValueError(f"Source folder {self.context_source.absolute()} is not a directory")
|
|
149
|
+
if not self.context_source.is_file() and self.path_type == 0:
|
|
150
|
+
raise ValueError(f"Source file {self.context_source.absolute()} is not a file")
|
|
151
|
+
|
|
152
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
153
|
+
from ._utils import update_hasher_for_source
|
|
154
|
+
|
|
155
|
+
update_hasher_for_source(self.context_source, hasher)
|
|
156
|
+
if self.image_dest:
|
|
157
|
+
hasher.update(self.image_dest.encode("utf-8"))
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@rich.repr.auto
|
|
161
|
+
@dataclass(frozen=True, repr=True)
|
|
162
|
+
class UVProject(Layer):
|
|
163
|
+
pyproject: Path
|
|
164
|
+
uvlock: Path
|
|
165
|
+
|
|
166
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
167
|
+
from ._utils import filehash_update
|
|
168
|
+
|
|
169
|
+
filehash_update(self.uvlock, hasher)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
@rich.repr.auto
|
|
173
|
+
@dataclass(frozen=True, repr=True)
|
|
174
|
+
class _DockerLines(Layer):
|
|
175
|
+
"""
|
|
176
|
+
This is an internal class and should only be used by the default images. It is not supported by most
|
|
177
|
+
builders so please don't use it.
|
|
178
|
+
"""
|
|
179
|
+
|
|
180
|
+
lines: Tuple[str, ...]
|
|
181
|
+
|
|
182
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
183
|
+
hasher.update("".join(self.lines).encode("utf-8"))
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@rich.repr.auto
|
|
187
|
+
@dataclass(frozen=True, repr=True)
|
|
188
|
+
class Env(Layer):
|
|
189
|
+
"""
|
|
190
|
+
This is an internal class and should only be used by the default images. It is not supported by most
|
|
191
|
+
builders so please don't use it.
|
|
192
|
+
"""
|
|
193
|
+
|
|
194
|
+
env_vars: Tuple[Tuple[str, str], ...] = field(default_factory=tuple)
|
|
195
|
+
|
|
196
|
+
def update_hash(self, hasher: hashlib._Hash):
|
|
197
|
+
txt = [f"{k}={v}" for k, v in self.env_vars]
|
|
198
|
+
hasher.update(" ".join(txt).encode("utf-8"))
|
|
199
|
+
|
|
200
|
+
@classmethod
|
|
201
|
+
def from_dict(cls, envs: Dict[str, str]) -> Env:
|
|
202
|
+
return cls(env_vars=tuple((k, v) for k, v in envs.items()))
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
Architecture = Literal["linux/amd64", "linux/arm64"]
|
|
206
|
+
|
|
207
|
+
_BASE_REGISTRY = "ghcr.io/unionai-oss"
|
|
208
|
+
_DEFAULT_IMAGE_NAME = "flyte"
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _detect_python_version() -> Tuple[int, int]:
|
|
212
|
+
"""
|
|
213
|
+
Detect the current Python version.
|
|
214
|
+
:return: Tuple of major and minor version
|
|
215
|
+
"""
|
|
216
|
+
return sys.version_info.major, sys.version_info.minor
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@dataclass(frozen=True, repr=True, eq=True)
|
|
220
|
+
class Image:
|
|
221
|
+
"""
|
|
222
|
+
This is a representation of Container Images, which can be used to create layered images programmatically.
|
|
223
|
+
|
|
224
|
+
Use by first calling one of the base constructor methods. These all begin with `from` or `default_`
|
|
225
|
+
The image can then be amended with additional layers using the various `with_*` methods.
|
|
226
|
+
|
|
227
|
+
Invariant for this class: The construction of Image objects must be doable everywhere. That is, if a
|
|
228
|
+
user has a custom image that is not accessible, calling .with_source_file on a file that doesn't exist, the
|
|
229
|
+
instantiation of the object itself must still go through. Further, the .identifier property of the image must
|
|
230
|
+
also still go through. This is because it may have been already built somewhere else.
|
|
231
|
+
Use validate() functions to check each layer for actual errors. These are invoked at actual
|
|
232
|
+
build time. See self.id for more information
|
|
233
|
+
"""
|
|
234
|
+
|
|
235
|
+
# These are base properties of an image
|
|
236
|
+
base_image: Optional[str] = field(default=None)
|
|
237
|
+
dockerfile: Optional[Path] = field(default=None)
|
|
238
|
+
registry: Optional[str] = field(default=None)
|
|
239
|
+
name: Optional[str] = field(default=None)
|
|
240
|
+
platform: Tuple[Architecture, ...] = field(default=("linux/amd64",))
|
|
241
|
+
tag: Optional[str] = field(default=None)
|
|
242
|
+
python_version: Tuple[int, int] = field(default_factory=_detect_python_version)
|
|
243
|
+
|
|
244
|
+
# For .auto() images. Don't compute an actual identifier.
|
|
245
|
+
_identifier_override: Optional[str] = field(default=None, init=False)
|
|
246
|
+
# This is set on default images. These images are built from the base Dockerfile in this library and shouldn't be
|
|
247
|
+
# modified with additional layers.
|
|
248
|
+
is_final: bool = field(default=False)
|
|
249
|
+
|
|
250
|
+
# Layers to be added to the image. In init, because frozen, but users shouldn't access, so underscore.
|
|
251
|
+
_layers: Tuple[Layer, ...] = field(default_factory=tuple)
|
|
252
|
+
|
|
253
|
+
_DEFAULT_IMAGE_PREFIXES: ClassVar = {
|
|
254
|
+
PYTHON_3_10: "py3.10-",
|
|
255
|
+
PYTHON_3_11: "py3.11-",
|
|
256
|
+
PYTHON_3_12: "py3.12-",
|
|
257
|
+
PYTHON_3_13: "py3.13-",
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
@cached_property
|
|
261
|
+
def identifier(self) -> str:
|
|
262
|
+
"""
|
|
263
|
+
This identifier is a hash of the layers and properties of the image. It is used to look up previously built
|
|
264
|
+
images. Why is this useful? For example, if a user has Image.from_uv_base().with_source_file("a/local/file"),
|
|
265
|
+
it's not necessarily the case that that file exists within the image (further commands may have removed/changed
|
|
266
|
+
it), and certainly not the case that the path to the file, inside the image (which is used as part of the layer
|
|
267
|
+
hash computation), is the same. That is, inside the image when a task runs, as we come across the same Image
|
|
268
|
+
declaration, we need a way of identifying the image and its uri, without hashing all the layers again. This
|
|
269
|
+
is what this identifier is for. See the ImageCache object for additional information.
|
|
270
|
+
|
|
271
|
+
:return: A unique identifier of the Image
|
|
272
|
+
"""
|
|
273
|
+
if self._identifier_override:
|
|
274
|
+
return self._identifier_override
|
|
275
|
+
|
|
276
|
+
# Only get the non-None values in the ImageSpec to ensure the hash is consistent
|
|
277
|
+
# across different SDK versions.
|
|
278
|
+
# Can potentially add a second hashing function to the Layer protocol, but relying on just asdict/str
|
|
279
|
+
# representation for now.
|
|
280
|
+
image_dict = asdict(self, dict_factory=lambda x: {k: v for (k, v) in x if v is not None and k != "_layers"})
|
|
281
|
+
layers_str_repr = "".join([layer._compute_identifier(layer) for layer in self._layers])
|
|
282
|
+
image_dict["layers"] = layers_str_repr
|
|
283
|
+
spec_bytes = image_dict.__str__().encode("utf-8")
|
|
284
|
+
return base64.urlsafe_b64encode(hashlib.md5(spec_bytes).digest()).decode("ascii").rstrip("=")
|
|
285
|
+
|
|
286
|
+
def validate(self):
|
|
287
|
+
for layer in self._layers:
|
|
288
|
+
layer.validate()
|
|
289
|
+
|
|
290
|
+
@classmethod
|
|
291
|
+
def _get_default_image_for(cls, python_version: Tuple[int, int], flyte_version: Optional[str] = None) -> Image:
|
|
292
|
+
# Would love a way to move this outside of this class (but still needs to be accessible via Image.auto())
|
|
293
|
+
# this default image definition may need to be updated once there is a released pypi version
|
|
294
|
+
preset_tag = None
|
|
295
|
+
if flyte_version:
|
|
296
|
+
preset_tag = flyte_version if flyte_version.startswith("v") else f"v{flyte_version}"
|
|
297
|
+
preset_tag = f"py{python_version[0]}.{python_version[1]}-{preset_tag}"
|
|
298
|
+
image = Image(
|
|
299
|
+
base_image=f"python:{python_version[0]}.{python_version[1]}-slim-bookworm",
|
|
300
|
+
registry=_BASE_REGISTRY,
|
|
301
|
+
name=_DEFAULT_IMAGE_NAME,
|
|
302
|
+
tag=preset_tag,
|
|
303
|
+
platform=("linux/amd64", "linux/arm64"),
|
|
304
|
+
)
|
|
305
|
+
labels_and_user = _DockerLines(
|
|
306
|
+
(
|
|
307
|
+
"LABEL org.opencontainers.image.authors='Union.AI <sales@union.ai>'",
|
|
308
|
+
"LABEL org.opencontainers.image.source=https://github.com/unionai/unionv2",
|
|
309
|
+
"RUN useradd --create-home --shell /bin/bash flytekit &&"
|
|
310
|
+
" chown -R flytekit /root && chown -R flytekit /home",
|
|
311
|
+
"WORKDIR /root",
|
|
312
|
+
)
|
|
313
|
+
)
|
|
314
|
+
image = image.clone(addl_layer=labels_and_user)
|
|
315
|
+
image = image.with_env_vars(
|
|
316
|
+
{
|
|
317
|
+
"VIRTUAL_ENV": "/opt/venv",
|
|
318
|
+
"PATH": "/opt/venv/bin:$PATH",
|
|
319
|
+
"PYTHONPATH": "/root",
|
|
320
|
+
"UV_LINK_MODE": "copy",
|
|
321
|
+
}
|
|
322
|
+
)
|
|
323
|
+
image = image.with_apt_packages(["build-essential", "ca-certificates"])
|
|
324
|
+
|
|
325
|
+
base_packages = ["kubernetes", "msgpack", "mashumaro"]
|
|
326
|
+
|
|
327
|
+
# Add in flyte library
|
|
328
|
+
if flyte_version:
|
|
329
|
+
base_packages.append(f"flyte=={flyte_version}")
|
|
330
|
+
image = image.with_pip_packages(base_packages)
|
|
331
|
+
else:
|
|
332
|
+
from flyte._version import __version__
|
|
333
|
+
|
|
334
|
+
if cls._is_editable_install() or (__version__ and "dev" in __version__):
|
|
335
|
+
image = image.with_pip_packages(base_packages)
|
|
336
|
+
image = image.with_local_v2()
|
|
337
|
+
else:
|
|
338
|
+
base_packages.append(f"flyte=={__version__}")
|
|
339
|
+
image = image.with_pip_packages(base_packages)
|
|
340
|
+
|
|
341
|
+
return image
|
|
342
|
+
|
|
343
|
+
@staticmethod
|
|
344
|
+
def _is_editable_install():
|
|
345
|
+
"""Internal hacky function to see if the current install is editable or not."""
|
|
346
|
+
curr = Path(__file__)
|
|
347
|
+
pyproject = curr.parent.parent.parent / "pyproject.toml"
|
|
348
|
+
return pyproject.exists()
|
|
349
|
+
|
|
350
|
+
@classmethod
|
|
351
|
+
def from_uv_debian(
|
|
352
|
+
cls,
|
|
353
|
+
registry: str,
|
|
354
|
+
name: str,
|
|
355
|
+
tag: Optional[str] = None,
|
|
356
|
+
python_version: Optional[Tuple[int, int]] = None,
|
|
357
|
+
arch: Union[Architecture, Tuple[Architecture, ...]] = "linux/amd64",
|
|
358
|
+
) -> Image:
|
|
359
|
+
"""
|
|
360
|
+
This creates a new debian-based base image.
|
|
361
|
+
If using the Union or docker builders, image will have uv available and a virtualenv created at /opt/venv.
|
|
362
|
+
|
|
363
|
+
:param registry: Registry to use for the image
|
|
364
|
+
:param name: Name of the image
|
|
365
|
+
:param tag: Tag to use for the image
|
|
366
|
+
:param python_version: Python version to use for the image
|
|
367
|
+
:param arch: Architecture to use for the image, default is linux/amd64
|
|
368
|
+
:return: Image
|
|
369
|
+
"""
|
|
370
|
+
base_image = "debian:bookworm-slim"
|
|
371
|
+
plat = arch if isinstance(arch, tuple) else (arch,)
|
|
372
|
+
if python_version is None:
|
|
373
|
+
python_version = _detect_python_version()
|
|
374
|
+
img = cls(
|
|
375
|
+
base_image=base_image, name=name, registry=registry, tag=tag, platform=plat, python_version=python_version
|
|
376
|
+
)
|
|
377
|
+
return img
|
|
378
|
+
|
|
379
|
+
@classmethod
|
|
380
|
+
def auto(
|
|
381
|
+
cls,
|
|
382
|
+
python_version: Optional[Tuple[int, int]] = None,
|
|
383
|
+
flyte_version: Optional[str] = None,
|
|
384
|
+
registry: Optional[str] = None,
|
|
385
|
+
name: Optional[str] = None,
|
|
386
|
+
) -> Image:
|
|
387
|
+
"""
|
|
388
|
+
Use this method to start using the default base image, built from this library's base Dockerfile
|
|
389
|
+
Default images are multi-arch amd/arm64
|
|
390
|
+
|
|
391
|
+
:param python_version: If not specified, will use the current Python version
|
|
392
|
+
:param flyte_version: Union version to use
|
|
393
|
+
:param registry: Registry to use for the image
|
|
394
|
+
:param name: Name of the image if you want to override the default name
|
|
395
|
+
|
|
396
|
+
:return: Image
|
|
397
|
+
"""
|
|
398
|
+
if python_version is None:
|
|
399
|
+
python_version = _detect_python_version()
|
|
400
|
+
|
|
401
|
+
base_image = cls._get_default_image_for(python_version=python_version, flyte_version=flyte_version)
|
|
402
|
+
if name is not None and registry is None:
|
|
403
|
+
raise ValueError("Both name and registry must be specified to override the default image name.")
|
|
404
|
+
|
|
405
|
+
if registry and name:
|
|
406
|
+
return base_image.clone(registry=registry, name=name)
|
|
407
|
+
|
|
408
|
+
# Set this to auto for all auto images because the meaning of "auto" can change (based on logic inside
|
|
409
|
+
# _get_default_image_for, acts differently in a running task container) so let's make sure it stays auto.
|
|
410
|
+
object.__setattr__(base_image, "_identifier_override", "auto")
|
|
411
|
+
return base_image
|
|
412
|
+
|
|
413
|
+
@classmethod
|
|
414
|
+
def from_prebuilt(cls, image_uri: str) -> Image:
|
|
415
|
+
"""
|
|
416
|
+
Use this method to start with a pre-built base image. This image must already exist in the registry of course.
|
|
417
|
+
|
|
418
|
+
:param image_uri: The full URI of the image, in the format <registry>/<name>:<tag>
|
|
419
|
+
:return:
|
|
420
|
+
"""
|
|
421
|
+
img = cls(base_image=image_uri)
|
|
422
|
+
return img
|
|
423
|
+
|
|
424
|
+
@classmethod
|
|
425
|
+
def from_uv_script(
|
|
426
|
+
cls,
|
|
427
|
+
script: Path | str,
|
|
428
|
+
*,
|
|
429
|
+
name: str,
|
|
430
|
+
registry: str | None = None,
|
|
431
|
+
python_version: Optional[Tuple[int, int]] = None,
|
|
432
|
+
arch: Union[Architecture, Tuple[Architecture, ...]] = "linux/amd64",
|
|
433
|
+
) -> Image:
|
|
434
|
+
"""
|
|
435
|
+
Use this method to create a new image with the specified uv script.
|
|
436
|
+
It uses the header of the script to determine the python version, dependencies to install.
|
|
437
|
+
The script must be a valid uv script, otherwise an error will be raised.
|
|
438
|
+
|
|
439
|
+
Usually the header of the script will look like this:
|
|
440
|
+
Example:
|
|
441
|
+
```python
|
|
442
|
+
#!/usr/bin/env -S uv run --script
|
|
443
|
+
# /// script
|
|
444
|
+
# requires-python = ">=3.12"
|
|
445
|
+
# dependencies = ["httpx"]
|
|
446
|
+
# ///
|
|
447
|
+
```
|
|
448
|
+
|
|
449
|
+
For more information on the uv script format, see the documentation:
|
|
450
|
+
<href="https://docs.astral.sh/uv/guides/scripts/#declaring-script-dependencies">
|
|
451
|
+
UV: Declaring script dependencies</href>
|
|
452
|
+
|
|
453
|
+
:param name: name of the image
|
|
454
|
+
:param registry: registry to use for the image
|
|
455
|
+
:param script: path to the uv script
|
|
456
|
+
:param arch: architecture to use for the image, default is linux/amd64, use tuple for multiple values
|
|
457
|
+
|
|
458
|
+
:return: Image
|
|
459
|
+
"""
|
|
460
|
+
from ._utils import parse_uv_script_file
|
|
461
|
+
|
|
462
|
+
if isinstance(script, str):
|
|
463
|
+
script = Path(script)
|
|
464
|
+
if not script.exists():
|
|
465
|
+
raise FileNotFoundError(f"UV script {script} does not exist")
|
|
466
|
+
if not script.is_file():
|
|
467
|
+
raise ValueError(f"UV script {script} is not a file")
|
|
468
|
+
if not script.suffix == ".py":
|
|
469
|
+
raise ValueError(f"UV script {script} must have a .py extension")
|
|
470
|
+
header = parse_uv_script_file(script)
|
|
471
|
+
if registry is None:
|
|
472
|
+
raise ValueError("registry must be specified")
|
|
473
|
+
img = cls.from_uv_debian(registry=registry, name=name, arch=arch, python_version=python_version)
|
|
474
|
+
if header.dependencies:
|
|
475
|
+
return img.with_pip_packages(header.dependencies)
|
|
476
|
+
# todo: override the _identifier_override to be the script name or a hash of the script contents
|
|
477
|
+
# This is needed because inside the image, the identifier will be computed to be something different.
|
|
478
|
+
return img
|
|
479
|
+
|
|
480
|
+
def clone(
|
|
481
|
+
self, registry: Optional[str] = None, name: Optional[str] = None, addl_layer: Optional[Layer] = None
|
|
482
|
+
) -> Image:
|
|
483
|
+
"""
|
|
484
|
+
Use this method to clone the current image and change the registry and name
|
|
485
|
+
|
|
486
|
+
:param registry: Registry to use for the image
|
|
487
|
+
:param name: Name of the image
|
|
488
|
+
|
|
489
|
+
:return:
|
|
490
|
+
"""
|
|
491
|
+
registry = registry if registry else self.registry
|
|
492
|
+
name = name if name else self.name
|
|
493
|
+
new_layers = (*self._layers, addl_layer) if addl_layer else self._layers
|
|
494
|
+
img = Image(
|
|
495
|
+
base_image=self.base_image,
|
|
496
|
+
dockerfile=self.dockerfile,
|
|
497
|
+
registry=registry,
|
|
498
|
+
name=name,
|
|
499
|
+
tag=self.tag,
|
|
500
|
+
platform=self.platform,
|
|
501
|
+
python_version=self.python_version,
|
|
502
|
+
is_final=self.is_final,
|
|
503
|
+
_layers=new_layers,
|
|
504
|
+
)
|
|
505
|
+
|
|
506
|
+
return img
|
|
507
|
+
|
|
508
|
+
@classmethod
|
|
509
|
+
def from_dockerfile(cls, file: Path, registry: str, name: str, tag: Optional[str] = None) -> Image:
|
|
510
|
+
"""
|
|
511
|
+
Use this method to create a new image with the specified dockerfile
|
|
512
|
+
|
|
513
|
+
:param file: path to the dockerfile
|
|
514
|
+
:param name: name of the image
|
|
515
|
+
:param registry: registry to use for the image
|
|
516
|
+
:param tag: tag to use for the image
|
|
517
|
+
|
|
518
|
+
:return:
|
|
519
|
+
"""
|
|
520
|
+
tag = tag or "latest"
|
|
521
|
+
img = cls(dockerfile=file, registry=registry, name=name, tag=tag)
|
|
522
|
+
|
|
523
|
+
return img
|
|
524
|
+
|
|
525
|
+
def _get_hash_digest(self) -> str:
|
|
526
|
+
"""
|
|
527
|
+
Returns the hash digest of the image, which is a combination of all the layers and properties of the image
|
|
528
|
+
"""
|
|
529
|
+
import hashlib
|
|
530
|
+
|
|
531
|
+
from ._utils import filehash_update
|
|
532
|
+
|
|
533
|
+
hasher = hashlib.md5()
|
|
534
|
+
if self.dockerfile:
|
|
535
|
+
# Note the location of the dockerfile shouldn't matter, only the contents
|
|
536
|
+
filehash_update(self.dockerfile, hasher)
|
|
537
|
+
if self._layers:
|
|
538
|
+
for layer in self._layers:
|
|
539
|
+
layer.update_hash(hasher)
|
|
540
|
+
return hasher.hexdigest()
|
|
541
|
+
|
|
542
|
+
@property
|
|
543
|
+
def _final_tag(self) -> str:
|
|
544
|
+
t = self.tag if self.tag else self._get_hash_digest()
|
|
545
|
+
return t or "latest"
|
|
546
|
+
|
|
547
|
+
@cached_property
|
|
548
|
+
def uri(self) -> str:
|
|
549
|
+
"""
|
|
550
|
+
Returns the URI of the image in the format <registry>/<name>:<tag>
|
|
551
|
+
"""
|
|
552
|
+
if self.registry and self.name:
|
|
553
|
+
tag = self._final_tag
|
|
554
|
+
return f"{self.registry}/{self.name}:{tag}"
|
|
555
|
+
elif self.name:
|
|
556
|
+
return f"{self.name}:{self._final_tag}"
|
|
557
|
+
elif self.base_image:
|
|
558
|
+
return self.base_image
|
|
559
|
+
|
|
560
|
+
raise ValueError("Image is not fully defined. Please set registry, name and tag.")
|
|
561
|
+
|
|
562
|
+
def with_workdir(self, workdir: str) -> Image:
|
|
563
|
+
"""
|
|
564
|
+
Use this method to create a new image with the specified working directory
|
|
565
|
+
This will override any existing working directory
|
|
566
|
+
|
|
567
|
+
:param workdir: working directory to use
|
|
568
|
+
:return:
|
|
569
|
+
"""
|
|
570
|
+
new_image = self.clone(addl_layer=WorkDir(workdir=workdir))
|
|
571
|
+
return new_image
|
|
572
|
+
|
|
573
|
+
def with_requirements(self, file: Path) -> Image:
|
|
574
|
+
"""
|
|
575
|
+
Use this method to create a new image with the specified requirements file layered on top of the current image
|
|
576
|
+
Cannot be used in conjunction with conda
|
|
577
|
+
|
|
578
|
+
:param file: path to the requirements file, must be a .txt file
|
|
579
|
+
:return:
|
|
580
|
+
"""
|
|
581
|
+
if not file.exists():
|
|
582
|
+
raise FileNotFoundError(f"Requirements file {file} does not exist")
|
|
583
|
+
if not file.is_file():
|
|
584
|
+
raise ValueError(f"Requirements file {file} is not a file")
|
|
585
|
+
if file.suffix != ".txt":
|
|
586
|
+
raise ValueError(f"Requirements file {file} must have a .txt extension")
|
|
587
|
+
new_image = self.clone(addl_layer=Requirements(file=file))
|
|
588
|
+
return new_image
|
|
589
|
+
|
|
590
|
+
def with_pip_packages(
|
|
591
|
+
self,
|
|
592
|
+
packages: Union[str, List[str], Tuple[str, ...]],
|
|
593
|
+
index_url: Optional[str] = None,
|
|
594
|
+
extra_index_urls: Union[str, List[str], Tuple[str, ...], None] = None,
|
|
595
|
+
pre: bool = False,
|
|
596
|
+
extra_args: Optional[str] = None,
|
|
597
|
+
) -> Image:
|
|
598
|
+
"""
|
|
599
|
+
Use this method to create a new image with the specified pip packages layered on top of the current image
|
|
600
|
+
Cannot be used in conjunction with conda
|
|
601
|
+
|
|
602
|
+
Example:
|
|
603
|
+
```python
|
|
604
|
+
@flyte.task(image=(flyte.Image
|
|
605
|
+
.ubuntu_python()
|
|
606
|
+
.with_pip_packages(["requests", "numpy"])))
|
|
607
|
+
def my_task(x: int) -> int:
|
|
608
|
+
import numpy as np
|
|
609
|
+
return np.sum([x, 1])
|
|
610
|
+
```
|
|
611
|
+
|
|
612
|
+
:param packages: list of pip packages to install, follows pip install syntax
|
|
613
|
+
:param index_url: index url to use for pip install, default is None
|
|
614
|
+
:param extra_index_urls: extra index urls to use for pip install, default is None
|
|
615
|
+
:param pre: whether to allow pre-release versions, default is False
|
|
616
|
+
:param extra_args: extra arguments to pass to pip install, default is None
|
|
617
|
+
# :param secret_mounts: todo
|
|
618
|
+
:param extra_args: extra arguments to pass to pip install, default is None
|
|
619
|
+
:return: Image
|
|
620
|
+
"""
|
|
621
|
+
|
|
622
|
+
new_packages: Optional[Tuple] = _ensure_tuple(packages) if packages else None
|
|
623
|
+
new_extra_index_urls: Optional[Tuple] = _ensure_tuple(extra_index_urls) if extra_index_urls else None
|
|
624
|
+
|
|
625
|
+
ll = PipPackages(
|
|
626
|
+
packages=new_packages,
|
|
627
|
+
index_url=index_url,
|
|
628
|
+
extra_index_urls=new_extra_index_urls,
|
|
629
|
+
pre=pre,
|
|
630
|
+
extra_args=extra_args,
|
|
631
|
+
)
|
|
632
|
+
new_image = self.clone(addl_layer=ll)
|
|
633
|
+
return new_image
|
|
634
|
+
|
|
635
|
+
def with_env_vars(self, env_vars: Dict[str, str]) -> Image:
|
|
636
|
+
"""
|
|
637
|
+
Use this method to create a new image with the specified environment variables layered on top of
|
|
638
|
+
the current image. Cannot be used in conjunction with conda
|
|
639
|
+
|
|
640
|
+
:param env_vars: dictionary of environment variables to set
|
|
641
|
+
:return: Image
|
|
642
|
+
"""
|
|
643
|
+
new_image = self.clone(addl_layer=Env.from_dict(env_vars))
|
|
644
|
+
return new_image
|
|
645
|
+
|
|
646
|
+
def with_source_folder(self, context_source: Path, image_dest: Optional[str] = None) -> Image:
|
|
647
|
+
"""
|
|
648
|
+
Use this method to create a new image with the specified local directory layered on top of the current image.
|
|
649
|
+
If dest is not specified, it will be copied to the working directory of the image
|
|
650
|
+
|
|
651
|
+
:param context_source: root folder of the source code from the build context to be copied
|
|
652
|
+
:param image_dest: destination folder in the image
|
|
653
|
+
:return: Image
|
|
654
|
+
"""
|
|
655
|
+
image_dest = image_dest if image_dest else "."
|
|
656
|
+
new_image = self.clone(addl_layer=CopyConfig(path_type=1, context_source=context_source, image_dest=image_dest))
|
|
657
|
+
return new_image
|
|
658
|
+
|
|
659
|
+
def with_source_file(self, context_source: Path, image_dest: Optional[str] = None) -> Image:
|
|
660
|
+
"""
|
|
661
|
+
Use this method to create a new image with the specified local file layered on top of the current image.
|
|
662
|
+
If dest is not specified, it will be copied to the working directory of the image
|
|
663
|
+
|
|
664
|
+
:param context_source: root folder of the source code from the build context to be copied
|
|
665
|
+
:param image_dest: destination folder in the image
|
|
666
|
+
:return: Image
|
|
667
|
+
"""
|
|
668
|
+
image_dest = image_dest if image_dest else "."
|
|
669
|
+
new_image = self.clone(addl_layer=CopyConfig(path_type=0, context_source=context_source, image_dest=image_dest))
|
|
670
|
+
return new_image
|
|
671
|
+
|
|
672
|
+
def with_uv_project(self, pyproject_file: Path) -> Image:
|
|
673
|
+
"""
|
|
674
|
+
Use this method to create a new image with the specified uv.lock file layered on top of the current image
|
|
675
|
+
Must have a corresponding pyproject.toml file in the same directory
|
|
676
|
+
Cannot be used in conjunction with conda
|
|
677
|
+
In the Union builders, using this will change the virtual env to /root/.venv
|
|
678
|
+
|
|
679
|
+
:param pyproject_file: path to the pyproject.toml file, needs to have a corresponding uv.lock file
|
|
680
|
+
:return:
|
|
681
|
+
"""
|
|
682
|
+
if not pyproject_file.exists():
|
|
683
|
+
raise FileNotFoundError(f"UVLock file {pyproject_file} does not exist")
|
|
684
|
+
if not pyproject_file.is_file():
|
|
685
|
+
raise ValueError(f"UVLock file {pyproject_file} is not a file")
|
|
686
|
+
lock = pyproject_file.parent / "uv.lock"
|
|
687
|
+
if not lock.exists():
|
|
688
|
+
raise ValueError(f"UVLock file {lock} does not exist")
|
|
689
|
+
new_image = self.clone(addl_layer=UVProject(pyproject=pyproject_file, uvlock=lock))
|
|
690
|
+
return new_image
|
|
691
|
+
|
|
692
|
+
def with_apt_packages(self, packages: Union[str, List[str], Tuple[str, ...]]) -> Image:
|
|
693
|
+
"""
|
|
694
|
+
Use this method to create a new image with the specified apt packages layered on top of the current image
|
|
695
|
+
|
|
696
|
+
:param packages: list of apt packages to install
|
|
697
|
+
:return: Image
|
|
698
|
+
"""
|
|
699
|
+
pkgs = _ensure_tuple(packages)
|
|
700
|
+
new_image = self.clone(addl_layer=AptPackages(packages=pkgs))
|
|
701
|
+
return new_image
|
|
702
|
+
|
|
703
|
+
def with_commands(self, commands: List[str]) -> Image:
|
|
704
|
+
"""
|
|
705
|
+
Use this method to create a new image with the specified commands layered on top of the current image
|
|
706
|
+
Be sure not to use RUN in your command.
|
|
707
|
+
|
|
708
|
+
:param commands: list of commands to run
|
|
709
|
+
:return: Image
|
|
710
|
+
"""
|
|
711
|
+
new_commands: Tuple = _ensure_tuple(commands)
|
|
712
|
+
new_image = self.clone(addl_layer=Commands(commands=new_commands))
|
|
713
|
+
return new_image
|
|
714
|
+
|
|
715
|
+
def with_local_v2(self) -> Image:
|
|
716
|
+
"""
|
|
717
|
+
Use this method to create a new image with the local v2 builder
|
|
718
|
+
This will override any existing builder
|
|
719
|
+
|
|
720
|
+
:return: Image
|
|
721
|
+
"""
|
|
722
|
+
dist_folder = Path(__file__).parent.parent.parent / "dist"
|
|
723
|
+
# Manually declare the CopyConfig instead of using with_source_folder so we can set the hashing
|
|
724
|
+
# used to compute the identifier. Can remove if we ever decide to expose the lambda in with_ commands
|
|
725
|
+
with_dist = self.clone(
|
|
726
|
+
addl_layer=CopyConfig(
|
|
727
|
+
path_type=1, context_source=dist_folder, image_dest=".", _compute_identifier=lambda x: "/dist"
|
|
728
|
+
)
|
|
729
|
+
)
|
|
730
|
+
|
|
731
|
+
return with_dist.with_commands(
|
|
732
|
+
[
|
|
733
|
+
"--mount=type=cache,sharing=locked,mode=0777,target=/root/.cache/uv,id=uv"
|
|
734
|
+
" --mount=from=uv,source=/uv,target=/usr/bin/uv"
|
|
735
|
+
" --mount=source=dist,target=/dist,type=bind"
|
|
736
|
+
" uv pip install --python $VIRTUALENV $(ls /dist/*whl)"
|
|
737
|
+
]
|
|
738
|
+
)
|
|
739
|
+
|
|
740
|
+
def __img_str__(self) -> str:
|
|
741
|
+
"""
|
|
742
|
+
For the current image only, print all the details if they are not None
|
|
743
|
+
"""
|
|
744
|
+
details = []
|
|
745
|
+
if self.base_image:
|
|
746
|
+
details.append(f"Base Image: {self.base_image}")
|
|
747
|
+
elif self.dockerfile:
|
|
748
|
+
details.append(f"Dockerfile: {self.dockerfile}")
|
|
749
|
+
if self.registry:
|
|
750
|
+
details.append(f"Registry: {self.registry}")
|
|
751
|
+
if self.name:
|
|
752
|
+
details.append(f"Name: {self.name}")
|
|
753
|
+
if self.platform:
|
|
754
|
+
details.append(f"Platform: {self.platform}")
|
|
755
|
+
|
|
756
|
+
if self.__getattribute__("_layers"):
|
|
757
|
+
for layer in self._layers:
|
|
758
|
+
details.append(f"Layer: {layer}")
|
|
759
|
+
|
|
760
|
+
return "\n".join(details)
|