flyte 0.0.1b3__py3-none-any.whl → 0.2.0a0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/__init__.py +20 -4
- flyte/_bin/runtime.py +33 -7
- flyte/_build.py +3 -2
- flyte/_cache/cache.py +1 -2
- flyte/_code_bundle/_packaging.py +1 -1
- flyte/_code_bundle/_utils.py +0 -16
- flyte/_code_bundle/bundle.py +43 -12
- flyte/_context.py +8 -2
- flyte/_deploy.py +56 -15
- flyte/_environment.py +45 -4
- flyte/_excepthook.py +37 -0
- flyte/_group.py +2 -1
- flyte/_image.py +8 -4
- flyte/_initialize.py +112 -254
- flyte/_interface.py +3 -3
- flyte/_internal/controllers/__init__.py +19 -6
- flyte/_internal/controllers/_local_controller.py +83 -8
- flyte/_internal/controllers/_trace.py +2 -1
- flyte/_internal/controllers/remote/__init__.py +27 -7
- flyte/_internal/controllers/remote/_action.py +7 -2
- flyte/_internal/controllers/remote/_client.py +5 -1
- flyte/_internal/controllers/remote/_controller.py +159 -26
- flyte/_internal/controllers/remote/_core.py +13 -5
- flyte/_internal/controllers/remote/_informer.py +4 -4
- flyte/_internal/controllers/remote/_service_protocol.py +6 -6
- flyte/_internal/imagebuild/docker_builder.py +12 -1
- flyte/_internal/imagebuild/image_builder.py +16 -11
- flyte/_internal/runtime/convert.py +164 -21
- flyte/_internal/runtime/entrypoints.py +1 -1
- flyte/_internal/runtime/io.py +3 -3
- flyte/_internal/runtime/task_serde.py +140 -20
- flyte/_internal/runtime/taskrunner.py +4 -3
- flyte/_internal/runtime/types_serde.py +1 -1
- flyte/_logging.py +12 -1
- flyte/_map.py +215 -0
- flyte/_pod.py +19 -0
- flyte/_protos/common/list_pb2.py +3 -3
- flyte/_protos/common/list_pb2.pyi +2 -0
- flyte/_protos/logs/dataplane/payload_pb2.py +28 -24
- flyte/_protos/logs/dataplane/payload_pb2.pyi +11 -2
- flyte/_protos/workflow/common_pb2.py +27 -0
- flyte/_protos/workflow/common_pb2.pyi +14 -0
- flyte/_protos/workflow/environment_pb2.py +29 -0
- flyte/_protos/workflow/environment_pb2.pyi +12 -0
- flyte/_protos/workflow/queue_service_pb2.py +40 -41
- flyte/_protos/workflow/queue_service_pb2.pyi +35 -30
- flyte/_protos/workflow/queue_service_pb2_grpc.py +15 -15
- flyte/_protos/workflow/run_definition_pb2.py +61 -61
- flyte/_protos/workflow/run_definition_pb2.pyi +8 -4
- flyte/_protos/workflow/run_service_pb2.py +20 -24
- flyte/_protos/workflow/run_service_pb2.pyi +2 -6
- flyte/_protos/workflow/state_service_pb2.py +36 -28
- flyte/_protos/workflow/state_service_pb2.pyi +19 -15
- flyte/_protos/workflow/state_service_pb2_grpc.py +28 -28
- flyte/_protos/workflow/task_definition_pb2.py +29 -22
- flyte/_protos/workflow/task_definition_pb2.pyi +21 -5
- flyte/_protos/workflow/task_service_pb2.py +27 -11
- flyte/_protos/workflow/task_service_pb2.pyi +29 -1
- flyte/_protos/workflow/task_service_pb2_grpc.py +34 -0
- flyte/_run.py +166 -95
- flyte/_task.py +110 -28
- flyte/_task_environment.py +55 -72
- flyte/_trace.py +6 -14
- flyte/_utils/__init__.py +6 -0
- flyte/_utils/async_cache.py +139 -0
- flyte/_utils/coro_management.py +0 -2
- flyte/_utils/helpers.py +45 -19
- flyte/_utils/org_discovery.py +57 -0
- flyte/_version.py +2 -2
- flyte/cli/__init__.py +3 -0
- flyte/cli/_abort.py +28 -0
- flyte/{_cli → cli}/_common.py +73 -23
- flyte/cli/_create.py +145 -0
- flyte/{_cli → cli}/_delete.py +4 -4
- flyte/{_cli → cli}/_deploy.py +26 -14
- flyte/cli/_gen.py +163 -0
- flyte/{_cli → cli}/_get.py +98 -23
- {union/_cli → flyte/cli}/_params.py +106 -147
- flyte/{_cli → cli}/_run.py +99 -20
- flyte/cli/main.py +166 -0
- flyte/config/__init__.py +3 -0
- flyte/config/_config.py +216 -0
- flyte/config/_internal.py +64 -0
- flyte/config/_reader.py +207 -0
- flyte/errors.py +29 -0
- flyte/extras/_container.py +33 -43
- flyte/io/__init__.py +17 -1
- flyte/io/_dir.py +2 -2
- flyte/io/_file.py +3 -4
- flyte/io/{structured_dataset → _structured_dataset}/basic_dfs.py +1 -1
- flyte/io/{structured_dataset → _structured_dataset}/structured_dataset.py +1 -1
- flyte/{_datastructures.py → models.py} +56 -7
- flyte/remote/__init__.py +2 -1
- flyte/remote/_client/_protocols.py +2 -0
- flyte/remote/_client/auth/_auth_utils.py +14 -0
- flyte/remote/_client/auth/_channel.py +34 -3
- flyte/remote/_client/auth/_token_client.py +3 -3
- flyte/remote/_client/controlplane.py +13 -13
- flyte/remote/_console.py +1 -1
- flyte/remote/_data.py +10 -6
- flyte/remote/_logs.py +89 -29
- flyte/remote/_project.py +8 -9
- flyte/remote/_run.py +228 -131
- flyte/remote/_secret.py +12 -12
- flyte/remote/_task.py +179 -15
- flyte/report/_report.py +4 -4
- flyte/storage/__init__.py +5 -0
- flyte/storage/_config.py +233 -0
- flyte/storage/_storage.py +23 -3
- flyte/syncify/__init__.py +56 -0
- flyte/syncify/_api.py +371 -0
- flyte/types/__init__.py +23 -0
- flyte/types/_interface.py +22 -7
- flyte/{io/pickle/transformer.py → types/_pickle.py} +2 -1
- flyte/types/_type_engine.py +95 -18
- flyte-0.2.0a0.dist-info/METADATA +249 -0
- flyte-0.2.0a0.dist-info/RECORD +218 -0
- {flyte-0.0.1b3.dist-info → flyte-0.2.0a0.dist-info}/entry_points.txt +1 -1
- flyte/_api_commons.py +0 -3
- flyte/_cli/__init__.py +0 -0
- flyte/_cli/_create.py +0 -42
- flyte/_cli/main.py +0 -72
- flyte/_internal/controllers/pbhash.py +0 -39
- flyte/io/_dataframe.py +0 -0
- flyte/io/pickle/__init__.py +0 -0
- flyte-0.0.1b3.dist-info/METADATA +0 -179
- flyte-0.0.1b3.dist-info/RECORD +0 -390
- union/__init__.py +0 -54
- union/_api_commons.py +0 -3
- union/_bin/__init__.py +0 -0
- union/_bin/runtime.py +0 -113
- union/_build.py +0 -25
- union/_cache/__init__.py +0 -12
- union/_cache/cache.py +0 -141
- union/_cache/defaults.py +0 -9
- union/_cache/policy_function_body.py +0 -42
- union/_cli/__init__.py +0 -0
- union/_cli/_common.py +0 -263
- union/_cli/_create.py +0 -40
- union/_cli/_delete.py +0 -23
- union/_cli/_deploy.py +0 -120
- union/_cli/_get.py +0 -162
- union/_cli/_run.py +0 -150
- union/_cli/main.py +0 -72
- union/_code_bundle/__init__.py +0 -8
- union/_code_bundle/_ignore.py +0 -113
- union/_code_bundle/_packaging.py +0 -187
- union/_code_bundle/_utils.py +0 -342
- union/_code_bundle/bundle.py +0 -176
- union/_context.py +0 -146
- union/_datastructures.py +0 -295
- union/_deploy.py +0 -185
- union/_doc.py +0 -29
- union/_docstring.py +0 -26
- union/_environment.py +0 -43
- union/_group.py +0 -31
- union/_hash.py +0 -23
- union/_image.py +0 -760
- union/_initialize.py +0 -585
- union/_interface.py +0 -84
- union/_internal/__init__.py +0 -3
- union/_internal/controllers/__init__.py +0 -77
- union/_internal/controllers/_local_controller.py +0 -77
- union/_internal/controllers/pbhash.py +0 -39
- union/_internal/controllers/remote/__init__.py +0 -40
- union/_internal/controllers/remote/_action.py +0 -131
- union/_internal/controllers/remote/_client.py +0 -43
- union/_internal/controllers/remote/_controller.py +0 -169
- union/_internal/controllers/remote/_core.py +0 -341
- union/_internal/controllers/remote/_informer.py +0 -260
- union/_internal/controllers/remote/_service_protocol.py +0 -44
- union/_internal/imagebuild/__init__.py +0 -11
- union/_internal/imagebuild/docker_builder.py +0 -416
- union/_internal/imagebuild/image_builder.py +0 -243
- union/_internal/imagebuild/remote_builder.py +0 -0
- union/_internal/resolvers/__init__.py +0 -0
- union/_internal/resolvers/_task_module.py +0 -31
- union/_internal/resolvers/common.py +0 -24
- union/_internal/resolvers/default.py +0 -27
- union/_internal/runtime/__init__.py +0 -0
- union/_internal/runtime/convert.py +0 -163
- union/_internal/runtime/entrypoints.py +0 -121
- union/_internal/runtime/io.py +0 -136
- union/_internal/runtime/resources_serde.py +0 -134
- union/_internal/runtime/task_serde.py +0 -202
- union/_internal/runtime/taskrunner.py +0 -179
- union/_internal/runtime/types_serde.py +0 -53
- union/_logging.py +0 -124
- union/_protos/__init__.py +0 -0
- union/_protos/common/authorization_pb2.py +0 -66
- union/_protos/common/authorization_pb2.pyi +0 -106
- union/_protos/common/identifier_pb2.py +0 -71
- union/_protos/common/identifier_pb2.pyi +0 -82
- union/_protos/common/identity_pb2.py +0 -48
- union/_protos/common/identity_pb2.pyi +0 -72
- union/_protos/common/identity_pb2_grpc.py +0 -4
- union/_protos/common/list_pb2.py +0 -36
- union/_protos/common/list_pb2.pyi +0 -69
- union/_protos/common/list_pb2_grpc.py +0 -4
- union/_protos/common/policy_pb2.py +0 -37
- union/_protos/common/policy_pb2.pyi +0 -27
- union/_protos/common/policy_pb2_grpc.py +0 -4
- union/_protos/common/role_pb2.py +0 -37
- union/_protos/common/role_pb2.pyi +0 -51
- union/_protos/common/role_pb2_grpc.py +0 -4
- union/_protos/common/runtime_version_pb2.py +0 -28
- union/_protos/common/runtime_version_pb2.pyi +0 -24
- union/_protos/common/runtime_version_pb2_grpc.py +0 -4
- union/_protos/logs/dataplane/payload_pb2.py +0 -96
- union/_protos/logs/dataplane/payload_pb2.pyi +0 -168
- union/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
- union/_protos/secret/definition_pb2.py +0 -49
- union/_protos/secret/definition_pb2.pyi +0 -93
- union/_protos/secret/definition_pb2_grpc.py +0 -4
- union/_protos/secret/payload_pb2.py +0 -62
- union/_protos/secret/payload_pb2.pyi +0 -94
- union/_protos/secret/payload_pb2_grpc.py +0 -4
- union/_protos/secret/secret_pb2.py +0 -38
- union/_protos/secret/secret_pb2.pyi +0 -6
- union/_protos/secret/secret_pb2_grpc.py +0 -198
- union/_protos/validate/validate/validate_pb2.py +0 -76
- union/_protos/workflow/node_execution_service_pb2.py +0 -26
- union/_protos/workflow/node_execution_service_pb2.pyi +0 -4
- union/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
- union/_protos/workflow/queue_service_pb2.py +0 -75
- union/_protos/workflow/queue_service_pb2.pyi +0 -103
- union/_protos/workflow/queue_service_pb2_grpc.py +0 -172
- union/_protos/workflow/run_definition_pb2.py +0 -100
- union/_protos/workflow/run_definition_pb2.pyi +0 -256
- union/_protos/workflow/run_definition_pb2_grpc.py +0 -4
- union/_protos/workflow/run_logs_service_pb2.py +0 -41
- union/_protos/workflow/run_logs_service_pb2.pyi +0 -28
- union/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
- union/_protos/workflow/run_service_pb2.py +0 -133
- union/_protos/workflow/run_service_pb2.pyi +0 -173
- union/_protos/workflow/run_service_pb2_grpc.py +0 -412
- union/_protos/workflow/state_service_pb2.py +0 -58
- union/_protos/workflow/state_service_pb2.pyi +0 -69
- union/_protos/workflow/state_service_pb2_grpc.py +0 -138
- union/_protos/workflow/task_definition_pb2.py +0 -72
- union/_protos/workflow/task_definition_pb2.pyi +0 -65
- union/_protos/workflow/task_definition_pb2_grpc.py +0 -4
- union/_protos/workflow/task_service_pb2.py +0 -44
- union/_protos/workflow/task_service_pb2.pyi +0 -31
- union/_protos/workflow/task_service_pb2_grpc.py +0 -104
- union/_resources.py +0 -226
- union/_retry.py +0 -32
- union/_reusable_environment.py +0 -25
- union/_run.py +0 -374
- union/_secret.py +0 -61
- union/_task.py +0 -354
- union/_task_environment.py +0 -186
- union/_timeout.py +0 -47
- union/_tools.py +0 -27
- union/_utils/__init__.py +0 -11
- union/_utils/asyn.py +0 -119
- union/_utils/file_handling.py +0 -71
- union/_utils/helpers.py +0 -46
- union/_utils/lazy_module.py +0 -54
- union/_utils/uv_script_parser.py +0 -49
- union/_version.py +0 -21
- union/connectors/__init__.py +0 -0
- union/errors.py +0 -128
- union/extras/__init__.py +0 -5
- union/extras/_container.py +0 -263
- union/io/__init__.py +0 -11
- union/io/_dataframe.py +0 -0
- union/io/_dir.py +0 -425
- union/io/_file.py +0 -418
- union/io/pickle/__init__.py +0 -0
- union/io/pickle/transformer.py +0 -117
- union/io/structured_dataset/__init__.py +0 -122
- union/io/structured_dataset/basic_dfs.py +0 -219
- union/io/structured_dataset/structured_dataset.py +0 -1057
- union/py.typed +0 -0
- union/remote/__init__.py +0 -23
- union/remote/_client/__init__.py +0 -0
- union/remote/_client/_protocols.py +0 -129
- union/remote/_client/auth/__init__.py +0 -12
- union/remote/_client/auth/_authenticators/__init__.py +0 -0
- union/remote/_client/auth/_authenticators/base.py +0 -391
- union/remote/_client/auth/_authenticators/client_credentials.py +0 -73
- union/remote/_client/auth/_authenticators/device_code.py +0 -120
- union/remote/_client/auth/_authenticators/external_command.py +0 -77
- union/remote/_client/auth/_authenticators/factory.py +0 -200
- union/remote/_client/auth/_authenticators/pkce.py +0 -515
- union/remote/_client/auth/_channel.py +0 -184
- union/remote/_client/auth/_client_config.py +0 -83
- union/remote/_client/auth/_default_html.py +0 -32
- union/remote/_client/auth/_grpc_utils/__init__.py +0 -0
- union/remote/_client/auth/_grpc_utils/auth_interceptor.py +0 -204
- union/remote/_client/auth/_grpc_utils/default_metadata_interceptor.py +0 -144
- union/remote/_client/auth/_keyring.py +0 -154
- union/remote/_client/auth/_token_client.py +0 -258
- union/remote/_client/auth/errors.py +0 -16
- union/remote/_client/controlplane.py +0 -86
- union/remote/_data.py +0 -149
- union/remote/_logs.py +0 -74
- union/remote/_project.py +0 -86
- union/remote/_run.py +0 -820
- union/remote/_secret.py +0 -132
- union/remote/_task.py +0 -193
- union/report/__init__.py +0 -3
- union/report/_report.py +0 -178
- union/report/_template.html +0 -124
- union/storage/__init__.py +0 -24
- union/storage/_remote_fs.py +0 -34
- union/storage/_storage.py +0 -247
- union/storage/_utils.py +0 -5
- union/types/__init__.py +0 -11
- union/types/_renderer.py +0 -162
- union/types/_string_literals.py +0 -120
- union/types/_type_engine.py +0 -2131
- union/types/_utils.py +0 -80
- /union/_protos/common/authorization_pb2_grpc.py → /flyte/_protos/workflow/common_pb2_grpc.py +0 -0
- /union/_protos/common/identifier_pb2_grpc.py → /flyte/_protos/workflow/environment_pb2_grpc.py +0 -0
- /flyte/io/{structured_dataset → _structured_dataset}/__init__.py +0 -0
- {flyte-0.0.1b3.dist-info → flyte-0.2.0a0.dist-info}/WHEEL +0 -0
- {flyte-0.0.1b3.dist-info → flyte-0.2.0a0.dist-info}/top_level.txt +0 -0
flyte/__init__.py
CHANGED
|
@@ -17,16 +17,15 @@ async def my_task():
|
|
|
17
17
|
"""
|
|
18
18
|
|
|
19
19
|
__all__ = [
|
|
20
|
-
"ABFS",
|
|
21
|
-
"GCS",
|
|
22
20
|
"GPU",
|
|
23
|
-
"S3",
|
|
24
21
|
"TPU",
|
|
25
22
|
"Cache",
|
|
26
23
|
"CachePolicy",
|
|
27
24
|
"CacheRequest",
|
|
28
25
|
"Device",
|
|
26
|
+
"Environment",
|
|
29
27
|
"Image",
|
|
28
|
+
"PodTemplate",
|
|
30
29
|
"Resources",
|
|
31
30
|
"RetryStrategy",
|
|
32
31
|
"ReusePolicy",
|
|
@@ -40,17 +39,25 @@ __all__ = [
|
|
|
40
39
|
"deploy",
|
|
41
40
|
"group",
|
|
42
41
|
"init",
|
|
42
|
+
"init_from_config",
|
|
43
|
+
"map",
|
|
43
44
|
"run",
|
|
44
45
|
"trace",
|
|
45
46
|
"with_runcontext",
|
|
46
47
|
]
|
|
47
48
|
|
|
49
|
+
import sys
|
|
50
|
+
|
|
48
51
|
from ._cache import Cache, CachePolicy, CacheRequest
|
|
49
52
|
from ._context import ctx
|
|
50
53
|
from ._deploy import deploy
|
|
54
|
+
from ._environment import Environment
|
|
55
|
+
from ._excepthook import custom_excepthook
|
|
51
56
|
from ._group import group
|
|
52
57
|
from ._image import Image
|
|
53
|
-
from ._initialize import
|
|
58
|
+
from ._initialize import init, init_from_config
|
|
59
|
+
from ._map import map
|
|
60
|
+
from ._pod import PodTemplate
|
|
54
61
|
from ._resources import GPU, TPU, Device, Resources
|
|
55
62
|
from ._retry import RetryStrategy
|
|
56
63
|
from ._reusable_environment import ReusePolicy
|
|
@@ -60,3 +67,12 @@ from ._task_environment import TaskEnvironment
|
|
|
60
67
|
from ._timeout import Timeout, TimeoutType
|
|
61
68
|
from ._trace import trace
|
|
62
69
|
from ._version import __version__
|
|
70
|
+
|
|
71
|
+
sys.excepthook = custom_excepthook
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def version() -> str:
|
|
75
|
+
"""
|
|
76
|
+
Returns the version of the Flyte SDK.
|
|
77
|
+
"""
|
|
78
|
+
return __version__
|
flyte/_bin/runtime.py
CHANGED
|
@@ -8,7 +8,7 @@ Refrain from importing any modules here. If you need to import any modules, do i
|
|
|
8
8
|
import asyncio
|
|
9
9
|
import os
|
|
10
10
|
import sys
|
|
11
|
-
from typing import List
|
|
11
|
+
from typing import Any, List
|
|
12
12
|
|
|
13
13
|
import click
|
|
14
14
|
|
|
@@ -27,6 +27,9 @@ ORG_NAME = "_U_ORG_NAME"
|
|
|
27
27
|
ENDPOINT_OVERRIDE = "_U_EP_OVERRIDE"
|
|
28
28
|
RUN_OUTPUT_BASE_DIR = "_U_RUN_BASE"
|
|
29
29
|
|
|
30
|
+
# TODO: Remove this after proper auth is implemented
|
|
31
|
+
_UNION_EAGER_API_KEY_ENV_VAR = "_UNION_EAGER_API_KEY"
|
|
32
|
+
|
|
30
33
|
|
|
31
34
|
@click.command("a0")
|
|
32
35
|
@click.option("--inputs", "-i", required=True)
|
|
@@ -73,12 +76,16 @@ def main(
|
|
|
73
76
|
):
|
|
74
77
|
sys.path.insert(0, ".")
|
|
75
78
|
|
|
79
|
+
import flyte
|
|
76
80
|
import flyte._utils as utils
|
|
77
|
-
from flyte.
|
|
78
|
-
from flyte._initialize import S3, initialize_in_cluster
|
|
81
|
+
from flyte._initialize import init
|
|
79
82
|
from flyte._internal.controllers import create_controller
|
|
80
83
|
from flyte._internal.imagebuild.image_builder import ImageCache
|
|
81
84
|
from flyte._internal.runtime.entrypoints import load_and_run_task
|
|
85
|
+
from flyte._logging import logger
|
|
86
|
+
from flyte.models import ActionID, Checkpoints, CodeBundle, RawDataPath
|
|
87
|
+
|
|
88
|
+
logger.info(f"Initializing flyte runtime - version {flyte.__version__}")
|
|
82
89
|
|
|
83
90
|
assert org, "Org is required for now"
|
|
84
91
|
assert project, "Project is required"
|
|
@@ -91,12 +98,31 @@ def main(
|
|
|
91
98
|
if name.startswith("{{"):
|
|
92
99
|
name = os.getenv("ACTION_NAME", "")
|
|
93
100
|
|
|
94
|
-
|
|
101
|
+
# Figure out how to connect
|
|
102
|
+
# This detection of api key is a hack for now.
|
|
103
|
+
controller_kwargs: dict[str, Any] = {"insecure": False}
|
|
104
|
+
if api_key := os.getenv(_UNION_EAGER_API_KEY_ENV_VAR):
|
|
105
|
+
logger.info("Using api key from environment")
|
|
106
|
+
controller_kwargs["api_key"] = api_key
|
|
107
|
+
else:
|
|
108
|
+
ep = os.environ.get(ENDPOINT_OVERRIDE, "host.docker.internal:8090")
|
|
109
|
+
controller_kwargs["endpoint"] = ep
|
|
110
|
+
if "localhost" in ep or "docker" in ep:
|
|
111
|
+
controller_kwargs["insecure"] = True
|
|
112
|
+
logger.debug(f"Using controller endpoint: {ep} with kwargs: {controller_kwargs}")
|
|
95
113
|
|
|
96
114
|
bundle = CodeBundle(tgz=tgz, pkl=pkl, destination=dest, computed_version=version)
|
|
97
|
-
#
|
|
98
|
-
|
|
99
|
-
|
|
115
|
+
# We init regular client here so that reference tasks can work
|
|
116
|
+
# Current reference tasks will not work with remote controller, because we create 2 different
|
|
117
|
+
# channels on different threads and this is not supported by grpcio or the auth system. It ends up leading
|
|
118
|
+
# File "src/python/grpcio/grpc/_cython/_cygrpc/aio/completion_queue.pyx.pxi", line 147,
|
|
119
|
+
# in grpc._cython.cygrpc.PollerCompletionQueue._handle_events
|
|
120
|
+
# BlockingIOError: [Errno 11] Resource temporarily unavailable
|
|
121
|
+
# init(org=org, project=project, domain=domain, **controller_kwargs)
|
|
122
|
+
# TODO solution is to use a single channel for both controller and reference tasks, but this requires a refactor
|
|
123
|
+
init()
|
|
124
|
+
# Controller is created with the same kwargs as init, so that it can be used to run tasks
|
|
125
|
+
controller = create_controller(ct="remote", **controller_kwargs)
|
|
100
126
|
|
|
101
127
|
ic = ImageCache.from_transport(image_cache) if image_cache else None
|
|
102
128
|
|
flyte/_build.py
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
from .
|
|
3
|
+
from flyte.syncify import syncify
|
|
4
|
+
|
|
4
5
|
from ._image import Image
|
|
5
6
|
|
|
6
7
|
|
|
7
|
-
@
|
|
8
|
+
@syncify
|
|
8
9
|
async def build(image: Image) -> str:
|
|
9
10
|
"""
|
|
10
11
|
Build an image. The existing async context will be used.
|
flyte/_cache/cache.py
CHANGED
|
@@ -14,10 +14,9 @@ from typing import (
|
|
|
14
14
|
import rich.repr
|
|
15
15
|
from typing_extensions import Literal, ParamSpec, TypeVar, get_args
|
|
16
16
|
|
|
17
|
-
from flyte._datastructures import CodeBundle
|
|
18
|
-
|
|
19
17
|
# if TYPE_CHECKING:
|
|
20
18
|
from flyte._image import Image
|
|
19
|
+
from flyte.models import CodeBundle
|
|
21
20
|
|
|
22
21
|
P = ParamSpec("P")
|
|
23
22
|
FuncOut = TypeVar("FuncOut")
|
flyte/_code_bundle/_packaging.py
CHANGED
|
@@ -91,7 +91,7 @@ def list_files_to_bundle(
|
|
|
91
91
|
ignore = IgnoreGroup(source, *ignores)
|
|
92
92
|
|
|
93
93
|
ls, ls_digest = ls_files(source, copy_style, deref_symlinks, ignore)
|
|
94
|
-
logger.debug(f"Hash
|
|
94
|
+
logger.debug(f"Hash of files to be included in the code bundle: {ls_digest}")
|
|
95
95
|
return ls, ls_digest
|
|
96
96
|
|
|
97
97
|
|
flyte/_code_bundle/_utils.py
CHANGED
|
@@ -14,7 +14,6 @@ import tempfile
|
|
|
14
14
|
import typing
|
|
15
15
|
from datetime import datetime, timezone
|
|
16
16
|
from functools import lru_cache
|
|
17
|
-
from pathlib import Path
|
|
18
17
|
from types import ModuleType
|
|
19
18
|
from typing import List, Literal, Optional, Tuple, Union
|
|
20
19
|
|
|
@@ -322,18 +321,3 @@ def hash_file(file_path: typing.Union[os.PathLike, str]) -> Tuple[bytes, str, in
|
|
|
322
321
|
size += len(chunk)
|
|
323
322
|
|
|
324
323
|
return h.digest(), h.hexdigest(), size
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
def _find_project_root(source_path) -> str:
|
|
328
|
-
"""
|
|
329
|
-
Find the root of the project.
|
|
330
|
-
The root of the project is considered to be the first ancestor from source_path that does
|
|
331
|
-
not contain a __init__.py file.
|
|
332
|
-
|
|
333
|
-
N.B.: This assumption only holds for regular packages (as opposed to namespace packages)
|
|
334
|
-
"""
|
|
335
|
-
# Start from the directory right above source_path
|
|
336
|
-
path = Path(source_path).parent.resolve()
|
|
337
|
-
while os.path.exists(os.path.join(path, "__init__.py")):
|
|
338
|
-
path = path.parent
|
|
339
|
-
return str(path)
|
flyte/_code_bundle/bundle.py
CHANGED
|
@@ -5,13 +5,13 @@ import os
|
|
|
5
5
|
import pathlib
|
|
6
6
|
import tempfile
|
|
7
7
|
from pathlib import Path
|
|
8
|
-
from typing import Type
|
|
8
|
+
from typing import ClassVar, Type
|
|
9
9
|
|
|
10
10
|
from flyteidl.core.tasks_pb2 import TaskTemplate
|
|
11
11
|
|
|
12
|
-
import flyte.storage as storage
|
|
13
|
-
from flyte._datastructures import CodeBundle
|
|
14
12
|
from flyte._logging import log, logger
|
|
13
|
+
from flyte._utils import AsyncLRUCache
|
|
14
|
+
from flyte.models import CodeBundle
|
|
15
15
|
|
|
16
16
|
from ._ignore import GitIgnore, Ignore, StandardIgnore
|
|
17
17
|
from ._packaging import create_bundle, list_files_to_bundle, print_ls_tree
|
|
@@ -21,10 +21,34 @@ _pickled_file_extension = ".pkl.gz"
|
|
|
21
21
|
_tar_file_extension = ".tar.gz"
|
|
22
22
|
|
|
23
23
|
|
|
24
|
+
class _PklCache:
|
|
25
|
+
_pkl_cache: ClassVar[AsyncLRUCache[str, str]] = AsyncLRUCache[str, str](maxsize=100)
|
|
26
|
+
|
|
27
|
+
@classmethod
|
|
28
|
+
async def put(cls, digest: str, upload_to_path: str, from_path: pathlib.Path) -> str:
|
|
29
|
+
"""
|
|
30
|
+
Get the pickled code bundle from the cache or build it if not present.
|
|
31
|
+
|
|
32
|
+
:param digest: The hash digest of the task template.
|
|
33
|
+
:param upload_to_path: The path to upload the pickled file to.
|
|
34
|
+
:param from_path: The path to read the pickled file from.
|
|
35
|
+
:return: CodeBundle object containing the pickled file path and the computed version.
|
|
36
|
+
"""
|
|
37
|
+
import flyte.storage as storage
|
|
38
|
+
|
|
39
|
+
async def put_data() -> str:
|
|
40
|
+
return await storage.put(str(from_path), to_path=str(upload_to_path))
|
|
41
|
+
|
|
42
|
+
return await cls._pkl_cache.get(
|
|
43
|
+
key=digest,
|
|
44
|
+
value_func=put_data,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
24
48
|
async def build_pkl_bundle(
|
|
25
49
|
o: TaskTemplate,
|
|
26
50
|
upload_to_controlplane: bool = True,
|
|
27
|
-
|
|
51
|
+
upload_from_dataplane_base_path: str | None = None,
|
|
28
52
|
copy_bundle_to: pathlib.Path | None = None,
|
|
29
53
|
) -> CodeBundle:
|
|
30
54
|
"""
|
|
@@ -36,16 +60,14 @@ async def build_pkl_bundle(
|
|
|
36
60
|
|
|
37
61
|
:param o: Object to be pickled. This is the task template.
|
|
38
62
|
:param upload_to_controlplane: Whether to upload the pickled file to the control plane or not
|
|
39
|
-
:param
|
|
63
|
+
:param upload_from_dataplane_base_path: If we are on the dataplane, this is the path where the
|
|
40
64
|
pickled file should be uploaded to. upload_to_controlplane has to be False in this case.
|
|
41
65
|
:param copy_bundle_to: If set, the bundle will be copied to this path. This is used for testing purposes.
|
|
42
66
|
:return: CodeBundle object containing the pickled file path and the computed version.
|
|
43
67
|
"""
|
|
44
68
|
import cloudpickle
|
|
45
69
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if upload_to_controlplane and upload_from_dataplane_path:
|
|
70
|
+
if upload_to_controlplane and upload_from_dataplane_base_path:
|
|
49
71
|
raise ValueError("Cannot upload to control plane and upload from dataplane path at the same time.")
|
|
50
72
|
|
|
51
73
|
logger.debug("Building pickled code bundle.")
|
|
@@ -61,10 +83,17 @@ async def build_pkl_bundle(
|
|
|
61
83
|
hash_digest, remote_path = await upload_file(dest)
|
|
62
84
|
return CodeBundle(pkl=remote_path, computed_version=hash_digest)
|
|
63
85
|
|
|
64
|
-
elif
|
|
65
|
-
|
|
86
|
+
elif upload_from_dataplane_base_path:
|
|
87
|
+
from flyte._internal.runtime import io
|
|
88
|
+
|
|
66
89
|
_, str_digest, _ = hash_file(file_path=dest)
|
|
67
|
-
|
|
90
|
+
upload_path = io.pkl_path(upload_from_dataplane_base_path, str_digest)
|
|
91
|
+
logger.debug(f"Uploading pickled code bundle to dataplane path {upload_path}.")
|
|
92
|
+
final_path = await _PklCache.put(
|
|
93
|
+
digest=str_digest,
|
|
94
|
+
upload_to_path=upload_path,
|
|
95
|
+
from_path=dest,
|
|
96
|
+
)
|
|
68
97
|
return CodeBundle(pkl=final_path, computed_version=str_digest)
|
|
69
98
|
|
|
70
99
|
else:
|
|
@@ -117,7 +146,7 @@ async def build_code_bundle(
|
|
|
117
146
|
logger.info(f"Code bundle created at {bundle_path}, size: {tar_size} MB, archive size: {archive_size} MB")
|
|
118
147
|
if not dryrun:
|
|
119
148
|
hash_digest, remote_path = await upload_file(bundle_path)
|
|
120
|
-
logger.
|
|
149
|
+
logger.debug(f"Code bundle uploaded to {remote_path}")
|
|
121
150
|
else:
|
|
122
151
|
remote_path = "na"
|
|
123
152
|
if copy_bundle_to:
|
|
@@ -138,6 +167,8 @@ async def download_bundle(bundle: CodeBundle) -> pathlib.Path:
|
|
|
138
167
|
|
|
139
168
|
:return: The path to the downloaded code bundle.
|
|
140
169
|
"""
|
|
170
|
+
import flyte.storage as storage
|
|
171
|
+
|
|
141
172
|
dest = pathlib.Path(bundle.destination)
|
|
142
173
|
if not dest.is_dir():
|
|
143
174
|
raise ValueError(f"Destination path should be a directory, found {dest}, {dest.stat()}")
|
flyte/_context.py
CHANGED
|
@@ -4,7 +4,8 @@ import contextvars
|
|
|
4
4
|
from dataclasses import dataclass, replace
|
|
5
5
|
from typing import TYPE_CHECKING, Awaitable, Callable, Optional, ParamSpec, TypeVar
|
|
6
6
|
|
|
7
|
-
from flyte.
|
|
7
|
+
from flyte._logging import logger
|
|
8
|
+
from flyte.models import GroupData, RawDataPath, TaskContext
|
|
8
9
|
|
|
9
10
|
if TYPE_CHECKING:
|
|
10
11
|
from flyte.report import Report
|
|
@@ -49,6 +50,7 @@ class Context:
|
|
|
49
50
|
raise ValueError("Cannot create a new context without contextdata.")
|
|
50
51
|
self._data = data
|
|
51
52
|
self._id = id(self) # Immutable unique identifier
|
|
53
|
+
self._token = None # Context variable token to restore the previous context
|
|
52
54
|
|
|
53
55
|
@property
|
|
54
56
|
def data(self) -> ContextData:
|
|
@@ -106,7 +108,11 @@ class Context:
|
|
|
106
108
|
|
|
107
109
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
108
110
|
"""Exit the context, restoring the previous context."""
|
|
109
|
-
|
|
111
|
+
try:
|
|
112
|
+
root_context_var.reset(self._token)
|
|
113
|
+
except Exception as e:
|
|
114
|
+
logger.warn(f"Failed to reset context: {e}")
|
|
115
|
+
raise e
|
|
110
116
|
|
|
111
117
|
async def __aenter__(self):
|
|
112
118
|
"""Async version of context entry."""
|
flyte/_deploy.py
CHANGED
|
@@ -6,11 +6,13 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
|
|
|
6
6
|
|
|
7
7
|
import rich.repr
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
from .
|
|
9
|
+
import flyte.errors
|
|
10
|
+
from flyte.models import SerializationContext
|
|
11
|
+
from flyte.syncify import syncify
|
|
12
|
+
|
|
11
13
|
from ._environment import Environment
|
|
12
14
|
from ._image import Image
|
|
13
|
-
from ._initialize import get_client, get_common_config,
|
|
15
|
+
from ._initialize import ensure_client, get_client, get_common_config, requires_initialization
|
|
14
16
|
from ._logging import logger
|
|
15
17
|
from ._task import TaskTemplate
|
|
16
18
|
from ._task_environment import TaskEnvironment
|
|
@@ -45,22 +47,55 @@ class Deployment:
|
|
|
45
47
|
)
|
|
46
48
|
return f"Deployment(envs=[{env_names}], tasks=[{task_names_versions}])"
|
|
47
49
|
|
|
50
|
+
def task_repr(self) -> List[List[Tuple[str, str]]]:
|
|
51
|
+
"""
|
|
52
|
+
Returns a detailed representation of the deployed tasks.
|
|
53
|
+
"""
|
|
54
|
+
tuples = []
|
|
55
|
+
if self.deployed_tasks:
|
|
56
|
+
for task in self.deployed_tasks:
|
|
57
|
+
tuples.append(
|
|
58
|
+
[
|
|
59
|
+
("name", task.task_template.id.name),
|
|
60
|
+
("version", task.task_template.id.version),
|
|
61
|
+
]
|
|
62
|
+
)
|
|
63
|
+
return tuples
|
|
64
|
+
|
|
65
|
+
def env_repr(self) -> List[List[Tuple[str, str]]]:
|
|
66
|
+
"""
|
|
67
|
+
Returns a detailed representation of the deployed environments.
|
|
68
|
+
"""
|
|
69
|
+
tuples = []
|
|
70
|
+
for env_name, env in self.envs.items():
|
|
71
|
+
tuples.append(
|
|
72
|
+
[
|
|
73
|
+
("environment", env_name),
|
|
74
|
+
("image", env.image.uri if isinstance(env.image, Image) else env.image or ""),
|
|
75
|
+
]
|
|
76
|
+
)
|
|
77
|
+
return tuples
|
|
78
|
+
|
|
48
79
|
|
|
49
|
-
@requires_client
|
|
50
80
|
async def _deploy_task(
|
|
51
81
|
task: TaskTemplate, serialization_context: SerializationContext, dryrun: bool = False
|
|
52
82
|
) -> task_definition_pb2.TaskSpec:
|
|
53
83
|
"""
|
|
54
84
|
Deploy the given task.
|
|
55
85
|
"""
|
|
86
|
+
ensure_client()
|
|
87
|
+
from ._internal.runtime.convert import convert_upload_default_inputs
|
|
56
88
|
from ._internal.runtime.task_serde import translate_task_to_wire
|
|
57
89
|
from ._protos.workflow import task_definition_pb2, task_service_pb2
|
|
58
90
|
|
|
59
91
|
image_uri = task.image.uri if isinstance(task.image, Image) else task.image
|
|
60
92
|
|
|
61
|
-
spec = translate_task_to_wire(task, serialization_context)
|
|
62
93
|
if dryrun:
|
|
63
|
-
return
|
|
94
|
+
return translate_task_to_wire(task, serialization_context)
|
|
95
|
+
|
|
96
|
+
default_inputs = await convert_upload_default_inputs(task.interface)
|
|
97
|
+
spec = translate_task_to_wire(task, serialization_context, default_inputs=default_inputs)
|
|
98
|
+
|
|
64
99
|
msg = f"Deploying task {task.name}, with image {image_uri} version {serialization_context.version}"
|
|
65
100
|
if spec.task_template.HasField("container") and spec.task_template.container.args:
|
|
66
101
|
msg += f" from {spec.task_template.container.args[-3]}.{spec.task_template.container.args[-1]}"
|
|
@@ -98,7 +133,7 @@ async def build_images(deployment: DeploymentPlan) -> ImageCache:
|
|
|
98
133
|
image_identifier_map = {}
|
|
99
134
|
for env_name, env in deployment.envs.items():
|
|
100
135
|
if not isinstance(env.image, str):
|
|
101
|
-
logger.
|
|
136
|
+
logger.warning(f"Building Image for environment {env_name}, image: {env.image}")
|
|
102
137
|
images.append(_build_image_bg(env_name, env.image))
|
|
103
138
|
|
|
104
139
|
elif env.image == "auto" and "auto" not in image_identifier_map:
|
|
@@ -107,7 +142,7 @@ async def build_images(deployment: DeploymentPlan) -> ImageCache:
|
|
|
107
142
|
final_images = await asyncio.gather(*images)
|
|
108
143
|
|
|
109
144
|
for env_name, image_uri in final_images:
|
|
110
|
-
logger.
|
|
145
|
+
logger.warning(f"Built Image for environment {env_name}, image: {image_uri}")
|
|
111
146
|
env = deployment.envs[env_name]
|
|
112
147
|
if isinstance(env.image, Image):
|
|
113
148
|
image_identifier_map[env.image.identifier] = env.image.uri
|
|
@@ -121,19 +156,24 @@ async def apply(deployment: DeploymentPlan, copy_style: CopyFiles, dryrun: bool
|
|
|
121
156
|
|
|
122
157
|
cfg = get_common_config()
|
|
123
158
|
image_cache = await build_images(deployment)
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
159
|
+
|
|
160
|
+
version = deployment.version
|
|
161
|
+
code_bundle = None
|
|
162
|
+
if copy_style == "none" and not version:
|
|
163
|
+
raise flyte.errors.DeploymentError("Version must be set when copy_style is none")
|
|
127
164
|
else:
|
|
128
165
|
code_bundle = await build_code_bundle(from_dir=cfg.root_dir, dryrun=dryrun, copy_style=copy_style)
|
|
129
|
-
|
|
166
|
+
version = version or code_bundle.computed_version
|
|
167
|
+
# TODO we should update the version to include the image cache digest and code bundle digest. This is
|
|
168
|
+
# to ensure that changes in image dependencies, cause an update to the deployment version.
|
|
169
|
+
# TODO Also hash the environment and tasks to ensure that changes in the environment or tasks
|
|
130
170
|
|
|
131
171
|
sc = SerializationContext(
|
|
132
172
|
project=cfg.project,
|
|
133
173
|
domain=cfg.domain,
|
|
134
174
|
org=cfg.org,
|
|
135
175
|
code_bundle=code_bundle,
|
|
136
|
-
version=
|
|
176
|
+
version=version,
|
|
137
177
|
image_cache=image_cache,
|
|
138
178
|
root_dir=cfg.root_dir,
|
|
139
179
|
)
|
|
@@ -141,6 +181,7 @@ async def apply(deployment: DeploymentPlan, copy_style: CopyFiles, dryrun: bool
|
|
|
141
181
|
tasks = []
|
|
142
182
|
for env_name, env in deployment.envs.items():
|
|
143
183
|
logger.info(f"Deploying environment {env_name}")
|
|
184
|
+
# TODO Make this pluggable based on the environment type
|
|
144
185
|
if isinstance(env, TaskEnvironment):
|
|
145
186
|
for task in env.tasks.values():
|
|
146
187
|
tasks.append(_deploy_task(task, dryrun=dryrun, serialization_context=sc))
|
|
@@ -161,7 +202,7 @@ def _recursive_discover(
|
|
|
161
202
|
if env.name in planned_envs:
|
|
162
203
|
continue
|
|
163
204
|
# Recursively discover dependent environments
|
|
164
|
-
for dependent_env in env.
|
|
205
|
+
for dependent_env in env.depends_on:
|
|
165
206
|
_recursive_discover(planned_envs, dependent_env)
|
|
166
207
|
# Add the environment to the existing envs
|
|
167
208
|
planned_envs[env.name] = env
|
|
@@ -175,7 +216,7 @@ def plan_deploy(*envs: Environment, version: Optional[str] = None) -> Deployment
|
|
|
175
216
|
return DeploymentPlan(planned_envs, version=version)
|
|
176
217
|
|
|
177
218
|
|
|
178
|
-
@
|
|
219
|
+
@syncify
|
|
179
220
|
async def deploy(
|
|
180
221
|
*envs: Environment,
|
|
181
222
|
dryrun: bool = False,
|
flyte/_environment.py
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import re
|
|
3
4
|
from dataclasses import dataclass, field
|
|
4
|
-
from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Union
|
|
5
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union
|
|
5
6
|
|
|
6
7
|
import rich.repr
|
|
7
8
|
|
|
@@ -14,6 +15,10 @@ if TYPE_CHECKING:
|
|
|
14
15
|
from kubernetes.client import V1PodTemplate
|
|
15
16
|
|
|
16
17
|
|
|
18
|
+
def is_snake_or_kebab_with_numbers(s: str) -> bool:
|
|
19
|
+
return re.fullmatch(r"^[a-z0-9]+([_-][a-z0-9]+)*$", s) is not None
|
|
20
|
+
|
|
21
|
+
|
|
17
22
|
@rich.repr.auto
|
|
18
23
|
@dataclass(init=True, repr=True)
|
|
19
24
|
class Environment:
|
|
@@ -23,12 +28,12 @@ class Environment:
|
|
|
23
28
|
:param resources: Resources to allocate for the environment.
|
|
24
29
|
:param env: Environment variables to set for the environment.
|
|
25
30
|
:param secrets: Secrets to inject into the environment.
|
|
26
|
-
:param
|
|
31
|
+
:param depends_on: Environment dependencies to hint, so when you deploy the environment, the dependencies are
|
|
27
32
|
also deployed. This is useful when you have a set of environments that depend on each other.
|
|
28
33
|
"""
|
|
29
34
|
|
|
30
35
|
name: str
|
|
31
|
-
|
|
36
|
+
depends_on: List[Environment] = field(default_factory=list)
|
|
32
37
|
pod_template: Optional[Union[str, "V1PodTemplate"]] = None
|
|
33
38
|
description: Optional[str] = None
|
|
34
39
|
secrets: Optional[SecretRequest] = None
|
|
@@ -36,8 +41,44 @@ class Environment:
|
|
|
36
41
|
resources: Optional[Resources] = None
|
|
37
42
|
image: Union[str, Image, Literal["auto"]] = "auto"
|
|
38
43
|
|
|
44
|
+
def __post_init__(self):
|
|
45
|
+
if not is_snake_or_kebab_with_numbers(self.name):
|
|
46
|
+
raise ValueError(f"Environment name '{self.name}' must be in snake_case or kebab-case format.")
|
|
47
|
+
|
|
39
48
|
def add_dependency(self, *env: Environment):
|
|
40
49
|
"""
|
|
41
50
|
Add a dependency to the environment.
|
|
42
51
|
"""
|
|
43
|
-
self.
|
|
52
|
+
self.depends_on.extend(env)
|
|
53
|
+
|
|
54
|
+
def clone_with(
|
|
55
|
+
self,
|
|
56
|
+
name: str,
|
|
57
|
+
image: Optional[Union[str, Image, Literal["auto"]]] = None,
|
|
58
|
+
resources: Optional[Resources] = None,
|
|
59
|
+
env: Optional[Dict[str, str]] = None,
|
|
60
|
+
secrets: Optional[SecretRequest] = None,
|
|
61
|
+
depends_on: Optional[List[Environment]] = None,
|
|
62
|
+
**kwargs: Any,
|
|
63
|
+
) -> Environment:
|
|
64
|
+
raise NotImplementedError
|
|
65
|
+
|
|
66
|
+
def _get_kwargs(self) -> Dict[str, Any]:
|
|
67
|
+
"""
|
|
68
|
+
Get the keyword arguments for the environment.
|
|
69
|
+
"""
|
|
70
|
+
kwargs: Dict[str, Any] = {
|
|
71
|
+
"depends_on": self.depends_on,
|
|
72
|
+
"image": self.image,
|
|
73
|
+
}
|
|
74
|
+
if self.resources is not None:
|
|
75
|
+
kwargs["resources"] = self.resources
|
|
76
|
+
if self.secrets is not None:
|
|
77
|
+
kwargs["secrets"] = self.secrets
|
|
78
|
+
if self.env is not None:
|
|
79
|
+
kwargs["env"] = self.env
|
|
80
|
+
if self.pod_template is not None:
|
|
81
|
+
kwargs["pod_template"] = self.pod_template
|
|
82
|
+
if self.description is not None:
|
|
83
|
+
kwargs["description"] = self.description
|
|
84
|
+
return kwargs
|
flyte/_excepthook.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
import traceback
|
|
4
|
+
|
|
5
|
+
from flyte._logging import logger
|
|
6
|
+
|
|
7
|
+
# Save the original excepthook so we can call it later
|
|
8
|
+
original_excepthook = sys.excepthook
|
|
9
|
+
|
|
10
|
+
# Filters: exclude frames where filename or function name contains these substrings
|
|
11
|
+
EXCLUDED_MODULE_SUBSTRINGS = ["_internal", "syncify"]
|
|
12
|
+
EXCLUDED_FILE_SUBSTRINGS = ["syncify", "_code_bundle"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def should_include_frame(frame: traceback.FrameSummary) -> bool:
|
|
16
|
+
return not (
|
|
17
|
+
any(sub in frame.name for sub in EXCLUDED_MODULE_SUBSTRINGS)
|
|
18
|
+
or any(sub in frame.filename for sub in EXCLUDED_FILE_SUBSTRINGS)
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def custom_excepthook(exc_type, exc_value, exc_tb):
|
|
23
|
+
"""
|
|
24
|
+
Custom exception hook to filter and format tracebacks.
|
|
25
|
+
If the logger's level is DEBUG, it uses the original excepthook.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
if logger.getEffectiveLevel() <= logging.DEBUG:
|
|
29
|
+
original_excepthook(exc_type, exc_value, exc_tb)
|
|
30
|
+
else:
|
|
31
|
+
# Extract and filter traceback
|
|
32
|
+
tb_list = traceback.extract_tb(exc_tb)
|
|
33
|
+
filtered_tb = [frame for frame in tb_list if should_include_frame(frame)]
|
|
34
|
+
# Print the filtered version (custom format)
|
|
35
|
+
print("Filtered traceback (most recent call last):")
|
|
36
|
+
print("".join(traceback.format_list(filtered_tb)))
|
|
37
|
+
print(f"{exc_type.__name__}: {exc_value}\n")
|
flyte/_group.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
from contextlib import contextmanager
|
|
2
2
|
|
|
3
3
|
from ._context import internal_ctx
|
|
4
|
-
from .
|
|
4
|
+
from .models import GroupData
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
@contextmanager
|
|
@@ -29,3 +29,4 @@ def group(name: str):
|
|
|
29
29
|
new_tctx = tctx.replace(group_data=GroupData(name))
|
|
30
30
|
with ctx.replace_task_context(new_tctx):
|
|
31
31
|
yield
|
|
32
|
+
# Exit the context and restore the previous context
|
flyte/_image.py
CHANGED
|
@@ -342,8 +342,7 @@ class Image:
|
|
|
342
342
|
"""Internal hacky function to see if the current install is editable or not."""
|
|
343
343
|
curr = Path(__file__)
|
|
344
344
|
pyproject = curr.parent.parent.parent / "pyproject.toml"
|
|
345
|
-
|
|
346
|
-
return pyproject.exists() and dist_folder
|
|
345
|
+
return pyproject.exists()
|
|
347
346
|
|
|
348
347
|
@classmethod
|
|
349
348
|
def from_uv_debian(
|
|
@@ -445,8 +444,7 @@ class Image:
|
|
|
445
444
|
```
|
|
446
445
|
|
|
447
446
|
For more information on the uv script format, see the documentation:
|
|
448
|
-
|
|
449
|
-
UV: Declaring script dependencies</href>
|
|
447
|
+
[UV: Declaring script dependencies](https://docs.astral.sh/uv/guides/scripts/#declaring-script-dependencies)
|
|
450
448
|
|
|
451
449
|
:param name: name of the image
|
|
452
450
|
:param registry: registry to use for the image
|
|
@@ -468,9 +466,15 @@ class Image:
|
|
|
468
466
|
header = parse_uv_script_file(script)
|
|
469
467
|
if registry is None:
|
|
470
468
|
raise ValueError("registry must be specified")
|
|
469
|
+
|
|
471
470
|
img = cls.from_uv_debian(registry=registry, name=name, arch=arch, python_version=python_version)
|
|
471
|
+
|
|
472
|
+
# add ca-certificates to the image by default
|
|
473
|
+
img = img.with_apt_packages(["ca-certificates"])
|
|
474
|
+
|
|
472
475
|
if header.dependencies:
|
|
473
476
|
return img.with_pip_packages(header.dependencies)
|
|
477
|
+
|
|
474
478
|
# todo: override the _identifier_override to be the script name or a hash of the script contents
|
|
475
479
|
# This is needed because inside the image, the identifier will be computed to be something different.
|
|
476
480
|
return img
|