flyte 2.0.0b23__py3-none-any.whl → 2.0.0b25__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/__init__.py +11 -2
- flyte/_cache/local_cache.py +4 -3
- flyte/_code_bundle/_utils.py +3 -3
- flyte/_code_bundle/bundle.py +12 -5
- flyte/_context.py +4 -1
- flyte/_custom_context.py +73 -0
- flyte/_deploy.py +31 -7
- flyte/_image.py +48 -16
- flyte/_initialize.py +69 -26
- flyte/_internal/controllers/_local_controller.py +1 -0
- flyte/_internal/controllers/_trace.py +1 -1
- flyte/_internal/controllers/remote/_action.py +9 -10
- flyte/_internal/controllers/remote/_client.py +1 -1
- flyte/_internal/controllers/remote/_controller.py +4 -2
- flyte/_internal/controllers/remote/_core.py +10 -13
- flyte/_internal/controllers/remote/_informer.py +3 -3
- flyte/_internal/controllers/remote/_service_protocol.py +7 -7
- flyte/_internal/imagebuild/docker_builder.py +45 -59
- flyte/_internal/imagebuild/remote_builder.py +51 -11
- flyte/_internal/imagebuild/utils.py +51 -3
- flyte/_internal/runtime/convert.py +39 -18
- flyte/_internal/runtime/io.py +8 -7
- flyte/_internal/runtime/resources_serde.py +20 -6
- flyte/_internal/runtime/reuse.py +1 -1
- flyte/_internal/runtime/task_serde.py +7 -10
- flyte/_internal/runtime/taskrunner.py +10 -1
- flyte/_internal/runtime/trigger_serde.py +13 -13
- flyte/_internal/runtime/types_serde.py +1 -1
- flyte/_keyring/file.py +2 -2
- flyte/_map.py +65 -13
- flyte/_pod.py +2 -2
- flyte/_resources.py +175 -31
- flyte/_run.py +37 -21
- flyte/_task.py +27 -6
- flyte/_task_environment.py +37 -10
- flyte/_utils/module_loader.py +2 -2
- flyte/_version.py +3 -3
- flyte/cli/_common.py +47 -5
- flyte/cli/_create.py +4 -0
- flyte/cli/_deploy.py +8 -0
- flyte/cli/_get.py +4 -0
- flyte/cli/_params.py +4 -4
- flyte/cli/_run.py +50 -7
- flyte/cli/_update.py +4 -3
- flyte/config/_config.py +2 -0
- flyte/config/_internal.py +1 -0
- flyte/config/_reader.py +3 -3
- flyte/errors.py +1 -1
- flyte/extend.py +4 -0
- flyte/extras/_container.py +6 -1
- flyte/git/_config.py +11 -9
- flyte/io/_dataframe/basic_dfs.py +1 -1
- flyte/io/_dataframe/dataframe.py +12 -8
- flyte/io/_dir.py +48 -15
- flyte/io/_file.py +48 -11
- flyte/models.py +12 -8
- flyte/remote/_action.py +18 -16
- flyte/remote/_client/_protocols.py +4 -3
- flyte/remote/_client/auth/_channel.py +1 -1
- flyte/remote/_client/controlplane.py +4 -8
- flyte/remote/_data.py +4 -3
- flyte/remote/_logs.py +3 -3
- flyte/remote/_run.py +5 -5
- flyte/remote/_secret.py +20 -13
- flyte/remote/_task.py +7 -8
- flyte/remote/_trigger.py +25 -27
- flyte/storage/_parallel_reader.py +274 -0
- flyte/storage/_storage.py +66 -2
- flyte/types/_interface.py +2 -2
- flyte/types/_pickle.py +1 -1
- flyte/types/_string_literals.py +8 -9
- flyte/types/_type_engine.py +25 -17
- flyte/types/_utils.py +1 -1
- {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/METADATA +2 -1
- flyte-2.0.0b25.dist-info/RECORD +184 -0
- flyte/_protos/__init__.py +0 -0
- flyte/_protos/common/authorization_pb2.py +0 -66
- flyte/_protos/common/authorization_pb2.pyi +0 -108
- flyte/_protos/common/authorization_pb2_grpc.py +0 -4
- flyte/_protos/common/identifier_pb2.py +0 -117
- flyte/_protos/common/identifier_pb2.pyi +0 -142
- flyte/_protos/common/identifier_pb2_grpc.py +0 -4
- flyte/_protos/common/identity_pb2.py +0 -48
- flyte/_protos/common/identity_pb2.pyi +0 -72
- flyte/_protos/common/identity_pb2_grpc.py +0 -4
- flyte/_protos/common/list_pb2.py +0 -36
- flyte/_protos/common/list_pb2.pyi +0 -71
- flyte/_protos/common/list_pb2_grpc.py +0 -4
- flyte/_protos/common/policy_pb2.py +0 -37
- flyte/_protos/common/policy_pb2.pyi +0 -27
- flyte/_protos/common/policy_pb2_grpc.py +0 -4
- flyte/_protos/common/role_pb2.py +0 -37
- flyte/_protos/common/role_pb2.pyi +0 -53
- flyte/_protos/common/role_pb2_grpc.py +0 -4
- flyte/_protos/common/runtime_version_pb2.py +0 -28
- flyte/_protos/common/runtime_version_pb2.pyi +0 -24
- flyte/_protos/common/runtime_version_pb2_grpc.py +0 -4
- flyte/_protos/imagebuilder/definition_pb2.py +0 -60
- flyte/_protos/imagebuilder/definition_pb2.pyi +0 -153
- flyte/_protos/imagebuilder/definition_pb2_grpc.py +0 -4
- flyte/_protos/imagebuilder/payload_pb2.py +0 -32
- flyte/_protos/imagebuilder/payload_pb2.pyi +0 -21
- flyte/_protos/imagebuilder/payload_pb2_grpc.py +0 -4
- flyte/_protos/imagebuilder/service_pb2.py +0 -29
- flyte/_protos/imagebuilder/service_pb2.pyi +0 -5
- flyte/_protos/imagebuilder/service_pb2_grpc.py +0 -66
- flyte/_protos/logs/dataplane/payload_pb2.py +0 -100
- flyte/_protos/logs/dataplane/payload_pb2.pyi +0 -177
- flyte/_protos/logs/dataplane/payload_pb2_grpc.py +0 -4
- flyte/_protos/secret/definition_pb2.py +0 -49
- flyte/_protos/secret/definition_pb2.pyi +0 -93
- flyte/_protos/secret/definition_pb2_grpc.py +0 -4
- flyte/_protos/secret/payload_pb2.py +0 -62
- flyte/_protos/secret/payload_pb2.pyi +0 -94
- flyte/_protos/secret/payload_pb2_grpc.py +0 -4
- flyte/_protos/secret/secret_pb2.py +0 -38
- flyte/_protos/secret/secret_pb2.pyi +0 -6
- flyte/_protos/secret/secret_pb2_grpc.py +0 -198
- flyte/_protos/validate/validate/validate_pb2.py +0 -76
- flyte/_protos/workflow/common_pb2.py +0 -38
- flyte/_protos/workflow/common_pb2.pyi +0 -63
- flyte/_protos/workflow/common_pb2_grpc.py +0 -4
- flyte/_protos/workflow/environment_pb2.py +0 -29
- flyte/_protos/workflow/environment_pb2.pyi +0 -12
- flyte/_protos/workflow/environment_pb2_grpc.py +0 -4
- flyte/_protos/workflow/node_execution_service_pb2.py +0 -26
- flyte/_protos/workflow/node_execution_service_pb2.pyi +0 -4
- flyte/_protos/workflow/node_execution_service_pb2_grpc.py +0 -32
- flyte/_protos/workflow/queue_service_pb2.py +0 -117
- flyte/_protos/workflow/queue_service_pb2.pyi +0 -182
- flyte/_protos/workflow/queue_service_pb2_grpc.py +0 -206
- flyte/_protos/workflow/run_definition_pb2.py +0 -123
- flyte/_protos/workflow/run_definition_pb2.pyi +0 -354
- flyte/_protos/workflow/run_definition_pb2_grpc.py +0 -4
- flyte/_protos/workflow/run_logs_service_pb2.py +0 -41
- flyte/_protos/workflow/run_logs_service_pb2.pyi +0 -28
- flyte/_protos/workflow/run_logs_service_pb2_grpc.py +0 -69
- flyte/_protos/workflow/run_service_pb2.py +0 -147
- flyte/_protos/workflow/run_service_pb2.pyi +0 -203
- flyte/_protos/workflow/run_service_pb2_grpc.py +0 -480
- flyte/_protos/workflow/state_service_pb2.py +0 -67
- flyte/_protos/workflow/state_service_pb2.pyi +0 -76
- flyte/_protos/workflow/state_service_pb2_grpc.py +0 -138
- flyte/_protos/workflow/task_definition_pb2.py +0 -86
- flyte/_protos/workflow/task_definition_pb2.pyi +0 -105
- flyte/_protos/workflow/task_definition_pb2_grpc.py +0 -4
- flyte/_protos/workflow/task_service_pb2.py +0 -61
- flyte/_protos/workflow/task_service_pb2.pyi +0 -62
- flyte/_protos/workflow/task_service_pb2_grpc.py +0 -138
- flyte/_protos/workflow/trigger_definition_pb2.py +0 -66
- flyte/_protos/workflow/trigger_definition_pb2.pyi +0 -117
- flyte/_protos/workflow/trigger_definition_pb2_grpc.py +0 -4
- flyte/_protos/workflow/trigger_service_pb2.py +0 -96
- flyte/_protos/workflow/trigger_service_pb2.pyi +0 -110
- flyte/_protos/workflow/trigger_service_pb2_grpc.py +0 -281
- flyte-2.0.0b23.dist-info/RECORD +0 -262
- {flyte-2.0.0b23.data → flyte-2.0.0b25.data}/scripts/debug.py +0 -0
- {flyte-2.0.0b23.data → flyte-2.0.0b25.data}/scripts/runtime.py +0 -0
- {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/WHEEL +0 -0
- {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/entry_points.txt +0 -0
- {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/licenses/LICENSE +0 -0
- {flyte-2.0.0b23.dist-info → flyte-2.0.0b25.dist-info}/top_level.txt +0 -0
flyte/_initialize.py
CHANGED
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import functools
|
|
4
4
|
import threading
|
|
5
5
|
import typing
|
|
6
|
-
from dataclasses import dataclass, replace
|
|
6
|
+
from dataclasses import dataclass, field, replace
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from typing import TYPE_CHECKING, Callable, List, Literal, Optional, TypeVar
|
|
9
9
|
|
|
@@ -41,6 +41,7 @@ class _InitConfig(CommonInit):
|
|
|
41
41
|
client: Optional[ClientSet] = None
|
|
42
42
|
storage: Optional[Storage] = None
|
|
43
43
|
image_builder: "ImageBuildEngine.ImageBuilderType" = "local"
|
|
44
|
+
images: typing.Dict[str, str] = field(default_factory=dict)
|
|
44
45
|
|
|
45
46
|
def replace(self, **kwargs) -> _InitConfig:
|
|
46
47
|
return replace(self, **kwargs)
|
|
@@ -141,6 +142,7 @@ async def init(
|
|
|
141
142
|
storage: Storage | None = None,
|
|
142
143
|
batch_size: int = 1000,
|
|
143
144
|
image_builder: ImageBuildEngine.ImageBuilderType = "local",
|
|
145
|
+
images: typing.Dict[str, str] | None = None,
|
|
144
146
|
source_config_path: Optional[Path] = None,
|
|
145
147
|
) -> None:
|
|
146
148
|
"""
|
|
@@ -170,40 +172,18 @@ async def init(
|
|
|
170
172
|
:param ca_cert_file_path: [optional] str Root Cert to be loaded and used to verify admin
|
|
171
173
|
:param http_proxy_url: [optional] HTTP Proxy to be used for OAuth requests
|
|
172
174
|
:param rpc_retries: [optional] int Number of times to retry the platform calls
|
|
173
|
-
:param audience: oauth2 audience for the token request. This is used to validate the token
|
|
174
175
|
:param insecure: insecure flag for the client
|
|
175
176
|
:param storage: Optional blob store (S3, GCS, Azure) configuration if needed to access (i.e. using Minio)
|
|
176
177
|
:param org: Optional organization override for the client. Should be set by auth instead.
|
|
177
178
|
:param batch_size: Optional batch size for operations that use listings, defaults to 1000, so limit larger than
|
|
178
179
|
batch_size will be split into multiple requests.
|
|
179
180
|
:param image_builder: Optional image builder configuration, if not provided, the default image builder will be used.
|
|
181
|
+
:param images: Optional dict of images that can be used by referencing the image name.
|
|
180
182
|
:param source_config_path: Optional path to the source configuration file (This is only used for documentation)
|
|
181
183
|
:return: None
|
|
182
184
|
"""
|
|
183
185
|
from flyte._utils import get_cwd_editable_install, org_from_endpoint, sanitize_endpoint
|
|
184
186
|
|
|
185
|
-
if endpoint or api_key:
|
|
186
|
-
if project is None:
|
|
187
|
-
raise ValueError(
|
|
188
|
-
"Project must be provided to initialize the client. "
|
|
189
|
-
"Please set 'project' in the 'task' section of your config file, "
|
|
190
|
-
"or pass it directly to flyte.init(project='your-project-name')."
|
|
191
|
-
)
|
|
192
|
-
|
|
193
|
-
if domain is None:
|
|
194
|
-
raise ValueError(
|
|
195
|
-
"Domain must be provided to initialize the client. "
|
|
196
|
-
"Please set 'domain' in the 'task' section of your config file, "
|
|
197
|
-
"or pass it directly to flyte.init(domain='your-domain-name')."
|
|
198
|
-
)
|
|
199
|
-
|
|
200
|
-
if org is None and org_from_endpoint(endpoint) is None:
|
|
201
|
-
raise ValueError(
|
|
202
|
-
"Organization must be provided to initialize the client. "
|
|
203
|
-
"Please set 'org' in the 'task' section of your config file, "
|
|
204
|
-
"or pass it directly to flyte.init(org='your-org-name')."
|
|
205
|
-
)
|
|
206
|
-
|
|
207
187
|
_initialize_logger(log_level=log_level)
|
|
208
188
|
|
|
209
189
|
global _init_config # noqa: PLW0603
|
|
@@ -238,7 +218,7 @@ async def init(
|
|
|
238
218
|
else:
|
|
239
219
|
logger.info("No editable install found, using current working directory as root directory.")
|
|
240
220
|
root_dir = Path.cwd()
|
|
241
|
-
|
|
221
|
+
|
|
242
222
|
_init_config = _InitConfig(
|
|
243
223
|
root_dir=root_dir,
|
|
244
224
|
project=project,
|
|
@@ -248,6 +228,7 @@ async def init(
|
|
|
248
228
|
org=org or org_from_endpoint(endpoint),
|
|
249
229
|
batch_size=batch_size,
|
|
250
230
|
image_builder=image_builder,
|
|
231
|
+
images=images or {},
|
|
251
232
|
source_config_path=source_config_path,
|
|
252
233
|
)
|
|
253
234
|
|
|
@@ -258,6 +239,7 @@ async def init_from_config(
|
|
|
258
239
|
root_dir: Path | None = None,
|
|
259
240
|
log_level: int | None = None,
|
|
260
241
|
storage: Storage | None = None,
|
|
242
|
+
images: tuple[str, ...] | None = None,
|
|
261
243
|
) -> None:
|
|
262
244
|
"""
|
|
263
245
|
Initialize the Flyte system using a configuration file or Config object. This method should be called before any
|
|
@@ -276,6 +258,7 @@ async def init_from_config(
|
|
|
276
258
|
from rich.highlighter import ReprHighlighter
|
|
277
259
|
|
|
278
260
|
import flyte.config as config
|
|
261
|
+
from flyte.cli._common import parse_images
|
|
279
262
|
|
|
280
263
|
cfg: config.Config
|
|
281
264
|
cfg_path: Optional[Path] = None
|
|
@@ -301,6 +284,9 @@ async def init_from_config(
|
|
|
301
284
|
|
|
302
285
|
logger.info(f"Flyte config initialized as {cfg}", extra={"highlighter": ReprHighlighter()})
|
|
303
286
|
|
|
287
|
+
# parse image, this will overwrite the image_refs set in the config file
|
|
288
|
+
parse_images(cfg, images)
|
|
289
|
+
|
|
304
290
|
await init.aio(
|
|
305
291
|
org=cfg.task.org,
|
|
306
292
|
project=cfg.task.project,
|
|
@@ -317,6 +303,7 @@ async def init_from_config(
|
|
|
317
303
|
root_dir=root_dir,
|
|
318
304
|
log_level=log_level,
|
|
319
305
|
image_builder=cfg.image.builder,
|
|
306
|
+
images=cfg.image.image_refs,
|
|
320
307
|
storage=storage,
|
|
321
308
|
source_config_path=cfg_path,
|
|
322
309
|
)
|
|
@@ -332,7 +319,7 @@ def _get_init_config() -> Optional[_InitConfig]:
|
|
|
332
319
|
return _init_config
|
|
333
320
|
|
|
334
321
|
|
|
335
|
-
def
|
|
322
|
+
def get_init_config() -> _InitConfig:
|
|
336
323
|
"""
|
|
337
324
|
Get the current initialization configuration. Thread-safe implementation.
|
|
338
325
|
|
|
@@ -490,6 +477,34 @@ def requires_initialization(func: T) -> T:
|
|
|
490
477
|
return typing.cast(T, wrapper)
|
|
491
478
|
|
|
492
479
|
|
|
480
|
+
def require_project_and_domain(func):
|
|
481
|
+
"""
|
|
482
|
+
Decorator that ensures the current Flyte configuration defines
|
|
483
|
+
both 'project' and 'domain'. Raises a clear error if not found.
|
|
484
|
+
"""
|
|
485
|
+
|
|
486
|
+
@functools.wraps(func)
|
|
487
|
+
def wrapper(*args, **kwargs):
|
|
488
|
+
cfg = get_init_config()
|
|
489
|
+
if cfg.project is None:
|
|
490
|
+
raise ValueError(
|
|
491
|
+
"Project must be provided to initialize the client. "
|
|
492
|
+
"Please set 'project' in the 'task' section of your config file, "
|
|
493
|
+
"or pass it directly to flyte.init(project='your-project-name')."
|
|
494
|
+
)
|
|
495
|
+
|
|
496
|
+
if cfg.domain is None:
|
|
497
|
+
raise ValueError(
|
|
498
|
+
"Domain must be provided to initialize the client. "
|
|
499
|
+
"Please set 'domain' in the 'task' section of your config file, "
|
|
500
|
+
"or pass it directly to flyte.init(domain='your-domain-name')."
|
|
501
|
+
)
|
|
502
|
+
|
|
503
|
+
return func(*args, **kwargs)
|
|
504
|
+
|
|
505
|
+
return wrapper
|
|
506
|
+
|
|
507
|
+
|
|
493
508
|
async def _init_for_testing(
|
|
494
509
|
project: str | None = None,
|
|
495
510
|
domain: str | None = None,
|
|
@@ -519,3 +534,31 @@ def replace_client(client):
|
|
|
519
534
|
|
|
520
535
|
with _init_lock:
|
|
521
536
|
_init_config = _init_config.replace(client=client)
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def current_domain() -> str:
|
|
540
|
+
"""
|
|
541
|
+
Returns the current domain from Runtime environment (on the cluster) or from the initialized configuration.
|
|
542
|
+
This is safe to be used during `deploy`, `run` and within `task` code.
|
|
543
|
+
|
|
544
|
+
NOTE: This will not work if you deploy a task to a domain and then run it in another domain.
|
|
545
|
+
|
|
546
|
+
Raises InitializationError if the configuration is not initialized or domain is not set.
|
|
547
|
+
:return: The current domain
|
|
548
|
+
"""
|
|
549
|
+
from ._context import ctx
|
|
550
|
+
|
|
551
|
+
tctx = ctx()
|
|
552
|
+
if tctx is not None:
|
|
553
|
+
domain = tctx.action.domain
|
|
554
|
+
if domain is not None:
|
|
555
|
+
return domain
|
|
556
|
+
|
|
557
|
+
cfg = _get_init_config()
|
|
558
|
+
if cfg is None or cfg.domain is None:
|
|
559
|
+
raise InitializationError(
|
|
560
|
+
"DomainNotInitializedError",
|
|
561
|
+
"user",
|
|
562
|
+
"Domain has not been initialized. Call flyte.init() with a valid domain before using this function.",
|
|
563
|
+
)
|
|
564
|
+
return cfg.domain
|
|
@@ -186,6 +186,7 @@ class LocalController:
|
|
|
186
186
|
tctx = ctx.data.task_context
|
|
187
187
|
if not tctx:
|
|
188
188
|
raise flyte.errors.NotInTaskContextError("BadContext", "Task context not initialized")
|
|
189
|
+
|
|
189
190
|
converted_inputs = convert.Inputs.empty()
|
|
190
191
|
if _interface.inputs:
|
|
191
192
|
converted_inputs = await convert.convert_from_native_to_inputs(_interface, *args, **kwargs)
|
|
@@ -3,16 +3,15 @@ from __future__ import annotations
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from typing import Literal, Optional
|
|
5
5
|
|
|
6
|
-
from
|
|
7
|
-
from
|
|
8
|
-
|
|
9
|
-
from
|
|
10
|
-
from flyte._protos.workflow import (
|
|
11
|
-
queue_service_pb2,
|
|
6
|
+
from flyteidl2.common import identifier_pb2
|
|
7
|
+
from flyteidl2.core import execution_pb2, interface_pb2
|
|
8
|
+
from flyteidl2.task import common_pb2, task_definition_pb2
|
|
9
|
+
from flyteidl2.workflow import (
|
|
12
10
|
run_definition_pb2,
|
|
13
11
|
state_service_pb2,
|
|
14
|
-
task_definition_pb2,
|
|
15
12
|
)
|
|
13
|
+
from google.protobuf import timestamp_pb2
|
|
14
|
+
|
|
16
15
|
from flyte.models import GroupData
|
|
17
16
|
|
|
18
17
|
ActionType = Literal["task", "trace"]
|
|
@@ -31,7 +30,7 @@ class Action:
|
|
|
31
30
|
friendly_name: str | None = None
|
|
32
31
|
group: GroupData | None = None
|
|
33
32
|
task: task_definition_pb2.TaskSpec | None = None
|
|
34
|
-
trace:
|
|
33
|
+
trace: run_definition_pb2.TraceAction | None = None
|
|
35
34
|
inputs_uri: str | None = None
|
|
36
35
|
run_output_base: str | None = None
|
|
37
36
|
realized_outputs_uri: str | None = None
|
|
@@ -198,12 +197,12 @@ class Action:
|
|
|
198
197
|
realized_outputs_uri=outputs_uri,
|
|
199
198
|
phase=run_definition_pb2.Phase.PHASE_SUCCEEDED,
|
|
200
199
|
run_output_base=run_output_base,
|
|
201
|
-
trace=
|
|
200
|
+
trace=run_definition_pb2.TraceAction(
|
|
202
201
|
name=friendly_name,
|
|
203
202
|
phase=run_definition_pb2.Phase.PHASE_SUCCEEDED,
|
|
204
203
|
start_time=st,
|
|
205
204
|
end_time=et,
|
|
206
|
-
outputs=
|
|
205
|
+
outputs=common_pb2.OutputReferences(
|
|
207
206
|
output_uri=outputs_uri,
|
|
208
207
|
report_uri=report_uri,
|
|
209
208
|
),
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
import grpc.aio
|
|
4
|
+
from flyteidl2.workflow import queue_service_pb2_grpc, state_service_pb2_grpc
|
|
4
5
|
|
|
5
|
-
from flyte._protos.workflow import queue_service_pb2_grpc, state_service_pb2_grpc
|
|
6
6
|
from flyte.remote import create_channel
|
|
7
7
|
|
|
8
8
|
from ._service_protocol import QueueService, StateService
|
|
@@ -9,6 +9,9 @@ from collections.abc import Callable
|
|
|
9
9
|
from pathlib import Path
|
|
10
10
|
from typing import Any, Awaitable, DefaultDict, Tuple, TypeVar
|
|
11
11
|
|
|
12
|
+
from flyteidl2.common import identifier_pb2
|
|
13
|
+
from flyteidl2.workflow import run_definition_pb2
|
|
14
|
+
|
|
12
15
|
import flyte
|
|
13
16
|
import flyte.errors
|
|
14
17
|
import flyte.storage as storage
|
|
@@ -22,8 +25,6 @@ from flyte._internal.runtime import convert, io
|
|
|
22
25
|
from flyte._internal.runtime.task_serde import translate_task_to_wire
|
|
23
26
|
from flyte._internal.runtime.types_serde import transform_native_to_typed_interface
|
|
24
27
|
from flyte._logging import logger
|
|
25
|
-
from flyte._protos.common import identifier_pb2
|
|
26
|
-
from flyte._protos.workflow import run_definition_pb2
|
|
27
28
|
from flyte._task import TaskTemplate
|
|
28
29
|
from flyte._utils.helpers import _selector_policy
|
|
29
30
|
from flyte.models import MAX_INLINE_IO_BYTES, ActionID, NativeInterface, SerializationContext
|
|
@@ -376,6 +377,7 @@ class RemoteController(Controller):
|
|
|
376
377
|
|
|
377
378
|
func_name = _func.__name__
|
|
378
379
|
invoke_seq_num = self.generate_task_call_sequence(_func, current_action_id)
|
|
380
|
+
|
|
379
381
|
inputs = await convert.convert_from_native_to_inputs(_interface, *args, **kwargs)
|
|
380
382
|
serialized_inputs = inputs.proto_inputs.SerializeToString(deterministic=True)
|
|
381
383
|
inputs_hash = convert.generate_inputs_hash_from_proto(inputs.proto_inputs)
|
|
@@ -9,15 +9,13 @@ from typing import Awaitable, Coroutine, Optional
|
|
|
9
9
|
|
|
10
10
|
import grpc.aio
|
|
11
11
|
from aiolimiter import AsyncLimiter
|
|
12
|
+
from flyteidl2.common import identifier_pb2
|
|
13
|
+
from flyteidl2.task import task_definition_pb2
|
|
14
|
+
from flyteidl2.workflow import queue_service_pb2, run_definition_pb2
|
|
12
15
|
from google.protobuf.wrappers_pb2 import StringValue
|
|
13
16
|
|
|
14
17
|
import flyte.errors
|
|
15
18
|
from flyte._logging import log, logger
|
|
16
|
-
from flyte._protos.common import identifier_pb2
|
|
17
|
-
from flyte._protos.workflow import (
|
|
18
|
-
queue_service_pb2,
|
|
19
|
-
task_definition_pb2,
|
|
20
|
-
)
|
|
21
19
|
|
|
22
20
|
from ._action import Action
|
|
23
21
|
from ._informer import InformerCache
|
|
@@ -303,11 +301,10 @@ class Controller:
|
|
|
303
301
|
async with self._rate_limiter:
|
|
304
302
|
logger.info(f"Cancelling action: {action.name}")
|
|
305
303
|
try:
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
# )
|
|
304
|
+
await self._queue_service.AbortQueuedAction(
|
|
305
|
+
queue_service_pb2.AbortQueuedActionRequest(action_id=action.action_id),
|
|
306
|
+
wait_for_ready=True,
|
|
307
|
+
)
|
|
311
308
|
logger.info(f"Successfully cancelled action: {action.name}")
|
|
312
309
|
except grpc.aio.AioRpcError as e:
|
|
313
310
|
if e.code() in [
|
|
@@ -330,8 +327,8 @@ class Controller:
|
|
|
330
327
|
"""
|
|
331
328
|
if not action.is_started():
|
|
332
329
|
async with self._rate_limiter:
|
|
333
|
-
task:
|
|
334
|
-
trace:
|
|
330
|
+
task: run_definition_pb2.TaskAction | None = None
|
|
331
|
+
trace: run_definition_pb2.TraceAction | None = None
|
|
335
332
|
if action.type == "task":
|
|
336
333
|
if action.task is None:
|
|
337
334
|
raise flyte.errors.RuntimeSystemError(
|
|
@@ -342,7 +339,7 @@ class Controller:
|
|
|
342
339
|
if action.cache_key:
|
|
343
340
|
cache_key = StringValue(value=action.cache_key)
|
|
344
341
|
|
|
345
|
-
task =
|
|
342
|
+
task = run_definition_pb2.TaskAction(
|
|
346
343
|
id=task_definition_pb2.TaskIdentifier(
|
|
347
344
|
version=action.task.task_template.id.version,
|
|
348
345
|
org=action.task.task_template.id.org,
|
|
@@ -5,10 +5,10 @@ from asyncio import Queue
|
|
|
5
5
|
from typing import AsyncIterator, Callable, Dict, Optional, Tuple, cast
|
|
6
6
|
|
|
7
7
|
import grpc.aio
|
|
8
|
+
from flyteidl2.common import identifier_pb2
|
|
9
|
+
from flyteidl2.workflow import run_definition_pb2, state_service_pb2
|
|
8
10
|
|
|
9
11
|
from flyte._logging import log, logger
|
|
10
|
-
from flyte._protos.common import identifier_pb2
|
|
11
|
-
from flyte._protos.workflow import run_definition_pb2, state_service_pb2
|
|
12
12
|
|
|
13
13
|
from ._action import Action
|
|
14
14
|
from ._service_protocol import StateService
|
|
@@ -373,7 +373,7 @@ class InformerCache:
|
|
|
373
373
|
"""Stop all informers and remove them from the cache"""
|
|
374
374
|
async with self._lock:
|
|
375
375
|
while self._cache:
|
|
376
|
-
|
|
376
|
+
_name, informer = self._cache.popitem()
|
|
377
377
|
try:
|
|
378
378
|
await informer.stop()
|
|
379
379
|
except asyncio.CancelledError:
|
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from typing import AsyncIterator, Protocol
|
|
4
4
|
|
|
5
|
-
from
|
|
5
|
+
from flyteidl2.workflow import queue_service_pb2, state_service_pb2
|
|
6
6
|
|
|
7
7
|
|
|
8
8
|
class StateService(Protocol):
|
|
@@ -28,12 +28,12 @@ class QueueService(Protocol):
|
|
|
28
28
|
) -> queue_service_pb2.EnqueueActionResponse:
|
|
29
29
|
"""Enqueue a task"""
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
31
|
+
async def AbortQueuedAction(
|
|
32
|
+
self,
|
|
33
|
+
req: queue_service_pb2.AbortQueuedActionRequest,
|
|
34
|
+
**kwargs,
|
|
35
|
+
) -> queue_service_pb2.AbortQueuedActionResponse:
|
|
36
|
+
"""Cancel an enqueued task"""
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
class ClientSet(Protocol):
|
|
@@ -6,7 +6,7 @@ import tempfile
|
|
|
6
6
|
import typing
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
from string import Template
|
|
9
|
-
from typing import ClassVar,
|
|
9
|
+
from typing import ClassVar, Optional, Protocol, cast
|
|
10
10
|
|
|
11
11
|
import aiofiles
|
|
12
12
|
import click
|
|
@@ -38,7 +38,7 @@ from flyte._internal.imagebuild.image_builder import (
|
|
|
38
38
|
LocalDockerCommandImageChecker,
|
|
39
39
|
LocalPodmanCommandImageChecker,
|
|
40
40
|
)
|
|
41
|
-
from flyte._internal.imagebuild.utils import copy_files_to_context
|
|
41
|
+
from flyte._internal.imagebuild.utils import copy_files_to_context, get_and_list_dockerignore
|
|
42
42
|
from flyte._logging import logger
|
|
43
43
|
|
|
44
44
|
_F_IMG_ID = "_F_IMG_ID"
|
|
@@ -263,24 +263,35 @@ class AptPackagesHandler:
|
|
|
263
263
|
|
|
264
264
|
class UVProjectHandler:
|
|
265
265
|
@staticmethod
|
|
266
|
-
async def handle(
|
|
266
|
+
async def handle(
|
|
267
|
+
layer: UVProject, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
|
|
268
|
+
) -> str:
|
|
267
269
|
secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
|
|
268
|
-
if layer.
|
|
270
|
+
if layer.project_install_mode == "dependencies_only":
|
|
271
|
+
pip_install_args = " ".join(layer.get_pip_install_args())
|
|
272
|
+
if "--no-install-project" not in pip_install_args:
|
|
273
|
+
pip_install_args += " --no-install-project"
|
|
269
274
|
# Only Copy pyproject.yaml and uv.lock.
|
|
270
275
|
pyproject_dst = copy_files_to_context(layer.pyproject, context_path)
|
|
271
276
|
uvlock_dst = copy_files_to_context(layer.uvlock, context_path)
|
|
272
277
|
delta = UV_LOCK_WITHOUT_PROJECT_INSTALL_TEMPLATE.substitute(
|
|
273
278
|
UV_LOCK_PATH=uvlock_dst.relative_to(context_path),
|
|
274
279
|
PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
|
|
275
|
-
PIP_INSTALL_ARGS=
|
|
280
|
+
PIP_INSTALL_ARGS=pip_install_args,
|
|
276
281
|
SECRET_MOUNT=secret_mounts,
|
|
277
282
|
)
|
|
278
283
|
else:
|
|
279
284
|
# Copy the entire project.
|
|
280
|
-
pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path)
|
|
281
|
-
|
|
282
|
-
|
|
285
|
+
pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path, docker_ignore_patterns)
|
|
286
|
+
|
|
287
|
+
# Make sure pyproject.toml and uv.lock files are not removed by docker ignore
|
|
288
|
+
uv_lock_context_path = pyproject_dst / "uv.lock"
|
|
289
|
+
pyproject_context_path = pyproject_dst / "pyproject.toml"
|
|
290
|
+
if not uv_lock_context_path.exists():
|
|
283
291
|
shutil.copy(layer.uvlock, pyproject_dst)
|
|
292
|
+
if not pyproject_context_path.exists():
|
|
293
|
+
shutil.copy(layer.pyproject, pyproject_dst)
|
|
294
|
+
|
|
284
295
|
delta = UV_LOCK_INSTALL_TEMPLATE.substitute(
|
|
285
296
|
PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
|
|
286
297
|
PIP_INSTALL_ARGS=" ".join(layer.get_pip_install_args()),
|
|
@@ -293,7 +304,9 @@ class UVProjectHandler:
|
|
|
293
304
|
|
|
294
305
|
class PoetryProjectHandler:
|
|
295
306
|
@staticmethod
|
|
296
|
-
async def handel(
|
|
307
|
+
async def handel(
|
|
308
|
+
layer: PoetryProject, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
|
|
309
|
+
) -> str:
|
|
297
310
|
secret_mounts = _get_secret_mounts_layer(layer.secret_mounts)
|
|
298
311
|
if layer.extra_args and "--no-root" in layer.extra_args:
|
|
299
312
|
# Only Copy pyproject.yaml and poetry.lock.
|
|
@@ -307,7 +320,16 @@ class PoetryProjectHandler:
|
|
|
307
320
|
)
|
|
308
321
|
else:
|
|
309
322
|
# Copy the entire project.
|
|
310
|
-
pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path)
|
|
323
|
+
pyproject_dst = copy_files_to_context(layer.pyproject.parent, context_path, docker_ignore_patterns)
|
|
324
|
+
|
|
325
|
+
# Make sure pyproject.toml and poetry.lock files are not removed by docker ignore
|
|
326
|
+
poetry_lock_context_path = pyproject_dst / "poetry.lock"
|
|
327
|
+
pyproject_context_path = pyproject_dst / "pyproject.toml"
|
|
328
|
+
if not poetry_lock_context_path.exists():
|
|
329
|
+
shutil.copy(layer.poetry_lock, pyproject_dst)
|
|
330
|
+
if not pyproject_context_path.exists():
|
|
331
|
+
shutil.copy(layer.pyproject, pyproject_dst)
|
|
332
|
+
|
|
311
333
|
delta = POETRY_LOCK_INSTALL_TEMPLATE.substitute(
|
|
312
334
|
PYPROJECT_PATH=pyproject_dst.relative_to(context_path),
|
|
313
335
|
POETRY_INSTALL_ARGS=layer.extra_args or "",
|
|
@@ -324,37 +346,9 @@ class DockerIgnoreHandler:
|
|
|
324
346
|
|
|
325
347
|
|
|
326
348
|
class CopyConfigHandler:
|
|
327
|
-
@staticmethod
|
|
328
|
-
def list_dockerignore(root_path: Optional[Path], docker_ignore_file_path: Optional[Path]) -> List[str]:
|
|
329
|
-
patterns: List[str] = []
|
|
330
|
-
dockerignore_path: Optional[Path] = None
|
|
331
|
-
if root_path:
|
|
332
|
-
dockerignore_path = root_path / ".dockerignore"
|
|
333
|
-
# DockerIgnore layer should be first priority
|
|
334
|
-
if docker_ignore_file_path:
|
|
335
|
-
dockerignore_path = docker_ignore_file_path
|
|
336
|
-
|
|
337
|
-
# Return empty list if no .dockerignore file found
|
|
338
|
-
if not dockerignore_path or not dockerignore_path.exists() or not dockerignore_path.is_file():
|
|
339
|
-
logger.info(f".dockerignore file not found at path: {dockerignore_path}")
|
|
340
|
-
return patterns
|
|
341
|
-
|
|
342
|
-
try:
|
|
343
|
-
with open(dockerignore_path, "r", encoding="utf-8") as f:
|
|
344
|
-
for line in f:
|
|
345
|
-
stripped_line = line.strip()
|
|
346
|
-
# Skip empty lines, whitespace-only lines, and comments
|
|
347
|
-
if not stripped_line or stripped_line.startswith("#"):
|
|
348
|
-
continue
|
|
349
|
-
patterns.append(stripped_line)
|
|
350
|
-
except Exception as e:
|
|
351
|
-
logger.error(f"Failed to read .dockerignore file at {dockerignore_path}: {e}")
|
|
352
|
-
return []
|
|
353
|
-
return patterns
|
|
354
|
-
|
|
355
349
|
@staticmethod
|
|
356
350
|
async def handle(
|
|
357
|
-
layer: CopyConfig, context_path: Path, dockerfile: str,
|
|
351
|
+
layer: CopyConfig, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
|
|
358
352
|
) -> str:
|
|
359
353
|
# Copy the source config file or directory to the context path
|
|
360
354
|
if layer.src.is_absolute() or ".." in str(layer.src):
|
|
@@ -370,11 +364,6 @@ class CopyConfigHandler:
|
|
|
370
364
|
shutil.copy(abs_path, dst_path)
|
|
371
365
|
elif layer.src.is_dir():
|
|
372
366
|
# Copy the entire directory
|
|
373
|
-
from flyte._initialize import _get_init_config
|
|
374
|
-
|
|
375
|
-
init_config = _get_init_config()
|
|
376
|
-
root_path = init_config.root_dir if init_config else None
|
|
377
|
-
docker_ignore_patterns = CopyConfigHandler.list_dockerignore(root_path, docker_ignore_file_path)
|
|
378
367
|
shutil.copytree(
|
|
379
368
|
abs_path, dst_path, dirs_exist_ok=True, ignore=shutil.ignore_patterns(*docker_ignore_patterns)
|
|
380
369
|
)
|
|
@@ -450,7 +439,7 @@ def _get_secret_mounts_layer(secrets: typing.Tuple[str | Secret, ...] | None) ->
|
|
|
450
439
|
|
|
451
440
|
|
|
452
441
|
async def _process_layer(
|
|
453
|
-
layer: Layer, context_path: Path, dockerfile: str,
|
|
442
|
+
layer: Layer, context_path: Path, dockerfile: str, docker_ignore_patterns: list[str] = []
|
|
454
443
|
) -> str:
|
|
455
444
|
match layer:
|
|
456
445
|
case PythonWheels():
|
|
@@ -483,15 +472,19 @@ async def _process_layer(
|
|
|
483
472
|
|
|
484
473
|
case UVProject():
|
|
485
474
|
# Handle UV project
|
|
486
|
-
dockerfile = await UVProjectHandler.handle(layer, context_path, dockerfile)
|
|
475
|
+
dockerfile = await UVProjectHandler.handle(layer, context_path, dockerfile, docker_ignore_patterns)
|
|
487
476
|
|
|
488
477
|
case PoetryProject():
|
|
489
478
|
# Handle Poetry project
|
|
490
|
-
dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile)
|
|
479
|
+
dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile, docker_ignore_patterns)
|
|
480
|
+
|
|
481
|
+
case PoetryProject():
|
|
482
|
+
# Handle Poetry project
|
|
483
|
+
dockerfile = await PoetryProjectHandler.handel(layer, context_path, dockerfile, docker_ignore_patterns)
|
|
491
484
|
|
|
492
485
|
case CopyConfig():
|
|
493
486
|
# Handle local files and folders
|
|
494
|
-
dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile,
|
|
487
|
+
dockerfile = await CopyConfigHandler.handle(layer, context_path, dockerfile, docker_ignore_patterns)
|
|
495
488
|
|
|
496
489
|
case Commands():
|
|
497
490
|
# Handle commands
|
|
@@ -618,15 +611,6 @@ class DockerImageBuilder(ImageBuilder):
|
|
|
618
611
|
else:
|
|
619
612
|
logger.info("Buildx builder already exists.")
|
|
620
613
|
|
|
621
|
-
def get_docker_ignore(self, image: Image) -> Optional[Path]:
|
|
622
|
-
"""Get the .dockerignore file path from the image layers."""
|
|
623
|
-
# Look for DockerIgnore layer in the image layers
|
|
624
|
-
for layer in image._layers:
|
|
625
|
-
if isinstance(layer, DockerIgnore) and layer.path.strip():
|
|
626
|
-
return Path(layer.path)
|
|
627
|
-
|
|
628
|
-
return None
|
|
629
|
-
|
|
630
614
|
async def _build_image(self, image: Image, *, push: bool = True, dry_run: bool = False) -> str:
|
|
631
615
|
"""
|
|
632
616
|
if default image (only base image and locked), raise an error, don't have a dockerfile
|
|
@@ -657,9 +641,11 @@ class DockerImageBuilder(ImageBuilder):
|
|
|
657
641
|
PYTHON_VERSION=f"{image.python_version[0]}.{image.python_version[1]}",
|
|
658
642
|
)
|
|
659
643
|
|
|
660
|
-
|
|
644
|
+
# Get .dockerignore file patterns first
|
|
645
|
+
docker_ignore_patterns = get_and_list_dockerignore(image)
|
|
646
|
+
|
|
661
647
|
for layer in image._layers:
|
|
662
|
-
dockerfile = await _process_layer(layer, tmp_path, dockerfile,
|
|
648
|
+
dockerfile = await _process_layer(layer, tmp_path, dockerfile, docker_ignore_patterns)
|
|
663
649
|
|
|
664
650
|
dockerfile += DOCKER_FILE_BASE_FOOTER.substitute(F_IMG_ID=image.uri)
|
|
665
651
|
|