flyte 2.0.0b2__py3-none-any.whl → 2.0.0b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/_internal/controllers/__init__.py +3 -1
- flyte/_internal/controllers/_local_controller.py +3 -1
- flyte/_internal/controllers/remote/_controller.py +26 -13
- flyte/_internal/runtime/io.py +48 -9
- flyte/_task.py +8 -2
- flyte/_task_environment.py +5 -1
- flyte/_version.py +2 -2
- flyte/errors.py +10 -0
- flyte/extras/_container.py +3 -1
- flyte/models.py +3 -0
- flyte/remote/_task.py +3 -1
- {flyte-2.0.0b2.dist-info → flyte-2.0.0b3.dist-info}/METADATA +1 -1
- {flyte-2.0.0b2.dist-info → flyte-2.0.0b3.dist-info}/RECORD +18 -18
- {flyte-2.0.0b2.data → flyte-2.0.0b3.data}/scripts/runtime.py +0 -0
- {flyte-2.0.0b2.dist-info → flyte-2.0.0b3.dist-info}/WHEEL +0 -0
- {flyte-2.0.0b2.dist-info → flyte-2.0.0b3.dist-info}/entry_points.txt +0 -0
- {flyte-2.0.0b2.dist-info → flyte-2.0.0b3.dist-info}/licenses/LICENSE +0 -0
- {flyte-2.0.0b2.dist-info → flyte-2.0.0b3.dist-info}/top_level.txt +0 -0
|
@@ -41,7 +41,9 @@ class Controller(Protocol):
|
|
|
41
41
|
"""
|
|
42
42
|
...
|
|
43
43
|
|
|
44
|
-
async def submit_task_ref(
|
|
44
|
+
async def submit_task_ref(
|
|
45
|
+
self, _task: task_definition_pb2.TaskDetails, max_inline_io_bytes: int, *args, **kwargs
|
|
46
|
+
) -> Any:
|
|
45
47
|
"""
|
|
46
48
|
Submit a task reference to the controller asynchronously and wait for the result. This is async and will block
|
|
47
49
|
the current coroutine until the result is available.
|
|
@@ -192,5 +192,7 @@ class LocalController:
|
|
|
192
192
|
assert info.start_time
|
|
193
193
|
assert info.end_time
|
|
194
194
|
|
|
195
|
-
async def submit_task_ref(
|
|
195
|
+
async def submit_task_ref(
|
|
196
|
+
self, _task: task_definition_pb2.TaskDetails, max_inline_io_bytes: int, *args, **kwargs
|
|
197
|
+
) -> Any:
|
|
196
198
|
raise flyte.errors.ReferenceTaskError("Reference tasks cannot be executed locally, only remotely.")
|
|
@@ -7,7 +7,7 @@ import threading
|
|
|
7
7
|
from collections import defaultdict
|
|
8
8
|
from collections.abc import Callable
|
|
9
9
|
from pathlib import Path
|
|
10
|
-
from typing import Any,
|
|
10
|
+
from typing import Any, Awaitable, DefaultDict, Tuple, TypeVar
|
|
11
11
|
|
|
12
12
|
import flyte
|
|
13
13
|
import flyte.errors
|
|
@@ -27,22 +27,30 @@ from flyte._protos.common import identifier_pb2
|
|
|
27
27
|
from flyte._protos.workflow import run_definition_pb2, task_definition_pb2
|
|
28
28
|
from flyte._task import TaskTemplate
|
|
29
29
|
from flyte._utils.helpers import _selector_policy
|
|
30
|
-
from flyte.models import ActionID, NativeInterface, SerializationContext
|
|
30
|
+
from flyte.models import MAX_INLINE_IO_BYTES, ActionID, NativeInterface, SerializationContext
|
|
31
31
|
|
|
32
32
|
R = TypeVar("R")
|
|
33
33
|
|
|
34
|
+
MAX_TRACE_BYTES = MAX_INLINE_IO_BYTES
|
|
34
35
|
|
|
35
|
-
|
|
36
|
+
|
|
37
|
+
async def upload_inputs_with_retry(serialized_inputs: bytes, inputs_uri: str, max_bytes: int) -> None:
|
|
36
38
|
"""
|
|
37
39
|
Upload inputs to the specified URI with error handling.
|
|
38
40
|
|
|
39
41
|
Args:
|
|
40
42
|
serialized_inputs: The serialized inputs to upload
|
|
41
43
|
inputs_uri: The destination URI
|
|
44
|
+
max_bytes: Maximum number of bytes to read from the input stream
|
|
42
45
|
|
|
43
46
|
Raises:
|
|
44
47
|
RuntimeSystemError: If the upload fails
|
|
45
48
|
"""
|
|
49
|
+
if len(serialized_inputs) > max_bytes:
|
|
50
|
+
raise flyte.errors.InlineIOMaxBytesBreached(
|
|
51
|
+
f"Inputs exceed max_bytes limit of {max_bytes / 1024 / 1024} MB,"
|
|
52
|
+
f" actual size: {len(serialized_inputs) / 1024 / 1024} MB"
|
|
53
|
+
)
|
|
46
54
|
try:
|
|
47
55
|
# TODO Add retry decorator to this
|
|
48
56
|
await storage.put_stream(serialized_inputs, to_path=inputs_uri)
|
|
@@ -80,19 +88,20 @@ async def handle_action_failure(action: Action, task_name: str) -> Exception:
|
|
|
80
88
|
return exc
|
|
81
89
|
|
|
82
90
|
|
|
83
|
-
async def load_and_convert_outputs(iface: NativeInterface, realized_outputs_uri: str) -> Any:
|
|
91
|
+
async def load_and_convert_outputs(iface: NativeInterface, realized_outputs_uri: str, max_bytes: int) -> Any:
|
|
84
92
|
"""
|
|
85
93
|
Load outputs from the given URI and convert them to native format.
|
|
86
94
|
|
|
87
95
|
Args:
|
|
88
96
|
iface: The Native interface
|
|
89
97
|
realized_outputs_uri: The URI where outputs are stored
|
|
98
|
+
max_bytes: Maximum number of bytes to read from the output file
|
|
90
99
|
|
|
91
100
|
Returns:
|
|
92
101
|
The converted native outputs
|
|
93
102
|
"""
|
|
94
103
|
outputs_file_path = io.outputs_path(realized_outputs_uri)
|
|
95
|
-
outputs = await io.load_outputs(outputs_file_path)
|
|
104
|
+
outputs = await io.load_outputs(outputs_file_path, max_bytes=max_bytes)
|
|
96
105
|
return await convert.convert_outputs_to_native(iface, outputs)
|
|
97
106
|
|
|
98
107
|
|
|
@@ -191,7 +200,7 @@ class RemoteController(Controller):
|
|
|
191
200
|
|
|
192
201
|
serialized_inputs = inputs.proto_inputs.SerializeToString(deterministic=True)
|
|
193
202
|
inputs_uri = io.inputs_path(sub_action_output_path)
|
|
194
|
-
await upload_inputs_with_retry(serialized_inputs, inputs_uri)
|
|
203
|
+
await upload_inputs_with_retry(serialized_inputs, inputs_uri, max_bytes=_task.max_inline_io_bytes)
|
|
195
204
|
|
|
196
205
|
md = task_spec.task_template.metadata
|
|
197
206
|
ignored_input_vars = []
|
|
@@ -254,7 +263,9 @@ class RemoteController(Controller):
|
|
|
254
263
|
"RuntimeError",
|
|
255
264
|
f"Task {n.action_id.name} did not return an output path, but the task has outputs defined.",
|
|
256
265
|
)
|
|
257
|
-
return await load_and_convert_outputs(
|
|
266
|
+
return await load_and_convert_outputs(
|
|
267
|
+
_task.native_interface, n.realized_outputs_uri, max_bytes=_task.max_inline_io_bytes
|
|
268
|
+
)
|
|
258
269
|
return None
|
|
259
270
|
|
|
260
271
|
async def submit(self, _task: TaskTemplate, *args, **kwargs) -> Any:
|
|
@@ -357,7 +368,7 @@ class RemoteController(Controller):
|
|
|
357
368
|
)
|
|
358
369
|
|
|
359
370
|
inputs_uri = io.inputs_path(sub_action_output_path)
|
|
360
|
-
await upload_inputs_with_retry(serialized_inputs, inputs_uri)
|
|
371
|
+
await upload_inputs_with_retry(serialized_inputs, inputs_uri, max_bytes=MAX_TRACE_BYTES)
|
|
361
372
|
# Clear to free memory
|
|
362
373
|
serialized_inputs = None # type: ignore
|
|
363
374
|
|
|
@@ -388,7 +399,7 @@ class RemoteController(Controller):
|
|
|
388
399
|
logger.warning(f"Action {prev_action.action_id.name} failed, but no error was found, re-running trace!")
|
|
389
400
|
elif prev_action.realized_outputs_uri is not None:
|
|
390
401
|
outputs_file_path = io.outputs_path(prev_action.realized_outputs_uri)
|
|
391
|
-
o = await io.load_outputs(outputs_file_path)
|
|
402
|
+
o = await io.load_outputs(outputs_file_path, max_bytes=MAX_TRACE_BYTES)
|
|
392
403
|
outputs = await convert.convert_outputs_to_native(_interface, o)
|
|
393
404
|
return (
|
|
394
405
|
TraceInfo(func_name, sub_action_id, _interface, inputs_uri, output=outputs),
|
|
@@ -421,7 +432,7 @@ class RemoteController(Controller):
|
|
|
421
432
|
f"Uploading outputs for {info.name} Outputs file path: {outputs_file_path}",
|
|
422
433
|
flush=True,
|
|
423
434
|
)
|
|
424
|
-
await io.upload_outputs(outputs, sub_run_output_path)
|
|
435
|
+
await io.upload_outputs(outputs, sub_run_output_path, max_bytes=MAX_TRACE_BYTES)
|
|
425
436
|
elif info.error:
|
|
426
437
|
err = convert.convert_from_native_to_error(info.error)
|
|
427
438
|
await io.upload_error(err.err, sub_run_output_path)
|
|
@@ -461,7 +472,9 @@ class RemoteController(Controller):
|
|
|
461
472
|
# If the action is cancelled, we need to cancel the action on the server as well
|
|
462
473
|
raise
|
|
463
474
|
|
|
464
|
-
async def submit_task_ref(
|
|
475
|
+
async def submit_task_ref(
|
|
476
|
+
self, _task: task_definition_pb2.TaskDetails, max_inline_io_bytes: int, *args, **kwargs
|
|
477
|
+
) -> Any:
|
|
465
478
|
ctx = internal_ctx()
|
|
466
479
|
tctx = ctx.data.task_context
|
|
467
480
|
if tctx is None:
|
|
@@ -482,7 +495,7 @@ class RemoteController(Controller):
|
|
|
482
495
|
|
|
483
496
|
serialized_inputs = inputs.proto_inputs.SerializeToString(deterministic=True)
|
|
484
497
|
inputs_uri = io.inputs_path(sub_action_output_path)
|
|
485
|
-
await upload_inputs_with_retry(serialized_inputs, inputs_uri)
|
|
498
|
+
await upload_inputs_with_retry(serialized_inputs, inputs_uri, max_inline_io_bytes)
|
|
486
499
|
# cache key - task name, task signature, inputs, cache version
|
|
487
500
|
cache_key = None
|
|
488
501
|
md = _task.spec.task_template.metadata
|
|
@@ -545,5 +558,5 @@ class RemoteController(Controller):
|
|
|
545
558
|
"RuntimeError",
|
|
546
559
|
f"Task {n.action_id.name} did not return an output path, but the task has outputs defined.",
|
|
547
560
|
)
|
|
548
|
-
return await load_and_convert_outputs(native_interface, n.realized_outputs_uri)
|
|
561
|
+
return await load_and_convert_outputs(native_interface, n.realized_outputs_uri, max_inline_io_bytes)
|
|
549
562
|
return None
|
flyte/_internal/runtime/io.py
CHANGED
|
@@ -5,14 +5,11 @@ It uses the storage module to handle the actual uploading and downloading of fil
|
|
|
5
5
|
TODO: Convert to use streaming apis
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
-
import logging
|
|
9
|
-
|
|
10
8
|
from flyteidl.core import errors_pb2, execution_pb2
|
|
11
9
|
|
|
12
10
|
import flyte.storage as storage
|
|
13
11
|
from flyte._protos.workflow import run_definition_pb2
|
|
14
12
|
|
|
15
|
-
from ..._logging import log
|
|
16
13
|
from .convert import Inputs, Outputs, _clean_error_code
|
|
17
14
|
|
|
18
15
|
# ------------------------------- CONSTANTS ------------------------------- #
|
|
@@ -55,11 +52,19 @@ async def upload_inputs(inputs: Inputs, input_path: str):
|
|
|
55
52
|
await storage.put_stream(data_iterable=inputs.proto_inputs.SerializeToString(), to_path=input_path)
|
|
56
53
|
|
|
57
54
|
|
|
58
|
-
async def upload_outputs(outputs: Outputs, output_path: str):
|
|
55
|
+
async def upload_outputs(outputs: Outputs, output_path: str, max_bytes: int = -1):
|
|
59
56
|
"""
|
|
60
57
|
:param outputs: Outputs
|
|
61
58
|
:param output_path: The path to upload the output file.
|
|
59
|
+
:param max_bytes: Maximum number of bytes to write to the output file. Default is -1, which means no limit.
|
|
62
60
|
"""
|
|
61
|
+
if max_bytes != -1 and outputs.proto_outputs.ByteSize() > max_bytes:
|
|
62
|
+
import flyte.errors
|
|
63
|
+
|
|
64
|
+
raise flyte.errors.InlineIOMaxBytesBreached(
|
|
65
|
+
f"Output file at {output_path} exceeds max_bytes limit of {max_bytes},"
|
|
66
|
+
f" size: {outputs.proto_outputs.ByteSize()}"
|
|
67
|
+
)
|
|
63
68
|
output_uri = outputs_path(output_path)
|
|
64
69
|
await storage.put_stream(data_iterable=outputs.proto_outputs.SerializeToString(), to_path=output_uri)
|
|
65
70
|
|
|
@@ -85,25 +90,59 @@ async def upload_error(err: execution_pb2.ExecutionError, output_prefix: str):
|
|
|
85
90
|
|
|
86
91
|
|
|
87
92
|
# ------------------------------- DOWNLOAD Methods ------------------------------- #
|
|
88
|
-
|
|
89
|
-
async def load_inputs(path: str) -> Inputs:
|
|
93
|
+
async def load_inputs(path: str, max_bytes: int = -1) -> Inputs:
|
|
90
94
|
"""
|
|
91
95
|
:param path: Input file to be downloaded
|
|
96
|
+
:param max_bytes: Maximum number of bytes to read from the input file. Default is -1, which means no limit.
|
|
92
97
|
:return: Inputs object
|
|
93
98
|
"""
|
|
94
99
|
lm = run_definition_pb2.Inputs()
|
|
95
|
-
|
|
100
|
+
|
|
101
|
+
if max_bytes == -1:
|
|
102
|
+
proto_str = b"".join([c async for c in storage.get_stream(path=path)])
|
|
103
|
+
else:
|
|
104
|
+
proto_bytes = []
|
|
105
|
+
total_bytes = 0
|
|
106
|
+
async for chunk in storage.get_stream(path=path):
|
|
107
|
+
if total_bytes + len(chunk) > max_bytes:
|
|
108
|
+
import flyte.errors
|
|
109
|
+
|
|
110
|
+
raise flyte.errors.InlineIOMaxBytesBreached(
|
|
111
|
+
f"Input file at {path} exceeds max_bytes limit of {max_bytes}"
|
|
112
|
+
)
|
|
113
|
+
proto_bytes.append(chunk)
|
|
114
|
+
total_bytes += len(chunk)
|
|
115
|
+
proto_str = b"".join(proto_bytes)
|
|
116
|
+
|
|
96
117
|
lm.ParseFromString(proto_str)
|
|
97
118
|
return Inputs(proto_inputs=lm)
|
|
98
119
|
|
|
99
120
|
|
|
100
|
-
async def load_outputs(path: str) -> Outputs:
|
|
121
|
+
async def load_outputs(path: str, max_bytes: int = -1) -> Outputs:
|
|
101
122
|
"""
|
|
102
123
|
:param path: output file to be loaded
|
|
124
|
+
:param max_bytes: Maximum number of bytes to read from the output file.
|
|
125
|
+
If -1, reads the entire file.
|
|
103
126
|
:return: Outputs object
|
|
104
127
|
"""
|
|
105
128
|
lm = run_definition_pb2.Outputs()
|
|
106
|
-
|
|
129
|
+
|
|
130
|
+
if max_bytes == -1:
|
|
131
|
+
proto_str = b"".join([c async for c in storage.get_stream(path=path)])
|
|
132
|
+
else:
|
|
133
|
+
proto_bytes = []
|
|
134
|
+
total_bytes = 0
|
|
135
|
+
async for chunk in storage.get_stream(path=path):
|
|
136
|
+
if total_bytes + len(chunk) > max_bytes:
|
|
137
|
+
import flyte.errors
|
|
138
|
+
|
|
139
|
+
raise flyte.errors.InlineIOMaxBytesBreached(
|
|
140
|
+
f"Output file at {path} exceeds max_bytes limit of {max_bytes}"
|
|
141
|
+
)
|
|
142
|
+
proto_bytes.append(chunk)
|
|
143
|
+
total_bytes += len(chunk)
|
|
144
|
+
proto_str = b"".join(proto_bytes)
|
|
145
|
+
|
|
107
146
|
lm.ParseFromString(proto_str)
|
|
108
147
|
return Outputs(proto_outputs=lm)
|
|
109
148
|
|
flyte/_task.py
CHANGED
|
@@ -33,7 +33,7 @@ from ._retry import RetryStrategy
|
|
|
33
33
|
from ._reusable_environment import ReusePolicy
|
|
34
34
|
from ._secret import SecretRequest
|
|
35
35
|
from ._timeout import TimeoutType
|
|
36
|
-
from .models import NativeInterface, SerializationContext
|
|
36
|
+
from .models import MAX_INLINE_IO_BYTES, NativeInterface, SerializationContext
|
|
37
37
|
|
|
38
38
|
if TYPE_CHECKING:
|
|
39
39
|
from flyteidl.core.tasks_pb2 import DataLoadingConfig
|
|
@@ -79,6 +79,8 @@ class TaskTemplate(Generic[P, R]):
|
|
|
79
79
|
:param env: Optional The environment variables to set for the task.
|
|
80
80
|
:param secrets: Optional The secrets that will be injected into the task at runtime.
|
|
81
81
|
:param timeout: Optional The timeout for the task.
|
|
82
|
+
:param max_inline_io_bytes: Maximum allowed size (in bytes) for all inputs and outputs passed directly to the task
|
|
83
|
+
(e.g., primitives, strings, dicts). Does not apply to files, directories, or dataframes.
|
|
82
84
|
"""
|
|
83
85
|
|
|
84
86
|
name: str
|
|
@@ -100,8 +102,8 @@ class TaskTemplate(Generic[P, R]):
|
|
|
100
102
|
report: bool = False
|
|
101
103
|
|
|
102
104
|
parent_env: Optional[weakref.ReferenceType[TaskEnvironment]] = None
|
|
103
|
-
local: bool = field(default=False, init=False)
|
|
104
105
|
ref: bool = field(default=False, init=False, repr=False, compare=False)
|
|
106
|
+
max_inline_io_bytes: int = MAX_INLINE_IO_BYTES
|
|
105
107
|
|
|
106
108
|
# Only used in python 3.10 and 3.11, where we cannot use markcoroutinefunction
|
|
107
109
|
_call_as_synchronous: bool = False
|
|
@@ -315,6 +317,7 @@ class TaskTemplate(Generic[P, R]):
|
|
|
315
317
|
reusable: Union[ReusePolicy, Literal["off"], None] = None,
|
|
316
318
|
env: Optional[Dict[str, str]] = None,
|
|
317
319
|
secrets: Optional[SecretRequest] = None,
|
|
320
|
+
max_inline_io_bytes: int | None = None,
|
|
318
321
|
**kwargs: Any,
|
|
319
322
|
) -> TaskTemplate:
|
|
320
323
|
"""
|
|
@@ -324,6 +327,8 @@ class TaskTemplate(Generic[P, R]):
|
|
|
324
327
|
cache = cache or self.cache
|
|
325
328
|
retries = retries or self.retries
|
|
326
329
|
timeout = timeout or self.timeout
|
|
330
|
+
max_inline_io_bytes = max_inline_io_bytes or self.max_inline_io_bytes
|
|
331
|
+
|
|
327
332
|
reusable = reusable or self.reusable
|
|
328
333
|
if reusable == "off":
|
|
329
334
|
reusable = None
|
|
@@ -371,6 +376,7 @@ class TaskTemplate(Generic[P, R]):
|
|
|
371
376
|
reusable=cast(Optional[ReusePolicy], reusable),
|
|
372
377
|
env=env,
|
|
373
378
|
secrets=secrets,
|
|
379
|
+
max_inline_io_bytes=max_inline_io_bytes,
|
|
374
380
|
)
|
|
375
381
|
|
|
376
382
|
|
flyte/_task_environment.py
CHANGED
|
@@ -27,7 +27,7 @@ from ._retry import RetryStrategy
|
|
|
27
27
|
from ._reusable_environment import ReusePolicy
|
|
28
28
|
from ._secret import SecretRequest
|
|
29
29
|
from ._task import AsyncFunctionTaskTemplate, TaskTemplate
|
|
30
|
-
from .models import NativeInterface
|
|
30
|
+
from .models import MAX_INLINE_IO_BYTES, NativeInterface
|
|
31
31
|
|
|
32
32
|
if TYPE_CHECKING:
|
|
33
33
|
from kubernetes.client import V1PodTemplate
|
|
@@ -141,6 +141,7 @@ class TaskEnvironment(Environment):
|
|
|
141
141
|
secrets: Optional[SecretRequest] = None,
|
|
142
142
|
pod_template: Optional[Union[str, "V1PodTemplate"]] = None,
|
|
143
143
|
report: bool = False,
|
|
144
|
+
max_inline_io_bytes: int = MAX_INLINE_IO_BYTES,
|
|
144
145
|
) -> Union[AsyncFunctionTaskTemplate, Callable[P, R]]:
|
|
145
146
|
"""
|
|
146
147
|
Decorate a function to be a task.
|
|
@@ -157,6 +158,8 @@ class TaskEnvironment(Environment):
|
|
|
157
158
|
:param pod_template: Optional The pod template for the task, if not provided the default pod template will be
|
|
158
159
|
used.
|
|
159
160
|
:param report: Optional Whether to generate the html report for the task, defaults to False.
|
|
161
|
+
:param max_inline_io_bytes: Maximum allowed size (in bytes) for all inputs and outputs passed directly to the
|
|
162
|
+
task (e.g., primitives, strings, dicts). Does not apply to files, directories, or dataframes.
|
|
160
163
|
"""
|
|
161
164
|
from ._task import P, R
|
|
162
165
|
|
|
@@ -208,6 +211,7 @@ class TaskEnvironment(Environment):
|
|
|
208
211
|
report=report,
|
|
209
212
|
friendly_name=friendly_name,
|
|
210
213
|
plugin_config=self.plugin_config,
|
|
214
|
+
max_inline_io_bytes=max_inline_io_bytes,
|
|
211
215
|
)
|
|
212
216
|
self._tasks[task_name] = tmpl
|
|
213
217
|
return tmpl
|
flyte/_version.py
CHANGED
|
@@ -17,5 +17,5 @@ __version__: str
|
|
|
17
17
|
__version_tuple__: VERSION_TUPLE
|
|
18
18
|
version_tuple: VERSION_TUPLE
|
|
19
19
|
|
|
20
|
-
__version__ = version = '2.0.
|
|
21
|
-
__version_tuple__ = version_tuple = (2, 0, 0, '
|
|
20
|
+
__version__ = version = '2.0.0b3'
|
|
21
|
+
__version_tuple__ = version_tuple = (2, 0, 0, 'b3')
|
flyte/errors.py
CHANGED
|
@@ -179,3 +179,13 @@ class ImageBuildError(RuntimeUserError):
|
|
|
179
179
|
|
|
180
180
|
def __init__(self, message: str):
|
|
181
181
|
super().__init__("ImageBuildError", message, "user")
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class InlineIOMaxBytesBreached(RuntimeUserError):
|
|
185
|
+
"""
|
|
186
|
+
This error is raised when the inline IO max bytes limit is breached.
|
|
187
|
+
This can be adjusted per task by setting max_inline_io_bytes in the task definition.
|
|
188
|
+
"""
|
|
189
|
+
|
|
190
|
+
def __init__(self, message: str):
|
|
191
|
+
super().__init__("InlineIOMaxBytesBreached", message, "user")
|
flyte/extras/_container.py
CHANGED
|
@@ -192,7 +192,9 @@ class ContainerTask(TaskTemplate):
|
|
|
192
192
|
microseconds=microseconds,
|
|
193
193
|
)
|
|
194
194
|
|
|
195
|
-
async def _convert_output_val_to_correct_type(
|
|
195
|
+
async def _convert_output_val_to_correct_type(
|
|
196
|
+
self, output_path: pathlib.Path, output_val: Any, output_type: Type
|
|
197
|
+
) -> Any:
|
|
196
198
|
import datetime
|
|
197
199
|
|
|
198
200
|
if issubclass(output_type, bool):
|
flyte/models.py
CHANGED
flyte/remote/_task.py
CHANGED
|
@@ -81,6 +81,7 @@ AutoVersioning = Literal["latest", "current"]
|
|
|
81
81
|
@dataclass
|
|
82
82
|
class TaskDetails:
|
|
83
83
|
pb2: task_definition_pb2.TaskDetails
|
|
84
|
+
max_inline_io_bytes: int = 10 * 1024 * 1024 # 10 MB
|
|
84
85
|
|
|
85
86
|
@classmethod
|
|
86
87
|
def get(
|
|
@@ -252,7 +253,7 @@ class TaskDetails:
|
|
|
252
253
|
|
|
253
254
|
controller = get_controller()
|
|
254
255
|
if controller:
|
|
255
|
-
return await controller.submit_task_ref(self.pb2, *args, **kwargs)
|
|
256
|
+
return await controller.submit_task_ref(self.pb2, self.max_inline_io_bytes, *args, **kwargs)
|
|
256
257
|
raise flyte.errors
|
|
257
258
|
|
|
258
259
|
def override(
|
|
@@ -267,6 +268,7 @@ class TaskDetails:
|
|
|
267
268
|
reusable: Union[flyte.ReusePolicy, Literal["auto"], None] = None,
|
|
268
269
|
env: Optional[Dict[str, str]] = None,
|
|
269
270
|
secrets: Optional[flyte.SecretRequest] = None,
|
|
271
|
+
max_inline_io_bytes: int | None = None,
|
|
270
272
|
**kwargs: Any,
|
|
271
273
|
) -> TaskDetails:
|
|
272
274
|
raise NotImplementedError
|
|
@@ -19,16 +19,16 @@ flyte/_retry.py,sha256=rfLv0MvWxzPByKESTglEmjPsytEAKiIvvmzlJxXwsfE,941
|
|
|
19
19
|
flyte/_reusable_environment.py,sha256=f8Y1GilUwGcXH4n2Fckrnx0SrZmhk3nCfoe-TqUKivI,3740
|
|
20
20
|
flyte/_run.py,sha256=HkTD3rHL34pAwvn1WPN6OXYmk-GWX0txLdRH1OIMvEA,24338
|
|
21
21
|
flyte/_secret.py,sha256=ogXmCNfYYIphV6p-2iiWmwr2cNUES5Cq01PPjY6uQNA,3217
|
|
22
|
-
flyte/_task.py,sha256=
|
|
23
|
-
flyte/_task_environment.py,sha256=
|
|
22
|
+
flyte/_task.py,sha256=rYR7SVGihfBp7UYrhdmqhHp0ngl2K7Za8IJfhv1K2oc,19402
|
|
23
|
+
flyte/_task_environment.py,sha256=Zpfr8gjwXg5KuCfIbT4s3l2mtJCFqDxXwv6ZStHWBuc,9840
|
|
24
24
|
flyte/_task_plugins.py,sha256=9MH3nFPOH_e8_92BT4sFk4oyAnj6GJFvaPYWaraX7yE,1037
|
|
25
25
|
flyte/_timeout.py,sha256=zx5sFcbYmjJAJbZWSGzzX-BpC9HC7Jfs35T7vVhKwkk,1571
|
|
26
26
|
flyte/_tools.py,sha256=tWb0sx3t3mm4jbaQVjCTc9y39oR_Ibo3z_KHToP3Lto,966
|
|
27
27
|
flyte/_trace.py,sha256=SSE1nzUgmVTS2xFNtchEOjEjlRavMOIInasXzY8i9lU,4911
|
|
28
|
-
flyte/_version.py,sha256=
|
|
29
|
-
flyte/errors.py,sha256=
|
|
28
|
+
flyte/_version.py,sha256=fn05QckBHXt6TeCLv-zTOz1tLPEdQLSISwMKmzhR-ks,519
|
|
29
|
+
flyte/errors.py,sha256=MsVZxq6oeIOXe6AVWpxqA3ZiyxCGM0A6UqnYuvmSUCU,5514
|
|
30
30
|
flyte/extend.py,sha256=GB4ZedGzKa30vYWRVPOdxEeK62xnUVFY4z2tD6H9eEw,376
|
|
31
|
-
flyte/models.py,sha256=
|
|
31
|
+
flyte/models.py,sha256=AJZXL8eRmZ2RHEvjL6VSdpTmgF5S1Ekh_tAKRn0b6mM,15321
|
|
32
32
|
flyte/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
33
|
flyte/_bin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
34
34
|
flyte/_bin/runtime.py,sha256=2jTy3ccvrJ__Xrfdo2t0Fxhsojc5o2zIxDHt98RE_eU,6475
|
|
@@ -42,13 +42,13 @@ flyte/_code_bundle/_packaging.py,sha256=5QUuea6kg9s-ebBg7gFAHaxOMchxR5MhTQ8KohWs
|
|
|
42
42
|
flyte/_code_bundle/_utils.py,sha256=qlAVmik9rLasfd1oNrCxhL870w5ntk5ZlNGeaKSKaAU,12028
|
|
43
43
|
flyte/_code_bundle/bundle.py,sha256=nUAwYTVAE3Z9dfgkBtsqCoKJImjSl4AicG36yweWHLc,8797
|
|
44
44
|
flyte/_internal/__init__.py,sha256=vjXgGzAAjy609YFkAy9_RVPuUlslsHSJBXCLNTVnqOY,136
|
|
45
|
-
flyte/_internal/controllers/__init__.py,sha256=
|
|
46
|
-
flyte/_internal/controllers/_local_controller.py,sha256=
|
|
45
|
+
flyte/_internal/controllers/__init__.py,sha256=TVAc4ydsldcIFmN3PW9-IX5UkKeD8oOmuIukIgEae9M,4341
|
|
46
|
+
flyte/_internal/controllers/_local_controller.py,sha256=__-eEira0k18DsBu1LBXeEjhFGFcp1Uai9K0YEBbwKM,7300
|
|
47
47
|
flyte/_internal/controllers/_trace.py,sha256=ywFg_M2nGrCKYLbh4iVdsVlRtPT1K2S-XZMvDJU8AKU,1499
|
|
48
48
|
flyte/_internal/controllers/remote/__init__.py,sha256=9_azH1eHLqY6VULpDugXi7Kf1kK1ODqEnsQ_3wM6IqU,1919
|
|
49
49
|
flyte/_internal/controllers/remote/_action.py,sha256=ENV1thRXllSpi2s4idL-vCVcmXQNS17hmP2lMDKzNdo,7397
|
|
50
50
|
flyte/_internal/controllers/remote/_client.py,sha256=HPbzbfaWZVv5wpOvKNtFXR6COiZDwd1cUJQqi60A7oU,1421
|
|
51
|
-
flyte/_internal/controllers/remote/_controller.py,sha256=
|
|
51
|
+
flyte/_internal/controllers/remote/_controller.py,sha256=80e7Sbvg0SKFvF5wln_8RlSkwaYmT1QUFYUao9BwvUA,24508
|
|
52
52
|
flyte/_internal/controllers/remote/_core.py,sha256=49l1X3YFyX-QYeSlvkOWdyEU2xc1Fr7I8qHvC3nQ6MA,19069
|
|
53
53
|
flyte/_internal/controllers/remote/_informer.py,sha256=w4p29_dzS_ns762eNBljvnbJLgCm36d1Ogo2ZkgV1yg,14418
|
|
54
54
|
flyte/_internal/controllers/remote/_service_protocol.py,sha256=B9qbIg6DiGeac-iSccLmX_AL2xUgX4ezNUOiAbSy4V0,1357
|
|
@@ -63,7 +63,7 @@ flyte/_internal/resolvers/default.py,sha256=nX4DHUYod1nRvEsl_vSgutQVEdExu2xL8pRk
|
|
|
63
63
|
flyte/_internal/runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
64
64
|
flyte/_internal/runtime/convert.py,sha256=yK5Fy25-CVSqTtWF8BuBel2jwlVoh8R5F4UhIMYpgmg,16086
|
|
65
65
|
flyte/_internal/runtime/entrypoints.py,sha256=9Ng-aQ45M-_MMWeOe9uGmgx69qO9b0xaMRiu542ZI9g,6581
|
|
66
|
-
flyte/_internal/runtime/io.py,sha256=
|
|
66
|
+
flyte/_internal/runtime/io.py,sha256=ysL7hMpfVumvsEYWOM-_VPa8MXn5_X_CZorKbOThyv4,5935
|
|
67
67
|
flyte/_internal/runtime/resources_serde.py,sha256=TObMVsSjVcQhcY8-nY81pbvrz7TP-adDD5xV-LqAaxM,4813
|
|
68
68
|
flyte/_internal/runtime/reuse.py,sha256=WEuBfC9tBezxaIXeUQDgnJfnRHiUmPK0S25nTOFle4E,4676
|
|
69
69
|
flyte/_internal/runtime/rusty.py,sha256=puMaa6aLaoR4Tl5xxZulC4AzY58nmGg-5ok4ydHHjdM,6145
|
|
@@ -172,7 +172,7 @@ flyte/config/_internal.py,sha256=LMcAtDjvTjf5bGlsJVxPuLxQQ82mLd00xK5-JlYGCi8,298
|
|
|
172
172
|
flyte/config/_reader.py,sha256=coidKV5CVODEnqDnsHZ9-VMC0UGXanXhAZE1SmYRhxI,7021
|
|
173
173
|
flyte/connectors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
174
174
|
flyte/extras/__init__.py,sha256=FhB0uK7H1Yo5De9vOuF7UGnezTKncj3u2Wo5uQdWN0g,74
|
|
175
|
-
flyte/extras/_container.py,sha256=
|
|
175
|
+
flyte/extras/_container.py,sha256=xfbxaGrx2JyQyrvQx1UWTzho2ion8_xujPW5V4uDEIs,11818
|
|
176
176
|
flyte/io/__init__.py,sha256=qF8jq_IKuocuGU0LAvoy_uxjMs4G-vXjDA9Prba0JGU,538
|
|
177
177
|
flyte/io/_dir.py,sha256=rih9CY1YjNX05bcAu5LG62Xoyij5GXAlv7jLyVF0je8,15310
|
|
178
178
|
flyte/io/_file.py,sha256=kp5700SKPy5htmMhm4hE2ybb99Ykny1b0Kwm3huCWXs,15572
|
|
@@ -187,7 +187,7 @@ flyte/remote/_logs.py,sha256=aDG18-uPVb2J3PxmqmAY1C0Z4Iv1P1agg-iF4nSQR4U,6709
|
|
|
187
187
|
flyte/remote/_project.py,sha256=CFNTGpgXU3X599tkJ_oxijs9zPzzCWOB6mAWn6WeDEU,2828
|
|
188
188
|
flyte/remote/_run.py,sha256=fpr_YcGZIv6K7Jt1if3-HHHVB2TVt_8AWcZ55rN_fgk,9750
|
|
189
189
|
flyte/remote/_secret.py,sha256=l5xeMS83uMcWWeSSTRsSZUNhS0N--1Dze09C-thSOQs,4341
|
|
190
|
-
flyte/remote/_task.py,sha256=
|
|
190
|
+
flyte/remote/_task.py,sha256=xHfWChnC-L7Hdp2IB9qDnbNyaOgM2TWOlnkmevoc2dA,15314
|
|
191
191
|
flyte/remote/_client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
192
192
|
flyte/remote/_client/_protocols.py,sha256=JyBWHs5WsVOxEDUyG9X7wPLDzzzjkoaNhJlU-X4YlN0,5599
|
|
193
193
|
flyte/remote/_client/controlplane.py,sha256=FsOfj4rO4MIMnYrpAT53F8q588VVf5t4sDuwoPuc840,3102
|
|
@@ -226,10 +226,10 @@ flyte/types/_renderer.py,sha256=ygcCo5l60lHufyQISFddZfWwLlQ8kJAKxUT_XnR_6dY,4818
|
|
|
226
226
|
flyte/types/_string_literals.py,sha256=NlG1xV8RSA-sZ-n-IFQCAsdB6jXJOAKkHWtnopxVVDk,4231
|
|
227
227
|
flyte/types/_type_engine.py,sha256=Tas_OXYddOi0nDuORjqan2SkJ96wKD8937I2l1bo8vk,97916
|
|
228
228
|
flyte/types/_utils.py,sha256=pbts9E1_2LTdLygAY0UYTLYJ8AsN3BZyviSXvrtcutc,2626
|
|
229
|
-
flyte-2.0.
|
|
230
|
-
flyte-2.0.
|
|
231
|
-
flyte-2.0.
|
|
232
|
-
flyte-2.0.
|
|
233
|
-
flyte-2.0.
|
|
234
|
-
flyte-2.0.
|
|
235
|
-
flyte-2.0.
|
|
229
|
+
flyte-2.0.0b3.data/scripts/runtime.py,sha256=2jTy3ccvrJ__Xrfdo2t0Fxhsojc5o2zIxDHt98RE_eU,6475
|
|
230
|
+
flyte-2.0.0b3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
231
|
+
flyte-2.0.0b3.dist-info/METADATA,sha256=DxTr4d_6Ohm1VRSqhqILMy_McBGsKB4ykk274bne-fM,9955
|
|
232
|
+
flyte-2.0.0b3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
233
|
+
flyte-2.0.0b3.dist-info/entry_points.txt,sha256=MIq2z5dBurdCJfpXfMKzgBv7sJOakKRYxr8G0cMiTrg,75
|
|
234
|
+
flyte-2.0.0b3.dist-info/top_level.txt,sha256=7dkyFbikvA12LEZEqawx8oDG1CMod6hTliPj7iWzgYo,6
|
|
235
|
+
flyte-2.0.0b3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|