flyte 0.2.0b26__py3-none-any.whl → 0.2.0b28__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flyte/_bin/runtime.py +6 -1
- flyte/_image.py +44 -9
- flyte/_internal/controllers/_local_controller.py +4 -2
- flyte/_internal/imagebuild/docker_builder.py +14 -5
- flyte/_internal/runtime/convert.py +8 -2
- flyte/_run.py +32 -3
- flyte/_task.py +2 -2
- flyte/_task_environment.py +6 -2
- flyte/_tools.py +13 -0
- flyte/_version.py +2 -2
- flyte/models.py +4 -2
- flyte/remote/_action.py +1 -1
- flyte/remote/_data.py +3 -3
- flyte/remote/_logs.py +8 -0
- flyte/types/_type_engine.py +4 -1
- {flyte-0.2.0b26.data → flyte-0.2.0b28.data}/scripts/runtime.py +6 -1
- {flyte-0.2.0b26.dist-info → flyte-0.2.0b28.dist-info}/METADATA +1 -1
- {flyte-0.2.0b26.dist-info → flyte-0.2.0b28.dist-info}/RECORD +21 -21
- {flyte-0.2.0b26.dist-info → flyte-0.2.0b28.dist-info}/WHEEL +0 -0
- {flyte-0.2.0b26.dist-info → flyte-0.2.0b28.dist-info}/entry_points.txt +0 -0
- {flyte-0.2.0b26.dist-info → flyte-0.2.0b28.dist-info}/top_level.txt +0 -0
flyte/_bin/runtime.py
CHANGED
|
@@ -82,6 +82,9 @@ def main(
|
|
|
82
82
|
):
|
|
83
83
|
sys.path.insert(0, ".")
|
|
84
84
|
|
|
85
|
+
import faulthandler
|
|
86
|
+
import signal
|
|
87
|
+
|
|
85
88
|
import flyte
|
|
86
89
|
import flyte._utils as utils
|
|
87
90
|
from flyte._initialize import init
|
|
@@ -91,8 +94,10 @@ def main(
|
|
|
91
94
|
from flyte._logging import logger
|
|
92
95
|
from flyte.models import ActionID, Checkpoints, CodeBundle, RawDataPath
|
|
93
96
|
|
|
94
|
-
logger.info(
|
|
97
|
+
logger.info("Registering faulthandler for SIGUSR1")
|
|
98
|
+
faulthandler.register(signal.SIGUSR1)
|
|
95
99
|
|
|
100
|
+
logger.info(f"Initializing flyte runtime - version {flyte.__version__}")
|
|
96
101
|
assert org, "Org is required for now"
|
|
97
102
|
assert project, "Project is required"
|
|
98
103
|
assert domain, "Domain is required"
|
flyte/_image.py
CHANGED
|
@@ -355,7 +355,11 @@ class Image:
|
|
|
355
355
|
|
|
356
356
|
@classmethod
|
|
357
357
|
def _get_default_image_for(
|
|
358
|
-
cls,
|
|
358
|
+
cls,
|
|
359
|
+
python_version: Tuple[int, int],
|
|
360
|
+
flyte_version: Optional[str] = None,
|
|
361
|
+
install_flyte: bool = True,
|
|
362
|
+
platform: Optional[Tuple[Architecture, ...]] = None,
|
|
359
363
|
) -> Image:
|
|
360
364
|
# Would love a way to move this outside of this class (but still needs to be accessible via Image.auto())
|
|
361
365
|
# this default image definition may need to be updated once there is a released pypi version
|
|
@@ -377,7 +381,7 @@ class Image:
|
|
|
377
381
|
base_image=f"python:{python_version[0]}.{python_version[1]}-slim-bookworm",
|
|
378
382
|
registry=_BASE_REGISTRY,
|
|
379
383
|
name=_DEFAULT_IMAGE_NAME,
|
|
380
|
-
platform=("linux/amd64", "linux/arm64"),
|
|
384
|
+
platform=("linux/amd64", "linux/arm64") if platform is None else platform,
|
|
381
385
|
)
|
|
382
386
|
labels_and_user = _DockerLines(
|
|
383
387
|
(
|
|
@@ -430,6 +434,7 @@ class Image:
|
|
|
430
434
|
install_flyte: bool = True,
|
|
431
435
|
registry: Optional[str] = None,
|
|
432
436
|
name: Optional[str] = None,
|
|
437
|
+
platform: Optional[Tuple[Architecture, ...]] = None,
|
|
433
438
|
) -> Image:
|
|
434
439
|
"""
|
|
435
440
|
Use this method to start using the default base image, built from this library's base Dockerfile
|
|
@@ -440,6 +445,8 @@ class Image:
|
|
|
440
445
|
:param install_flyte: If True, will install the flyte library in the image
|
|
441
446
|
:param registry: Registry to use for the image
|
|
442
447
|
:param name: Name of the image if you want to override the default name
|
|
448
|
+
:param platform: Platform to use for the image, default is linux/amd64, use tuple for multiple values
|
|
449
|
+
Example: ("linux/amd64", "linux/arm64")
|
|
443
450
|
|
|
444
451
|
:return: Image
|
|
445
452
|
"""
|
|
@@ -447,7 +454,10 @@ class Image:
|
|
|
447
454
|
python_version = _detect_python_version()
|
|
448
455
|
|
|
449
456
|
base_image = cls._get_default_image_for(
|
|
450
|
-
python_version=python_version,
|
|
457
|
+
python_version=python_version,
|
|
458
|
+
flyte_version=flyte_version,
|
|
459
|
+
install_flyte=install_flyte,
|
|
460
|
+
platform=platform,
|
|
451
461
|
)
|
|
452
462
|
|
|
453
463
|
if registry and name:
|
|
@@ -481,7 +491,7 @@ class Image:
|
|
|
481
491
|
extra_index_urls: Union[str, List[str], Tuple[str, ...], None] = None,
|
|
482
492
|
pre: bool = False,
|
|
483
493
|
extra_args: Optional[str] = None,
|
|
484
|
-
|
|
494
|
+
platform: Optional[Tuple[Architecture, ...]] = None,
|
|
485
495
|
) -> Image:
|
|
486
496
|
"""
|
|
487
497
|
Use this method to create a new image with the specified uv script.
|
|
@@ -506,7 +516,7 @@ class Image:
|
|
|
506
516
|
:param python_version: Python version to use for the image, if not specified, will use the current Python
|
|
507
517
|
version
|
|
508
518
|
:param script: path to the uv script
|
|
509
|
-
:param
|
|
519
|
+
:param platform: architecture to use for the image, default is linux/amd64, use tuple for multiple values
|
|
510
520
|
:param python_version: Python version for the image, if not specified, will use the current Python version
|
|
511
521
|
:param index_url: index url to use for pip install, default is None
|
|
512
522
|
:param extra_index_urls: extra index urls to use for pip install, default is None
|
|
@@ -530,7 +540,7 @@ class Image:
|
|
|
530
540
|
raise ValueError("registry must be specified")
|
|
531
541
|
|
|
532
542
|
# todo: arch
|
|
533
|
-
img = cls.from_debian_base(registry=registry, name=name, python_version=python_version)
|
|
543
|
+
img = cls.from_debian_base(registry=registry, name=name, python_version=python_version, platform=platform)
|
|
534
544
|
|
|
535
545
|
# add ca-certificates to the image by default
|
|
536
546
|
img = img.with_apt_packages("ca-certificates")
|
|
@@ -565,6 +575,13 @@ class Image:
|
|
|
565
575
|
|
|
566
576
|
:return:
|
|
567
577
|
"""
|
|
578
|
+
if addl_layer and self.dockerfile:
|
|
579
|
+
# We don't know how to inspect dockerfiles to know what kind it is (OS, python version, uv vs poetry, etc)
|
|
580
|
+
# so there's no guarantee any of the layering logic will work.
|
|
581
|
+
raise ValueError(
|
|
582
|
+
"Flyte current cannot add additional layers to a Dockerfile-based Image."
|
|
583
|
+
" Please amend the dockerfile directly."
|
|
584
|
+
)
|
|
568
585
|
registry = registry if registry else self.registry
|
|
569
586
|
name = name if name else self.name
|
|
570
587
|
if addl_layer and (not name):
|
|
@@ -585,17 +602,35 @@ class Image:
|
|
|
585
602
|
return img
|
|
586
603
|
|
|
587
604
|
@classmethod
|
|
588
|
-
def from_dockerfile(
|
|
605
|
+
def from_dockerfile(
|
|
606
|
+
cls, file: Path, registry: str, name: str, platform: Union[Architecture, Tuple[Architecture, ...], None] = None
|
|
607
|
+
) -> Image:
|
|
589
608
|
"""
|
|
590
|
-
Use this method to create a new image with the specified dockerfile
|
|
609
|
+
Use this method to create a new image with the specified dockerfile. Note you cannot use additional layers
|
|
610
|
+
after this, as the system doesn't attempt to parse/understand the Dockerfile, and what kind of setup it has
|
|
611
|
+
(python version, uv vs poetry, etc), so please put all logic into the dockerfile itself.
|
|
612
|
+
|
|
613
|
+
Also since Python sees paths as from the calling directory, please use Path objects with absolute paths. The
|
|
614
|
+
context for the builder will be the directory where the dockerfile is located.
|
|
591
615
|
|
|
592
616
|
:param file: path to the dockerfile
|
|
593
617
|
:param name: name of the image
|
|
594
618
|
:param registry: registry to use for the image
|
|
619
|
+
:param platform: architecture to use for the image, default is linux/amd64, use tuple for multiple values
|
|
620
|
+
Example: ("linux/amd64", "linux/arm64")
|
|
595
621
|
|
|
596
622
|
:return:
|
|
597
623
|
"""
|
|
598
|
-
|
|
624
|
+
platform = _ensure_tuple(platform) if platform else None
|
|
625
|
+
file = file.absolute() # for clarity when debugging
|
|
626
|
+
kwargs = {
|
|
627
|
+
"dockerfile": file,
|
|
628
|
+
"registry": registry,
|
|
629
|
+
"name": name,
|
|
630
|
+
}
|
|
631
|
+
if platform:
|
|
632
|
+
kwargs["platform"] = platform
|
|
633
|
+
img = cls._new(**kwargs)
|
|
599
634
|
|
|
600
635
|
return img
|
|
601
636
|
|
|
@@ -106,10 +106,12 @@ class LocalController:
|
|
|
106
106
|
raise exc
|
|
107
107
|
else:
|
|
108
108
|
raise flyte.errors.RuntimeSystemError("BadError", "Unknown error")
|
|
109
|
-
if _task.native_interface.outputs
|
|
109
|
+
if _task.native_interface.outputs:
|
|
110
|
+
if out is None:
|
|
111
|
+
raise flyte.errors.RuntimeSystemError("BadOutput", "Task output not captured.")
|
|
110
112
|
result = await convert.convert_outputs_to_native(_task.native_interface, out)
|
|
111
113
|
return result
|
|
112
|
-
return
|
|
114
|
+
return None
|
|
113
115
|
|
|
114
116
|
def submit_sync(self, _task: TaskTemplate, *args, **kwargs) -> concurrent.futures.Future:
|
|
115
117
|
name = threading.current_thread().name + f"PID:{os.getpid()}"
|
|
@@ -305,14 +305,14 @@ class DockerImageBuilder(ImageBuilder):
|
|
|
305
305
|
return [LocalDockerCommandImageChecker, LocalPodmanCommandImageChecker, DockerAPIImageChecker]
|
|
306
306
|
|
|
307
307
|
async def build_image(self, image: Image, dry_run: bool = False) -> str:
|
|
308
|
-
if len(image._layers) == 0:
|
|
309
|
-
logger.warning("No layers to build, returning the image URI as is.")
|
|
310
|
-
return image.uri
|
|
311
|
-
|
|
312
308
|
if image.dockerfile:
|
|
313
309
|
# If a dockerfile is provided, use it directly
|
|
314
310
|
return await self._build_from_dockerfile(image, push=True)
|
|
315
311
|
|
|
312
|
+
if len(image._layers) == 0:
|
|
313
|
+
logger.warning("No layers to build, returning the image URI as is.")
|
|
314
|
+
return image.uri
|
|
315
|
+
|
|
316
316
|
return await self._build_image(
|
|
317
317
|
image,
|
|
318
318
|
push=True,
|
|
@@ -323,18 +323,27 @@ class DockerImageBuilder(ImageBuilder):
|
|
|
323
323
|
"""
|
|
324
324
|
Build the image from a provided Dockerfile.
|
|
325
325
|
"""
|
|
326
|
+
assert image.dockerfile # for mypy
|
|
327
|
+
|
|
326
328
|
command = [
|
|
327
329
|
"docker",
|
|
330
|
+
"buildx",
|
|
328
331
|
"build",
|
|
332
|
+
"--builder",
|
|
333
|
+
DockerImageBuilder._builder_name,
|
|
334
|
+
"-f",
|
|
335
|
+
str(image.dockerfile),
|
|
329
336
|
"--tag",
|
|
330
337
|
f"{image.uri}",
|
|
331
338
|
"--platform",
|
|
332
339
|
",".join(image.platform),
|
|
333
|
-
|
|
340
|
+
str(image.dockerfile.parent.absolute()), # Use the parent directory of the Dockerfile as the context
|
|
334
341
|
]
|
|
335
342
|
|
|
336
343
|
if image.registry and push:
|
|
337
344
|
command.append("--push")
|
|
345
|
+
else:
|
|
346
|
+
command.append("--load")
|
|
338
347
|
|
|
339
348
|
concat_command = " ".join(command)
|
|
340
349
|
logger.debug(f"Build command: {concat_command}")
|
|
@@ -120,13 +120,19 @@ async def convert_from_native_to_inputs(interface: NativeInterface, *args, **kwa
|
|
|
120
120
|
for input_name, (input_type, default_value) in interface.inputs.items():
|
|
121
121
|
if input_name in kwargs:
|
|
122
122
|
type_hints[input_name] = input_type
|
|
123
|
-
elif (
|
|
124
|
-
default_value is None and
|
|
123
|
+
elif (
|
|
124
|
+
(default_value is not None and default_value is not inspect.Signature.empty)
|
|
125
|
+
or (default_value is None and is_optional_type(input_type))
|
|
126
|
+
or input_type is None
|
|
125
127
|
):
|
|
126
128
|
if default_value == NativeInterface.has_default:
|
|
127
129
|
if interface._remote_defaults is None or input_name not in interface._remote_defaults:
|
|
128
130
|
raise ValueError(f"Input '{input_name}' has a default value but it is not set in the interface.")
|
|
129
131
|
already_converted_kwargs[input_name] = interface._remote_defaults[input_name]
|
|
132
|
+
elif input_type is None:
|
|
133
|
+
# If the type is None, we assume it's a placeholder for no type
|
|
134
|
+
kwargs[input_name] = None
|
|
135
|
+
type_hints[input_name] = NoneType
|
|
130
136
|
else:
|
|
131
137
|
kwargs[input_name] = default_value
|
|
132
138
|
type_hints[input_name] = input_type
|
flyte/_run.py
CHANGED
|
@@ -376,9 +376,10 @@ class _Runner:
|
|
|
376
376
|
raise err
|
|
377
377
|
return outputs
|
|
378
378
|
|
|
379
|
-
async def _run_local(self, obj: TaskTemplate[P, R], *args: P.args, **kwargs: P.kwargs) ->
|
|
379
|
+
async def _run_local(self, obj: TaskTemplate[P, R], *args: P.args, **kwargs: P.kwargs) -> Run:
|
|
380
380
|
from flyte._internal.controllers import create_controller
|
|
381
381
|
from flyte._internal.controllers._local_controller import LocalController
|
|
382
|
+
from flyte.remote import Run
|
|
382
383
|
from flyte.report import Report
|
|
383
384
|
|
|
384
385
|
controller = cast(LocalController, create_controller("local"))
|
|
@@ -409,9 +410,37 @@ class _Runner:
|
|
|
409
410
|
if obj._call_as_synchronous:
|
|
410
411
|
fut = controller.submit_sync(obj, *args, **kwargs)
|
|
411
412
|
awaitable = asyncio.wrap_future(fut)
|
|
412
|
-
|
|
413
|
+
outputs = await awaitable
|
|
413
414
|
else:
|
|
414
|
-
|
|
415
|
+
outputs = await controller.submit(obj, *args, **kwargs)
|
|
416
|
+
|
|
417
|
+
class _LocalRun(Run):
|
|
418
|
+
def __init__(self, outputs: Tuple[Any, ...] | Any):
|
|
419
|
+
from flyte._protos.workflow import run_definition_pb2
|
|
420
|
+
|
|
421
|
+
self._outputs = outputs
|
|
422
|
+
super().__init__(
|
|
423
|
+
pb2=run_definition_pb2.Run(
|
|
424
|
+
action=run_definition_pb2.Action(
|
|
425
|
+
id=run_definition_pb2.ActionIdentifier(
|
|
426
|
+
name="a0",
|
|
427
|
+
run=run_definition_pb2.RunIdentifier(name="dry-run"),
|
|
428
|
+
)
|
|
429
|
+
)
|
|
430
|
+
)
|
|
431
|
+
)
|
|
432
|
+
|
|
433
|
+
@property
|
|
434
|
+
def url(self) -> str:
|
|
435
|
+
return "local-run"
|
|
436
|
+
|
|
437
|
+
def wait(self, quiet: bool = False, wait_for: Literal["terminal", "running"] = "terminal"):
|
|
438
|
+
pass
|
|
439
|
+
|
|
440
|
+
def outputs(self) -> R:
|
|
441
|
+
return cast(R, self._outputs)
|
|
442
|
+
|
|
443
|
+
return _LocalRun(outputs)
|
|
415
444
|
|
|
416
445
|
@syncify
|
|
417
446
|
async def run(
|
flyte/_task.py
CHANGED
|
@@ -20,8 +20,6 @@ from typing import (
|
|
|
20
20
|
Union,
|
|
21
21
|
)
|
|
22
22
|
|
|
23
|
-
from flyteidl.core.tasks_pb2 import DataLoadingConfig
|
|
24
|
-
|
|
25
23
|
from flyte._pod import PodTemplate
|
|
26
24
|
from flyte.errors import RuntimeSystemError, RuntimeUserError
|
|
27
25
|
|
|
@@ -37,6 +35,8 @@ from ._timeout import TimeoutType
|
|
|
37
35
|
from .models import NativeInterface, SerializationContext
|
|
38
36
|
|
|
39
37
|
if TYPE_CHECKING:
|
|
38
|
+
from flyteidl.core.tasks_pb2 import DataLoadingConfig
|
|
39
|
+
|
|
40
40
|
from ._task_environment import TaskEnvironment
|
|
41
41
|
|
|
42
42
|
P = ParamSpec("P") # capture the function's parameters
|
flyte/_task_environment.py
CHANGED
|
@@ -90,7 +90,11 @@ class TaskEnvironment(Environment):
|
|
|
90
90
|
|
|
91
91
|
"""
|
|
92
92
|
cache = kwargs.pop("cache", None)
|
|
93
|
-
reusable =
|
|
93
|
+
reusable = None
|
|
94
|
+
reusable_set = False
|
|
95
|
+
if "reusable" in kwargs:
|
|
96
|
+
reusable_set = True
|
|
97
|
+
reusable = kwargs.pop("reusable", None)
|
|
94
98
|
|
|
95
99
|
# validate unknown kwargs if needed
|
|
96
100
|
if kwargs:
|
|
@@ -106,7 +110,7 @@ class TaskEnvironment(Environment):
|
|
|
106
110
|
kwargs["cache"] = cache
|
|
107
111
|
if env is not None:
|
|
108
112
|
kwargs["env"] = env
|
|
109
|
-
if
|
|
113
|
+
if reusable_set:
|
|
110
114
|
kwargs["reusable"] = reusable
|
|
111
115
|
if secrets is not None:
|
|
112
116
|
kwargs["secrets"] = secrets
|
flyte/_tools.py
CHANGED
|
@@ -25,3 +25,16 @@ def is_in_cluster() -> bool:
|
|
|
25
25
|
if os.getenv("_UN_CLS"):
|
|
26
26
|
return True
|
|
27
27
|
return False
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def ipywidgets_check() -> bool:
|
|
31
|
+
"""
|
|
32
|
+
Check if the interface is running in IPython with ipywidgets support.
|
|
33
|
+
:return: True if running in IPython with ipywidgets support, False otherwise.
|
|
34
|
+
"""
|
|
35
|
+
try:
|
|
36
|
+
import ipywidgets # noqa: F401
|
|
37
|
+
|
|
38
|
+
return True
|
|
39
|
+
except (ImportError, NameError):
|
|
40
|
+
return False
|
flyte/_version.py
CHANGED
|
@@ -17,5 +17,5 @@ __version__: str
|
|
|
17
17
|
__version_tuple__: VERSION_TUPLE
|
|
18
18
|
version_tuple: VERSION_TUPLE
|
|
19
19
|
|
|
20
|
-
__version__ = version = '0.2.
|
|
21
|
-
__version_tuple__ = version_tuple = (0, 2, 0, '
|
|
20
|
+
__version__ = version = '0.2.0b28'
|
|
21
|
+
__version_tuple__ = version_tuple = (0, 2, 0, 'b28')
|
flyte/models.py
CHANGED
|
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|
|
3
3
|
import inspect
|
|
4
4
|
import os
|
|
5
5
|
import pathlib
|
|
6
|
-
import tempfile
|
|
7
6
|
from dataclasses import dataclass, field, replace
|
|
8
7
|
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Literal, Optional, Tuple, Type
|
|
9
8
|
|
|
@@ -12,7 +11,6 @@ import rich.repr
|
|
|
12
11
|
from flyte._docstring import Docstring
|
|
13
12
|
from flyte._interface import extract_return_annotation
|
|
14
13
|
from flyte._logging import logger
|
|
15
|
-
from flyte._utils.helpers import base36_encode
|
|
16
14
|
|
|
17
15
|
if TYPE_CHECKING:
|
|
18
16
|
from flyteidl.core import literals_pb2
|
|
@@ -65,6 +63,8 @@ class ActionID:
|
|
|
65
63
|
"""Make a deterministic name"""
|
|
66
64
|
import hashlib
|
|
67
65
|
|
|
66
|
+
from flyte._utils.helpers import base36_encode
|
|
67
|
+
|
|
68
68
|
components = f"{self.name}-{input_hash}-{task_hash}-{task_call_seq}" + (f"-{group}" if group else "")
|
|
69
69
|
logger.debug(f"----- Generating sub-action ID from components: {components}")
|
|
70
70
|
# has the components into something deterministic
|
|
@@ -89,6 +89,8 @@ class RawDataPath:
|
|
|
89
89
|
Create a new context attribute object, with local path given. Will be created if it doesn't exist.
|
|
90
90
|
:return: Path to the temporary directory
|
|
91
91
|
"""
|
|
92
|
+
import tempfile
|
|
93
|
+
|
|
92
94
|
match local_folder:
|
|
93
95
|
case pathlib.Path():
|
|
94
96
|
local_folder.mkdir(parents=True, exist_ok=True)
|
flyte/remote/_action.py
CHANGED
|
@@ -345,7 +345,7 @@ class Action:
|
|
|
345
345
|
else:
|
|
346
346
|
console.print(
|
|
347
347
|
f"[bold red]Run '{self.run_name}' exited unsuccessfully in state {ad.phase}"
|
|
348
|
-
f"with error: {ad.error_info}[/bold red]"
|
|
348
|
+
f" with error: {ad.error_info}[/bold red]"
|
|
349
349
|
)
|
|
350
350
|
break
|
|
351
351
|
except asyncio.CancelledError:
|
flyte/remote/_data.py
CHANGED
|
@@ -79,16 +79,16 @@ async def _upload_single_file(
|
|
|
79
79
|
except grpc.aio.AioRpcError as e:
|
|
80
80
|
if e.code() == grpc.StatusCode.NOT_FOUND:
|
|
81
81
|
raise RuntimeSystemError(
|
|
82
|
-
"NotFound", f"Failed to get signed url for {fp}, please check your project and domain."
|
|
82
|
+
"NotFound", f"Failed to get signed url for {fp}, please check your project and domain: {e.details()}"
|
|
83
83
|
)
|
|
84
84
|
elif e.code() == grpc.StatusCode.PERMISSION_DENIED:
|
|
85
85
|
raise RuntimeSystemError(
|
|
86
|
-
"PermissionDenied", f"Failed to get signed url for {fp}, please check your permissions."
|
|
86
|
+
"PermissionDenied", f"Failed to get signed url for {fp}, please check your permissions: {e.details()}"
|
|
87
87
|
)
|
|
88
88
|
elif e.code() == grpc.StatusCode.UNAVAILABLE:
|
|
89
89
|
raise InitializationError("EndpointUnavailable", "user", "Service is unavailable.")
|
|
90
90
|
else:
|
|
91
|
-
raise RuntimeSystemError(e.code().value, f"Failed to get signed url for {fp}.")
|
|
91
|
+
raise RuntimeSystemError(e.code().value, f"Failed to get signed url for {fp}: {e.details()}")
|
|
92
92
|
except Exception as e:
|
|
93
93
|
raise RuntimeSystemError(type(e).__name__, f"Failed to get signed url for {fp}.") from e
|
|
94
94
|
logger.debug(f"Uploading to {make_hyperlink('signed url', resp.signed_url)} for {fp}")
|
flyte/remote/_logs.py
CHANGED
|
@@ -10,8 +10,10 @@ from rich.panel import Panel
|
|
|
10
10
|
from rich.text import Text
|
|
11
11
|
|
|
12
12
|
from flyte._initialize import ensure_client, get_client
|
|
13
|
+
from flyte._logging import logger
|
|
13
14
|
from flyte._protos.logs.dataplane import payload_pb2
|
|
14
15
|
from flyte._protos.workflow import run_definition_pb2, run_logs_service_pb2
|
|
16
|
+
from flyte._tools import ipython_check, ipywidgets_check
|
|
15
17
|
from flyte.errors import LogsNotYetAvailableError
|
|
16
18
|
from flyte.syncify import syncify
|
|
17
19
|
|
|
@@ -158,6 +160,12 @@ class Logs:
|
|
|
158
160
|
"""
|
|
159
161
|
if attempt < 1:
|
|
160
162
|
raise ValueError("Attempt number must be greater than 0.")
|
|
163
|
+
|
|
164
|
+
if ipython_check():
|
|
165
|
+
if not ipywidgets_check():
|
|
166
|
+
logger.warning("IPython widgets is not available, defaulting to console output.")
|
|
167
|
+
raw = True
|
|
168
|
+
|
|
161
169
|
if raw:
|
|
162
170
|
console = Console()
|
|
163
171
|
async for line in cls.tail.aio(action_id=action_id, attempt=attempt):
|
flyte/types/_type_engine.py
CHANGED
|
@@ -16,7 +16,7 @@ import typing
|
|
|
16
16
|
from abc import ABC, abstractmethod
|
|
17
17
|
from collections import OrderedDict
|
|
18
18
|
from functools import lru_cache
|
|
19
|
-
from types import GenericAlias
|
|
19
|
+
from types import GenericAlias, NoneType
|
|
20
20
|
from typing import Any, Dict, NamedTuple, Optional, Type, cast
|
|
21
21
|
|
|
22
22
|
import msgpack
|
|
@@ -307,6 +307,9 @@ class SimpleTransformer(TypeTransformer[T]):
|
|
|
307
307
|
expected_python_type = get_underlying_type(expected_python_type)
|
|
308
308
|
|
|
309
309
|
if expected_python_type is not self._type:
|
|
310
|
+
if expected_python_type is None and issubclass(self._type, NoneType):
|
|
311
|
+
# If the expected type is NoneType, we can return None
|
|
312
|
+
return None
|
|
310
313
|
raise TypeTransformerFailedError(
|
|
311
314
|
f"Cannot convert to type {expected_python_type}, only {self._type} is supported"
|
|
312
315
|
)
|
|
@@ -82,6 +82,9 @@ def main(
|
|
|
82
82
|
):
|
|
83
83
|
sys.path.insert(0, ".")
|
|
84
84
|
|
|
85
|
+
import faulthandler
|
|
86
|
+
import signal
|
|
87
|
+
|
|
85
88
|
import flyte
|
|
86
89
|
import flyte._utils as utils
|
|
87
90
|
from flyte._initialize import init
|
|
@@ -91,8 +94,10 @@ def main(
|
|
|
91
94
|
from flyte._logging import logger
|
|
92
95
|
from flyte.models import ActionID, Checkpoints, CodeBundle, RawDataPath
|
|
93
96
|
|
|
94
|
-
logger.info(
|
|
97
|
+
logger.info("Registering faulthandler for SIGUSR1")
|
|
98
|
+
faulthandler.register(signal.SIGUSR1)
|
|
95
99
|
|
|
100
|
+
logger.info(f"Initializing flyte runtime - version {flyte.__version__}")
|
|
96
101
|
assert org, "Org is required for now"
|
|
97
102
|
assert project, "Project is required"
|
|
98
103
|
assert domain, "Domain is required"
|
|
@@ -8,7 +8,7 @@ flyte/_environment.py,sha256=oKVXLBX0ky2eE_wjBdzvQGI_2LiT2Nbx58ur7GMt50c,3231
|
|
|
8
8
|
flyte/_excepthook.py,sha256=nXts84rzEg6-7RtFarbKzOsRZTQR4plnbWVIFMAEprs,1310
|
|
9
9
|
flyte/_group.py,sha256=7o1j16sZyUmYB50mOiq1ui4TBAKhRpDqLakV8Ya1kw4,803
|
|
10
10
|
flyte/_hash.py,sha256=Of_Zl_DzzzF2jp4ZsLm-3o-xJFCCJ8_GubmLI1htx78,504
|
|
11
|
-
flyte/_image.py,sha256=
|
|
11
|
+
flyte/_image.py,sha256=RIjkZrj4q0-KVpYW_Z0Zd14Rn9WIQqOPG6DUGDH0ZKo,33258
|
|
12
12
|
flyte/_initialize.py,sha256=xKl_LYMluRt21wWqa6RTKuLo0_DCbSaTfUk27_brtNk,18232
|
|
13
13
|
flyte/_interface.py,sha256=1B9zIwFDjiVp_3l_mk8EpA4g3Re-6DUBEBi9z9vDvPs,3504
|
|
14
14
|
flyte/_logging.py,sha256=_yNo-Nx2yzh0MLoZGbnIYHGKei4wtQmSGM0lE30Ev7w,3662
|
|
@@ -17,21 +17,21 @@ flyte/_pod.py,sha256=--72b0c6IkOEbBwZPLmgl-ll-j7ECfG-kh75LzBnNN8,1068
|
|
|
17
17
|
flyte/_resources.py,sha256=L2JuvQDlMo1JLJeUmJPRwtWbunhR2xJEhFgQW5yc72c,9690
|
|
18
18
|
flyte/_retry.py,sha256=rfLv0MvWxzPByKESTglEmjPsytEAKiIvvmzlJxXwsfE,941
|
|
19
19
|
flyte/_reusable_environment.py,sha256=f8Y1GilUwGcXH4n2Fckrnx0SrZmhk3nCfoe-TqUKivI,3740
|
|
20
|
-
flyte/_run.py,sha256=
|
|
20
|
+
flyte/_run.py,sha256=htq4zyltmaWL-B_IkLX5joSzfB-pLTHQ5DtI49c220I,24247
|
|
21
21
|
flyte/_secret.py,sha256=ogXmCNfYYIphV6p-2iiWmwr2cNUES5Cq01PPjY6uQNA,3217
|
|
22
|
-
flyte/_task.py,sha256=
|
|
23
|
-
flyte/_task_environment.py,sha256=
|
|
22
|
+
flyte/_task.py,sha256=aZpnLLMZLNmPPS6-_pafBEgOtvg7Z-kwI1fhexfI8JM,18018
|
|
23
|
+
flyte/_task_environment.py,sha256=qYcTG9452a_POueQCHqkTafN4HG8Xo7KkBPwSMkKRRU,8536
|
|
24
24
|
flyte/_task_plugins.py,sha256=9MH3nFPOH_e8_92BT4sFk4oyAnj6GJFvaPYWaraX7yE,1037
|
|
25
25
|
flyte/_timeout.py,sha256=zx5sFcbYmjJAJbZWSGzzX-BpC9HC7Jfs35T7vVhKwkk,1571
|
|
26
|
-
flyte/_tools.py,sha256=
|
|
26
|
+
flyte/_tools.py,sha256=tWb0sx3t3mm4jbaQVjCTc9y39oR_Ibo3z_KHToP3Lto,966
|
|
27
27
|
flyte/_trace.py,sha256=C788bgoSc3st8kE8Cae2xegnLx2CT6uuRKKfaDrDUys,5122
|
|
28
|
-
flyte/_version.py,sha256=
|
|
28
|
+
flyte/_version.py,sha256=17NKFMLoy5Yz1urofNe-xHRTlzSKuLO6mP2yK1P7AxI,521
|
|
29
29
|
flyte/errors.py,sha256=IUXHuEgszis9ZyhBUOSSlZx7VPQk0BF4N875yj1FBKo,4963
|
|
30
30
|
flyte/extend.py,sha256=GB4ZedGzKa30vYWRVPOdxEeK62xnUVFY4z2tD6H9eEw,376
|
|
31
|
-
flyte/models.py,sha256=
|
|
31
|
+
flyte/models.py,sha256=g8QBIF9JjrNxDR_R_esAFG72hrq3fFV42bb7Nm0E0Zo,15225
|
|
32
32
|
flyte/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
33
33
|
flyte/_bin/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
34
|
-
flyte/_bin/runtime.py,sha256=
|
|
34
|
+
flyte/_bin/runtime.py,sha256=2jTy3ccvrJ__Xrfdo2t0Fxhsojc5o2zIxDHt98RE_eU,6475
|
|
35
35
|
flyte/_cache/__init__.py,sha256=zhdO5UuHQRdzn8GHmSN40nrxfAmI4ihDRuHZM11U84Y,305
|
|
36
36
|
flyte/_cache/cache.py,sha256=ErhWzzJdEjTIuEF4f-r6IBgko-3Al9iUs1Eq4O42TUE,5021
|
|
37
37
|
flyte/_cache/defaults.py,sha256=gzJZW0QJPUfd2OPnGpv3tzIfwPtgFjAKoie3NP1P97U,217
|
|
@@ -43,7 +43,7 @@ flyte/_code_bundle/_utils.py,sha256=b0s3ZVKSRwaa_2CMTCqt2iRrUvTTW3FmlyqCD9k5BS0,
|
|
|
43
43
|
flyte/_code_bundle/bundle.py,sha256=nUAwYTVAE3Z9dfgkBtsqCoKJImjSl4AicG36yweWHLc,8797
|
|
44
44
|
flyte/_internal/__init__.py,sha256=vjXgGzAAjy609YFkAy9_RVPuUlslsHSJBXCLNTVnqOY,136
|
|
45
45
|
flyte/_internal/controllers/__init__.py,sha256=5CBnS9lb1VFMzZuRXUiaPhlN3G9qh7Aq9kTwxW5hsRw,4301
|
|
46
|
-
flyte/_internal/controllers/_local_controller.py,sha256=
|
|
46
|
+
flyte/_internal/controllers/_local_controller.py,sha256=6zyNxr8PRYS9hKiOwmt4YgNgrOqT_woe8Xb6Ec4KrlE,7229
|
|
47
47
|
flyte/_internal/controllers/_trace.py,sha256=biI-lXSIe3gXuWI-KT6T-jTtojQCQ7BLOHTCG3J6MQc,1145
|
|
48
48
|
flyte/_internal/controllers/remote/__init__.py,sha256=9_azH1eHLqY6VULpDugXi7Kf1kK1ODqEnsQ_3wM6IqU,1919
|
|
49
49
|
flyte/_internal/controllers/remote/_action.py,sha256=tB3GfcB4-PcqaupeoFKxqkodFuNCaruHbXHFGkPaTFM,6040
|
|
@@ -53,7 +53,7 @@ flyte/_internal/controllers/remote/_core.py,sha256=FJe1ZAWu_0w5SQUFgAHY4Hjjf_Aju
|
|
|
53
53
|
flyte/_internal/controllers/remote/_informer.py,sha256=StiPcQLLW0h36uEBhKsupMY79EeFCKA3QQzvv2IyvRo,14188
|
|
54
54
|
flyte/_internal/controllers/remote/_service_protocol.py,sha256=B9qbIg6DiGeac-iSccLmX_AL2xUgX4ezNUOiAbSy4V0,1357
|
|
55
55
|
flyte/_internal/imagebuild/__init__.py,sha256=dwXdJ1jMhw9RF8itF7jkPLanvX1yCviSns7hE5eoIts,102
|
|
56
|
-
flyte/_internal/imagebuild/docker_builder.py,sha256=
|
|
56
|
+
flyte/_internal/imagebuild/docker_builder.py,sha256=idA3HNmYrsX0qaYjmiJVJEuwYYjZfYoIEMB7aogwWYs,16468
|
|
57
57
|
flyte/_internal/imagebuild/image_builder.py,sha256=dXBXl62qcPabus6dR3eP8P9mBGNhpZHZ2Xm12AymKkk,11150
|
|
58
58
|
flyte/_internal/imagebuild/remote_builder.py,sha256=JBeek4B4a6tUeyAD8dgQKVv4myMdeERqmxMStABVWN0,10452
|
|
59
59
|
flyte/_internal/resolvers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -61,7 +61,7 @@ flyte/_internal/resolvers/_task_module.py,sha256=jwy1QYygUK7xmpCZLt1SPTfJCkfox3C
|
|
|
61
61
|
flyte/_internal/resolvers/common.py,sha256=ADQLRoyGsJ4vuUkitffMGrMKKjy0vpk6X53g4FuKDLc,993
|
|
62
62
|
flyte/_internal/resolvers/default.py,sha256=nX4DHUYod1nRvEsl_vSgutQVEdExu2xL8pRkyi4VWbY,981
|
|
63
63
|
flyte/_internal/runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
64
|
-
flyte/_internal/runtime/convert.py,sha256=
|
|
64
|
+
flyte/_internal/runtime/convert.py,sha256=yK5Fy25-CVSqTtWF8BuBel2jwlVoh8R5F4UhIMYpgmg,16086
|
|
65
65
|
flyte/_internal/runtime/entrypoints.py,sha256=9Ng-aQ45M-_MMWeOe9uGmgx69qO9b0xaMRiu542ZI9g,6581
|
|
66
66
|
flyte/_internal/runtime/io.py,sha256=Lgdy4iPjlKjUO-V_AkoPZff6lywaFjZUG-PErRukmx4,4248
|
|
67
67
|
flyte/_internal/runtime/resources_serde.py,sha256=TObMVsSjVcQhcY8-nY81pbvrz7TP-adDD5xV-LqAaxM,4813
|
|
@@ -180,10 +180,10 @@ flyte/io/_structured_dataset/__init__.py,sha256=69ixVV9OEXiLiQ6SV2S8tEC7dVQe7YTt
|
|
|
180
180
|
flyte/io/_structured_dataset/basic_dfs.py,sha256=D0QzcaMBO_R2s9Oi9mDqiykkBp0kgi-eI_A3w92vUkw,7980
|
|
181
181
|
flyte/io/_structured_dataset/structured_dataset.py,sha256=ddRjz36RhNxIy0gakzdLStBzoo4cAOgXbNqiqt5YhMI,52645
|
|
182
182
|
flyte/remote/__init__.py,sha256=y9eke9JzEJkygk8eKZjSj44fJGlyepuW4i-j6lbCAPY,617
|
|
183
|
-
flyte/remote/_action.py,sha256=
|
|
183
|
+
flyte/remote/_action.py,sha256=r2rLmJoPUIOCUoBUy4RG_1mMYmC4uiI1tKXuZZ9iC9k,23564
|
|
184
184
|
flyte/remote/_console.py,sha256=avmELJPx8nQMAVPrHlh6jEIRPjrMwFpdZjJsWOOa9rE,660
|
|
185
|
-
flyte/remote/_data.py,sha256=
|
|
186
|
-
flyte/remote/_logs.py,sha256=
|
|
185
|
+
flyte/remote/_data.py,sha256=zYXXlqEvPdsC44Gm7rP_hQjRgVe3EFfhZNEWKF0p4MQ,6163
|
|
186
|
+
flyte/remote/_logs.py,sha256=xBx4ozfY-NBMw3uD5o5YLsI4no10zGroNMcx1Oj0ef0,6689
|
|
187
187
|
flyte/remote/_project.py,sha256=CFNTGpgXU3X599tkJ_oxijs9zPzzCWOB6mAWn6WeDEU,2828
|
|
188
188
|
flyte/remote/_run.py,sha256=HzpoDthSw50vuT-gmzyzdkf28H4SyBgMr0d6bFSmyNU,9747
|
|
189
189
|
flyte/remote/_secret.py,sha256=l5xeMS83uMcWWeSSTRsSZUNhS0N--1Dze09C-thSOQs,4341
|
|
@@ -224,11 +224,11 @@ flyte/types/_interface.py,sha256=5y9EC5r897xz03Hh0ltF8QVGKMfMfAznws-hKSEO4Go,167
|
|
|
224
224
|
flyte/types/_pickle.py,sha256=PjdR66OTDMZ3OYq6GvM_Ua0cIo5t2XQaIjmpJ9xo4Ys,4050
|
|
225
225
|
flyte/types/_renderer.py,sha256=ygcCo5l60lHufyQISFddZfWwLlQ8kJAKxUT_XnR_6dY,4818
|
|
226
226
|
flyte/types/_string_literals.py,sha256=NlG1xV8RSA-sZ-n-IFQCAsdB6jXJOAKkHWtnopxVVDk,4231
|
|
227
|
-
flyte/types/_type_engine.py,sha256=
|
|
227
|
+
flyte/types/_type_engine.py,sha256=QBH-XNwyBHvKP8PjI_BaEPneonfoeNS86aMld87YLMA,97278
|
|
228
228
|
flyte/types/_utils.py,sha256=pbts9E1_2LTdLygAY0UYTLYJ8AsN3BZyviSXvrtcutc,2626
|
|
229
|
-
flyte-0.2.
|
|
230
|
-
flyte-0.2.
|
|
231
|
-
flyte-0.2.
|
|
232
|
-
flyte-0.2.
|
|
233
|
-
flyte-0.2.
|
|
234
|
-
flyte-0.2.
|
|
229
|
+
flyte-0.2.0b28.data/scripts/runtime.py,sha256=2jTy3ccvrJ__Xrfdo2t0Fxhsojc5o2zIxDHt98RE_eU,6475
|
|
230
|
+
flyte-0.2.0b28.dist-info/METADATA,sha256=EL-2mFQ2imGb8w-oib02h7qXZdE1Pcen75jRIeiJSyI,5857
|
|
231
|
+
flyte-0.2.0b28.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
232
|
+
flyte-0.2.0b28.dist-info/entry_points.txt,sha256=MIq2z5dBurdCJfpXfMKzgBv7sJOakKRYxr8G0cMiTrg,75
|
|
233
|
+
flyte-0.2.0b28.dist-info/top_level.txt,sha256=7dkyFbikvA12LEZEqawx8oDG1CMod6hTliPj7iWzgYo,6
|
|
234
|
+
flyte-0.2.0b28.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|