flyte 0.2.0b17__py3-none-any.whl → 0.2.0b19__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of flyte might be problematic. Click here for more details.
- flyte/_bin/runtime.py +10 -2
- flyte/_code_bundle/bundle.py +2 -2
- flyte/_deploy.py +2 -3
- flyte/_image.py +100 -64
- flyte/_initialize.py +9 -1
- flyte/_internal/imagebuild/__init__.py +4 -0
- flyte/_internal/imagebuild/docker_builder.py +57 -24
- flyte/_internal/imagebuild/image_builder.py +69 -42
- flyte/_internal/imagebuild/remote_builder.py +259 -0
- flyte/_protos/imagebuilder/definition_pb2.py +59 -0
- flyte/_protos/imagebuilder/definition_pb2.pyi +140 -0
- flyte/_protos/imagebuilder/definition_pb2_grpc.py +4 -0
- flyte/_protos/imagebuilder/payload_pb2.py +32 -0
- flyte/_protos/imagebuilder/payload_pb2.pyi +21 -0
- flyte/_protos/imagebuilder/payload_pb2_grpc.py +4 -0
- flyte/_protos/imagebuilder/service_pb2.py +29 -0
- flyte/_protos/imagebuilder/service_pb2.pyi +5 -0
- flyte/_protos/imagebuilder/service_pb2_grpc.py +66 -0
- flyte/_run.py +71 -8
- flyte/_task_environment.py +1 -0
- flyte/_version.py +2 -2
- flyte/cli/__init__.py +9 -0
- flyte/cli/_build.py +1 -1
- flyte/cli/_common.py +14 -11
- flyte/cli/_create.py +15 -0
- flyte/cli/_deploy.py +2 -2
- flyte/cli/_get.py +9 -6
- flyte/cli/_run.py +1 -0
- flyte/cli/main.py +8 -0
- flyte/config/_config.py +30 -2
- flyte/config/_internal.py +8 -0
- flyte/config/_reader.py +0 -3
- flyte/extras/_container.py +2 -2
- flyte/remote/_data.py +2 -0
- flyte/remote/_run.py +10 -4
- flyte/remote/_task.py +35 -7
- flyte/types/_type_engine.py +2 -4
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/METADATA +1 -1
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/RECORD +42 -33
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/WHEEL +0 -0
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/entry_points.txt +0 -0
- {flyte-0.2.0b17.dist-info → flyte-0.2.0b19.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
from flyte._protos.validate.validate import validate_pb2 as _validate_pb2
|
|
2
|
+
from google.protobuf.internal import containers as _containers
|
|
3
|
+
from google.protobuf import descriptor as _descriptor
|
|
4
|
+
from google.protobuf import message as _message
|
|
5
|
+
from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union
|
|
6
|
+
|
|
7
|
+
DESCRIPTOR: _descriptor.FileDescriptor
|
|
8
|
+
|
|
9
|
+
class ImageIdentifier(_message.Message):
|
|
10
|
+
__slots__ = ["name"]
|
|
11
|
+
NAME_FIELD_NUMBER: _ClassVar[int]
|
|
12
|
+
name: str
|
|
13
|
+
def __init__(self, name: _Optional[str] = ...) -> None: ...
|
|
14
|
+
|
|
15
|
+
class Image(_message.Message):
|
|
16
|
+
__slots__ = ["id", "fqin"]
|
|
17
|
+
ID_FIELD_NUMBER: _ClassVar[int]
|
|
18
|
+
FQIN_FIELD_NUMBER: _ClassVar[int]
|
|
19
|
+
id: ImageIdentifier
|
|
20
|
+
fqin: str
|
|
21
|
+
def __init__(self, id: _Optional[_Union[ImageIdentifier, _Mapping]] = ..., fqin: _Optional[str] = ...) -> None: ...
|
|
22
|
+
|
|
23
|
+
class AptPackages(_message.Message):
|
|
24
|
+
__slots__ = ["packages"]
|
|
25
|
+
PACKAGES_FIELD_NUMBER: _ClassVar[int]
|
|
26
|
+
packages: _containers.RepeatedScalarFieldContainer[str]
|
|
27
|
+
def __init__(self, packages: _Optional[_Iterable[str]] = ...) -> None: ...
|
|
28
|
+
|
|
29
|
+
class PipOptions(_message.Message):
|
|
30
|
+
__slots__ = ["index_url", "extra_index_urls", "pre", "extra_args"]
|
|
31
|
+
INDEX_URL_FIELD_NUMBER: _ClassVar[int]
|
|
32
|
+
EXTRA_INDEX_URLS_FIELD_NUMBER: _ClassVar[int]
|
|
33
|
+
PRE_FIELD_NUMBER: _ClassVar[int]
|
|
34
|
+
EXTRA_ARGS_FIELD_NUMBER: _ClassVar[int]
|
|
35
|
+
index_url: str
|
|
36
|
+
extra_index_urls: _containers.RepeatedScalarFieldContainer[str]
|
|
37
|
+
pre: bool
|
|
38
|
+
extra_args: str
|
|
39
|
+
def __init__(self, index_url: _Optional[str] = ..., extra_index_urls: _Optional[_Iterable[str]] = ..., pre: bool = ..., extra_args: _Optional[str] = ...) -> None: ...
|
|
40
|
+
|
|
41
|
+
class PipPackages(_message.Message):
|
|
42
|
+
__slots__ = ["packages", "options"]
|
|
43
|
+
PACKAGES_FIELD_NUMBER: _ClassVar[int]
|
|
44
|
+
OPTIONS_FIELD_NUMBER: _ClassVar[int]
|
|
45
|
+
packages: _containers.RepeatedScalarFieldContainer[str]
|
|
46
|
+
options: PipOptions
|
|
47
|
+
def __init__(self, packages: _Optional[_Iterable[str]] = ..., options: _Optional[_Union[PipOptions, _Mapping]] = ...) -> None: ...
|
|
48
|
+
|
|
49
|
+
class Requirements(_message.Message):
|
|
50
|
+
__slots__ = ["file", "options"]
|
|
51
|
+
FILE_FIELD_NUMBER: _ClassVar[int]
|
|
52
|
+
OPTIONS_FIELD_NUMBER: _ClassVar[int]
|
|
53
|
+
file: str
|
|
54
|
+
options: PipOptions
|
|
55
|
+
def __init__(self, file: _Optional[str] = ..., options: _Optional[_Union[PipOptions, _Mapping]] = ...) -> None: ...
|
|
56
|
+
|
|
57
|
+
class PythonWheels(_message.Message):
|
|
58
|
+
__slots__ = ["dir", "options"]
|
|
59
|
+
DIR_FIELD_NUMBER: _ClassVar[int]
|
|
60
|
+
OPTIONS_FIELD_NUMBER: _ClassVar[int]
|
|
61
|
+
dir: str
|
|
62
|
+
options: PipOptions
|
|
63
|
+
def __init__(self, dir: _Optional[str] = ..., options: _Optional[_Union[PipOptions, _Mapping]] = ...) -> None: ...
|
|
64
|
+
|
|
65
|
+
class UVProject(_message.Message):
|
|
66
|
+
__slots__ = ["pyproject", "uvlock", "options"]
|
|
67
|
+
PYPROJECT_FIELD_NUMBER: _ClassVar[int]
|
|
68
|
+
UVLOCK_FIELD_NUMBER: _ClassVar[int]
|
|
69
|
+
OPTIONS_FIELD_NUMBER: _ClassVar[int]
|
|
70
|
+
pyproject: str
|
|
71
|
+
uvlock: str
|
|
72
|
+
options: PipOptions
|
|
73
|
+
def __init__(self, pyproject: _Optional[str] = ..., uvlock: _Optional[str] = ..., options: _Optional[_Union[PipOptions, _Mapping]] = ...) -> None: ...
|
|
74
|
+
|
|
75
|
+
class Commands(_message.Message):
|
|
76
|
+
__slots__ = ["cmd"]
|
|
77
|
+
CMD_FIELD_NUMBER: _ClassVar[int]
|
|
78
|
+
cmd: _containers.RepeatedScalarFieldContainer[str]
|
|
79
|
+
def __init__(self, cmd: _Optional[_Iterable[str]] = ...) -> None: ...
|
|
80
|
+
|
|
81
|
+
class WorkDir(_message.Message):
|
|
82
|
+
__slots__ = ["workdir"]
|
|
83
|
+
WORKDIR_FIELD_NUMBER: _ClassVar[int]
|
|
84
|
+
workdir: str
|
|
85
|
+
def __init__(self, workdir: _Optional[str] = ...) -> None: ...
|
|
86
|
+
|
|
87
|
+
class CopyConfig(_message.Message):
|
|
88
|
+
__slots__ = ["src", "dst"]
|
|
89
|
+
SRC_FIELD_NUMBER: _ClassVar[int]
|
|
90
|
+
DST_FIELD_NUMBER: _ClassVar[int]
|
|
91
|
+
src: str
|
|
92
|
+
dst: str
|
|
93
|
+
def __init__(self, src: _Optional[str] = ..., dst: _Optional[str] = ...) -> None: ...
|
|
94
|
+
|
|
95
|
+
class Env(_message.Message):
|
|
96
|
+
__slots__ = ["env_variables"]
|
|
97
|
+
class EnvVariablesEntry(_message.Message):
|
|
98
|
+
__slots__ = ["key", "value"]
|
|
99
|
+
KEY_FIELD_NUMBER: _ClassVar[int]
|
|
100
|
+
VALUE_FIELD_NUMBER: _ClassVar[int]
|
|
101
|
+
key: str
|
|
102
|
+
value: str
|
|
103
|
+
def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
|
|
104
|
+
ENV_VARIABLES_FIELD_NUMBER: _ClassVar[int]
|
|
105
|
+
env_variables: _containers.ScalarMap[str, str]
|
|
106
|
+
def __init__(self, env_variables: _Optional[_Mapping[str, str]] = ...) -> None: ...
|
|
107
|
+
|
|
108
|
+
class Layer(_message.Message):
|
|
109
|
+
__slots__ = ["apt_packages", "pip_packages", "commands", "requirements", "python_wheels", "workdir", "copy_config", "uv_project", "env"]
|
|
110
|
+
APT_PACKAGES_FIELD_NUMBER: _ClassVar[int]
|
|
111
|
+
PIP_PACKAGES_FIELD_NUMBER: _ClassVar[int]
|
|
112
|
+
COMMANDS_FIELD_NUMBER: _ClassVar[int]
|
|
113
|
+
REQUIREMENTS_FIELD_NUMBER: _ClassVar[int]
|
|
114
|
+
PYTHON_WHEELS_FIELD_NUMBER: _ClassVar[int]
|
|
115
|
+
WORKDIR_FIELD_NUMBER: _ClassVar[int]
|
|
116
|
+
COPY_CONFIG_FIELD_NUMBER: _ClassVar[int]
|
|
117
|
+
UV_PROJECT_FIELD_NUMBER: _ClassVar[int]
|
|
118
|
+
ENV_FIELD_NUMBER: _ClassVar[int]
|
|
119
|
+
apt_packages: AptPackages
|
|
120
|
+
pip_packages: PipPackages
|
|
121
|
+
commands: Commands
|
|
122
|
+
requirements: Requirements
|
|
123
|
+
python_wheels: PythonWheels
|
|
124
|
+
workdir: WorkDir
|
|
125
|
+
copy_config: CopyConfig
|
|
126
|
+
uv_project: UVProject
|
|
127
|
+
env: Env
|
|
128
|
+
def __init__(self, apt_packages: _Optional[_Union[AptPackages, _Mapping]] = ..., pip_packages: _Optional[_Union[PipPackages, _Mapping]] = ..., commands: _Optional[_Union[Commands, _Mapping]] = ..., requirements: _Optional[_Union[Requirements, _Mapping]] = ..., python_wheels: _Optional[_Union[PythonWheels, _Mapping]] = ..., workdir: _Optional[_Union[WorkDir, _Mapping]] = ..., copy_config: _Optional[_Union[CopyConfig, _Mapping]] = ..., uv_project: _Optional[_Union[UVProject, _Mapping]] = ..., env: _Optional[_Union[Env, _Mapping]] = ...) -> None: ...
|
|
129
|
+
|
|
130
|
+
class ImageSpec(_message.Message):
|
|
131
|
+
__slots__ = ["base_image", "python_version", "layers", "platform"]
|
|
132
|
+
BASE_IMAGE_FIELD_NUMBER: _ClassVar[int]
|
|
133
|
+
PYTHON_VERSION_FIELD_NUMBER: _ClassVar[int]
|
|
134
|
+
LAYERS_FIELD_NUMBER: _ClassVar[int]
|
|
135
|
+
PLATFORM_FIELD_NUMBER: _ClassVar[int]
|
|
136
|
+
base_image: str
|
|
137
|
+
python_version: str
|
|
138
|
+
layers: _containers.RepeatedCompositeFieldContainer[Layer]
|
|
139
|
+
platform: _containers.RepeatedScalarFieldContainer[str]
|
|
140
|
+
def __init__(self, base_image: _Optional[str] = ..., python_version: _Optional[str] = ..., layers: _Optional[_Iterable[_Union[Layer, _Mapping]]] = ..., platform: _Optional[_Iterable[str]] = ...) -> None: ...
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
|
+
# source: imagebuilder/payload.proto
|
|
4
|
+
"""Generated protocol buffer code."""
|
|
5
|
+
from google.protobuf import descriptor as _descriptor
|
|
6
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
7
|
+
from google.protobuf import symbol_database as _symbol_database
|
|
8
|
+
from google.protobuf.internal import builder as _builder
|
|
9
|
+
# @@protoc_insertion_point(imports)
|
|
10
|
+
|
|
11
|
+
_sym_db = _symbol_database.Default()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
from flyte._protos.imagebuilder import definition_pb2 as imagebuilder_dot_definition__pb2
|
|
15
|
+
from flyte._protos.validate.validate import validate_pb2 as validate_dot_validate__pb2
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1aimagebuilder/payload.proto\x12\x15\x63loudidl.imagebuilder\x1a\x1dimagebuilder/definition.proto\x1a\x17validate/validate.proto\"w\n\x0fGetImageRequest\x12@\n\x02id\x18\x01 \x01(\x0b\x32&.cloudidl.imagebuilder.ImageIdentifierB\x08\xfa\x42\x05\x8a\x01\x02\x10\x01R\x02id\x12\"\n\x0corganization\x18\x02 \x01(\tR\x0corganization\"F\n\x10GetImageResponse\x12\x32\n\x05image\x18\x01 \x01(\x0b\x32\x1c.cloudidl.imagebuilder.ImageR\x05imageB\xd1\x01\n\x19\x63om.cloudidl.imagebuilderB\x0cPayloadProtoH\x02P\x01Z/github.com/unionai/cloud/gen/pb-go/imagebuilder\xa2\x02\x03\x43IX\xaa\x02\x15\x43loudidl.Imagebuilder\xca\x02\x15\x43loudidl\\Imagebuilder\xe2\x02!Cloudidl\\Imagebuilder\\GPBMetadata\xea\x02\x16\x43loudidl::Imagebuilderb\x06proto3')
|
|
19
|
+
|
|
20
|
+
_globals = globals()
|
|
21
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
22
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'imagebuilder.payload_pb2', _globals)
|
|
23
|
+
if _descriptor._USE_C_DESCRIPTORS == False:
|
|
24
|
+
DESCRIPTOR._options = None
|
|
25
|
+
DESCRIPTOR._serialized_options = b'\n\031com.cloudidl.imagebuilderB\014PayloadProtoH\002P\001Z/github.com/unionai/cloud/gen/pb-go/imagebuilder\242\002\003CIX\252\002\025Cloudidl.Imagebuilder\312\002\025Cloudidl\\Imagebuilder\342\002!Cloudidl\\Imagebuilder\\GPBMetadata\352\002\026Cloudidl::Imagebuilder'
|
|
26
|
+
_GETIMAGEREQUEST.fields_by_name['id']._options = None
|
|
27
|
+
_GETIMAGEREQUEST.fields_by_name['id']._serialized_options = b'\372B\005\212\001\002\020\001'
|
|
28
|
+
_globals['_GETIMAGEREQUEST']._serialized_start=109
|
|
29
|
+
_globals['_GETIMAGEREQUEST']._serialized_end=228
|
|
30
|
+
_globals['_GETIMAGERESPONSE']._serialized_start=230
|
|
31
|
+
_globals['_GETIMAGERESPONSE']._serialized_end=300
|
|
32
|
+
# @@protoc_insertion_point(module_scope)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from flyte._protos.imagebuilder import definition_pb2 as _definition_pb2
|
|
2
|
+
from flyte._protos.validate.validate import validate_pb2 as _validate_pb2
|
|
3
|
+
from google.protobuf import descriptor as _descriptor
|
|
4
|
+
from google.protobuf import message as _message
|
|
5
|
+
from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union
|
|
6
|
+
|
|
7
|
+
DESCRIPTOR: _descriptor.FileDescriptor
|
|
8
|
+
|
|
9
|
+
class GetImageRequest(_message.Message):
|
|
10
|
+
__slots__ = ["id", "organization"]
|
|
11
|
+
ID_FIELD_NUMBER: _ClassVar[int]
|
|
12
|
+
ORGANIZATION_FIELD_NUMBER: _ClassVar[int]
|
|
13
|
+
id: _definition_pb2.ImageIdentifier
|
|
14
|
+
organization: str
|
|
15
|
+
def __init__(self, id: _Optional[_Union[_definition_pb2.ImageIdentifier, _Mapping]] = ..., organization: _Optional[str] = ...) -> None: ...
|
|
16
|
+
|
|
17
|
+
class GetImageResponse(_message.Message):
|
|
18
|
+
__slots__ = ["image"]
|
|
19
|
+
IMAGE_FIELD_NUMBER: _ClassVar[int]
|
|
20
|
+
image: _definition_pb2.Image
|
|
21
|
+
def __init__(self, image: _Optional[_Union[_definition_pb2.Image, _Mapping]] = ...) -> None: ...
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
|
3
|
+
# source: imagebuilder/service.proto
|
|
4
|
+
"""Generated protocol buffer code."""
|
|
5
|
+
from google.protobuf import descriptor as _descriptor
|
|
6
|
+
from google.protobuf import descriptor_pool as _descriptor_pool
|
|
7
|
+
from google.protobuf import symbol_database as _symbol_database
|
|
8
|
+
from google.protobuf.internal import builder as _builder
|
|
9
|
+
# @@protoc_insertion_point(imports)
|
|
10
|
+
|
|
11
|
+
_sym_db = _symbol_database.Default()
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
from flyte._protos.imagebuilder import payload_pb2 as imagebuilder_dot_payload__pb2
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1aimagebuilder/service.proto\x12\x15\x63loudidl.imagebuilder\x1a\x1aimagebuilder/payload.proto2p\n\x0cImageService\x12`\n\x08GetImage\x12&.cloudidl.imagebuilder.GetImageRequest\x1a\'.cloudidl.imagebuilder.GetImageResponse\"\x03\x90\x02\x01\x42\xd1\x01\n\x19\x63om.cloudidl.imagebuilderB\x0cServiceProtoH\x02P\x01Z/github.com/unionai/cloud/gen/pb-go/imagebuilder\xa2\x02\x03\x43IX\xaa\x02\x15\x43loudidl.Imagebuilder\xca\x02\x15\x43loudidl\\Imagebuilder\xe2\x02!Cloudidl\\Imagebuilder\\GPBMetadata\xea\x02\x16\x43loudidl::Imagebuilderb\x06proto3')
|
|
18
|
+
|
|
19
|
+
_globals = globals()
|
|
20
|
+
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
|
21
|
+
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'imagebuilder.service_pb2', _globals)
|
|
22
|
+
if _descriptor._USE_C_DESCRIPTORS == False:
|
|
23
|
+
DESCRIPTOR._options = None
|
|
24
|
+
DESCRIPTOR._serialized_options = b'\n\031com.cloudidl.imagebuilderB\014ServiceProtoH\002P\001Z/github.com/unionai/cloud/gen/pb-go/imagebuilder\242\002\003CIX\252\002\025Cloudidl.Imagebuilder\312\002\025Cloudidl\\Imagebuilder\342\002!Cloudidl\\Imagebuilder\\GPBMetadata\352\002\026Cloudidl::Imagebuilder'
|
|
25
|
+
_IMAGESERVICE.methods_by_name['GetImage']._options = None
|
|
26
|
+
_IMAGESERVICE.methods_by_name['GetImage']._serialized_options = b'\220\002\001'
|
|
27
|
+
_globals['_IMAGESERVICE']._serialized_start=81
|
|
28
|
+
_globals['_IMAGESERVICE']._serialized_end=193
|
|
29
|
+
# @@protoc_insertion_point(module_scope)
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
|
2
|
+
"""Client and server classes corresponding to protobuf-defined services."""
|
|
3
|
+
import grpc
|
|
4
|
+
|
|
5
|
+
from flyte._protos.imagebuilder import payload_pb2 as imagebuilder_dot_payload__pb2
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ImageServiceStub(object):
|
|
9
|
+
"""Missing associated documentation comment in .proto file."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, channel):
|
|
12
|
+
"""Constructor.
|
|
13
|
+
|
|
14
|
+
Args:
|
|
15
|
+
channel: A grpc.Channel.
|
|
16
|
+
"""
|
|
17
|
+
self.GetImage = channel.unary_unary(
|
|
18
|
+
'/cloudidl.imagebuilder.ImageService/GetImage',
|
|
19
|
+
request_serializer=imagebuilder_dot_payload__pb2.GetImageRequest.SerializeToString,
|
|
20
|
+
response_deserializer=imagebuilder_dot_payload__pb2.GetImageResponse.FromString,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class ImageServiceServicer(object):
|
|
25
|
+
"""Missing associated documentation comment in .proto file."""
|
|
26
|
+
|
|
27
|
+
def GetImage(self, request, context):
|
|
28
|
+
"""Missing associated documentation comment in .proto file."""
|
|
29
|
+
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
|
30
|
+
context.set_details('Method not implemented!')
|
|
31
|
+
raise NotImplementedError('Method not implemented!')
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def add_ImageServiceServicer_to_server(servicer, server):
|
|
35
|
+
rpc_method_handlers = {
|
|
36
|
+
'GetImage': grpc.unary_unary_rpc_method_handler(
|
|
37
|
+
servicer.GetImage,
|
|
38
|
+
request_deserializer=imagebuilder_dot_payload__pb2.GetImageRequest.FromString,
|
|
39
|
+
response_serializer=imagebuilder_dot_payload__pb2.GetImageResponse.SerializeToString,
|
|
40
|
+
),
|
|
41
|
+
}
|
|
42
|
+
generic_handler = grpc.method_handlers_generic_handler(
|
|
43
|
+
'cloudidl.imagebuilder.ImageService', rpc_method_handlers)
|
|
44
|
+
server.add_generic_rpc_handlers((generic_handler,))
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
# This class is part of an EXPERIMENTAL API.
|
|
48
|
+
class ImageService(object):
|
|
49
|
+
"""Missing associated documentation comment in .proto file."""
|
|
50
|
+
|
|
51
|
+
@staticmethod
|
|
52
|
+
def GetImage(request,
|
|
53
|
+
target,
|
|
54
|
+
options=(),
|
|
55
|
+
channel_credentials=None,
|
|
56
|
+
call_credentials=None,
|
|
57
|
+
insecure=False,
|
|
58
|
+
compression=None,
|
|
59
|
+
wait_for_ready=None,
|
|
60
|
+
timeout=None,
|
|
61
|
+
metadata=None):
|
|
62
|
+
return grpc.experimental.unary_unary(request, target, '/cloudidl.imagebuilder.ImageService/GetImage',
|
|
63
|
+
imagebuilder_dot_payload__pb2.GetImageRequest.SerializeToString,
|
|
64
|
+
imagebuilder_dot_payload__pb2.GetImageResponse.FromString,
|
|
65
|
+
options, channel_credentials,
|
|
66
|
+
insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
|
flyte/_run.py
CHANGED
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import pathlib
|
|
5
5
|
import uuid
|
|
6
|
-
from typing import TYPE_CHECKING, Any, Literal, Optional, Tuple, Union, cast
|
|
6
|
+
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
|
|
7
7
|
|
|
8
8
|
import flyte.errors
|
|
9
9
|
from flyte.errors import InitializationError
|
|
@@ -43,7 +43,7 @@ async def _get_code_bundle_for_run(name: str) -> CodeBundle | None:
|
|
|
43
43
|
|
|
44
44
|
run = await Run.get.aio(name=name)
|
|
45
45
|
if run:
|
|
46
|
-
run_details = await run.details()
|
|
46
|
+
run_details = await run.details.aio()
|
|
47
47
|
spec = run_details.action_details.pb2.resolved_task_spec
|
|
48
48
|
return extract_code_bundle(spec)
|
|
49
49
|
return None
|
|
@@ -64,6 +64,13 @@ class _Runner:
|
|
|
64
64
|
metadata_path: str | None = None,
|
|
65
65
|
run_base_dir: str | None = None,
|
|
66
66
|
overwrite_cache: bool = False,
|
|
67
|
+
project: str | None = None,
|
|
68
|
+
domain: str | None = None,
|
|
69
|
+
env: Dict[str, str] | None = None,
|
|
70
|
+
labels: Dict[str, str] | None = None,
|
|
71
|
+
annotations: Dict[str, str] | None = None,
|
|
72
|
+
interruptible: bool = False,
|
|
73
|
+
log_level: int | None = None,
|
|
67
74
|
):
|
|
68
75
|
init_config = _get_init_config()
|
|
69
76
|
client = init_config.client if init_config else None
|
|
@@ -83,10 +90,19 @@ class _Runner:
|
|
|
83
90
|
self._metadata_path = metadata_path or "/tmp"
|
|
84
91
|
self._run_base_dir = run_base_dir or "/tmp/base"
|
|
85
92
|
self._overwrite_cache = overwrite_cache
|
|
93
|
+
self._project = project
|
|
94
|
+
self._domain = domain
|
|
95
|
+
self._env = env
|
|
96
|
+
self._labels = labels
|
|
97
|
+
self._annotations = annotations
|
|
98
|
+
self._interruptible = interruptible
|
|
99
|
+
self._log_level = log_level
|
|
86
100
|
|
|
87
101
|
@requires_initialization
|
|
88
102
|
async def _run_remote(self, obj: TaskTemplate[P, R] | LazyEntity, *args: P.args, **kwargs: P.kwargs) -> Run:
|
|
89
103
|
import grpc
|
|
104
|
+
from flyteidl.core import literals_pb2
|
|
105
|
+
from google.protobuf import wrappers_pb2
|
|
90
106
|
|
|
91
107
|
from flyte.remote import Run
|
|
92
108
|
from flyte.remote._task import LazyEntity
|
|
@@ -99,6 +115,8 @@ class _Runner:
|
|
|
99
115
|
from ._protos.workflow import run_definition_pb2, run_service_pb2
|
|
100
116
|
|
|
101
117
|
cfg = get_common_config()
|
|
118
|
+
project = self._project or cfg.project
|
|
119
|
+
domain = self._domain or cfg.domain
|
|
102
120
|
|
|
103
121
|
if isinstance(obj, LazyEntity):
|
|
104
122
|
task = await obj.fetch.aio()
|
|
@@ -141,6 +159,12 @@ class _Runner:
|
|
|
141
159
|
task_spec = translate_task_to_wire(obj, s_ctx)
|
|
142
160
|
inputs = await convert_from_native_to_inputs(obj.native_interface, *args, **kwargs)
|
|
143
161
|
|
|
162
|
+
env = self._env or {}
|
|
163
|
+
if self._log_level:
|
|
164
|
+
env["LOG_LEVEL"] = str(self._log_level)
|
|
165
|
+
else:
|
|
166
|
+
env["LOG_LEVEL"] = str(logger.getEffectiveLevel())
|
|
167
|
+
|
|
144
168
|
if not self._dry_run:
|
|
145
169
|
if get_client() is None:
|
|
146
170
|
# This can only happen, if the user forces flyte.run(mode="remote") without initializing the client
|
|
@@ -154,28 +178,38 @@ class _Runner:
|
|
|
154
178
|
project_id = None
|
|
155
179
|
if self._name:
|
|
156
180
|
run_id = run_definition_pb2.RunIdentifier(
|
|
157
|
-
project=
|
|
158
|
-
domain=
|
|
181
|
+
project=project,
|
|
182
|
+
domain=domain,
|
|
159
183
|
org=cfg.org,
|
|
160
184
|
name=self._name if self._name else None,
|
|
161
185
|
)
|
|
162
186
|
else:
|
|
163
187
|
project_id = identifier_pb2.ProjectIdentifier(
|
|
164
|
-
name=
|
|
165
|
-
domain=
|
|
188
|
+
name=project,
|
|
189
|
+
domain=domain,
|
|
166
190
|
organization=cfg.org,
|
|
167
191
|
)
|
|
168
192
|
# Fill in task id inside the task template if it's not provided.
|
|
169
193
|
# Maybe this should be done here, or the backend.
|
|
170
194
|
if task_spec.task_template.id.project == "":
|
|
171
|
-
task_spec.task_template.id.project =
|
|
195
|
+
task_spec.task_template.id.project = project if project else ""
|
|
172
196
|
if task_spec.task_template.id.domain == "":
|
|
173
|
-
task_spec.task_template.id.domain =
|
|
197
|
+
task_spec.task_template.id.domain = domain if domain else ""
|
|
174
198
|
if task_spec.task_template.id.org == "":
|
|
175
199
|
task_spec.task_template.id.org = cfg.org if cfg.org else ""
|
|
176
200
|
if task_spec.task_template.id.version == "":
|
|
177
201
|
task_spec.task_template.id.version = version
|
|
178
202
|
|
|
203
|
+
kv_pairs: List[literals_pb2.KeyValuePair] = []
|
|
204
|
+
for k, v in env.items():
|
|
205
|
+
if not isinstance(v, str):
|
|
206
|
+
raise ValueError(f"Environment variable {k} must be a string, got {type(v)}")
|
|
207
|
+
kv_pairs.append(literals_pb2.KeyValuePair(key=k, value=v))
|
|
208
|
+
|
|
209
|
+
env_kv = run_definition_pb2.Envs(values=kv_pairs)
|
|
210
|
+
annotations = run_definition_pb2.Annotations(values=self._annotations)
|
|
211
|
+
labels = run_definition_pb2.Labels(values=self._labels)
|
|
212
|
+
|
|
179
213
|
try:
|
|
180
214
|
resp = await get_client().run_service.CreateRun(
|
|
181
215
|
run_service_pb2.CreateRunRequest(
|
|
@@ -185,6 +219,10 @@ class _Runner:
|
|
|
185
219
|
inputs=inputs.proto_inputs,
|
|
186
220
|
run_spec=run_definition_pb2.RunSpec(
|
|
187
221
|
overwrite_cache=self._overwrite_cache,
|
|
222
|
+
interruptible=wrappers_pb2.BoolValue(value=self._interruptible),
|
|
223
|
+
annotations=annotations,
|
|
224
|
+
labels=labels,
|
|
225
|
+
envs=env_kv,
|
|
188
226
|
),
|
|
189
227
|
),
|
|
190
228
|
)
|
|
@@ -418,6 +456,13 @@ def with_runcontext(
|
|
|
418
456
|
raw_data_path: str | None = None,
|
|
419
457
|
run_base_dir: str | None = None,
|
|
420
458
|
overwrite_cache: bool = False,
|
|
459
|
+
project: str | None = None,
|
|
460
|
+
domain: str | None = None,
|
|
461
|
+
env: Dict[str, str] | None = None,
|
|
462
|
+
labels: Dict[str, str] | None = None,
|
|
463
|
+
annotations: Dict[str, str] | None = None,
|
|
464
|
+
interruptible: bool = False,
|
|
465
|
+
log_level: int | None = None,
|
|
421
466
|
) -> _Runner:
|
|
422
467
|
"""
|
|
423
468
|
Launch a new run with the given parameters as the context.
|
|
@@ -449,6 +494,16 @@ def with_runcontext(
|
|
|
449
494
|
and can be used to store raw data in specific locations. TODO coming soon for remote runs as well.
|
|
450
495
|
:param run_base_dir: Optional The base directory to use for the run. This is used to store the metadata for the run,
|
|
451
496
|
that is passed between tasks.
|
|
497
|
+
:param overwrite_cache: Optional If true, the cache will be overwritten for the run
|
|
498
|
+
:param project: Optional The project to use for the run
|
|
499
|
+
:param domain: Optional The domain to use for the run
|
|
500
|
+
:param env: Optional Environment variables to set for the run
|
|
501
|
+
:param labels: Optional Labels to set for the run
|
|
502
|
+
:param annotations: Optional Annotations to set for the run
|
|
503
|
+
:param interruptible: Optional If true, the run can be interrupted by the user.
|
|
504
|
+
:param log_level: Optional Log level to set for the run. If not provided, it will be set to the default log level
|
|
505
|
+
set using `flyte.init()`
|
|
506
|
+
|
|
452
507
|
:return: runner
|
|
453
508
|
"""
|
|
454
509
|
if mode == "hybrid" and not name and not run_base_dir:
|
|
@@ -464,6 +519,14 @@ def with_runcontext(
|
|
|
464
519
|
interactive_mode=interactive_mode,
|
|
465
520
|
raw_data_path=raw_data_path,
|
|
466
521
|
run_base_dir=run_base_dir,
|
|
522
|
+
overwrite_cache=overwrite_cache,
|
|
523
|
+
env=env,
|
|
524
|
+
labels=labels,
|
|
525
|
+
annotations=annotations,
|
|
526
|
+
interruptible=interruptible,
|
|
527
|
+
project=project,
|
|
528
|
+
domain=domain,
|
|
529
|
+
log_level=log_level,
|
|
467
530
|
)
|
|
468
531
|
|
|
469
532
|
|
flyte/_task_environment.py
CHANGED
|
@@ -180,4 +180,5 @@ class TaskEnvironment(Environment):
|
|
|
180
180
|
if task.name in self._tasks:
|
|
181
181
|
raise ValueError(f"Task {task.name} already exists in the environment. Task names should be unique.")
|
|
182
182
|
self._tasks[task.name] = task
|
|
183
|
+
task.parent_env = weakref.ref(self)
|
|
183
184
|
return task
|
flyte/_version.py
CHANGED
|
@@ -17,5 +17,5 @@ __version__: str
|
|
|
17
17
|
__version_tuple__: VERSION_TUPLE
|
|
18
18
|
version_tuple: VERSION_TUPLE
|
|
19
19
|
|
|
20
|
-
__version__ = version = '0.2.
|
|
21
|
-
__version_tuple__ = version_tuple = (0, 2, 0, '
|
|
20
|
+
__version__ = version = '0.2.0b19'
|
|
21
|
+
__version_tuple__ = version_tuple = (0, 2, 0, 'b19')
|
flyte/cli/__init__.py
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
import os
|
|
2
|
+
|
|
1
3
|
from flyte.cli.main import main
|
|
2
4
|
|
|
3
5
|
__all__ = ["main"]
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# Set GRPC_VERBOSITY to NONE if not already set to silence unwanted output
|
|
9
|
+
# This addresses the issue with grpcio >=1.68.0 causing unwanted output
|
|
10
|
+
# https://github.com/flyteorg/flyte/issues/6082
|
|
11
|
+
if "GRPC_VERBOSITY" not in os.environ:
|
|
12
|
+
os.environ["GRPC_VERBOSITY"] = "NONE"
|
flyte/cli/_build.py
CHANGED
|
@@ -54,7 +54,7 @@ class BuildEnvCommand(click.Command):
|
|
|
54
54
|
with console.status("Building...", spinner="dots"):
|
|
55
55
|
image_cache = flyte.build_images(self.obj)
|
|
56
56
|
|
|
57
|
-
console.print(common.get_table("Images", image_cache.repr()))
|
|
57
|
+
console.print(common.get_table("Images", image_cache.repr(), simple=obj.simple))
|
|
58
58
|
|
|
59
59
|
|
|
60
60
|
class EnvPerFileGroup(common.ObjectsPerFileGroup):
|
flyte/cli/_common.py
CHANGED
|
@@ -24,7 +24,6 @@ from flyte.config import Config
|
|
|
24
24
|
PREFERRED_BORDER_COLOR = "dim cyan"
|
|
25
25
|
PREFERRED_ACCENT_COLOR = "bold #FFD700"
|
|
26
26
|
HEADER_STYLE = f"{PREFERRED_ACCENT_COLOR} on black"
|
|
27
|
-
PANELS = False
|
|
28
27
|
|
|
29
28
|
PROJECT_OPTION = click.Option(
|
|
30
29
|
param_decls=["-p", "--project"],
|
|
@@ -78,6 +77,7 @@ class CLIConfig:
|
|
|
78
77
|
endpoint: str | None = None
|
|
79
78
|
insecure: bool = False
|
|
80
79
|
org: str | None = None
|
|
80
|
+
simple: bool = False
|
|
81
81
|
|
|
82
82
|
def replace(self, **kwargs) -> CLIConfig:
|
|
83
83
|
"""
|
|
@@ -296,17 +296,20 @@ class FileGroup(GroupBase):
|
|
|
296
296
|
raise NotImplementedError
|
|
297
297
|
|
|
298
298
|
|
|
299
|
-
def get_table(title: str, vals: Iterable[Any]) -> Table:
|
|
299
|
+
def get_table(title: str, vals: Iterable[Any], simple: bool = False) -> Table:
|
|
300
300
|
"""
|
|
301
301
|
Get a table from a list of values.
|
|
302
302
|
"""
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
303
|
+
if simple:
|
|
304
|
+
table = Table(title, box=None)
|
|
305
|
+
else:
|
|
306
|
+
table = Table(
|
|
307
|
+
title=title,
|
|
308
|
+
box=rich.box.SQUARE_DOUBLE_HEAD,
|
|
309
|
+
header_style=HEADER_STYLE,
|
|
310
|
+
show_header=True,
|
|
311
|
+
border_style=PREFERRED_BORDER_COLOR,
|
|
312
|
+
)
|
|
310
313
|
headers = None
|
|
311
314
|
has_rich_repr = False
|
|
312
315
|
for p in vals:
|
|
@@ -323,11 +326,11 @@ def get_table(title: str, vals: Iterable[Any]) -> Table:
|
|
|
323
326
|
return table
|
|
324
327
|
|
|
325
328
|
|
|
326
|
-
def get_panel(title: str, renderable: Any) -> Panel:
|
|
329
|
+
def get_panel(title: str, renderable: Any, simple: bool = False) -> Panel:
|
|
327
330
|
"""
|
|
328
331
|
Get a panel from a list of values.
|
|
329
332
|
"""
|
|
330
|
-
if
|
|
333
|
+
if simple:
|
|
331
334
|
return renderable
|
|
332
335
|
return Panel.fit(
|
|
333
336
|
renderable,
|
flyte/cli/_create.py
CHANGED
|
@@ -87,6 +87,14 @@ def secret(
|
|
|
87
87
|
help="Force overwrite of the configuration file if it already exists.",
|
|
88
88
|
show_default=True,
|
|
89
89
|
)
|
|
90
|
+
@click.option(
|
|
91
|
+
"--image-builder",
|
|
92
|
+
"--builder",
|
|
93
|
+
type=click.Choice(["local", "remote"]),
|
|
94
|
+
default="local",
|
|
95
|
+
help="Image builder to use for building images. Defaults to 'local'.",
|
|
96
|
+
show_default=True,
|
|
97
|
+
)
|
|
90
98
|
def config(
|
|
91
99
|
output: str,
|
|
92
100
|
endpoint: str | None = None,
|
|
@@ -95,6 +103,7 @@ def config(
|
|
|
95
103
|
project: str | None = None,
|
|
96
104
|
domain: str | None = None,
|
|
97
105
|
force: bool = False,
|
|
106
|
+
image_builder: str | None = None,
|
|
98
107
|
):
|
|
99
108
|
"""
|
|
100
109
|
Creates a configuration file for Flyte CLI.
|
|
@@ -131,6 +140,10 @@ def config(
|
|
|
131
140
|
if domain:
|
|
132
141
|
task["domain"] = domain
|
|
133
142
|
|
|
143
|
+
image: Dict[str, str] = {}
|
|
144
|
+
if image_builder:
|
|
145
|
+
image["builder"] = image_builder
|
|
146
|
+
|
|
134
147
|
if not admin and not task:
|
|
135
148
|
raise click.BadParameter("At least one of --endpoint or --org must be provided.")
|
|
136
149
|
|
|
@@ -140,6 +153,8 @@ def config(
|
|
|
140
153
|
d["admin"] = admin
|
|
141
154
|
if task:
|
|
142
155
|
d["task"] = task
|
|
156
|
+
if image:
|
|
157
|
+
d["image"] = image
|
|
143
158
|
yaml.dump(d, f)
|
|
144
159
|
|
|
145
160
|
click.echo(f"Config file written to {output_path}")
|
flyte/cli/_deploy.py
CHANGED
|
@@ -78,8 +78,8 @@ class DeployEnvCommand(click.Command):
|
|
|
78
78
|
version=self.deploy_args.version,
|
|
79
79
|
)
|
|
80
80
|
|
|
81
|
-
console.print(common.get_table("Environments", deployment.env_repr()))
|
|
82
|
-
console.print(common.get_table("Tasks", deployment.task_repr()))
|
|
81
|
+
console.print(common.get_table("Environments", deployment.env_repr(), simple=obj.simple))
|
|
82
|
+
console.print(common.get_table("Tasks", deployment.task_repr(), simple=obj.simple))
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
class EnvPerFileGroup(common.ObjectsPerFileGroup):
|