indexify 0.3.5__py3-none-any.whl → 0.3.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indexify/cli/cli.py +2 -2
- indexify/executor/downloader.py +2 -3
- indexify/executor/executor.py +2 -3
- indexify/executor/function_executor/function_executor.py +3 -4
- indexify/executor/function_executor/health_checker.py +2 -3
- indexify/executor/function_executor/invocation_state_client.py +3 -4
- indexify/executor/function_executor/single_task_runner.py +2 -4
- indexify/executor/function_executor/task_input.py +1 -1
- indexify/executor/function_executor/task_output.py +1 -1
- indexify/executor/task_fetcher.py +1 -2
- indexify/executor/task_reporter.py +3 -4
- {indexify-0.3.5.dist-info → indexify-0.3.7.dist-info}/METADATA +3 -4
- indexify-0.3.7.dist-info/RECORD +25 -0
- indexify-0.3.7.dist-info/entry_points.txt +3 -0
- indexify/function_executor/README.md +0 -18
- indexify/function_executor/handlers/run_function/function_inputs_loader.py +0 -53
- indexify/function_executor/handlers/run_function/handler.py +0 -126
- indexify/function_executor/handlers/run_function/request_validator.py +0 -26
- indexify/function_executor/handlers/run_function/response_helper.py +0 -96
- indexify/function_executor/info.py +0 -16
- indexify/function_executor/initialize_request_validator.py +0 -21
- indexify/function_executor/invocation_state/invocation_state_proxy_server.py +0 -170
- indexify/function_executor/invocation_state/proxied_invocation_state.py +0 -22
- indexify/function_executor/invocation_state/response_validator.py +0 -29
- indexify/function_executor/main.py +0 -51
- indexify/function_executor/proto/function_executor.proto +0 -148
- indexify/function_executor/proto/function_executor_pb2.py +0 -73
- indexify/function_executor/proto/function_executor_pb2.pyi +0 -247
- indexify/function_executor/proto/function_executor_pb2_grpc.py +0 -307
- indexify/function_executor/proto/message_validator.py +0 -38
- indexify/function_executor/proto/server_configuration.py +0 -19
- indexify/function_executor/server.py +0 -29
- indexify/function_executor/service.py +0 -145
- indexify/utils/README.md +0 -3
- indexify/utils/http_client.py +0 -88
- indexify/utils/logging.py +0 -66
- indexify-0.3.5.dist-info/RECORD +0 -47
- indexify-0.3.5.dist-info/entry_points.txt +0 -4
- {indexify-0.3.5.dist-info → indexify-0.3.7.dist-info}/WHEEL +0 -0
@@ -1,247 +0,0 @@
|
|
1
|
-
from typing import ClassVar as _ClassVar
|
2
|
-
from typing import Iterable as _Iterable
|
3
|
-
from typing import Mapping as _Mapping
|
4
|
-
from typing import Optional as _Optional
|
5
|
-
from typing import Union as _Union
|
6
|
-
|
7
|
-
from google.protobuf import descriptor as _descriptor
|
8
|
-
from google.protobuf import message as _message
|
9
|
-
from google.protobuf.internal import containers as _containers
|
10
|
-
|
11
|
-
DESCRIPTOR: _descriptor.FileDescriptor
|
12
|
-
|
13
|
-
class SerializedObject(_message.Message):
|
14
|
-
__slots__ = ("bytes", "string", "content_type")
|
15
|
-
BYTES_FIELD_NUMBER: _ClassVar[int]
|
16
|
-
STRING_FIELD_NUMBER: _ClassVar[int]
|
17
|
-
CONTENT_TYPE_FIELD_NUMBER: _ClassVar[int]
|
18
|
-
bytes: bytes
|
19
|
-
string: str
|
20
|
-
content_type: str
|
21
|
-
def __init__(
|
22
|
-
self,
|
23
|
-
bytes: _Optional[bytes] = ...,
|
24
|
-
string: _Optional[str] = ...,
|
25
|
-
content_type: _Optional[str] = ...,
|
26
|
-
) -> None: ...
|
27
|
-
|
28
|
-
class InitializeRequest(_message.Message):
|
29
|
-
__slots__ = ("namespace", "graph_name", "graph_version", "function_name", "graph")
|
30
|
-
NAMESPACE_FIELD_NUMBER: _ClassVar[int]
|
31
|
-
GRAPH_NAME_FIELD_NUMBER: _ClassVar[int]
|
32
|
-
GRAPH_VERSION_FIELD_NUMBER: _ClassVar[int]
|
33
|
-
FUNCTION_NAME_FIELD_NUMBER: _ClassVar[int]
|
34
|
-
GRAPH_FIELD_NUMBER: _ClassVar[int]
|
35
|
-
namespace: str
|
36
|
-
graph_name: str
|
37
|
-
graph_version: str
|
38
|
-
function_name: str
|
39
|
-
graph: SerializedObject
|
40
|
-
def __init__(
|
41
|
-
self,
|
42
|
-
namespace: _Optional[str] = ...,
|
43
|
-
graph_name: _Optional[str] = ...,
|
44
|
-
graph_version: _Optional[str] = ...,
|
45
|
-
function_name: _Optional[str] = ...,
|
46
|
-
graph: _Optional[_Union[SerializedObject, _Mapping]] = ...,
|
47
|
-
) -> None: ...
|
48
|
-
|
49
|
-
class InitializeResponse(_message.Message):
|
50
|
-
__slots__ = ("success", "customer_error")
|
51
|
-
SUCCESS_FIELD_NUMBER: _ClassVar[int]
|
52
|
-
CUSTOMER_ERROR_FIELD_NUMBER: _ClassVar[int]
|
53
|
-
success: bool
|
54
|
-
customer_error: str
|
55
|
-
def __init__(
|
56
|
-
self, success: bool = ..., customer_error: _Optional[str] = ...
|
57
|
-
) -> None: ...
|
58
|
-
|
59
|
-
class SetInvocationStateRequest(_message.Message):
|
60
|
-
__slots__ = ("key", "value")
|
61
|
-
KEY_FIELD_NUMBER: _ClassVar[int]
|
62
|
-
VALUE_FIELD_NUMBER: _ClassVar[int]
|
63
|
-
key: str
|
64
|
-
value: SerializedObject
|
65
|
-
def __init__(
|
66
|
-
self,
|
67
|
-
key: _Optional[str] = ...,
|
68
|
-
value: _Optional[_Union[SerializedObject, _Mapping]] = ...,
|
69
|
-
) -> None: ...
|
70
|
-
|
71
|
-
class SetInvocationStateResponse(_message.Message):
|
72
|
-
__slots__ = ()
|
73
|
-
def __init__(self) -> None: ...
|
74
|
-
|
75
|
-
class GetInvocationStateRequest(_message.Message):
|
76
|
-
__slots__ = ("key",)
|
77
|
-
KEY_FIELD_NUMBER: _ClassVar[int]
|
78
|
-
key: str
|
79
|
-
def __init__(self, key: _Optional[str] = ...) -> None: ...
|
80
|
-
|
81
|
-
class GetInvocationStateResponse(_message.Message):
|
82
|
-
__slots__ = ("key", "value")
|
83
|
-
KEY_FIELD_NUMBER: _ClassVar[int]
|
84
|
-
VALUE_FIELD_NUMBER: _ClassVar[int]
|
85
|
-
key: str
|
86
|
-
value: SerializedObject
|
87
|
-
def __init__(
|
88
|
-
self,
|
89
|
-
key: _Optional[str] = ...,
|
90
|
-
value: _Optional[_Union[SerializedObject, _Mapping]] = ...,
|
91
|
-
) -> None: ...
|
92
|
-
|
93
|
-
class InvocationStateRequest(_message.Message):
|
94
|
-
__slots__ = ("request_id", "task_id", "set", "get")
|
95
|
-
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
|
96
|
-
TASK_ID_FIELD_NUMBER: _ClassVar[int]
|
97
|
-
SET_FIELD_NUMBER: _ClassVar[int]
|
98
|
-
GET_FIELD_NUMBER: _ClassVar[int]
|
99
|
-
request_id: str
|
100
|
-
task_id: str
|
101
|
-
set: SetInvocationStateRequest
|
102
|
-
get: GetInvocationStateRequest
|
103
|
-
def __init__(
|
104
|
-
self,
|
105
|
-
request_id: _Optional[str] = ...,
|
106
|
-
task_id: _Optional[str] = ...,
|
107
|
-
set: _Optional[_Union[SetInvocationStateRequest, _Mapping]] = ...,
|
108
|
-
get: _Optional[_Union[GetInvocationStateRequest, _Mapping]] = ...,
|
109
|
-
) -> None: ...
|
110
|
-
|
111
|
-
class InvocationStateResponse(_message.Message):
|
112
|
-
__slots__ = ("request_id", "success", "set", "get")
|
113
|
-
REQUEST_ID_FIELD_NUMBER: _ClassVar[int]
|
114
|
-
SUCCESS_FIELD_NUMBER: _ClassVar[int]
|
115
|
-
SET_FIELD_NUMBER: _ClassVar[int]
|
116
|
-
GET_FIELD_NUMBER: _ClassVar[int]
|
117
|
-
request_id: str
|
118
|
-
success: bool
|
119
|
-
set: SetInvocationStateResponse
|
120
|
-
get: GetInvocationStateResponse
|
121
|
-
def __init__(
|
122
|
-
self,
|
123
|
-
request_id: _Optional[str] = ...,
|
124
|
-
success: bool = ...,
|
125
|
-
set: _Optional[_Union[SetInvocationStateResponse, _Mapping]] = ...,
|
126
|
-
get: _Optional[_Union[GetInvocationStateResponse, _Mapping]] = ...,
|
127
|
-
) -> None: ...
|
128
|
-
|
129
|
-
class FunctionOutput(_message.Message):
|
130
|
-
__slots__ = ("outputs",)
|
131
|
-
OUTPUTS_FIELD_NUMBER: _ClassVar[int]
|
132
|
-
outputs: _containers.RepeatedCompositeFieldContainer[SerializedObject]
|
133
|
-
def __init__(
|
134
|
-
self, outputs: _Optional[_Iterable[_Union[SerializedObject, _Mapping]]] = ...
|
135
|
-
) -> None: ...
|
136
|
-
|
137
|
-
class RouterOutput(_message.Message):
|
138
|
-
__slots__ = ("edges",)
|
139
|
-
EDGES_FIELD_NUMBER: _ClassVar[int]
|
140
|
-
edges: _containers.RepeatedScalarFieldContainer[str]
|
141
|
-
def __init__(self, edges: _Optional[_Iterable[str]] = ...) -> None: ...
|
142
|
-
|
143
|
-
class RunTaskRequest(_message.Message):
|
144
|
-
__slots__ = (
|
145
|
-
"namespace",
|
146
|
-
"graph_name",
|
147
|
-
"graph_version",
|
148
|
-
"function_name",
|
149
|
-
"graph_invocation_id",
|
150
|
-
"task_id",
|
151
|
-
"function_input",
|
152
|
-
"function_init_value",
|
153
|
-
)
|
154
|
-
NAMESPACE_FIELD_NUMBER: _ClassVar[int]
|
155
|
-
GRAPH_NAME_FIELD_NUMBER: _ClassVar[int]
|
156
|
-
GRAPH_VERSION_FIELD_NUMBER: _ClassVar[int]
|
157
|
-
FUNCTION_NAME_FIELD_NUMBER: _ClassVar[int]
|
158
|
-
GRAPH_INVOCATION_ID_FIELD_NUMBER: _ClassVar[int]
|
159
|
-
TASK_ID_FIELD_NUMBER: _ClassVar[int]
|
160
|
-
FUNCTION_INPUT_FIELD_NUMBER: _ClassVar[int]
|
161
|
-
FUNCTION_INIT_VALUE_FIELD_NUMBER: _ClassVar[int]
|
162
|
-
namespace: str
|
163
|
-
graph_name: str
|
164
|
-
graph_version: str
|
165
|
-
function_name: str
|
166
|
-
graph_invocation_id: str
|
167
|
-
task_id: str
|
168
|
-
function_input: SerializedObject
|
169
|
-
function_init_value: SerializedObject
|
170
|
-
def __init__(
|
171
|
-
self,
|
172
|
-
namespace: _Optional[str] = ...,
|
173
|
-
graph_name: _Optional[str] = ...,
|
174
|
-
graph_version: _Optional[str] = ...,
|
175
|
-
function_name: _Optional[str] = ...,
|
176
|
-
graph_invocation_id: _Optional[str] = ...,
|
177
|
-
task_id: _Optional[str] = ...,
|
178
|
-
function_input: _Optional[_Union[SerializedObject, _Mapping]] = ...,
|
179
|
-
function_init_value: _Optional[_Union[SerializedObject, _Mapping]] = ...,
|
180
|
-
) -> None: ...
|
181
|
-
|
182
|
-
class RunTaskResponse(_message.Message):
|
183
|
-
__slots__ = (
|
184
|
-
"task_id",
|
185
|
-
"function_output",
|
186
|
-
"router_output",
|
187
|
-
"stdout",
|
188
|
-
"stderr",
|
189
|
-
"is_reducer",
|
190
|
-
"success",
|
191
|
-
)
|
192
|
-
TASK_ID_FIELD_NUMBER: _ClassVar[int]
|
193
|
-
FUNCTION_OUTPUT_FIELD_NUMBER: _ClassVar[int]
|
194
|
-
ROUTER_OUTPUT_FIELD_NUMBER: _ClassVar[int]
|
195
|
-
STDOUT_FIELD_NUMBER: _ClassVar[int]
|
196
|
-
STDERR_FIELD_NUMBER: _ClassVar[int]
|
197
|
-
IS_REDUCER_FIELD_NUMBER: _ClassVar[int]
|
198
|
-
SUCCESS_FIELD_NUMBER: _ClassVar[int]
|
199
|
-
task_id: str
|
200
|
-
function_output: FunctionOutput
|
201
|
-
router_output: RouterOutput
|
202
|
-
stdout: str
|
203
|
-
stderr: str
|
204
|
-
is_reducer: bool
|
205
|
-
success: bool
|
206
|
-
def __init__(
|
207
|
-
self,
|
208
|
-
task_id: _Optional[str] = ...,
|
209
|
-
function_output: _Optional[_Union[FunctionOutput, _Mapping]] = ...,
|
210
|
-
router_output: _Optional[_Union[RouterOutput, _Mapping]] = ...,
|
211
|
-
stdout: _Optional[str] = ...,
|
212
|
-
stderr: _Optional[str] = ...,
|
213
|
-
is_reducer: bool = ...,
|
214
|
-
success: bool = ...,
|
215
|
-
) -> None: ...
|
216
|
-
|
217
|
-
class HealthCheckRequest(_message.Message):
|
218
|
-
__slots__ = ()
|
219
|
-
def __init__(self) -> None: ...
|
220
|
-
|
221
|
-
class HealthCheckResponse(_message.Message):
|
222
|
-
__slots__ = ("healthy",)
|
223
|
-
HEALTHY_FIELD_NUMBER: _ClassVar[int]
|
224
|
-
healthy: bool
|
225
|
-
def __init__(self, healthy: bool = ...) -> None: ...
|
226
|
-
|
227
|
-
class InfoRequest(_message.Message):
|
228
|
-
__slots__ = ()
|
229
|
-
def __init__(self) -> None: ...
|
230
|
-
|
231
|
-
class InfoResponse(_message.Message):
|
232
|
-
__slots__ = ("version", "sdk_version", "sdk_language", "sdk_language_version")
|
233
|
-
VERSION_FIELD_NUMBER: _ClassVar[int]
|
234
|
-
SDK_VERSION_FIELD_NUMBER: _ClassVar[int]
|
235
|
-
SDK_LANGUAGE_FIELD_NUMBER: _ClassVar[int]
|
236
|
-
SDK_LANGUAGE_VERSION_FIELD_NUMBER: _ClassVar[int]
|
237
|
-
version: str
|
238
|
-
sdk_version: str
|
239
|
-
sdk_language: str
|
240
|
-
sdk_language_version: str
|
241
|
-
def __init__(
|
242
|
-
self,
|
243
|
-
version: _Optional[str] = ...,
|
244
|
-
sdk_version: _Optional[str] = ...,
|
245
|
-
sdk_language: _Optional[str] = ...,
|
246
|
-
sdk_language_version: _Optional[str] = ...,
|
247
|
-
) -> None: ...
|
@@ -1,307 +0,0 @@
|
|
1
|
-
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
|
2
|
-
"""Client and server classes corresponding to protobuf-defined services."""
|
3
|
-
import warnings
|
4
|
-
|
5
|
-
import grpc
|
6
|
-
|
7
|
-
from indexify.function_executor.proto import (
|
8
|
-
function_executor_pb2 as indexify_dot_function__executor_dot_proto_dot_function__executor__pb2,
|
9
|
-
)
|
10
|
-
|
11
|
-
GRPC_GENERATED_VERSION = "1.68.1"
|
12
|
-
GRPC_VERSION = grpc.__version__
|
13
|
-
_version_not_supported = False
|
14
|
-
|
15
|
-
try:
|
16
|
-
from grpc._utilities import first_version_is_lower
|
17
|
-
|
18
|
-
_version_not_supported = first_version_is_lower(
|
19
|
-
GRPC_VERSION, GRPC_GENERATED_VERSION
|
20
|
-
)
|
21
|
-
except ImportError:
|
22
|
-
_version_not_supported = True
|
23
|
-
|
24
|
-
if _version_not_supported:
|
25
|
-
raise RuntimeError(
|
26
|
-
f"The grpc package installed is at version {GRPC_VERSION},"
|
27
|
-
+ f" but the generated code in indexify/function_executor/proto/function_executor_pb2_grpc.py depends on"
|
28
|
-
+ f" grpcio>={GRPC_GENERATED_VERSION}."
|
29
|
-
+ f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}"
|
30
|
-
+ f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}."
|
31
|
-
)
|
32
|
-
|
33
|
-
|
34
|
-
class FunctionExecutorStub(object):
|
35
|
-
"""Missing associated documentation comment in .proto file."""
|
36
|
-
|
37
|
-
def __init__(self, channel):
|
38
|
-
"""Constructor.
|
39
|
-
|
40
|
-
Args:
|
41
|
-
channel: A grpc.Channel.
|
42
|
-
"""
|
43
|
-
self.initialize = channel.unary_unary(
|
44
|
-
"/function_executor_service.FunctionExecutor/initialize",
|
45
|
-
request_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InitializeRequest.SerializeToString,
|
46
|
-
response_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InitializeResponse.FromString,
|
47
|
-
_registered_method=True,
|
48
|
-
)
|
49
|
-
self.initialize_invocation_state_server = channel.stream_stream(
|
50
|
-
"/function_executor_service.FunctionExecutor/initialize_invocation_state_server",
|
51
|
-
request_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InvocationStateResponse.SerializeToString,
|
52
|
-
response_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InvocationStateRequest.FromString,
|
53
|
-
_registered_method=True,
|
54
|
-
)
|
55
|
-
self.run_task = channel.unary_unary(
|
56
|
-
"/function_executor_service.FunctionExecutor/run_task",
|
57
|
-
request_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.RunTaskRequest.SerializeToString,
|
58
|
-
response_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.RunTaskResponse.FromString,
|
59
|
-
_registered_method=True,
|
60
|
-
)
|
61
|
-
self.check_health = channel.unary_unary(
|
62
|
-
"/function_executor_service.FunctionExecutor/check_health",
|
63
|
-
request_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckRequest.SerializeToString,
|
64
|
-
response_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckResponse.FromString,
|
65
|
-
_registered_method=True,
|
66
|
-
)
|
67
|
-
self.get_info = channel.unary_unary(
|
68
|
-
"/function_executor_service.FunctionExecutor/get_info",
|
69
|
-
request_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoRequest.SerializeToString,
|
70
|
-
response_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoResponse.FromString,
|
71
|
-
_registered_method=True,
|
72
|
-
)
|
73
|
-
|
74
|
-
|
75
|
-
class FunctionExecutorServicer(object):
|
76
|
-
"""Missing associated documentation comment in .proto file."""
|
77
|
-
|
78
|
-
def initialize(self, request, context):
|
79
|
-
"""Initializes the Function Executor to run tasks
|
80
|
-
for a particular function. This method is called only
|
81
|
-
once per Function Executor as it can only run a single function.
|
82
|
-
It should be called before calling RunTask for the function.
|
83
|
-
"""
|
84
|
-
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
85
|
-
context.set_details("Method not implemented!")
|
86
|
-
raise NotImplementedError("Method not implemented!")
|
87
|
-
|
88
|
-
def initialize_invocation_state_server(self, request_iterator, context):
|
89
|
-
"""Initializes a server that sends requests to the client to perform actions on
|
90
|
-
a task's graph invocation state. This method is called only once per Function Executor
|
91
|
-
It should be called before calling RunTask for the function.
|
92
|
-
"""
|
93
|
-
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
94
|
-
context.set_details("Method not implemented!")
|
95
|
-
raise NotImplementedError("Method not implemented!")
|
96
|
-
|
97
|
-
def run_task(self, request, context):
|
98
|
-
"""Executes the task defined in the request.
|
99
|
-
Multiple tasks can be running in parallel.
|
100
|
-
"""
|
101
|
-
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
102
|
-
context.set_details("Method not implemented!")
|
103
|
-
raise NotImplementedError("Method not implemented!")
|
104
|
-
|
105
|
-
def check_health(self, request, context):
|
106
|
-
"""Health check method to check if the Function Executor is healthy."""
|
107
|
-
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
108
|
-
context.set_details("Method not implemented!")
|
109
|
-
raise NotImplementedError("Method not implemented!")
|
110
|
-
|
111
|
-
def get_info(self, request, context):
|
112
|
-
"""Information about this Function Executor."""
|
113
|
-
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
114
|
-
context.set_details("Method not implemented!")
|
115
|
-
raise NotImplementedError("Method not implemented!")
|
116
|
-
|
117
|
-
|
118
|
-
def add_FunctionExecutorServicer_to_server(servicer, server):
|
119
|
-
rpc_method_handlers = {
|
120
|
-
"initialize": grpc.unary_unary_rpc_method_handler(
|
121
|
-
servicer.initialize,
|
122
|
-
request_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InitializeRequest.FromString,
|
123
|
-
response_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InitializeResponse.SerializeToString,
|
124
|
-
),
|
125
|
-
"initialize_invocation_state_server": grpc.stream_stream_rpc_method_handler(
|
126
|
-
servicer.initialize_invocation_state_server,
|
127
|
-
request_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InvocationStateResponse.FromString,
|
128
|
-
response_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InvocationStateRequest.SerializeToString,
|
129
|
-
),
|
130
|
-
"run_task": grpc.unary_unary_rpc_method_handler(
|
131
|
-
servicer.run_task,
|
132
|
-
request_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.RunTaskRequest.FromString,
|
133
|
-
response_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.RunTaskResponse.SerializeToString,
|
134
|
-
),
|
135
|
-
"check_health": grpc.unary_unary_rpc_method_handler(
|
136
|
-
servicer.check_health,
|
137
|
-
request_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckRequest.FromString,
|
138
|
-
response_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckResponse.SerializeToString,
|
139
|
-
),
|
140
|
-
"get_info": grpc.unary_unary_rpc_method_handler(
|
141
|
-
servicer.get_info,
|
142
|
-
request_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoRequest.FromString,
|
143
|
-
response_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoResponse.SerializeToString,
|
144
|
-
),
|
145
|
-
}
|
146
|
-
generic_handler = grpc.method_handlers_generic_handler(
|
147
|
-
"function_executor_service.FunctionExecutor", rpc_method_handlers
|
148
|
-
)
|
149
|
-
server.add_generic_rpc_handlers((generic_handler,))
|
150
|
-
server.add_registered_method_handlers(
|
151
|
-
"function_executor_service.FunctionExecutor", rpc_method_handlers
|
152
|
-
)
|
153
|
-
|
154
|
-
|
155
|
-
# This class is part of an EXPERIMENTAL API.
|
156
|
-
class FunctionExecutor(object):
|
157
|
-
"""Missing associated documentation comment in .proto file."""
|
158
|
-
|
159
|
-
@staticmethod
|
160
|
-
def initialize(
|
161
|
-
request,
|
162
|
-
target,
|
163
|
-
options=(),
|
164
|
-
channel_credentials=None,
|
165
|
-
call_credentials=None,
|
166
|
-
insecure=False,
|
167
|
-
compression=None,
|
168
|
-
wait_for_ready=None,
|
169
|
-
timeout=None,
|
170
|
-
metadata=None,
|
171
|
-
):
|
172
|
-
return grpc.experimental.unary_unary(
|
173
|
-
request,
|
174
|
-
target,
|
175
|
-
"/function_executor_service.FunctionExecutor/initialize",
|
176
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InitializeRequest.SerializeToString,
|
177
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InitializeResponse.FromString,
|
178
|
-
options,
|
179
|
-
channel_credentials,
|
180
|
-
insecure,
|
181
|
-
call_credentials,
|
182
|
-
compression,
|
183
|
-
wait_for_ready,
|
184
|
-
timeout,
|
185
|
-
metadata,
|
186
|
-
_registered_method=True,
|
187
|
-
)
|
188
|
-
|
189
|
-
@staticmethod
|
190
|
-
def initialize_invocation_state_server(
|
191
|
-
request_iterator,
|
192
|
-
target,
|
193
|
-
options=(),
|
194
|
-
channel_credentials=None,
|
195
|
-
call_credentials=None,
|
196
|
-
insecure=False,
|
197
|
-
compression=None,
|
198
|
-
wait_for_ready=None,
|
199
|
-
timeout=None,
|
200
|
-
metadata=None,
|
201
|
-
):
|
202
|
-
return grpc.experimental.stream_stream(
|
203
|
-
request_iterator,
|
204
|
-
target,
|
205
|
-
"/function_executor_service.FunctionExecutor/initialize_invocation_state_server",
|
206
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InvocationStateResponse.SerializeToString,
|
207
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InvocationStateRequest.FromString,
|
208
|
-
options,
|
209
|
-
channel_credentials,
|
210
|
-
insecure,
|
211
|
-
call_credentials,
|
212
|
-
compression,
|
213
|
-
wait_for_ready,
|
214
|
-
timeout,
|
215
|
-
metadata,
|
216
|
-
_registered_method=True,
|
217
|
-
)
|
218
|
-
|
219
|
-
@staticmethod
|
220
|
-
def run_task(
|
221
|
-
request,
|
222
|
-
target,
|
223
|
-
options=(),
|
224
|
-
channel_credentials=None,
|
225
|
-
call_credentials=None,
|
226
|
-
insecure=False,
|
227
|
-
compression=None,
|
228
|
-
wait_for_ready=None,
|
229
|
-
timeout=None,
|
230
|
-
metadata=None,
|
231
|
-
):
|
232
|
-
return grpc.experimental.unary_unary(
|
233
|
-
request,
|
234
|
-
target,
|
235
|
-
"/function_executor_service.FunctionExecutor/run_task",
|
236
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.RunTaskRequest.SerializeToString,
|
237
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.RunTaskResponse.FromString,
|
238
|
-
options,
|
239
|
-
channel_credentials,
|
240
|
-
insecure,
|
241
|
-
call_credentials,
|
242
|
-
compression,
|
243
|
-
wait_for_ready,
|
244
|
-
timeout,
|
245
|
-
metadata,
|
246
|
-
_registered_method=True,
|
247
|
-
)
|
248
|
-
|
249
|
-
@staticmethod
|
250
|
-
def check_health(
|
251
|
-
request,
|
252
|
-
target,
|
253
|
-
options=(),
|
254
|
-
channel_credentials=None,
|
255
|
-
call_credentials=None,
|
256
|
-
insecure=False,
|
257
|
-
compression=None,
|
258
|
-
wait_for_ready=None,
|
259
|
-
timeout=None,
|
260
|
-
metadata=None,
|
261
|
-
):
|
262
|
-
return grpc.experimental.unary_unary(
|
263
|
-
request,
|
264
|
-
target,
|
265
|
-
"/function_executor_service.FunctionExecutor/check_health",
|
266
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckRequest.SerializeToString,
|
267
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckResponse.FromString,
|
268
|
-
options,
|
269
|
-
channel_credentials,
|
270
|
-
insecure,
|
271
|
-
call_credentials,
|
272
|
-
compression,
|
273
|
-
wait_for_ready,
|
274
|
-
timeout,
|
275
|
-
metadata,
|
276
|
-
_registered_method=True,
|
277
|
-
)
|
278
|
-
|
279
|
-
@staticmethod
|
280
|
-
def get_info(
|
281
|
-
request,
|
282
|
-
target,
|
283
|
-
options=(),
|
284
|
-
channel_credentials=None,
|
285
|
-
call_credentials=None,
|
286
|
-
insecure=False,
|
287
|
-
compression=None,
|
288
|
-
wait_for_ready=None,
|
289
|
-
timeout=None,
|
290
|
-
metadata=None,
|
291
|
-
):
|
292
|
-
return grpc.experimental.unary_unary(
|
293
|
-
request,
|
294
|
-
target,
|
295
|
-
"/function_executor_service.FunctionExecutor/get_info",
|
296
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoRequest.SerializeToString,
|
297
|
-
indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoResponse.FromString,
|
298
|
-
options,
|
299
|
-
channel_credentials,
|
300
|
-
insecure,
|
301
|
-
call_credentials,
|
302
|
-
compression,
|
303
|
-
wait_for_ready,
|
304
|
-
timeout,
|
305
|
-
metadata,
|
306
|
-
_registered_method=True,
|
307
|
-
)
|
@@ -1,38 +0,0 @@
|
|
1
|
-
from typing import Any
|
2
|
-
|
3
|
-
from .function_executor_pb2 import SerializedObject
|
4
|
-
|
5
|
-
|
6
|
-
class MessageValidator:
|
7
|
-
def __init__(self, message: Any):
|
8
|
-
self._message = message
|
9
|
-
|
10
|
-
def required_field(self, field_name: str) -> "MessageValidator":
|
11
|
-
if not self._message.HasField(field_name):
|
12
|
-
raise ValueError(
|
13
|
-
f"Field '{field_name}' is required in {type(self._message).__name__}"
|
14
|
-
)
|
15
|
-
return self
|
16
|
-
|
17
|
-
def required_serialized_object(self, field_name: str) -> "MessageValidator":
|
18
|
-
"""Validates the SerializedObject.
|
19
|
-
|
20
|
-
Raises: ValueError: If the SerializedObject is invalid or not present."""
|
21
|
-
self.required_field(field_name)
|
22
|
-
return self.optional_serialized_object(field_name)
|
23
|
-
|
24
|
-
def optional_serialized_object(self, field_name: str) -> "MessageValidator":
|
25
|
-
"""Validates the SerializedObject.
|
26
|
-
|
27
|
-
Raises: ValueError: If the SerializedObject is invalid."""
|
28
|
-
if not self._message.HasField(field_name):
|
29
|
-
return self
|
30
|
-
|
31
|
-
serializedObject: SerializedObject = getattr(self._message, field_name)
|
32
|
-
if not serializedObject.HasField("string") and not serializedObject.HasField(
|
33
|
-
"bytes"
|
34
|
-
):
|
35
|
-
raise ValueError("oneof 'data' is required in SerializedObject")
|
36
|
-
if not serializedObject.HasField("content_type"):
|
37
|
-
raise ValueError("Field 'content_type' is required in SerializedObject")
|
38
|
-
return self
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# We send function inputs and outputs over gRPC.
|
2
|
-
# -1 means unlimited. We don't want to limit the size of data customers are using.
|
3
|
-
# The effective max message size in this case is about 1.9 GB, see the max payload test.
|
4
|
-
# This is due to internal hard gRPC limits. When we want to increase the message sizes
|
5
|
-
# we'll have to implement chunking for large messages.
|
6
|
-
_MAX_GRPC_MESSAGE_LENGTH = -1
|
7
|
-
# Disable port reuse: fail if multiple Function Executor Servers attempt to bind to the
|
8
|
-
# same port. This happens when Indexify users misconfigure the Servers. Disabling the port
|
9
|
-
# reuse results in a clear error message on Server startup instead of obscure errors later
|
10
|
-
# while Indexify cluster is serving tasks.
|
11
|
-
# If we don't disable port reuse then a random Server gets the requests so wrong tasks get
|
12
|
-
# routed to wrong servers.
|
13
|
-
_REUSE_SERVER_PORT = 0
|
14
|
-
|
15
|
-
GRPC_SERVER_OPTIONS = [
|
16
|
-
("grpc.max_receive_message_length", _MAX_GRPC_MESSAGE_LENGTH),
|
17
|
-
("grpc.max_send_message_length", _MAX_GRPC_MESSAGE_LENGTH),
|
18
|
-
("grpc.so_reuseport", _REUSE_SERVER_PORT),
|
19
|
-
]
|
@@ -1,29 +0,0 @@
|
|
1
|
-
from concurrent.futures import ThreadPoolExecutor
|
2
|
-
|
3
|
-
import grpc
|
4
|
-
|
5
|
-
from .proto.function_executor_pb2_grpc import add_FunctionExecutorServicer_to_server
|
6
|
-
from .proto.server_configuration import GRPC_SERVER_OPTIONS
|
7
|
-
from .service import Service
|
8
|
-
|
9
|
-
# Temporary limit until we have a better way to control this.
|
10
|
-
# This limits the number of concurrent tasks that Function Executor can run.
|
11
|
-
MAX_RPC_CONCURRENCY = 100
|
12
|
-
|
13
|
-
|
14
|
-
class Server:
|
15
|
-
def __init__(self, server_address: str, service: Service):
|
16
|
-
self._server_address: str = server_address
|
17
|
-
self._service: Service = service
|
18
|
-
|
19
|
-
def run(self):
|
20
|
-
"""Runs Function Executor Service at the configured address."""
|
21
|
-
server = grpc.server(
|
22
|
-
thread_pool=ThreadPoolExecutor(max_workers=MAX_RPC_CONCURRENCY),
|
23
|
-
maximum_concurrent_rpcs=MAX_RPC_CONCURRENCY,
|
24
|
-
options=GRPC_SERVER_OPTIONS,
|
25
|
-
)
|
26
|
-
add_FunctionExecutorServicer_to_server(self._service, server)
|
27
|
-
server.add_insecure_port(self._server_address)
|
28
|
-
server.start()
|
29
|
-
server.wait_for_termination()
|