indexify 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indexify/cli/cli.py +6 -4
- indexify/executor/downloader.py +2 -3
- indexify/executor/executor.py +8 -7
- indexify/executor/function_executor/function_executor.py +3 -4
- indexify/executor/function_executor/health_checker.py +4 -4
- indexify/executor/function_executor/invocation_state_client.py +3 -4
- indexify/{function_executor/proto/configuration.py → executor/function_executor/server/client_configuration.py} +7 -11
- indexify/executor/function_executor/server/subprocess_function_executor_server.py +1 -2
- indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py +0 -5
- indexify/executor/function_executor/single_task_runner.py +2 -4
- indexify/executor/function_executor/task_input.py +1 -1
- indexify/executor/function_executor/task_output.py +1 -1
- indexify/executor/task_fetcher.py +5 -5
- indexify/executor/task_reporter.py +3 -4
- {indexify-0.3.4.dist-info → indexify-0.3.6.dist-info}/METADATA +3 -4
- indexify-0.3.6.dist-info/RECORD +25 -0
- indexify-0.3.6.dist-info/entry_points.txt +3 -0
- indexify/function_executor/README.md +0 -18
- indexify/function_executor/handlers/run_function/function_inputs_loader.py +0 -53
- indexify/function_executor/handlers/run_function/handler.py +0 -126
- indexify/function_executor/handlers/run_function/request_validator.py +0 -26
- indexify/function_executor/handlers/run_function/response_helper.py +0 -96
- indexify/function_executor/initialize_request_validator.py +0 -21
- indexify/function_executor/invocation_state/invocation_state_proxy_server.py +0 -170
- indexify/function_executor/invocation_state/proxied_invocation_state.py +0 -22
- indexify/function_executor/invocation_state/response_validator.py +0 -29
- indexify/function_executor/main.py +0 -50
- indexify/function_executor/proto/function_executor.proto +0 -130
- indexify/function_executor/proto/function_executor_pb2.py +0 -69
- indexify/function_executor/proto/function_executor_pb2.pyi +0 -225
- indexify/function_executor/proto/function_executor_pb2_grpc.py +0 -260
- indexify/function_executor/proto/message_validator.py +0 -38
- indexify/function_executor/server.py +0 -29
- indexify/function_executor/service.py +0 -133
- indexify/utils/README.md +0 -3
- indexify/utils/http_client.py +0 -88
- indexify/utils/logging.py +0 -66
- indexify-0.3.4.dist-info/RECORD +0 -45
- indexify-0.3.4.dist-info/entry_points.txt +0 -4
- {indexify-0.3.4.dist-info → indexify-0.3.6.dist-info}/WHEEL +0 -0
@@ -1,96 +0,0 @@
|
|
1
|
-
from typing import List
|
2
|
-
|
3
|
-
from tensorlake.functions_sdk.data_objects import TensorlakeData
|
4
|
-
from tensorlake.functions_sdk.functions import FunctionCallResult, RouterCallResult
|
5
|
-
from tensorlake.functions_sdk.object_serializer import get_serializer
|
6
|
-
|
7
|
-
from ...proto.function_executor_pb2 import (
|
8
|
-
FunctionOutput,
|
9
|
-
RouterOutput,
|
10
|
-
RunTaskResponse,
|
11
|
-
SerializedObject,
|
12
|
-
)
|
13
|
-
|
14
|
-
|
15
|
-
class ResponseHelper:
|
16
|
-
"""Helper class for generating RunFunctionResponse."""
|
17
|
-
|
18
|
-
def __init__(self, task_id: str):
|
19
|
-
self._task_id = task_id
|
20
|
-
|
21
|
-
def function_response(
|
22
|
-
self,
|
23
|
-
result: FunctionCallResult,
|
24
|
-
is_reducer: bool,
|
25
|
-
stdout: str = "",
|
26
|
-
stderr: str = "",
|
27
|
-
) -> RunTaskResponse:
|
28
|
-
if result.traceback_msg is None:
|
29
|
-
return RunTaskResponse(
|
30
|
-
task_id=self._task_id,
|
31
|
-
function_output=self._to_function_output(result.ser_outputs),
|
32
|
-
router_output=None,
|
33
|
-
stdout=stdout,
|
34
|
-
stderr=stderr,
|
35
|
-
is_reducer=is_reducer,
|
36
|
-
success=True,
|
37
|
-
)
|
38
|
-
else:
|
39
|
-
return self.failure_response(
|
40
|
-
message=result.traceback_msg,
|
41
|
-
stdout=stdout,
|
42
|
-
stderr=stderr,
|
43
|
-
)
|
44
|
-
|
45
|
-
def router_response(
|
46
|
-
self,
|
47
|
-
result: RouterCallResult,
|
48
|
-
stdout: str = "",
|
49
|
-
stderr: str = "",
|
50
|
-
) -> RunTaskResponse:
|
51
|
-
if result.traceback_msg is None:
|
52
|
-
return RunTaskResponse(
|
53
|
-
task_id=self._task_id,
|
54
|
-
function_output=None,
|
55
|
-
router_output=RouterOutput(edges=result.edges),
|
56
|
-
stdout=stdout,
|
57
|
-
stderr=stderr,
|
58
|
-
is_reducer=False,
|
59
|
-
success=True,
|
60
|
-
)
|
61
|
-
else:
|
62
|
-
return self.failure_response(
|
63
|
-
message=result.traceback_msg,
|
64
|
-
stdout=stdout,
|
65
|
-
stderr=stderr,
|
66
|
-
)
|
67
|
-
|
68
|
-
def failure_response(
|
69
|
-
self, message: str, stdout: str, stderr: str
|
70
|
-
) -> RunTaskResponse:
|
71
|
-
stderr = "\n".join([stderr, message])
|
72
|
-
return RunTaskResponse(
|
73
|
-
task_id=self._task_id,
|
74
|
-
function_output=None,
|
75
|
-
router_output=None,
|
76
|
-
stdout=stdout,
|
77
|
-
stderr=stderr,
|
78
|
-
is_reducer=False,
|
79
|
-
success=False,
|
80
|
-
)
|
81
|
-
|
82
|
-
def _to_function_output(self, outputs: List[TensorlakeData]) -> FunctionOutput:
|
83
|
-
output = FunctionOutput(outputs=[])
|
84
|
-
for ix_data in outputs:
|
85
|
-
serialized_object: SerializedObject = SerializedObject(
|
86
|
-
content_type=get_serializer(ix_data.encoder).content_type,
|
87
|
-
)
|
88
|
-
if isinstance(ix_data.payload, bytes):
|
89
|
-
serialized_object.bytes = ix_data.payload
|
90
|
-
elif isinstance(ix_data.payload, str):
|
91
|
-
serialized_object.string = ix_data.payload
|
92
|
-
else:
|
93
|
-
raise ValueError(f"Unsupported payload type: {type(ix_data.payload)}")
|
94
|
-
|
95
|
-
output.outputs.append(serialized_object)
|
96
|
-
return output
|
@@ -1,21 +0,0 @@
|
|
1
|
-
from .proto.function_executor_pb2 import InitializeRequest
|
2
|
-
from .proto.message_validator import MessageValidator
|
3
|
-
|
4
|
-
|
5
|
-
class InitializeRequestValidator:
|
6
|
-
def __init__(self, request: InitializeRequest):
|
7
|
-
self._request = request
|
8
|
-
self._message_validator = MessageValidator(request)
|
9
|
-
|
10
|
-
def check(self):
|
11
|
-
"""Validates the request.
|
12
|
-
|
13
|
-
Raises: ValueError: If the request is invalid.
|
14
|
-
"""
|
15
|
-
(
|
16
|
-
self._message_validator.required_field("namespace")
|
17
|
-
.required_field("graph_name")
|
18
|
-
.required_field("graph_version")
|
19
|
-
.required_field("function_name")
|
20
|
-
.required_serialized_object("graph")
|
21
|
-
)
|
@@ -1,170 +0,0 @@
|
|
1
|
-
import queue
|
2
|
-
import threading
|
3
|
-
from typing import Any, Iterator, Optional
|
4
|
-
|
5
|
-
from tensorlake.functions_sdk.object_serializer import (
|
6
|
-
CloudPickleSerializer,
|
7
|
-
get_serializer,
|
8
|
-
)
|
9
|
-
|
10
|
-
from ..proto.function_executor_pb2 import (
|
11
|
-
GetInvocationStateRequest,
|
12
|
-
InvocationStateRequest,
|
13
|
-
InvocationStateResponse,
|
14
|
-
SerializedObject,
|
15
|
-
SetInvocationStateRequest,
|
16
|
-
)
|
17
|
-
from .response_validator import ResponseValidator
|
18
|
-
|
19
|
-
|
20
|
-
class InvocationStateProxyServer:
|
21
|
-
"""A gRPC server that proxies InvocationState calls to the gRPC client.
|
22
|
-
|
23
|
-
The gRPC client is responsible for the actual implementation of the InvocationState.
|
24
|
-
We do the proxying to remove authorization logic and credentials from Function Executor.
|
25
|
-
This improves security posture of Function Executor because it may run untrusted code.
|
26
|
-
"""
|
27
|
-
|
28
|
-
def __init__(
|
29
|
-
self, client_responses: Iterator[InvocationStateResponse], logger: Any
|
30
|
-
):
|
31
|
-
self._client_responses: Iterator[InvocationStateResponse] = client_responses
|
32
|
-
self._logger: Any = logger.bind(module=__name__)
|
33
|
-
self._reciever_thread: threading.Thread = threading.Thread(
|
34
|
-
target=self._reciever
|
35
|
-
)
|
36
|
-
self._request_queue: queue.SimpleQueue = queue.SimpleQueue()
|
37
|
-
# This lock protects everything below.
|
38
|
-
self._lock: threading.Lock = threading.Lock()
|
39
|
-
# Python supports big integers natively so we don't need
|
40
|
-
# to be worried about interger overflows.
|
41
|
-
self._request_seq_num: int = 0
|
42
|
-
# Request ID -> Client Response.
|
43
|
-
self._response_map: dict[str, InvocationStateResponse] = {}
|
44
|
-
self._new_response: threading.Condition = threading.Condition(self._lock)
|
45
|
-
|
46
|
-
def run(self) -> Iterator[InvocationStateRequest]:
|
47
|
-
# There's no need to implement shutdown of the server and its threads because
|
48
|
-
# the server lives while the Function Executor process lives.
|
49
|
-
self._reciever_thread.start()
|
50
|
-
yield from self._sender()
|
51
|
-
|
52
|
-
def _reciever(self) -> None:
|
53
|
-
self._logger.info("reciever thread started")
|
54
|
-
try:
|
55
|
-
for response in self._client_responses:
|
56
|
-
validator = ResponseValidator(response)
|
57
|
-
try:
|
58
|
-
validator.check()
|
59
|
-
except ValueError as e:
|
60
|
-
self._logger.error("invalid response from the client", exc_info=e)
|
61
|
-
continue
|
62
|
-
|
63
|
-
with self._lock:
|
64
|
-
self._response_map[response.request_id] = response
|
65
|
-
self._new_response.notify_all()
|
66
|
-
except Exception as e:
|
67
|
-
self._logger.error("error in reciever thread, exiting", exc_info=e)
|
68
|
-
|
69
|
-
def _sender(self) -> Iterator[InvocationStateRequest]:
|
70
|
-
while True:
|
71
|
-
yield self._request_queue.get()
|
72
|
-
with self._lock:
|
73
|
-
# Wait until we get a response for the request.
|
74
|
-
# This allows to ensure a serialized order of reads and writes so
|
75
|
-
# we can avoid a read returning not previously written value.
|
76
|
-
self._new_response.wait()
|
77
|
-
|
78
|
-
def set(self, task_id: str, key: str, value: Any) -> None:
|
79
|
-
with self._lock:
|
80
|
-
request_id: str = str(self._request_seq_num)
|
81
|
-
self._request_seq_num += 1
|
82
|
-
|
83
|
-
# We currently use CloudPickleSerializer for function inputs,
|
84
|
-
# outputs and invocation state values. This provides consistent UX.
|
85
|
-
request = InvocationStateRequest(
|
86
|
-
request_id=request_id,
|
87
|
-
task_id=task_id,
|
88
|
-
set=SetInvocationStateRequest(
|
89
|
-
key=key,
|
90
|
-
value=SerializedObject(
|
91
|
-
content_type=CloudPickleSerializer.content_type,
|
92
|
-
bytes=CloudPickleSerializer.serialize(value),
|
93
|
-
),
|
94
|
-
),
|
95
|
-
)
|
96
|
-
self._request_queue.put(request)
|
97
|
-
while request_id not in self._response_map:
|
98
|
-
self._new_response.wait()
|
99
|
-
|
100
|
-
response: InvocationStateResponse = self._response_map.pop(request_id)
|
101
|
-
if response.request_id != request_id:
|
102
|
-
self._logger.error(
|
103
|
-
"response request_id doesn't match actual request_id",
|
104
|
-
request_id=request_id,
|
105
|
-
response=response,
|
106
|
-
)
|
107
|
-
raise RuntimeError(
|
108
|
-
"response request_id doesn't match actual request_id"
|
109
|
-
)
|
110
|
-
if not response.HasField("set"):
|
111
|
-
self._logger.error(
|
112
|
-
"set response is missing in the client response",
|
113
|
-
request_id=request_id,
|
114
|
-
response=response,
|
115
|
-
)
|
116
|
-
raise RuntimeError("set response is missing in the client response")
|
117
|
-
if not response.success:
|
118
|
-
self._logger.error(
|
119
|
-
"failed to set the invocation state for key",
|
120
|
-
key=key,
|
121
|
-
)
|
122
|
-
raise RuntimeError("failed to set the invocation state for key")
|
123
|
-
|
124
|
-
def get(self, task_id: str, key: str) -> Optional[Any]:
|
125
|
-
with self._lock:
|
126
|
-
request_id: str = str(self._request_seq_num)
|
127
|
-
self._request_seq_num += 1
|
128
|
-
|
129
|
-
request = InvocationStateRequest(
|
130
|
-
request_id=request_id,
|
131
|
-
task_id=task_id,
|
132
|
-
get=GetInvocationStateRequest(
|
133
|
-
key=key,
|
134
|
-
),
|
135
|
-
)
|
136
|
-
self._request_queue.put(request)
|
137
|
-
while request_id not in self._response_map:
|
138
|
-
self._new_response.wait()
|
139
|
-
|
140
|
-
response: InvocationStateResponse = self._response_map.pop(request_id)
|
141
|
-
if response.request_id != request_id:
|
142
|
-
self._logger.error(
|
143
|
-
"response request_id doesn't match actual request_id",
|
144
|
-
request_id=request_id,
|
145
|
-
response=response,
|
146
|
-
)
|
147
|
-
raise RuntimeError(
|
148
|
-
"response request_id doesn't match actual request_id"
|
149
|
-
)
|
150
|
-
if not response.HasField("get"):
|
151
|
-
self._logger.error(
|
152
|
-
"get response is missing in the client response",
|
153
|
-
request_id=request_id,
|
154
|
-
response=response,
|
155
|
-
)
|
156
|
-
raise RuntimeError("get response is missing in the client response")
|
157
|
-
if not response.success:
|
158
|
-
self._logger.error(
|
159
|
-
"failed to get the invocation state for key",
|
160
|
-
key=key,
|
161
|
-
)
|
162
|
-
raise RuntimeError("failed to get the invocation state for key")
|
163
|
-
if not response.get.HasField("value"):
|
164
|
-
return None
|
165
|
-
|
166
|
-
return get_serializer(response.get.value.content_type).deserialize(
|
167
|
-
response.get.value.bytes
|
168
|
-
if response.get.value.HasField("bytes")
|
169
|
-
else response.get.value.string
|
170
|
-
)
|
@@ -1,22 +0,0 @@
|
|
1
|
-
from typing import Any, Optional
|
2
|
-
|
3
|
-
from tensorlake.functions_sdk.invocation_state.invocation_state import InvocationState
|
4
|
-
|
5
|
-
from .invocation_state_proxy_server import InvocationStateProxyServer
|
6
|
-
|
7
|
-
|
8
|
-
class ProxiedInvocationState(InvocationState):
|
9
|
-
"""InvocationState that proxies the calls via InvocationStateProxyServer."""
|
10
|
-
|
11
|
-
def __init__(self, task_id: str, proxy_server: InvocationStateProxyServer):
|
12
|
-
self._task_id: str = task_id
|
13
|
-
self._proxy_server: InvocationStateProxyServer = proxy_server
|
14
|
-
|
15
|
-
def set(self, key: str, value: Any) -> None:
|
16
|
-
"""Set a key-value pair."""
|
17
|
-
self._proxy_server.set(self._task_id, key, value)
|
18
|
-
|
19
|
-
def get(self, key: str, default: Optional[Any] = None) -> Optional[Any]:
|
20
|
-
"""Get a value by key. If the key does not exist, return the default value."""
|
21
|
-
value: Optional[Any] = self._proxy_server.get(self._task_id, key)
|
22
|
-
return default if value is None else value
|
@@ -1,29 +0,0 @@
|
|
1
|
-
from ..proto.function_executor_pb2 import InvocationStateResponse
|
2
|
-
from ..proto.message_validator import MessageValidator
|
3
|
-
|
4
|
-
|
5
|
-
class ResponseValidator(MessageValidator):
|
6
|
-
def __init__(self, response: InvocationStateResponse):
|
7
|
-
self._response = response
|
8
|
-
|
9
|
-
def check(self):
|
10
|
-
"""Validates the request.
|
11
|
-
|
12
|
-
Raises: ValueError: If the response is invalid.
|
13
|
-
"""
|
14
|
-
(
|
15
|
-
MessageValidator(self._response)
|
16
|
-
.required_field("request_id")
|
17
|
-
.required_field("success")
|
18
|
-
)
|
19
|
-
|
20
|
-
if self._response.HasField("set"):
|
21
|
-
pass
|
22
|
-
elif self._response.HasField("get"):
|
23
|
-
(
|
24
|
-
MessageValidator(self._response.get)
|
25
|
-
.required_field("key")
|
26
|
-
.optional_serialized_object("value")
|
27
|
-
)
|
28
|
-
else:
|
29
|
-
raise ValueError(f"Unknown response type: {self._response}")
|
@@ -1,50 +0,0 @@
|
|
1
|
-
from indexify.utils.logging import (
|
2
|
-
configure_development_mode_logging,
|
3
|
-
configure_logging_early,
|
4
|
-
configure_production_mode_logging,
|
5
|
-
)
|
6
|
-
|
7
|
-
configure_logging_early()
|
8
|
-
|
9
|
-
import argparse
|
10
|
-
|
11
|
-
import structlog
|
12
|
-
|
13
|
-
from .server import Server
|
14
|
-
from .service import Service
|
15
|
-
|
16
|
-
logger = structlog.get_logger(module=__name__)
|
17
|
-
|
18
|
-
|
19
|
-
def validate_args(args):
|
20
|
-
if args.address is None:
|
21
|
-
logger.error("--address argument is required")
|
22
|
-
exit(1)
|
23
|
-
|
24
|
-
|
25
|
-
def main():
|
26
|
-
parser = argparse.ArgumentParser(
|
27
|
-
description="Runs Function Executor with the specified API server address"
|
28
|
-
)
|
29
|
-
parser.add_argument("--address", help="API server address to listen on", type=str)
|
30
|
-
parser.add_argument(
|
31
|
-
"-d", "--dev", help="Run in development mode", action="store_true"
|
32
|
-
)
|
33
|
-
args = parser.parse_args()
|
34
|
-
|
35
|
-
if args.dev:
|
36
|
-
configure_development_mode_logging()
|
37
|
-
else:
|
38
|
-
configure_production_mode_logging()
|
39
|
-
validate_args(args)
|
40
|
-
|
41
|
-
logger.info("starting function executor server", address=args.address)
|
42
|
-
|
43
|
-
Server(
|
44
|
-
server_address=args.address,
|
45
|
-
service=Service(),
|
46
|
-
).run()
|
47
|
-
|
48
|
-
|
49
|
-
if __name__ == "__main__":
|
50
|
-
main()
|
@@ -1,130 +0,0 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
syntax = "proto3";
|
4
|
-
|
5
|
-
package function_executor_service;
|
6
|
-
|
7
|
-
// The messages should not use any Python SDK objects. Only Function Executor implemented
|
8
|
-
// in Python is allowed to import Python SDK to run customer functions. This ensures that
|
9
|
-
// all the other components can be written in any language.
|
10
|
-
|
11
|
-
message SerializedObject {
|
12
|
-
oneof data {
|
13
|
-
// Set bytes_data if the object is serialized as bytes.
|
14
|
-
bytes bytes = 1;
|
15
|
-
// Set string_data if the object is serialized as string.
|
16
|
-
string string = 2;
|
17
|
-
}
|
18
|
-
// The content type determines the serializer used to serialize the object.
|
19
|
-
optional string content_type = 3;
|
20
|
-
}
|
21
|
-
|
22
|
-
// InitializeRequest contains information about the function
|
23
|
-
// that Function Executor is going to run the tasks for.
|
24
|
-
message InitializeRequest {
|
25
|
-
optional string namespace = 1;
|
26
|
-
optional string graph_name = 2;
|
27
|
-
optional string graph_version = 3;
|
28
|
-
optional string function_name = 5;
|
29
|
-
optional SerializedObject graph = 7;
|
30
|
-
}
|
31
|
-
|
32
|
-
message InitializeResponse {
|
33
|
-
optional bool success = 1;
|
34
|
-
optional string customer_error = 2;
|
35
|
-
}
|
36
|
-
|
37
|
-
message SetInvocationStateRequest {
|
38
|
-
optional string key = 1;
|
39
|
-
optional SerializedObject value = 2;
|
40
|
-
}
|
41
|
-
|
42
|
-
message SetInvocationStateResponse {}
|
43
|
-
|
44
|
-
message GetInvocationStateRequest {
|
45
|
-
optional string key = 1;
|
46
|
-
}
|
47
|
-
|
48
|
-
message GetInvocationStateResponse {
|
49
|
-
optional string key = 1;
|
50
|
-
optional SerializedObject value = 2;
|
51
|
-
}
|
52
|
-
|
53
|
-
// InvocationStateRequest is sent by RPC Server to the client
|
54
|
-
// to perform actions on a task's graph invocation state.
|
55
|
-
message InvocationStateRequest {
|
56
|
-
// The ID of the request sent by the client.
|
57
|
-
// Must be unique per Function Executor.
|
58
|
-
optional string request_id = 1;
|
59
|
-
// The ID of the task initiated the request.
|
60
|
-
optional string task_id = 2;
|
61
|
-
oneof request {
|
62
|
-
SetInvocationStateRequest set = 3;
|
63
|
-
GetInvocationStateRequest get = 4;
|
64
|
-
}
|
65
|
-
}
|
66
|
-
|
67
|
-
// InvocationStateResponse is sent by RPC client to the Server.
|
68
|
-
// A response contains the result of the action performed on the
|
69
|
-
// task's graph invocation state.
|
70
|
-
message InvocationStateResponse {
|
71
|
-
// The id of the request this response is for.
|
72
|
-
optional string request_id = 1;
|
73
|
-
optional bool success = 2;
|
74
|
-
oneof response {
|
75
|
-
SetInvocationStateResponse set = 3;
|
76
|
-
GetInvocationStateResponse get = 4;
|
77
|
-
}
|
78
|
-
}
|
79
|
-
|
80
|
-
message FunctionOutput {
|
81
|
-
repeated SerializedObject outputs = 1;
|
82
|
-
}
|
83
|
-
|
84
|
-
message RouterOutput {
|
85
|
-
repeated string edges = 1;
|
86
|
-
}
|
87
|
-
|
88
|
-
message RunTaskRequest {
|
89
|
-
optional string namespace = 1;
|
90
|
-
optional string graph_name = 2;
|
91
|
-
optional string graph_version = 3;
|
92
|
-
optional string function_name = 4;
|
93
|
-
optional string graph_invocation_id = 5;
|
94
|
-
optional string task_id = 6;
|
95
|
-
optional SerializedObject function_input = 7;
|
96
|
-
optional SerializedObject function_init_value = 8;
|
97
|
-
}
|
98
|
-
|
99
|
-
message RunTaskResponse {
|
100
|
-
optional string task_id = 1;
|
101
|
-
optional FunctionOutput function_output = 2;
|
102
|
-
optional RouterOutput router_output = 3;
|
103
|
-
optional string stdout = 4;
|
104
|
-
optional string stderr = 5;
|
105
|
-
optional bool is_reducer = 6;
|
106
|
-
optional bool success = 7;
|
107
|
-
}
|
108
|
-
|
109
|
-
message HealthCheckRequest {}
|
110
|
-
|
111
|
-
message HealthCheckResponse {
|
112
|
-
optional bool healthy = 1;
|
113
|
-
}
|
114
|
-
|
115
|
-
service FunctionExecutor {
|
116
|
-
// Initializes the Function Executor to run tasks
|
117
|
-
// for a particular function. This method is called only
|
118
|
-
// once per Function Executor as it can only run a single function.
|
119
|
-
// It should be called before calling RunTask for the function.
|
120
|
-
rpc initialize(InitializeRequest) returns (InitializeResponse);
|
121
|
-
// Initializes a server that sends requests to the client to perform actions on
|
122
|
-
// a task's graph invocation state. This method is called only once per Function Executor
|
123
|
-
// It should be called before calling RunTask for the function.
|
124
|
-
rpc initialize_invocation_state_server(stream InvocationStateResponse) returns (stream InvocationStateRequest);
|
125
|
-
// Executes the task defined in the request.
|
126
|
-
// Multiple tasks can be running in parallel.
|
127
|
-
rpc run_task(RunTaskRequest) returns (RunTaskResponse);
|
128
|
-
// Health check method to check if the Function Executor is healthy.
|
129
|
-
rpc check_health(HealthCheckRequest) returns (HealthCheckResponse);
|
130
|
-
}
|
@@ -1,69 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
3
|
-
# NO CHECKED-IN PROTOBUF GENCODE
|
4
|
-
# source: indexify/function_executor/proto/function_executor.proto
|
5
|
-
# Protobuf Python Version: 5.28.1
|
6
|
-
"""Generated protocol buffer code."""
|
7
|
-
from google.protobuf import descriptor as _descriptor
|
8
|
-
from google.protobuf import descriptor_pool as _descriptor_pool
|
9
|
-
from google.protobuf import runtime_version as _runtime_version
|
10
|
-
from google.protobuf import symbol_database as _symbol_database
|
11
|
-
from google.protobuf.internal import builder as _builder
|
12
|
-
|
13
|
-
_runtime_version.ValidateProtobufRuntimeVersion(
|
14
|
-
_runtime_version.Domain.PUBLIC,
|
15
|
-
5,
|
16
|
-
28,
|
17
|
-
1,
|
18
|
-
"",
|
19
|
-
"indexify/function_executor/proto/function_executor.proto",
|
20
|
-
)
|
21
|
-
# @@protoc_insertion_point(imports)
|
22
|
-
|
23
|
-
_sym_db = _symbol_database.Default()
|
24
|
-
|
25
|
-
|
26
|
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
|
27
|
-
b'\n8indexify/function_executor/proto/function_executor.proto\x12\x19\x66unction_executor_service"i\n\x10SerializedObject\x12\x0f\n\x05\x62ytes\x18\x01 \x01(\x0cH\x00\x12\x10\n\x06string\x18\x02 \x01(\tH\x00\x12\x19\n\x0c\x63ontent_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x06\n\x04\x64\x61taB\x0f\n\r_content_type"\x88\x02\n\x11InitializeRequest\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x03\x88\x01\x01\x12?\n\x05graph\x18\x07 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x04\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x08\n\x06_graph"f\n\x12InitializeResponse\x12\x14\n\x07success\x18\x01 \x01(\x08H\x00\x88\x01\x01\x12\x1b\n\x0e\x63ustomer_error\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_successB\x11\n\x0f_customer_error"\x80\x01\n\x19SetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\x1c\n\x1aSetInvocationStateResponse"5\n\x19GetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_key"\x81\x01\n\x1aGetInvocationStateResponse\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\xf7\x01\n\x16InvocationStateRequest\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07task_id\x18\x02 \x01(\tH\x02\x88\x01\x01\x12\x43\n\x03set\x18\x03 \x01(\x0b\x32\x34.function_executor_service.SetInvocationStateRequestH\x00\x12\x43\n\x03get\x18\x04 \x01(\x0b\x32\x34.function_executor_service.GetInvocationStateRequestH\x00\x42\t\n\x07requestB\r\n\x0b_request_idB\n\n\x08_task_id"\xfb\x01\n\x17InvocationStateResponse\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07success\x18\x02 \x01(\x08H\x02\x88\x01\x01\x12\x44\n\x03set\x18\x03 \x01(\x0b\x32\x35.function_executor_service.SetInvocationStateResponseH\x00\x12\x44\n\x03get\x18\x04 \x01(\x0b\x32\x35.function_executor_service.GetInvocationStateResponseH\x00\x42\n\n\x08responseB\r\n\x0b_request_idB\n\n\x08_success"N\n\x0e\x46unctionOutput\x12<\n\x07outputs\x18\x01 \x03(\x0b\x32+.function_executor_service.SerializedObject"\x1d\n\x0cRouterOutput\x12\r\n\x05\x65\x64ges\x18\x01 \x03(\t"\xda\x03\n\x0eRunTaskRequest\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x14\n\x07task_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12H\n\x0e\x66unction_input\x18\x07 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x06\x88\x01\x01\x12M\n\x13\x66unction_init_value\x18\x08 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x07\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_task_idB\x11\n\x0f_function_inputB\x16\n\x14_function_init_value"\xf1\x02\n\x0fRunTaskResponse\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12G\n\x0f\x66unction_output\x18\x02 \x01(\x0b\x32).function_executor_service.FunctionOutputH\x01\x88\x01\x01\x12\x43\n\rrouter_output\x18\x03 \x01(\x0b\x32\'.function_executor_service.RouterOutputH\x02\x88\x01\x01\x12\x13\n\x06stdout\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06stderr\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x17\n\nis_reducer\x18\x06 \x01(\x08H\x05\x88\x01\x01\x12\x14\n\x07success\x18\x07 \x01(\x08H\x06\x88\x01\x01\x42\n\n\x08_task_idB\x12\n\x10_function_outputB\x10\n\x0e_router_outputB\t\n\x07_stdoutB\t\n\x07_stderrB\r\n\x0b_is_reducerB\n\n\x08_success"\x14\n\x12HealthCheckRequest"7\n\x13HealthCheckResponse\x12\x14\n\x07healthy\x18\x01 \x01(\x08H\x00\x88\x01\x01\x42\n\n\x08_healthy2\xe1\x03\n\x10\x46unctionExecutor\x12i\n\ninitialize\x12,.function_executor_service.InitializeRequest\x1a-.function_executor_service.InitializeResponse\x12\x8f\x01\n"initialize_invocation_state_server\x12\x32.function_executor_service.InvocationStateResponse\x1a\x31.function_executor_service.InvocationStateRequest(\x01\x30\x01\x12\x61\n\x08run_task\x12).function_executor_service.RunTaskRequest\x1a*.function_executor_service.RunTaskResponse\x12m\n\x0c\x63heck_health\x12-.function_executor_service.HealthCheckRequest\x1a..function_executor_service.HealthCheckResponseb\x06proto3'
|
28
|
-
)
|
29
|
-
|
30
|
-
_globals = globals()
|
31
|
-
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
32
|
-
_builder.BuildTopDescriptorsAndMessages(
|
33
|
-
DESCRIPTOR, "indexify.function_executor.proto.function_executor_pb2", _globals
|
34
|
-
)
|
35
|
-
if not _descriptor._USE_C_DESCRIPTORS:
|
36
|
-
DESCRIPTOR._loaded_options = None
|
37
|
-
_globals["_SERIALIZEDOBJECT"]._serialized_start = 87
|
38
|
-
_globals["_SERIALIZEDOBJECT"]._serialized_end = 192
|
39
|
-
_globals["_INITIALIZEREQUEST"]._serialized_start = 195
|
40
|
-
_globals["_INITIALIZEREQUEST"]._serialized_end = 459
|
41
|
-
_globals["_INITIALIZERESPONSE"]._serialized_start = 461
|
42
|
-
_globals["_INITIALIZERESPONSE"]._serialized_end = 563
|
43
|
-
_globals["_SETINVOCATIONSTATEREQUEST"]._serialized_start = 566
|
44
|
-
_globals["_SETINVOCATIONSTATEREQUEST"]._serialized_end = 694
|
45
|
-
_globals["_SETINVOCATIONSTATERESPONSE"]._serialized_start = 696
|
46
|
-
_globals["_SETINVOCATIONSTATERESPONSE"]._serialized_end = 724
|
47
|
-
_globals["_GETINVOCATIONSTATEREQUEST"]._serialized_start = 726
|
48
|
-
_globals["_GETINVOCATIONSTATEREQUEST"]._serialized_end = 779
|
49
|
-
_globals["_GETINVOCATIONSTATERESPONSE"]._serialized_start = 782
|
50
|
-
_globals["_GETINVOCATIONSTATERESPONSE"]._serialized_end = 911
|
51
|
-
_globals["_INVOCATIONSTATEREQUEST"]._serialized_start = 914
|
52
|
-
_globals["_INVOCATIONSTATEREQUEST"]._serialized_end = 1161
|
53
|
-
_globals["_INVOCATIONSTATERESPONSE"]._serialized_start = 1164
|
54
|
-
_globals["_INVOCATIONSTATERESPONSE"]._serialized_end = 1415
|
55
|
-
_globals["_FUNCTIONOUTPUT"]._serialized_start = 1417
|
56
|
-
_globals["_FUNCTIONOUTPUT"]._serialized_end = 1495
|
57
|
-
_globals["_ROUTEROUTPUT"]._serialized_start = 1497
|
58
|
-
_globals["_ROUTEROUTPUT"]._serialized_end = 1526
|
59
|
-
_globals["_RUNTASKREQUEST"]._serialized_start = 1529
|
60
|
-
_globals["_RUNTASKREQUEST"]._serialized_end = 2003
|
61
|
-
_globals["_RUNTASKRESPONSE"]._serialized_start = 2006
|
62
|
-
_globals["_RUNTASKRESPONSE"]._serialized_end = 2375
|
63
|
-
_globals["_HEALTHCHECKREQUEST"]._serialized_start = 2377
|
64
|
-
_globals["_HEALTHCHECKREQUEST"]._serialized_end = 2397
|
65
|
-
_globals["_HEALTHCHECKRESPONSE"]._serialized_start = 2399
|
66
|
-
_globals["_HEALTHCHECKRESPONSE"]._serialized_end = 2454
|
67
|
-
_globals["_FUNCTIONEXECUTOR"]._serialized_start = 2457
|
68
|
-
_globals["_FUNCTIONEXECUTOR"]._serialized_end = 2938
|
69
|
-
# @@protoc_insertion_point(module_scope)
|