indexify 0.3.18__py3-none-any.whl → 0.3.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indexify/cli/cli.py +15 -17
- indexify/executor/api_objects.py +12 -0
- indexify/executor/blob_store/blob_store.py +69 -0
- indexify/executor/blob_store/local_fs_blob_store.py +48 -0
- indexify/executor/blob_store/metrics/blob_store.py +33 -0
- indexify/executor/blob_store/s3_blob_store.py +85 -0
- indexify/executor/downloader.py +149 -25
- indexify/executor/executor.py +77 -41
- indexify/executor/function_executor/function_executor.py +24 -11
- indexify/executor/function_executor/function_executor_state.py +9 -1
- indexify/executor/function_executor/function_executor_states_container.py +8 -1
- indexify/executor/function_executor/function_executor_status.py +4 -0
- indexify/executor/function_executor/health_checker.py +7 -2
- indexify/executor/function_executor/invocation_state_client.py +4 -2
- indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py +6 -0
- indexify/executor/function_executor/single_task_runner.py +15 -11
- indexify/executor/function_executor/task_output.py +36 -2
- indexify/executor/grpc/channel_manager.py +4 -3
- indexify/executor/grpc/function_executor_controller.py +391 -0
- indexify/executor/grpc/metrics/state_reconciler.py +17 -0
- indexify/executor/grpc/metrics/task_controller.py +8 -0
- indexify/executor/grpc/state_reconciler.py +324 -217
- indexify/executor/grpc/state_reporter.py +52 -41
- indexify/executor/grpc/task_controller.py +492 -0
- indexify/executor/metrics/task_reporter.py +14 -0
- indexify/executor/task_reporter.py +115 -6
- indexify/executor/task_runner.py +1 -0
- indexify/proto/executor_api.proto +91 -7
- indexify/proto/executor_api_pb2.py +49 -37
- indexify/proto/executor_api_pb2.pyi +158 -3
- indexify/proto/executor_api_pb2_grpc.py +47 -0
- {indexify-0.3.18.dist-info → indexify-0.3.20.dist-info}/METADATA +2 -1
- {indexify-0.3.18.dist-info → indexify-0.3.20.dist-info}/RECORD +35 -27
- {indexify-0.3.18.dist-info → indexify-0.3.20.dist-info}/WHEEL +0 -0
- {indexify-0.3.18.dist-info → indexify-0.3.20.dist-info}/entry_points.txt +0 -0
@@ -7,14 +7,28 @@ from httpx import Timeout
|
|
7
7
|
from tensorlake.function_executor.proto.function_executor_pb2 import FunctionOutput
|
8
8
|
from tensorlake.utils.http_client import get_httpx_client
|
9
9
|
|
10
|
+
from indexify.proto.executor_api_pb2 import (
|
11
|
+
DataPayload,
|
12
|
+
DataPayloadEncoding,
|
13
|
+
OutputEncoding,
|
14
|
+
ReportTaskOutcomeRequest,
|
15
|
+
TaskOutcome,
|
16
|
+
)
|
17
|
+
from indexify.proto.executor_api_pb2_grpc import ExecutorAPIStub
|
18
|
+
|
10
19
|
from .api_objects import (
|
11
20
|
TASK_OUTCOME_FAILURE,
|
12
21
|
TASK_OUTCOME_SUCCESS,
|
22
|
+
IngestFnOutputsResponse,
|
13
23
|
RouterOutput,
|
14
24
|
TaskResult,
|
15
25
|
)
|
16
26
|
from .function_executor.task_output import TaskOutput
|
27
|
+
from .grpc.channel_manager import ChannelManager
|
17
28
|
from .metrics.task_reporter import (
|
29
|
+
metric_report_task_outcome_errors,
|
30
|
+
metric_report_task_outcome_latency,
|
31
|
+
metric_report_task_outcome_rpcs,
|
18
32
|
metric_server_ingest_files_errors,
|
19
33
|
metric_server_ingest_files_latency,
|
20
34
|
metric_server_ingest_files_requests,
|
@@ -45,7 +59,11 @@ class TaskOutputSummary:
|
|
45
59
|
|
46
60
|
class TaskReporter:
|
47
61
|
def __init__(
|
48
|
-
self,
|
62
|
+
self,
|
63
|
+
base_url: str,
|
64
|
+
executor_id: str,
|
65
|
+
channel_manager: ChannelManager,
|
66
|
+
config_path: Optional[str] = None,
|
49
67
|
):
|
50
68
|
self._base_url = base_url
|
51
69
|
self._executor_id = executor_id
|
@@ -56,8 +74,9 @@ class TaskReporter:
|
|
56
74
|
# Creating a new async client for each request fixes this but it
|
57
75
|
# results in not reusing established TCP connections to server.
|
58
76
|
self._client = get_httpx_client(config_path, make_async=False)
|
77
|
+
self._channel_manager = channel_manager
|
59
78
|
|
60
|
-
async def shutdown(self):
|
79
|
+
async def shutdown(self) -> None:
|
61
80
|
"""Shuts down the task reporter.
|
62
81
|
|
63
82
|
Task reporter stops reporting all task outcomes to the Server.
|
@@ -66,7 +85,7 @@ class TaskReporter:
|
|
66
85
|
"""
|
67
86
|
self._is_shutdown = True
|
68
87
|
|
69
|
-
async def report(self, output: TaskOutput, logger: Any):
|
88
|
+
async def report(self, output: TaskOutput, logger: Any) -> None:
|
70
89
|
"""Reports result of the supplied task."""
|
71
90
|
logger = logger.bind(module=__name__)
|
72
91
|
|
@@ -103,18 +122,19 @@ class TaskReporter:
|
|
103
122
|
"files": output_files if len(output_files) > 0 else FORCE_MULTIPART,
|
104
123
|
}
|
105
124
|
|
125
|
+
# TODO: Instead of uploading the files to server, upload them to S3.
|
106
126
|
start_time = time.time()
|
107
127
|
with metric_server_ingest_files_latency.time():
|
108
128
|
metric_server_ingest_files_requests.inc()
|
109
129
|
# Run in a separate thread to not block the main event loop.
|
110
130
|
response = await asyncio.to_thread(
|
111
131
|
self._client.post,
|
112
|
-
url=f"{self._base_url}/internal/
|
132
|
+
url=f"{self._base_url}/internal/ingest_fn_outputs",
|
113
133
|
**kwargs,
|
114
134
|
)
|
115
135
|
end_time = time.time()
|
116
136
|
logger.info(
|
117
|
-
"
|
137
|
+
"files uploaded",
|
118
138
|
response_time=end_time - start_time,
|
119
139
|
response_code=response.status_code,
|
120
140
|
)
|
@@ -125,11 +145,79 @@ class TaskReporter:
|
|
125
145
|
metric_server_ingest_files_errors.inc()
|
126
146
|
# Caller catches and logs the exception.
|
127
147
|
raise Exception(
|
128
|
-
"failed to
|
148
|
+
"failed to upload files. "
|
129
149
|
f"Response code: {response.status_code}. "
|
130
150
|
f"Response text: '{response.text}'."
|
131
151
|
) from e
|
132
152
|
|
153
|
+
# TODO: If the files are uploaded successfully,
|
154
|
+
# we should record that so that if we fail to report
|
155
|
+
# the task outcome, we don't retry the upload.
|
156
|
+
# This will save us some time and resources.
|
157
|
+
|
158
|
+
ingested_files_response = response.json()
|
159
|
+
ingested_files = IngestFnOutputsResponse.model_validate(ingested_files_response)
|
160
|
+
fn_outputs = []
|
161
|
+
for data_payload in ingested_files.data_payloads:
|
162
|
+
fn_outputs.append(
|
163
|
+
DataPayload(
|
164
|
+
path=data_payload.path, # TODO: stop using this deprecated field once Server side migration is done.
|
165
|
+
uri=data_payload.path,
|
166
|
+
size=data_payload.size,
|
167
|
+
sha256_hash=data_payload.sha256_hash,
|
168
|
+
encoding=_to_grpc_data_payload_encoding(output),
|
169
|
+
encoding_version=0,
|
170
|
+
)
|
171
|
+
)
|
172
|
+
stdout, stderr = None, None
|
173
|
+
if ingested_files.stdout:
|
174
|
+
stdout = DataPayload(
|
175
|
+
path=ingested_files.stdout.path, # TODO: stop using this deprecated field once Server side migration is done.
|
176
|
+
uri=ingested_files.stdout.path,
|
177
|
+
size=ingested_files.stdout.size,
|
178
|
+
sha256_hash=ingested_files.stdout.sha256_hash,
|
179
|
+
encoding=DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_TEXT,
|
180
|
+
encoding_version=0,
|
181
|
+
)
|
182
|
+
if ingested_files.stderr:
|
183
|
+
stderr = DataPayload(
|
184
|
+
path=ingested_files.stderr.path, # TODO: stop using this deprecated field once Server side migration is done.
|
185
|
+
uri=ingested_files.stderr.path,
|
186
|
+
size=ingested_files.stderr.size,
|
187
|
+
sha256_hash=ingested_files.stderr.sha256_hash,
|
188
|
+
encoding=DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_TEXT,
|
189
|
+
encoding_version=0,
|
190
|
+
)
|
191
|
+
|
192
|
+
request = ReportTaskOutcomeRequest(
|
193
|
+
task_id=output.task_id,
|
194
|
+
namespace=output.namespace,
|
195
|
+
graph_name=output.graph_name,
|
196
|
+
function_name=output.function_name,
|
197
|
+
graph_invocation_id=output.graph_invocation_id,
|
198
|
+
outcome=_to_grpc_task_outcome(output),
|
199
|
+
invocation_id=output.graph_invocation_id,
|
200
|
+
executor_id=self._executor_id,
|
201
|
+
reducer=output.reducer,
|
202
|
+
next_functions=(output.router_output.edges if output.router_output else []),
|
203
|
+
fn_outputs=fn_outputs,
|
204
|
+
stdout=stdout,
|
205
|
+
stderr=stderr,
|
206
|
+
output_encoding=_to_grpc_output_encoding(output),
|
207
|
+
output_encoding_version=0,
|
208
|
+
)
|
209
|
+
try:
|
210
|
+
stub = ExecutorAPIStub(await self._channel_manager.get_channel())
|
211
|
+
with (
|
212
|
+
metric_report_task_outcome_latency.time(),
|
213
|
+
metric_report_task_outcome_errors.count_exceptions(),
|
214
|
+
):
|
215
|
+
metric_report_task_outcome_rpcs.inc()
|
216
|
+
await stub.report_task_outcome(request, timeout=5.0)
|
217
|
+
except Exception as e:
|
218
|
+
logger.error("failed to report task outcome", error=e)
|
219
|
+
raise e
|
220
|
+
|
133
221
|
def _process_task_output(
|
134
222
|
self, output: TaskOutput
|
135
223
|
) -> Tuple[TaskResult, List[Any], TaskOutputSummary]:
|
@@ -246,3 +334,24 @@ def _process_stderr(
|
|
246
334
|
)
|
247
335
|
summary.stderr_count += 1
|
248
336
|
summary.stderr_total_bytes += len(stderr)
|
337
|
+
|
338
|
+
|
339
|
+
def _to_grpc_task_outcome(task_output: TaskOutput) -> TaskOutcome:
|
340
|
+
if task_output.success:
|
341
|
+
return TaskOutcome.TASK_OUTCOME_SUCCESS
|
342
|
+
else:
|
343
|
+
return TaskOutcome.TASK_OUTCOME_FAILURE
|
344
|
+
|
345
|
+
|
346
|
+
def _to_grpc_output_encoding(task_output: TaskOutput) -> OutputEncoding:
|
347
|
+
if task_output.output_encoding == "json":
|
348
|
+
return OutputEncoding.OUTPUT_ENCODING_JSON
|
349
|
+
else:
|
350
|
+
return OutputEncoding.OUTPUT_ENCODING_PICKLE
|
351
|
+
|
352
|
+
|
353
|
+
def _to_grpc_data_payload_encoding(task_output: TaskOutput) -> DataPayloadEncoding:
|
354
|
+
if task_output.output_encoding == "json":
|
355
|
+
return DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_JSON
|
356
|
+
else:
|
357
|
+
return DataPayloadEncoding.DATA_PAYLOAD_ENCODING_BINARY_PICKLE
|
indexify/executor/task_runner.py
CHANGED
@@ -4,7 +4,29 @@ syntax = "proto3";
|
|
4
4
|
// Existing clients won't find the service if the package name changes.
|
5
5
|
package executor_api_pb;
|
6
6
|
|
7
|
-
// =====
|
7
|
+
// ===== DataPayload =====
|
8
|
+
enum DataPayloadEncoding {
|
9
|
+
DATA_PAYLOAD_ENCODING_UNKNOWN = 0;
|
10
|
+
// These encodings are currently mapping 1:1 to mime types.
|
11
|
+
// TODO: use SDK specific encodings becase 1:1 mapping might not work in the future.
|
12
|
+
DATA_PAYLOAD_ENCODING_UTF8_JSON = 1;
|
13
|
+
DATA_PAYLOAD_ENCODING_UTF8_TEXT = 2;
|
14
|
+
DATA_PAYLOAD_ENCODING_BINARY_PICKLE = 3;
|
15
|
+
}
|
16
|
+
|
17
|
+
message DataPayload {
|
18
|
+
optional string path = 1; // deprecated, TODO: remove when URI us used everywhere
|
19
|
+
optional uint64 size = 2;
|
20
|
+
optional string sha256_hash = 3;
|
21
|
+
// URI of the data.
|
22
|
+
// S3 URI if the data is stored in S3.
|
23
|
+
// Starts with "file://"" prefix if the data is stored on a local file system.
|
24
|
+
optional string uri = 4;
|
25
|
+
optional DataPayloadEncoding encoding = 5;
|
26
|
+
optional uint64 encoding_version = 6;
|
27
|
+
}
|
28
|
+
|
29
|
+
// ===== report_executor_state RPC =====
|
8
30
|
|
9
31
|
enum GPUModel {
|
10
32
|
GPU_MODEL_UNKNOWN = 0;
|
@@ -54,7 +76,10 @@ enum FunctionExecutorStatus {
|
|
54
76
|
FUNCTION_EXECUTOR_STATUS_RUNNING_TASK = 5;
|
55
77
|
FUNCTION_EXECUTOR_STATUS_UNHEALTHY = 6;
|
56
78
|
FUNCTION_EXECUTOR_STATUS_STOPPING = 7;
|
79
|
+
// FE is stopped but can be started up.
|
57
80
|
FUNCTION_EXECUTOR_STATUS_STOPPED = 8;
|
81
|
+
// FE is stopped forever, all resources are freed.
|
82
|
+
FUNCTION_EXECUTOR_STATUS_SHUTDOWN = 9;
|
58
83
|
}
|
59
84
|
|
60
85
|
// Immutable information that identifies and describes a Function Executor.
|
@@ -67,11 +92,18 @@ message FunctionExecutorDescription {
|
|
67
92
|
optional string image_uri = 6;
|
68
93
|
repeated string secret_names = 7;
|
69
94
|
optional HostResources resource_limits = 8;
|
95
|
+
// Timeout for customer code duration during FE creation.
|
96
|
+
optional uint32 customer_code_timeout_ms = 9;
|
97
|
+
optional DataPayload graph = 10;
|
70
98
|
}
|
71
99
|
|
72
100
|
message FunctionExecutorState {
|
73
101
|
optional FunctionExecutorDescription description = 1;
|
74
102
|
optional FunctionExecutorStatus status = 2;
|
103
|
+
// Human readable message clarifying the status.
|
104
|
+
// Currently it contains error message from customer code
|
105
|
+
// if status is FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR.
|
106
|
+
optional string status_message = 3;
|
75
107
|
}
|
76
108
|
|
77
109
|
enum ExecutorStatus {
|
@@ -103,6 +135,9 @@ message ExecutorState {
|
|
103
135
|
repeated FunctionExecutorState function_executor_states = 9;
|
104
136
|
map<string, string> labels = 10;
|
105
137
|
optional string state_hash = 11;
|
138
|
+
// Server supplied clock value of the latest desired executor state that was
|
139
|
+
// reconciled by Executor. Not included into state_hash.
|
140
|
+
optional uint64 server_clock = 12;
|
106
141
|
}
|
107
142
|
|
108
143
|
// A message sent by Executor to report its up to date state to Server.
|
@@ -114,7 +149,7 @@ message ReportExecutorStateRequest {
|
|
114
149
|
message ReportExecutorStateResponse {
|
115
150
|
}
|
116
151
|
|
117
|
-
// =====
|
152
|
+
// ===== get_desired_executor_states RPC =====
|
118
153
|
message Task {
|
119
154
|
optional string id = 1;
|
120
155
|
optional string namespace = 2;
|
@@ -122,9 +157,15 @@ message Task {
|
|
122
157
|
optional string graph_version = 4;
|
123
158
|
optional string function_name = 5;
|
124
159
|
optional string graph_invocation_id = 6;
|
125
|
-
optional string input_key = 8;
|
126
|
-
optional string reducer_output_key = 9;
|
127
|
-
optional
|
160
|
+
optional string input_key = 8; // deprecated. TODO: remove when input is used everywhere
|
161
|
+
optional string reducer_output_key = 9; // deprecated. TODO: remove when reducer_input is used everywhere
|
162
|
+
optional uint32 timeout_ms = 10;
|
163
|
+
optional DataPayload input = 11;
|
164
|
+
optional DataPayload reducer_input = 12;
|
165
|
+
// URI prefix for the output payloads.
|
166
|
+
// S3 URI if the data is stored in S3.
|
167
|
+
// Starts with "file://"" prefix followed by an absolute directory path if the data is stored on a local file system.
|
168
|
+
optional string output_payload_uri_prefix = 13;
|
128
169
|
}
|
129
170
|
|
130
171
|
message TaskAllocation {
|
@@ -147,6 +188,49 @@ message DesiredExecutorState {
|
|
147
188
|
optional uint64 clock = 3;
|
148
189
|
}
|
149
190
|
|
191
|
+
// ===== report_task_outcome RPC =====
|
192
|
+
enum TaskOutcome {
|
193
|
+
TASK_OUTCOME_UNKNOWN = 0;
|
194
|
+
TASK_OUTCOME_SUCCESS = 1;
|
195
|
+
TASK_OUTCOME_FAILURE = 2;
|
196
|
+
}
|
197
|
+
|
198
|
+
enum OutputEncoding {
|
199
|
+
OUTPUT_ENCODING_UNKNOWN = 0;
|
200
|
+
OUTPUT_ENCODING_JSON = 1;
|
201
|
+
OUTPUT_ENCODING_PICKLE = 2;
|
202
|
+
OUTPUT_ENCODING_BINARY = 3;
|
203
|
+
}
|
204
|
+
|
205
|
+
message ReportTaskOutcomeRequest {
|
206
|
+
optional string task_id = 1;
|
207
|
+
optional string namespace = 2;
|
208
|
+
optional string graph_name = 3;
|
209
|
+
optional string function_name = 4;
|
210
|
+
optional string graph_invocation_id = 6;
|
211
|
+
optional TaskOutcome outcome = 7;
|
212
|
+
optional string invocation_id = 8; // deprecated. TODO: remove when graph_invocation_id is used everywhere
|
213
|
+
optional string executor_id = 9;
|
214
|
+
optional bool reducer = 10;
|
215
|
+
|
216
|
+
// Edges that the function wants the invocation to be routed to.
|
217
|
+
// Previously called router_edges.
|
218
|
+
repeated string next_functions = 11;
|
219
|
+
// Outputs of the function.
|
220
|
+
repeated DataPayload fn_outputs = 12;
|
221
|
+
// Standard output and error streams of the function.
|
222
|
+
optional DataPayload stdout = 14;
|
223
|
+
optional DataPayload stderr = 15;
|
224
|
+
// Output encoding of all the outputs of a function have to be same.
|
225
|
+
optional OutputEncoding output_encoding = 13; // deprecated. TODO: remove when DataPayload.encoding is used everywhere
|
226
|
+
// This allows us to change how we encode the output from functions
|
227
|
+
// and serialize them into storage.
|
228
|
+
optional uint64 output_encoding_version = 5; // deprecated. TODO: remove when DataPayload.encoding_version is used everywhere
|
229
|
+
}
|
230
|
+
|
231
|
+
message ReportTaskOutcomeResponse {
|
232
|
+
}
|
233
|
+
|
150
234
|
// Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
|
151
235
|
// Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
|
152
236
|
//
|
@@ -165,6 +249,6 @@ service ExecutorAPI {
|
|
165
249
|
// Deprecated HTTP API is used to download the serialized graph and task inputs.
|
166
250
|
rpc get_desired_executor_states(GetDesiredExecutorStatesRequest) returns (stream DesiredExecutorState) {}
|
167
251
|
|
168
|
-
//
|
169
|
-
|
252
|
+
// Report the outcome of a task.
|
253
|
+
rpc report_task_outcome(ReportTaskOutcomeRequest) returns (ReportTaskOutcomeResponse) {}
|
170
254
|
}
|
@@ -19,7 +19,7 @@ _sym_db = _symbol_database.Default()
|
|
19
19
|
|
20
20
|
|
21
21
|
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
|
22
|
-
b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xed\
|
22
|
+
b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"\x87\x02\n\x0b\x44\x61taPayload\x12\x11\n\x04path\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x11\n\x04size\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x18\n\x0bsha256_hash\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x10\n\x03uri\x18\x04 \x01(\tH\x03\x88\x01\x01\x12;\n\x08\x65ncoding\x18\x05 \x01(\x0e\x32$.executor_api_pb.DataPayloadEncodingH\x04\x88\x01\x01\x12\x1d\n\x10\x65ncoding_version\x18\x06 \x01(\x04H\x05\x88\x01\x01\x42\x07\n\x05_pathB\x07\n\x05_sizeB\x0e\n\x0c_sha256_hashB\x06\n\x04_uriB\x0b\n\t_encodingB\x13\n\x11_encoding_version"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xed\x03\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12<\n\x0fresource_limits\x18\x08 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12%\n\x18\x63ustomer_code_timeout_ms\x18\t \x01(\rH\x07\x88\x01\x01\x12\x30\n\x05graph\x18\n \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x08\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x12\n\x10_resource_limitsB\x1b\n\x19_customer_code_timeout_msB\x08\n\x06_graph"\xe8\x01\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x12\x1b\n\x0estatus_message\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_statusB\x11\n\x0f_status_message"\xcb\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1d\n\x10\x64\x65velopment_mode\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06\x66lavor\x18\x04 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorFlavorH\x03\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x05\x88\x01\x01\x12;\n\x0e\x66ree_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cserver_clock\x18\x0c \x01(\x04H\x08\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x13\n\x11_development_modeB\x0b\n\t_hostnameB\t\n\x07_flavorB\n\n\x08_versionB\t\n\x07_statusB\x11\n\x0f_free_resourcesB\r\n\x0b_state_hashB\x0f\n\r_server_clock"l\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x42\x11\n\x0f_executor_state"\x1d\n\x1bReportExecutorStateResponse"\xd6\x04\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x16\n\tinput_key\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x1f\n\x12reducer_output_key\x18\t \x01(\tH\x07\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\rH\x08\x88\x01\x01\x12\x30\n\x05input\x18\x0b \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\t\x88\x01\x01\x12\x38\n\rreducer_input\x18\x0c \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\r \x01(\tH\x0b\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\x0c\n\n_input_keyB\x15\n\x13_reducer_output_keyB\r\n\x0b_timeout_msB\x08\n\x06_inputB\x10\n\x0e_reducer_inputB\x1c\n\x1a_output_payload_uri_prefix"\x7f\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_task"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock"\x87\x06\n\x18ReportTaskOutcomeRequest\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x04\x88\x01\x01\x12\x32\n\x07outcome\x18\x07 \x01(\x0e\x32\x1c.executor_api_pb.TaskOutcomeH\x05\x88\x01\x01\x12\x1a\n\rinvocation_id\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0b\x65xecutor_id\x18\t \x01(\tH\x07\x88\x01\x01\x12\x14\n\x07reducer\x18\n \x01(\x08H\x08\x88\x01\x01\x12\x16\n\x0enext_functions\x18\x0b \x03(\t\x12\x30\n\nfn_outputs\x18\x0c \x03(\x0b\x32\x1c.executor_api_pb.DataPayload\x12\x31\n\x06stdout\x18\x0e \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\t\x88\x01\x01\x12\x31\n\x06stderr\x18\x0f \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12=\n\x0foutput_encoding\x18\r \x01(\x0e\x32\x1f.executor_api_pb.OutputEncodingH\x0b\x88\x01\x01\x12$\n\x17output_encoding_version\x18\x05 \x01(\x04H\x0c\x88\x01\x01\x42\n\n\x08_task_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_outcomeB\x10\n\x0e_invocation_idB\x0e\n\x0c_executor_idB\n\n\x08_reducerB\t\n\x07_stdoutB\t\n\x07_stderrB\x12\n\x10_output_encodingB\x1a\n\x18_output_encoding_version"\x1b\n\x19ReportTaskOutcomeResponse*\xab\x01\n\x13\x44\x61taPayloadEncoding\x12!\n\x1d\x44\x41TA_PAYLOAD_ENCODING_UNKNOWN\x10\x00\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_JSON\x10\x01\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_TEXT\x10\x02\x12\'\n#DATA_PAYLOAD_ENCODING_BINARY_PICKLE\x10\x03*\x86\x03\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12"\n\x1eGPU_MODEL_NVIDIA_TESLA_T4_16GB\x10\n\x12$\n GPU_MODEL_NVIDIA_TESLA_V100_16GB\x10\x14\x12\x1d\n\x19GPU_MODEL_NVIDIA_A10_24GB\x10\x1e\x12\x1f\n\x1bGPU_MODEL_NVIDIA_A6000_48GB\x10(\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_40GB\x10\x32\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_80GB\x10\x33\x12"\n\x1eGPU_MODEL_NVIDIA_A100_PCI_40GB\x10\x34\x12#\n\x1fGPU_MODEL_NVIDIA_H100_SXM5_80GB\x10<\x12"\n\x1eGPU_MODEL_NVIDIA_H100_PCI_80GB\x10=\x12"\n\x1eGPU_MODEL_NVIDIA_RTX_6000_24GB\x10>*\xca\x03\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12(\n$FUNCTION_EXECUTOR_STATUS_STARTING_UP\x10\x01\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR\x10\x02\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_PLATFORM_ERROR\x10\x03\x12!\n\x1d\x46UNCTION_EXECUTOR_STATUS_IDLE\x10\x04\x12)\n%FUNCTION_EXECUTOR_STATUS_RUNNING_TASK\x10\x05\x12&\n"FUNCTION_EXECUTOR_STATUS_UNHEALTHY\x10\x06\x12%\n!FUNCTION_EXECUTOR_STATUS_STOPPING\x10\x07\x12$\n FUNCTION_EXECUTOR_STATUS_STOPPED\x10\x08\x12%\n!FUNCTION_EXECUTOR_STATUS_SHUTDOWN\x10\t*\xc3\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1c\n\x18\x45XECUTOR_STATUS_STOPPING\x10\x04\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x05*d\n\x0e\x45xecutorFlavor\x12\x1b\n\x17\x45XECUTOR_FLAVOR_UNKNOWN\x10\x00\x12\x17\n\x13\x45XECUTOR_FLAVOR_OSS\x10\x01\x12\x1c\n\x18\x45XECUTOR_FLAVOR_PLATFORM\x10\x02*[\n\x0bTaskOutcome\x12\x18\n\x14TASK_OUTCOME_UNKNOWN\x10\x00\x12\x18\n\x14TASK_OUTCOME_SUCCESS\x10\x01\x12\x18\n\x14TASK_OUTCOME_FAILURE\x10\x02*\x7f\n\x0eOutputEncoding\x12\x1b\n\x17OUTPUT_ENCODING_UNKNOWN\x10\x00\x12\x18\n\x14OUTPUT_ENCODING_JSON\x10\x01\x12\x1a\n\x16OUTPUT_ENCODING_PICKLE\x10\x02\x12\x1a\n\x16OUTPUT_ENCODING_BINARY\x10\x03\x32\xef\x02\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x12n\n\x13report_task_outcome\x12).executor_api_pb.ReportTaskOutcomeRequest\x1a*.executor_api_pb.ReportTaskOutcomeResponse"\x00\x62\x06proto3'
|
23
23
|
)
|
24
24
|
|
25
25
|
_globals = globals()
|
@@ -31,40 +31,52 @@ if not _descriptor._USE_C_DESCRIPTORS:
|
|
31
31
|
DESCRIPTOR._loaded_options = None
|
32
32
|
_globals["_EXECUTORSTATE_LABELSENTRY"]._loaded_options = None
|
33
33
|
_globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_options = b"8\001"
|
34
|
-
_globals["
|
35
|
-
_globals["
|
36
|
-
_globals["
|
37
|
-
_globals["
|
38
|
-
_globals["
|
39
|
-
_globals["
|
40
|
-
_globals["
|
41
|
-
_globals["
|
42
|
-
_globals["
|
43
|
-
_globals["
|
44
|
-
_globals["
|
45
|
-
_globals["
|
46
|
-
_globals["
|
47
|
-
_globals["
|
48
|
-
_globals["
|
49
|
-
_globals["
|
50
|
-
_globals["
|
51
|
-
_globals["
|
52
|
-
_globals["
|
53
|
-
_globals["
|
54
|
-
_globals["
|
55
|
-
_globals["
|
56
|
-
_globals["
|
57
|
-
_globals["
|
58
|
-
_globals["
|
59
|
-
_globals["
|
60
|
-
_globals["
|
61
|
-
_globals["
|
62
|
-
_globals["
|
63
|
-
_globals["
|
64
|
-
_globals["
|
65
|
-
_globals["
|
66
|
-
_globals["
|
67
|
-
_globals["
|
68
|
-
_globals["
|
69
|
-
_globals["
|
34
|
+
_globals["_DATAPAYLOADENCODING"]._serialized_start = 4203
|
35
|
+
_globals["_DATAPAYLOADENCODING"]._serialized_end = 4374
|
36
|
+
_globals["_GPUMODEL"]._serialized_start = 4377
|
37
|
+
_globals["_GPUMODEL"]._serialized_end = 4767
|
38
|
+
_globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 4770
|
39
|
+
_globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 5228
|
40
|
+
_globals["_EXECUTORSTATUS"]._serialized_start = 5231
|
41
|
+
_globals["_EXECUTORSTATUS"]._serialized_end = 5426
|
42
|
+
_globals["_EXECUTORFLAVOR"]._serialized_start = 5428
|
43
|
+
_globals["_EXECUTORFLAVOR"]._serialized_end = 5528
|
44
|
+
_globals["_TASKOUTCOME"]._serialized_start = 5530
|
45
|
+
_globals["_TASKOUTCOME"]._serialized_end = 5621
|
46
|
+
_globals["_OUTPUTENCODING"]._serialized_start = 5623
|
47
|
+
_globals["_OUTPUTENCODING"]._serialized_end = 5750
|
48
|
+
_globals["_DATAPAYLOAD"]._serialized_start = 55
|
49
|
+
_globals["_DATAPAYLOAD"]._serialized_end = 318
|
50
|
+
_globals["_GPURESOURCES"]._serialized_start = 320
|
51
|
+
_globals["_GPURESOURCES"]._serialized_end = 421
|
52
|
+
_globals["_HOSTRESOURCES"]._serialized_start = 424
|
53
|
+
_globals["_HOSTRESOURCES"]._serialized_end = 618
|
54
|
+
_globals["_ALLOWEDFUNCTION"]._serialized_start = 621
|
55
|
+
_globals["_ALLOWEDFUNCTION"]._serialized_end = 808
|
56
|
+
_globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_start = 811
|
57
|
+
_globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_end = 1304
|
58
|
+
_globals["_FUNCTIONEXECUTORSTATE"]._serialized_start = 1307
|
59
|
+
_globals["_FUNCTIONEXECUTORSTATE"]._serialized_end = 1539
|
60
|
+
_globals["_EXECUTORSTATE"]._serialized_start = 1542
|
61
|
+
_globals["_EXECUTORSTATE"]._serialized_end = 2257
|
62
|
+
_globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_start = 2077
|
63
|
+
_globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_end = 2122
|
64
|
+
_globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_start = 2259
|
65
|
+
_globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_end = 2367
|
66
|
+
_globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_start = 2369
|
67
|
+
_globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_end = 2398
|
68
|
+
_globals["_TASK"]._serialized_start = 2401
|
69
|
+
_globals["_TASK"]._serialized_end = 2999
|
70
|
+
_globals["_TASKALLOCATION"]._serialized_start = 3001
|
71
|
+
_globals["_TASKALLOCATION"]._serialized_end = 3128
|
72
|
+
_globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_start = 3130
|
73
|
+
_globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 3205
|
74
|
+
_globals["_DESIREDEXECUTORSTATE"]._serialized_start = 3208
|
75
|
+
_globals["_DESIREDEXECUTORSTATE"]._serialized_end = 3393
|
76
|
+
_globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_start = 3396
|
77
|
+
_globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_end = 4171
|
78
|
+
_globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_start = 4173
|
79
|
+
_globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_end = 4200
|
80
|
+
_globals["_EXECUTORAPI"]._serialized_start = 5753
|
81
|
+
_globals["_EXECUTORAPI"]._serialized_end = 6120
|
70
82
|
# @@protoc_insertion_point(module_scope)
|