indexify 0.3.19__py3-none-any.whl → 0.3.20__py3-none-any.whl

indexify/executor/task_reporter.py

@@ -9,6 +9,7 @@ from tensorlake.utils.http_client import get_httpx_client
 
 from indexify.proto.executor_api_pb2 import (
     DataPayload,
+    DataPayloadEncoding,
     OutputEncoding,
     ReportTaskOutcomeRequest,
     TaskOutcome,
@@ -75,7 +76,7 @@ class TaskReporter:
         self._client = get_httpx_client(config_path, make_async=False)
         self._channel_manager = channel_manager
 
-    async def shutdown(self):
+    async def shutdown(self) -> None:
         """Shuts down the task reporter.
 
         Task reporter stops reporting all task outcomes to the Server.
@@ -84,7 +85,7 @@ class TaskReporter:
         """
         self._is_shutdown = True
 
-    async def report(self, output: TaskOutput, logger: Any):
+    async def report(self, output: TaskOutput, logger: Any) -> None:
         """Reports result of the supplied task."""
         logger = logger.bind(module=__name__)
 
@@ -121,6 +122,7 @@ class TaskReporter:
             "files": output_files if len(output_files) > 0 else FORCE_MULTIPART,
         }
 
+        # TODO: Instead of uploading the files to server, upload them to S3.
         start_time = time.time()
         with metric_server_ingest_files_latency.time():
             metric_server_ingest_files_requests.inc()
@@ -159,23 +161,32 @@ class TaskReporter:
         for data_payload in ingested_files.data_payloads:
             fn_outputs.append(
                 DataPayload(
-                    path=data_payload.path,
+                    path=data_payload.path,  # TODO: stop using this deprecated field once Server side migration is done.
+                    uri=data_payload.path,
                     size=data_payload.size,
                     sha256_hash=data_payload.sha256_hash,
+                    encoding=_to_grpc_data_payload_encoding(output),
+                    encoding_version=0,
                 )
             )
         stdout, stderr = None, None
         if ingested_files.stdout:
             stdout = DataPayload(
-                path=ingested_files.stdout.path,
+                path=ingested_files.stdout.path,  # TODO: stop using this deprecated field once Server side migration is done.
+                uri=ingested_files.stdout.path,
                 size=ingested_files.stdout.size,
                 sha256_hash=ingested_files.stdout.sha256_hash,
+                encoding=DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_TEXT,
+                encoding_version=0,
             )
         if ingested_files.stderr:
             stderr = DataPayload(
-                path=ingested_files.stderr.path,
+                path=ingested_files.stderr.path,  # TODO: stop using this deprecated field once Server side migration is done.
+                uri=ingested_files.stderr.path,
                 size=ingested_files.stderr.size,
                 sha256_hash=ingested_files.stderr.sha256_hash,
+                encoding=DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_TEXT,
+                encoding_version=0,
             )
 
         request = ReportTaskOutcomeRequest(
@@ -337,3 +348,10 @@ def _to_grpc_output_encoding(task_output: TaskOutput) -> OutputEncoding:
         return OutputEncoding.OUTPUT_ENCODING_JSON
     else:
         return OutputEncoding.OUTPUT_ENCODING_PICKLE
+
+
+def _to_grpc_data_payload_encoding(task_output: TaskOutput) -> DataPayloadEncoding:
+    if task_output.output_encoding == "json":
+        return DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_JSON
+    else:
+        return DataPayloadEncoding.DATA_PAYLOAD_ENCODING_BINARY_PICKLE

indexify/proto/executor_api.proto

@@ -4,6 +4,28 @@ syntax = "proto3";
 // Existing clients won't find the service if the package name changes.
 package executor_api_pb;
 
+// ===== DataPayload =====
+enum DataPayloadEncoding {
+    DATA_PAYLOAD_ENCODING_UNKNOWN = 0;
+    // These encodings are currently mapping 1:1 to mime types.
+    // TODO: use SDK specific encodings becase 1:1 mapping might not work in the future.
+    DATA_PAYLOAD_ENCODING_UTF8_JSON = 1;
+    DATA_PAYLOAD_ENCODING_UTF8_TEXT = 2;
+    DATA_PAYLOAD_ENCODING_BINARY_PICKLE = 3;
+}
+
+message DataPayload {
+    optional string path = 1; // deprecated, TODO: remove when URI us used everywhere
+    optional uint64 size = 2;
+    optional string sha256_hash = 3;
+    // URI of the data.
+    // S3 URI if the data is stored in S3.
+    // Starts with "file://"" prefix if the data is stored on a local file system.
+    optional string uri = 4;
+    optional DataPayloadEncoding encoding = 5;
+    optional uint64 encoding_version = 6;
+}
+
 // ===== report_executor_state RPC =====
 
 enum GPUModel {
@@ -72,6 +94,7 @@ message FunctionExecutorDescription {
     optional HostResources resource_limits = 8;
     // Timeout for customer code duration during FE creation.
     optional uint32 customer_code_timeout_ms = 9;
+    optional DataPayload graph = 10;
 }
 
 message FunctionExecutorState {
@@ -112,6 +135,9 @@ message ExecutorState {
     repeated FunctionExecutorState function_executor_states = 9;
     map<string, string> labels = 10;
     optional string state_hash = 11;
+    // Server supplied clock value of the latest desired executor state that was
+    // reconciled by Executor. Not included into state_hash.
+    optional uint64 server_clock = 12;
 }
 
 // A message sent by Executor to report its up to date state to Server.
@@ -131,9 +157,15 @@ message Task {
     optional string graph_version = 4;
     optional string function_name = 5;
     optional string graph_invocation_id = 6;
-    optional string input_key = 8;
-    optional string reducer_output_key = 9;
+    optional string input_key = 8; // deprecated. TODO: remove when input is used everywhere
+    optional string reducer_output_key = 9; // deprecated. TODO: remove when reducer_input is used everywhere
     optional uint32 timeout_ms = 10;
+    optional DataPayload input = 11;
+    optional DataPayload reducer_input = 12;
+    // URI prefix for the output payloads.
+    // S3 URI if the data is stored in S3.
+    // Starts with "file://"" prefix followed by an absolute directory path if the data is stored on a local file system.
+    optional string output_payload_uri_prefix = 13;
 }
 
 message TaskAllocation {
@@ -163,12 +195,6 @@ enum TaskOutcome {
     TASK_OUTCOME_FAILURE = 2;
 }
 
-message DataPayload {
-    optional string path = 1;
-    optional uint64 size = 2;
-    optional string sha256_hash = 3;
-}
-
 enum OutputEncoding {
     OUTPUT_ENCODING_UNKNOWN = 0;
     OUTPUT_ENCODING_JSON = 1;
@@ -183,7 +209,7 @@ message ReportTaskOutcomeRequest {
     optional string function_name = 4;
     optional string graph_invocation_id = 6;
     optional TaskOutcome outcome = 7;
-    optional string invocation_id = 8;
+    optional string invocation_id = 8; // deprecated. TODO: remove when graph_invocation_id is used everywhere
     optional string executor_id = 9;
     optional bool reducer = 10;
 
@@ -196,10 +222,10 @@ message ReportTaskOutcomeRequest {
     optional DataPayload stdout = 14;
     optional DataPayload stderr = 15;
     // Output encoding of all the outputs of a function have to be same.
-    optional OutputEncoding output_encoding = 13;
+    optional OutputEncoding output_encoding = 13; // deprecated. TODO: remove when DataPayload.encoding is used everywhere
     // This allows us to change how we encode the output from functions
     // and serialize them into storage.
-    optional uint64 output_encoding_version = 5;
+    optional uint64 output_encoding_version = 5;  // deprecated. TODO: remove when DataPayload.encoding_version is used everywhere
 }
 
 message ReportTaskOutcomeResponse {

indexify/proto/executor_api_pb2.py

@@ -19,7 +19,7 @@ _sym_db = _symbol_database.Default()
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xb1\x03\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12<\n\x0fresource_limits\x18\x08 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12%\n\x18\x63ustomer_code_timeout_ms\x18\t \x01(\rH\x07\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x12\n\x10_resource_limitsB\x1b\n\x19_customer_code_timeout_ms"\xe8\x01\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x12\x1b\n\x0estatus_message\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_statusB\x11\n\x0f_status_message"\x9f\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1d\n\x10\x64\x65velopment_mode\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06\x66lavor\x18\x04 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorFlavorH\x03\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x05\x88\x01\x01\x12;\n\x0e\x66ree_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x07\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x13\n\x11_development_modeB\x0b\n\t_hostnameB\t\n\x07_flavorB\n\n\x08_versionB\t\n\x07_statusB\x11\n\x0f_free_resourcesB\r\n\x0b_state_hash"l\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x42\x11\n\x0f_executor_state"\x1d\n\x1bReportExecutorStateResponse"\x88\x03\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x16\n\tinput_key\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x1f\n\x12reducer_output_key\x18\t \x01(\tH\x07\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\rH\x08\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\x0c\n\n_input_keyB\x15\n\x13_reducer_output_keyB\r\n\x0b_timeout_ms"\x7f\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_task"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock"o\n\x0b\x44\x61taPayload\x12\x11\n\x04path\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x11\n\x04size\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x18\n\x0bsha256_hash\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x07\n\x05_pathB\x07\n\x05_sizeB\x0e\n\x0c_sha256_hash"\x87\x06\n\x18ReportTaskOutcomeRequest\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x04\x88\x01\x01\x12\x32\n\x07outcome\x18\x07 \x01(\x0e\x32\x1c.executor_api_pb.TaskOutcomeH\x05\x88\x01\x01\x12\x1a\n\rinvocation_id\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0b\x65xecutor_id\x18\t \x01(\tH\x07\x88\x01\x01\x12\x14\n\x07reducer\x18\n \x01(\x08H\x08\x88\x01\x01\x12\x16\n\x0enext_functions\x18\x0b \x03(\t\x12\x30\n\nfn_outputs\x18\x0c \x03(\x0b\x32\x1c.executor_api_pb.DataPayload\x12\x31\n\x06stdout\x18\x0e \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\t\x88\x01\x01\x12\x31\n\x06stderr\x18\x0f \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12=\n\x0foutput_encoding\x18\r \x01(\x0e\x32\x1f.executor_api_pb.OutputEncodingH\x0b\x88\x01\x01\x12$\n\x17output_encoding_version\x18\x05 \x01(\x04H\x0c\x88\x01\x01\x42\n\n\x08_task_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_outcomeB\x10\n\x0e_invocation_idB\x0e\n\x0c_executor_idB\n\n\x08_reducerB\t\n\x07_stdoutB\t\n\x07_stderrB\x12\n\x10_output_encodingB\x1a\n\x18_output_encoding_version"\x1b\n\x19ReportTaskOutcomeResponse*\x86\x03\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12"\n\x1eGPU_MODEL_NVIDIA_TESLA_T4_16GB\x10\n\x12$\n GPU_MODEL_NVIDIA_TESLA_V100_16GB\x10\x14\x12\x1d\n\x19GPU_MODEL_NVIDIA_A10_24GB\x10\x1e\x12\x1f\n\x1bGPU_MODEL_NVIDIA_A6000_48GB\x10(\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_40GB\x10\x32\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_80GB\x10\x33\x12"\n\x1eGPU_MODEL_NVIDIA_A100_PCI_40GB\x10\x34\x12#\n\x1fGPU_MODEL_NVIDIA_H100_SXM5_80GB\x10<\x12"\n\x1eGPU_MODEL_NVIDIA_H100_PCI_80GB\x10=\x12"\n\x1eGPU_MODEL_NVIDIA_RTX_6000_24GB\x10>*\xca\x03\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12(\n$FUNCTION_EXECUTOR_STATUS_STARTING_UP\x10\x01\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR\x10\x02\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_PLATFORM_ERROR\x10\x03\x12!\n\x1d\x46UNCTION_EXECUTOR_STATUS_IDLE\x10\x04\x12)\n%FUNCTION_EXECUTOR_STATUS_RUNNING_TASK\x10\x05\x12&\n"FUNCTION_EXECUTOR_STATUS_UNHEALTHY\x10\x06\x12%\n!FUNCTION_EXECUTOR_STATUS_STOPPING\x10\x07\x12$\n FUNCTION_EXECUTOR_STATUS_STOPPED\x10\x08\x12%\n!FUNCTION_EXECUTOR_STATUS_SHUTDOWN\x10\t*\xc3\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1c\n\x18\x45XECUTOR_STATUS_STOPPING\x10\x04\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x05*d\n\x0e\x45xecutorFlavor\x12\x1b\n\x17\x45XECUTOR_FLAVOR_UNKNOWN\x10\x00\x12\x17\n\x13\x45XECUTOR_FLAVOR_OSS\x10\x01\x12\x1c\n\x18\x45XECUTOR_FLAVOR_PLATFORM\x10\x02*[\n\x0bTaskOutcome\x12\x18\n\x14TASK_OUTCOME_UNKNOWN\x10\x00\x12\x18\n\x14TASK_OUTCOME_SUCCESS\x10\x01\x12\x18\n\x14TASK_OUTCOME_FAILURE\x10\x02*\x7f\n\x0eOutputEncoding\x12\x1b\n\x17OUTPUT_ENCODING_UNKNOWN\x10\x00\x12\x18\n\x14OUTPUT_ENCODING_JSON\x10\x01\x12\x1a\n\x16OUTPUT_ENCODING_PICKLE\x10\x02\x12\x1a\n\x16OUTPUT_ENCODING_BINARY\x10\x03\x32\xef\x02\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x12n\n\x13report_task_outcome\x12).executor_api_pb.ReportTaskOutcomeRequest\x1a*.executor_api_pb.ReportTaskOutcomeResponse"\x00\x62\x06proto3'
+    b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"\x87\x02\n\x0b\x44\x61taPayload\x12\x11\n\x04path\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x11\n\x04size\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x18\n\x0bsha256_hash\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x10\n\x03uri\x18\x04 \x01(\tH\x03\x88\x01\x01\x12;\n\x08\x65ncoding\x18\x05 \x01(\x0e\x32$.executor_api_pb.DataPayloadEncodingH\x04\x88\x01\x01\x12\x1d\n\x10\x65ncoding_version\x18\x06 \x01(\x04H\x05\x88\x01\x01\x42\x07\n\x05_pathB\x07\n\x05_sizeB\x0e\n\x0c_sha256_hashB\x06\n\x04_uriB\x0b\n\t_encodingB\x13\n\x11_encoding_version"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xed\x03\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12<\n\x0fresource_limits\x18\x08 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12%\n\x18\x63ustomer_code_timeout_ms\x18\t \x01(\rH\x07\x88\x01\x01\x12\x30\n\x05graph\x18\n \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x08\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x12\n\x10_resource_limitsB\x1b\n\x19_customer_code_timeout_msB\x08\n\x06_graph"\xe8\x01\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x12\x1b\n\x0estatus_message\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_statusB\x11\n\x0f_status_message"\xcb\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1d\n\x10\x64\x65velopment_mode\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06\x66lavor\x18\x04 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorFlavorH\x03\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x05\x88\x01\x01\x12;\n\x0e\x66ree_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cserver_clock\x18\x0c \x01(\x04H\x08\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x13\n\x11_development_modeB\x0b\n\t_hostnameB\t\n\x07_flavorB\n\n\x08_versionB\t\n\x07_statusB\x11\n\x0f_free_resourcesB\r\n\x0b_state_hashB\x0f\n\r_server_clock"l\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x42\x11\n\x0f_executor_state"\x1d\n\x1bReportExecutorStateResponse"\xd6\x04\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x16\n\tinput_key\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x1f\n\x12reducer_output_key\x18\t \x01(\tH\x07\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\rH\x08\x88\x01\x01\x12\x30\n\x05input\x18\x0b \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\t\x88\x01\x01\x12\x38\n\rreducer_input\x18\x0c \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\r \x01(\tH\x0b\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\x0c\n\n_input_keyB\x15\n\x13_reducer_output_keyB\r\n\x0b_timeout_msB\x08\n\x06_inputB\x10\n\x0e_reducer_inputB\x1c\n\x1a_output_payload_uri_prefix"\x7f\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_task"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock"\x87\x06\n\x18ReportTaskOutcomeRequest\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x04\x88\x01\x01\x12\x32\n\x07outcome\x18\x07 \x01(\x0e\x32\x1c.executor_api_pb.TaskOutcomeH\x05\x88\x01\x01\x12\x1a\n\rinvocation_id\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0b\x65xecutor_id\x18\t \x01(\tH\x07\x88\x01\x01\x12\x14\n\x07reducer\x18\n \x01(\x08H\x08\x88\x01\x01\x12\x16\n\x0enext_functions\x18\x0b \x03(\t\x12\x30\n\nfn_outputs\x18\x0c \x03(\x0b\x32\x1c.executor_api_pb.DataPayload\x12\x31\n\x06stdout\x18\x0e \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\t\x88\x01\x01\x12\x31\n\x06stderr\x18\x0f \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12=\n\x0foutput_encoding\x18\r \x01(\x0e\x32\x1f.executor_api_pb.OutputEncodingH\x0b\x88\x01\x01\x12$\n\x17output_encoding_version\x18\x05 \x01(\x04H\x0c\x88\x01\x01\x42\n\n\x08_task_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_outcomeB\x10\n\x0e_invocation_idB\x0e\n\x0c_executor_idB\n\n\x08_reducerB\t\n\x07_stdoutB\t\n\x07_stderrB\x12\n\x10_output_encodingB\x1a\n\x18_output_encoding_version"\x1b\n\x19ReportTaskOutcomeResponse*\xab\x01\n\x13\x44\x61taPayloadEncoding\x12!\n\x1d\x44\x41TA_PAYLOAD_ENCODING_UNKNOWN\x10\x00\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_JSON\x10\x01\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_TEXT\x10\x02\x12\'\n#DATA_PAYLOAD_ENCODING_BINARY_PICKLE\x10\x03*\x86\x03\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12"\n\x1eGPU_MODEL_NVIDIA_TESLA_T4_16GB\x10\n\x12$\n GPU_MODEL_NVIDIA_TESLA_V100_16GB\x10\x14\x12\x1d\n\x19GPU_MODEL_NVIDIA_A10_24GB\x10\x1e\x12\x1f\n\x1bGPU_MODEL_NVIDIA_A6000_48GB\x10(\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_40GB\x10\x32\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_80GB\x10\x33\x12"\n\x1eGPU_MODEL_NVIDIA_A100_PCI_40GB\x10\x34\x12#\n\x1fGPU_MODEL_NVIDIA_H100_SXM5_80GB\x10<\x12"\n\x1eGPU_MODEL_NVIDIA_H100_PCI_80GB\x10=\x12"\n\x1eGPU_MODEL_NVIDIA_RTX_6000_24GB\x10>*\xca\x03\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12(\n$FUNCTION_EXECUTOR_STATUS_STARTING_UP\x10\x01\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR\x10\x02\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_PLATFORM_ERROR\x10\x03\x12!\n\x1d\x46UNCTION_EXECUTOR_STATUS_IDLE\x10\x04\x12)\n%FUNCTION_EXECUTOR_STATUS_RUNNING_TASK\x10\x05\x12&\n"FUNCTION_EXECUTOR_STATUS_UNHEALTHY\x10\x06\x12%\n!FUNCTION_EXECUTOR_STATUS_STOPPING\x10\x07\x12$\n FUNCTION_EXECUTOR_STATUS_STOPPED\x10\x08\x12%\n!FUNCTION_EXECUTOR_STATUS_SHUTDOWN\x10\t*\xc3\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1c\n\x18\x45XECUTOR_STATUS_STOPPING\x10\x04\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x05*d\n\x0e\x45xecutorFlavor\x12\x1b\n\x17\x45XECUTOR_FLAVOR_UNKNOWN\x10\x00\x12\x17\n\x13\x45XECUTOR_FLAVOR_OSS\x10\x01\x12\x1c\n\x18\x45XECUTOR_FLAVOR_PLATFORM\x10\x02*[\n\x0bTaskOutcome\x12\x18\n\x14TASK_OUTCOME_UNKNOWN\x10\x00\x12\x18\n\x14TASK_OUTCOME_SUCCESS\x10\x01\x12\x18\n\x14TASK_OUTCOME_FAILURE\x10\x02*\x7f\n\x0eOutputEncoding\x12\x1b\n\x17OUTPUT_ENCODING_UNKNOWN\x10\x00\x12\x18\n\x14OUTPUT_ENCODING_JSON\x10\x01\x12\x1a\n\x16OUTPUT_ENCODING_PICKLE\x10\x02\x12\x1a\n\x16OUTPUT_ENCODING_BINARY\x10\x03\x32\xef\x02\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x12n\n\x13report_task_outcome\x12).executor_api_pb.ReportTaskOutcomeRequest\x1a*.executor_api_pb.ReportTaskOutcomeResponse"\x00\x62\x06proto3'
 )
 
 _globals = globals()
@@ -31,50 +31,52 @@ if not _descriptor._USE_C_DESCRIPTORS:
     DESCRIPTOR._loaded_options = None
     _globals["_EXECUTORSTATE_LABELSENTRY"]._loaded_options = None
     _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_options = b"8\001"
-    _globals["_GPUMODEL"]._serialized_start = 3740
-    _globals["_GPUMODEL"]._serialized_end = 4130
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 4133
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 4591
-    _globals["_EXECUTORSTATUS"]._serialized_start = 4594
-    _globals["_EXECUTORSTATUS"]._serialized_end = 4789
-    _globals["_EXECUTORFLAVOR"]._serialized_start = 4791
-    _globals["_EXECUTORFLAVOR"]._serialized_end = 4891
-    _globals["_TASKOUTCOME"]._serialized_start = 4893
-    _globals["_TASKOUTCOME"]._serialized_end = 4984
-    _globals["_OUTPUTENCODING"]._serialized_start = 4986
-    _globals["_OUTPUTENCODING"]._serialized_end = 5113
-    _globals["_GPURESOURCES"]._serialized_start = 54
-    _globals["_GPURESOURCES"]._serialized_end = 155
-    _globals["_HOSTRESOURCES"]._serialized_start = 158
-    _globals["_HOSTRESOURCES"]._serialized_end = 352
-    _globals["_ALLOWEDFUNCTION"]._serialized_start = 355
-    _globals["_ALLOWEDFUNCTION"]._serialized_end = 542
-    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_start = 545
-    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_end = 978
-    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_start = 981
-    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_end = 1213
-    _globals["_EXECUTORSTATE"]._serialized_start = 1216
-    _globals["_EXECUTORSTATE"]._serialized_end = 1887
-    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_start = 1724
-    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_end = 1769
-    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_start = 1889
-    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_end = 1997
-    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_start = 1999
-    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_end = 2028
-    _globals["_TASK"]._serialized_start = 2031
-    _globals["_TASK"]._serialized_end = 2423
-    _globals["_TASKALLOCATION"]._serialized_start = 2425
-    _globals["_TASKALLOCATION"]._serialized_end = 2552
-    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_start = 2554
-    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 2629
-    _globals["_DESIREDEXECUTORSTATE"]._serialized_start = 2632
-    _globals["_DESIREDEXECUTORSTATE"]._serialized_end = 2817
-    _globals["_DATAPAYLOAD"]._serialized_start = 2819
-    _globals["_DATAPAYLOAD"]._serialized_end = 2930
-    _globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_start = 2933
-    _globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_end = 3708
-    _globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_start = 3710
-    _globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_end = 3737
-    _globals["_EXECUTORAPI"]._serialized_start = 5116
-    _globals["_EXECUTORAPI"]._serialized_end = 5483
+    _globals["_DATAPAYLOADENCODING"]._serialized_start = 4203
+    _globals["_DATAPAYLOADENCODING"]._serialized_end = 4374
+    _globals["_GPUMODEL"]._serialized_start = 4377
+    _globals["_GPUMODEL"]._serialized_end = 4767
+    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 4770
+    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 5228
+    _globals["_EXECUTORSTATUS"]._serialized_start = 5231
+    _globals["_EXECUTORSTATUS"]._serialized_end = 5426
+    _globals["_EXECUTORFLAVOR"]._serialized_start = 5428
+    _globals["_EXECUTORFLAVOR"]._serialized_end = 5528
+    _globals["_TASKOUTCOME"]._serialized_start = 5530
+    _globals["_TASKOUTCOME"]._serialized_end = 5621
+    _globals["_OUTPUTENCODING"]._serialized_start = 5623
+    _globals["_OUTPUTENCODING"]._serialized_end = 5750
+    _globals["_DATAPAYLOAD"]._serialized_start = 55
+    _globals["_DATAPAYLOAD"]._serialized_end = 318
+    _globals["_GPURESOURCES"]._serialized_start = 320
+    _globals["_GPURESOURCES"]._serialized_end = 421
+    _globals["_HOSTRESOURCES"]._serialized_start = 424
+    _globals["_HOSTRESOURCES"]._serialized_end = 618
+    _globals["_ALLOWEDFUNCTION"]._serialized_start = 621
+    _globals["_ALLOWEDFUNCTION"]._serialized_end = 808
+    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_start = 811
+    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_end = 1304
+    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_start = 1307
+    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_end = 1539
+    _globals["_EXECUTORSTATE"]._serialized_start = 1542
+    _globals["_EXECUTORSTATE"]._serialized_end = 2257
+    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_start = 2077
+    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_end = 2122
+    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_start = 2259
+    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_end = 2367
+    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_start = 2369
+    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_end = 2398
+    _globals["_TASK"]._serialized_start = 2401
+    _globals["_TASK"]._serialized_end = 2999
+    _globals["_TASKALLOCATION"]._serialized_start = 3001
+    _globals["_TASKALLOCATION"]._serialized_end = 3128
+    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_start = 3130
+    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 3205
+    _globals["_DESIREDEXECUTORSTATE"]._serialized_start = 3208
+    _globals["_DESIREDEXECUTORSTATE"]._serialized_end = 3393
+    _globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_start = 3396
+    _globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_end = 4171
+    _globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_start = 4173
+    _globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_end = 4200
+    _globals["_EXECUTORAPI"]._serialized_start = 5753
+    _globals["_EXECUTORAPI"]._serialized_end = 6120
 # @@protoc_insertion_point(module_scope)

indexify/proto/executor_api_pb2.pyi

@@ -11,6 +11,13 @@ from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
 
 DESCRIPTOR: _descriptor.FileDescriptor
 
+class DataPayloadEncoding(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    DATA_PAYLOAD_ENCODING_UNKNOWN: _ClassVar[DataPayloadEncoding]
+    DATA_PAYLOAD_ENCODING_UTF8_JSON: _ClassVar[DataPayloadEncoding]
+    DATA_PAYLOAD_ENCODING_UTF8_TEXT: _ClassVar[DataPayloadEncoding]
+    DATA_PAYLOAD_ENCODING_BINARY_PICKLE: _ClassVar[DataPayloadEncoding]
+
 class GPUModel(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
     GPU_MODEL_UNKNOWN: _ClassVar[GPUModel]
@@ -70,6 +77,10 @@ class OutputEncoding(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     OUTPUT_ENCODING_PICKLE: _ClassVar[OutputEncoding]
     OUTPUT_ENCODING_BINARY: _ClassVar[OutputEncoding]
 
+DATA_PAYLOAD_ENCODING_UNKNOWN: DataPayloadEncoding
+DATA_PAYLOAD_ENCODING_UTF8_JSON: DataPayloadEncoding
+DATA_PAYLOAD_ENCODING_UTF8_TEXT: DataPayloadEncoding
+DATA_PAYLOAD_ENCODING_BINARY_PICKLE: DataPayloadEncoding
 GPU_MODEL_UNKNOWN: GPUModel
 GPU_MODEL_NVIDIA_TESLA_T4_16GB: GPUModel
 GPU_MODEL_NVIDIA_TESLA_V100_16GB: GPUModel
@@ -108,6 +119,30 @@ OUTPUT_ENCODING_JSON: OutputEncoding
 OUTPUT_ENCODING_PICKLE: OutputEncoding
 OUTPUT_ENCODING_BINARY: OutputEncoding
 
+class DataPayload(_message.Message):
+    __slots__ = ("path", "size", "sha256_hash", "uri", "encoding", "encoding_version")
+    PATH_FIELD_NUMBER: _ClassVar[int]
+    SIZE_FIELD_NUMBER: _ClassVar[int]
+    SHA256_HASH_FIELD_NUMBER: _ClassVar[int]
+    URI_FIELD_NUMBER: _ClassVar[int]
+    ENCODING_FIELD_NUMBER: _ClassVar[int]
+    ENCODING_VERSION_FIELD_NUMBER: _ClassVar[int]
+    path: str
+    size: int
+    sha256_hash: str
+    uri: str
+    encoding: DataPayloadEncoding
+    encoding_version: int
+    def __init__(
+        self,
+        path: _Optional[str] = ...,
+        size: _Optional[int] = ...,
+        sha256_hash: _Optional[str] = ...,
+        uri: _Optional[str] = ...,
+        encoding: _Optional[_Union[DataPayloadEncoding, str]] = ...,
+        encoding_version: _Optional[int] = ...,
+    ) -> None: ...
+
 class GPUResources(_message.Message):
     __slots__ = ("count", "model")
     COUNT_FIELD_NUMBER: _ClassVar[int]
@@ -165,6 +200,7 @@ class FunctionExecutorDescription(_message.Message):
         "secret_names",
         "resource_limits",
         "customer_code_timeout_ms",
+        "graph",
     )
     ID_FIELD_NUMBER: _ClassVar[int]
     NAMESPACE_FIELD_NUMBER: _ClassVar[int]
@@ -175,6 +211,7 @@ class FunctionExecutorDescription(_message.Message):
     SECRET_NAMES_FIELD_NUMBER: _ClassVar[int]
     RESOURCE_LIMITS_FIELD_NUMBER: _ClassVar[int]
     CUSTOMER_CODE_TIMEOUT_MS_FIELD_NUMBER: _ClassVar[int]
+    GRAPH_FIELD_NUMBER: _ClassVar[int]
     id: str
     namespace: str
     graph_name: str
@@ -184,6 +221,7 @@ class FunctionExecutorDescription(_message.Message):
     secret_names: _containers.RepeatedScalarFieldContainer[str]
     resource_limits: HostResources
     customer_code_timeout_ms: int
+    graph: DataPayload
     def __init__(
         self,
         id: _Optional[str] = ...,
@@ -195,6 +233,7 @@ class FunctionExecutorDescription(_message.Message):
         secret_names: _Optional[_Iterable[str]] = ...,
         resource_limits: _Optional[_Union[HostResources, _Mapping]] = ...,
         customer_code_timeout_ms: _Optional[int] = ...,
+        graph: _Optional[_Union[DataPayload, _Mapping]] = ...,
     ) -> None: ...
 
 class FunctionExecutorState(_message.Message):
@@ -225,6 +264,7 @@ class ExecutorState(_message.Message):
         "function_executor_states",
         "labels",
         "state_hash",
+        "server_clock",
     )
 
     class LabelsEntry(_message.Message):
@@ -248,6 +288,7 @@ class ExecutorState(_message.Message):
     FUNCTION_EXECUTOR_STATES_FIELD_NUMBER: _ClassVar[int]
     LABELS_FIELD_NUMBER: _ClassVar[int]
     STATE_HASH_FIELD_NUMBER: _ClassVar[int]
+    SERVER_CLOCK_FIELD_NUMBER: _ClassVar[int]
     executor_id: str
     development_mode: bool
     hostname: str
@@ -261,6 +302,7 @@ class ExecutorState(_message.Message):
     ]
     labels: _containers.ScalarMap[str, str]
     state_hash: str
+    server_clock: int
     def __init__(
         self,
         executor_id: _Optional[str] = ...,
@@ -278,6 +320,7 @@ class ExecutorState(_message.Message):
         ] = ...,
         labels: _Optional[_Mapping[str, str]] = ...,
         state_hash: _Optional[str] = ...,
+        server_clock: _Optional[int] = ...,
     ) -> None: ...
 
 class ReportExecutorStateRequest(_message.Message):
@@ -303,6 +346,9 @@ class Task(_message.Message):
         "input_key",
         "reducer_output_key",
         "timeout_ms",
+        "input",
+        "reducer_input",
+        "output_payload_uri_prefix",
     )
     ID_FIELD_NUMBER: _ClassVar[int]
     NAMESPACE_FIELD_NUMBER: _ClassVar[int]
@@ -313,6 +359,9 @@ class Task(_message.Message):
     INPUT_KEY_FIELD_NUMBER: _ClassVar[int]
     REDUCER_OUTPUT_KEY_FIELD_NUMBER: _ClassVar[int]
     TIMEOUT_MS_FIELD_NUMBER: _ClassVar[int]
+    INPUT_FIELD_NUMBER: _ClassVar[int]
+    REDUCER_INPUT_FIELD_NUMBER: _ClassVar[int]
+    OUTPUT_PAYLOAD_URI_PREFIX_FIELD_NUMBER: _ClassVar[int]
     id: str
     namespace: str
     graph_name: str
@@ -322,6 +371,9 @@ class Task(_message.Message):
     input_key: str
     reducer_output_key: str
     timeout_ms: int
+    input: DataPayload
+    reducer_input: DataPayload
+    output_payload_uri_prefix: str
     def __init__(
         self,
         id: _Optional[str] = ...,
@@ -333,6 +385,9 @@ class Task(_message.Message):
         input_key: _Optional[str] = ...,
         reducer_output_key: _Optional[str] = ...,
         timeout_ms: _Optional[int] = ...,
+        input: _Optional[_Union[DataPayload, _Mapping]] = ...,
+        reducer_input: _Optional[_Union[DataPayload, _Mapping]] = ...,
+        output_payload_uri_prefix: _Optional[str] = ...,
     ) -> None: ...
 
 class TaskAllocation(_message.Message):
@@ -372,21 +427,6 @@ class DesiredExecutorState(_message.Message):
         clock: _Optional[int] = ...,
     ) -> None: ...
 
-class DataPayload(_message.Message):
-    __slots__ = ("path", "size", "sha256_hash")
-    PATH_FIELD_NUMBER: _ClassVar[int]
-    SIZE_FIELD_NUMBER: _ClassVar[int]
-    SHA256_HASH_FIELD_NUMBER: _ClassVar[int]
-    path: str
-    size: int
-    sha256_hash: str
-    def __init__(
-        self,
-        path: _Optional[str] = ...,
-        size: _Optional[int] = ...,
-        sha256_hash: _Optional[str] = ...,
-    ) -> None: ...
-
 class ReportTaskOutcomeRequest(_message.Message):
     __slots__ = (
         "task_id",

{indexify-0.3.19.dist-info → indexify-0.3.20.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.19
+Version: 0.3.20
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
+Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: tensorlake (>=0.1)

{indexify-0.3.19.dist-info → indexify-0.3.20.dist-info}/RECORD

@@ -1,15 +1,19 @@
-indexify/cli/cli.py,sha256=TgXC_ZZPT-MJA7CWH297Y4_5M5SDt2QslPUoMotp3Cc,8708
+indexify/cli/cli.py,sha256=L-QVyRO-nGazfrdIbhxYvxMwhEnNjJ4H32Lw9pz464I,9345
 indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
 indexify/executor/api_objects.py,sha256=qKQMEjr18xIq7yjpnsLPnWXP4KsVQ9O76EsbYoaL_qQ,1507
-indexify/executor/downloader.py,sha256=59FD2LNUQvl6Ul6m59uTsgE0FWU5t6C2XflaaDcc2FA,9459
-indexify/executor/executor.py,sha256=jdsp9vP2q8y6rIAIRmwfTXBGW3D5cgaFjfy_tHetJ3k,16035
+indexify/executor/blob_store/blob_store.py,sha256=XViw_KRfFSNqwcFYwMZixZF-EYCjXK2AQHdt0xh4UVo,2368
+indexify/executor/blob_store/local_fs_blob_store.py,sha256=hDZXrcjKmYoH7ob0AAZ-7Zz1aoksxD8ArQEzmUSjkY4,1807
+indexify/executor/blob_store/metrics/blob_store.py,sha256=5_xiPREeHWFtxFh1NupDsF8zP4pmUPgLNNn-UE9Uzvc,1008
+indexify/executor/blob_store/s3_blob_store.py,sha256=4Hn1r5XpiNdfa-buRge0hR36j8j3xmuQC-7noOoOFo0,3319
+indexify/executor/downloader.py,sha256=0tRQY8jT03-R36bBo5MsoLjxkoehWCY9-61fsoW3Gng,13507
+indexify/executor/executor.py,sha256=uZiERFDKBQLmfPgun6bxEN7JE-BEEqmdGwawLjTAjf0,16499
 indexify/executor/executor_flavor.py,sha256=uilzDQVVYlQGR1MVnrUC4NevUActDWHdnJkr38M6kTk,118
-indexify/executor/function_executor/function_executor.py,sha256=rmqJrz4yihIefVhfbIIu3K6HFjKU3eEvEy2xFICYPp4,11881
+indexify/executor/function_executor/function_executor.py,sha256=agfUxzSQ-2TqkpMhW3OvOSMF_EhpemetaL3_dYp29Ro,11888
 indexify/executor/function_executor/function_executor_state.py,sha256=ljPm1IrRMJ8hFklwvFp7Xax2HMpUIOHm0DwOxxMcy7U,4336
-indexify/executor/function_executor/function_executor_states_container.py,sha256=wKehM_GXv3i0WkKNS72JVxXq6s60iqiOCX_Gm0qa9pw,3546
-indexify/executor/function_executor/function_executor_status.py,sha256=FUu5PcFC-kkVSCYGwzcEmvpPytoM--XUZ9ylVjv79S4,3416
-indexify/executor/function_executor/health_checker.py,sha256=Fvd1gmrcjyJqP-8vcsUxfnTHQIMNlHeMWCS70PAVr9E,6095
-indexify/executor/function_executor/invocation_state_client.py,sha256=p-xgM4__cHR1ApvMV9hShrGWee_Je0VDhICZUGjpQY4,9644
+indexify/executor/function_executor/function_executor_states_container.py,sha256=MFtHNMkzL3zzoa4YdWaw6X21dzo3P4vdEyJ_OKAuTlw,3766
+indexify/executor/function_executor/function_executor_status.py,sha256=Ms8tHG0wlw__pToeQIfBV6SO9c4tPu3UQgJAwXUkg2M,3597
+indexify/executor/function_executor/health_checker.py,sha256=EVA65Rn7eAC7HXzhZgScMDc57ODnvFopopdye7j1hfU,6254
+indexify/executor/function_executor/invocation_state_client.py,sha256=VTpeNxxfsa0ej20Q_ker5RZVdHiu59HWd5qNOjo6DBQ,9800
 indexify/executor/function_executor/metrics/function_executor.py,sha256=TDksxLRJr-P9ZKhF2Orsaxzzb4lVIBxFEjd_9Zv53Ng,6313
 indexify/executor/function_executor/metrics/function_executor_state.py,sha256=qheMhnoiYLiZB7ky5EyegfDy4Mr0Zh83bOE0gJ38YmU,1607
 indexify/executor/function_executor/metrics/function_executor_state_container.py,sha256=6rrAfml-TivjkHatCM4BLY7jmVs523Wzb6QIysncc-0,302
@@ -23,15 +27,16 @@ indexify/executor/function_executor/server/subprocess_function_executor_server.p
 indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=g1AUbhOoPsdhp_50Ayahdyv1Ix5-nEBE8orOQfkATpM,4470
 indexify/executor/function_executor/single_task_runner.py,sha256=OY3a2znwtuv0_Pqn9r5ScGhekNn1OquznKmaGnQL71k,13979
 indexify/executor/function_executor/task_input.py,sha256=wSrHR4m0juiGClQyeVdhRC37QzDt6Rrjq-ZXJkfBi9k,584
-indexify/executor/function_executor/task_output.py,sha256=kBRy7eSYzxZ3FYQ9PhA56VtBmhJ7ywePICMx_Vx5ipo,3011
-indexify/executor/grpc/channel_manager.py,sha256=THamn5VghCxRkXDlu2WEXtC6-SNKGc0xoa718bw9A4k,6257
-indexify/executor/grpc/completed_tasks_container.py,sha256=0JhWBUyEQXurVWZ1UtxoPDLXceIrcnnTBlSmfG3v0Cs,898
-indexify/executor/grpc/function_executor_controller.py,sha256=pfxMgyxLR2FFbLjyY5Yib8RSaI_6YeQOdf_VscXLqN0,18645
+indexify/executor/function_executor/task_output.py,sha256=snh25Ynj7uss1SUIeivpDYWdRkODoDpdPrQrGvnjyIs,3077
+indexify/executor/grpc/channel_manager.py,sha256=ihDkLoiGBLfSmoA2szbntjCfL3E_NDf5LABRXE7YRec,6330
+indexify/executor/grpc/function_executor_controller.py,sha256=b_xqiXsA79yIMCHtRJv0i0px6QMToIj9nCTc_km9fHk,16070
 indexify/executor/grpc/metrics/channel_manager.py,sha256=k-WArgklmP5WhjcmFmrgRblB7yc3XlaOXO8owRyV-mw,649
+indexify/executor/grpc/metrics/state_reconciler.py,sha256=0aI2IM4XztKxFa7NCxYSLafw_iiej3p07yEiKyewXIM,585
 indexify/executor/grpc/metrics/state_reporter.py,sha256=GggBEjMzQUYIG95LtTS4fUg1u9jYowkaXoUXppAXucs,543
-indexify/executor/grpc/state_reconciler.py,sha256=XRaZxHy2bX-zIzbJCBg3io_ZlslnGeG61dOk0fhYdJw,12866
-indexify/executor/grpc/state_reporter.py,sha256=7Z3pFqtXucxqAGX3o004uVG6tYWflsnKx1H2ldfnGT8,10043
-indexify/executor/grpc/task_controller.py,sha256=fsUiABPuZhGAjdpSKm0AVNIyoRxlJVFd4XDo3GzXEIQ,18483
+indexify/executor/grpc/metrics/task_controller.py,sha256=9Nm86nGxL2rZ3rAORB0_CBdO--Fe4MBrewVW4CqGyOU,222
+indexify/executor/grpc/state_reconciler.py,sha256=Yx_r1oUBGWjHWqUSUhap2ALZpz_o1Ue-SdL7BOgWQQU,19273
+indexify/executor/grpc/state_reporter.py,sha256=iixvl1nzJ_SfQhTUJpBEfiVlFPvOxQxvZyqT5Szk43Q,10503
+indexify/executor/grpc/task_controller.py,sha256=IQkxKYfj8TLhmoLi4xDVjEKBZlK6ttP1w2a9kOlX58A,20277
 indexify/executor/metrics/downloader.py,sha256=lctPh8xjkXeLEFJnl1hNrD1yEhLhIl5sggsR4Yoe_Zc,2746
 indexify/executor/metrics/executor.py,sha256=ua-Vv_k1CB4juJdF7tEBQbBMksqWAA3iXKKMKXZUCLk,2369
 indexify/executor/metrics/task_fetcher.py,sha256=iJEwCLzYr2cuz7hRvNiqaa2nvQP4OrA0hm0iJY0YKG0,736
@@ -48,13 +53,13 @@ indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFs
 indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpxWDfed1VHtKKnwvLQ8-frpG98,425
 indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
 indexify/executor/task_fetcher.py,sha256=p3iEsWyGi0ZMPAv0183smzOUD1KycQ_dXsyd9mpB9IU,3529
-indexify/executor/task_reporter.py,sha256=59Bgsjflxzo936KMOj70lGpDhwbUlQDuqqfHEy16kcw,11274
+indexify/executor/task_reporter.py,sha256=AbestQmKxpJo0pfJWnB_9ziOnAdFmMScvWzFTeyG6X4,12382
 indexify/executor/task_runner.py,sha256=2-f6Uupu_aVPg34G27yuiPTC6eE4XnT-qizpzPryzDI,7134
-indexify/proto/executor_api.proto,sha256=16jEP4VxSaEPWVBx3h2_rel3ltYdgW4SBLvRl-iUe78,8319
-indexify/proto/executor_api_pb2.py,sha256=dW4zAMZ-YU8aeU77-5ciQ0bTNWDt7v4m6CGQ2VdQZZI,12874
-indexify/proto/executor_api_pb2.pyi,sha256=6WHUUjsn5y7jJot_y_h3NXy2L04E8Xo-zpO9_tZJcAA,16851
+indexify/proto/executor_api.proto,sha256=K1lwFmk042GA1tp8s633FZJVg6Fi8f8LtAuFj8Gz7XU,9930
+indexify/proto/executor_api_pb2.py,sha256=5y570_FIgc6WFhHVAKWFieMuUhyKBA7rPJJ4DJ5hcCM,14054
+indexify/proto/executor_api_pb2.pyi,sha256=5eJJJjPNdTMSttNUOtzGwADbASsCh7138de_Y3l8uq4,18612
 indexify/proto/executor_api_pb2_grpc.py,sha256=GGiDtyQlA2382E_ZyKUBYcWNEJHH_RlulieStKfkJXI,9514
-indexify-0.3.19.dist-info/METADATA,sha256=gii0ukZNSxaQ4jF1J1NtsQOSbisC4lCbIV7hjXSFiqk,1158
-indexify-0.3.19.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.19.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
-indexify-0.3.19.dist-info/RECORD,,
+indexify-0.3.20.dist-info/METADATA,sha256=rg0FBC-z-e_Ft09YgQ5ldhaS3u5XKtmqxNFFMa9074A,1198
+indexify-0.3.20.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.20.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
+indexify-0.3.20.dist-info/RECORD,,

indexify/executor/grpc/completed_tasks_container.py

@@ -1,26 +0,0 @@
-import asyncio
-from typing import List, Set
-
-
-class CompletedTasksContainer:
-    """An asyncio concurrent container for the completed task IDs."""
-
-    def __init__(self):
-        # The fields below are protected by the lock.
-        self._lock: asyncio.Lock = asyncio.Lock()
-        self._completed_task_ids: Set[str] = set()
-
-    async def add(self, task_id: str) -> None:
-        """Add a task to the container."""
-        async with self._lock:
-            self._completed_task_ids.add(task_id)
-
-    async def contains(self, task_id: str) -> bool:
-        """Check if the task is in the container."""
-        async with self._lock:
-            return task_id in self._completed_task_ids
-
-    async def replace(self, task_ids: List[str]) -> None:
-        """Replaces the task IDs with the supplied task IDs."""
-        async with self._lock:
-            self._completed_task_ids = set(task_ids)