indexify 0.3.21__py3-none-any.whl → 0.3.23__py3-none-any.whl

indexify/cli/cli.py

@@ -102,9 +102,6 @@ def executor(
     executor_cache: Optional[str] = typer.Option(
         "~/.indexify/executor_cache", help="Path to the executor cache directory"
     ),
-    executor_id: Optional[str] = typer.Option(
-        None, help="ID of the executor, if not provided, a random ID will be generated"
-    ),
     # Registred ports range ends at 49151.
     ports: Tuple[int, int] = typer.Option(
         (50000, 51000),
@@ -153,18 +150,12 @@ def executor(
                 "At least one function must be specified when not running in development mode"
             )
 
-    if executor_id is None:
-        executor_id = nanoid.generate()
-    elif not re.compile(r"^[a-zA-Z0-9_-]{10,}$").match(executor_id):
-        raise typer.BadParameter(
-            "--executor-id should be at least 10 characters long and only include characters _-[0-9][a-z][A-Z]"
-        )
-
     kv_labels: Dict[str, str] = {}
     for label in labels:
         key, value = label.split("=")
         kv_labels[key] = value
 
+    executor_id: str = nanoid.generate()
     executor_version = version("indexify")
     logger = structlog.get_logger(module=__name__, executor_id=executor_id)
 

indexify/executor/api_objects.py

@@ -10,6 +10,25 @@ class DataPayload(BaseModel):
     content_type: Optional[str] = None
 
 
+class NodeGPU(BaseModel):
+    count: int
+    model: str
+
+
+class TaskResources(BaseModel):
+    cpus: float
+    memory_mb: int
+    ephemeral_disk_mb: int
+    gpu: Optional[NodeGPU] = None
+
+
+class TaskRetryPolicy(BaseModel):
+    max_retries: int
+    initial_delay_sec: float
+    max_delay_sec: float
+    delay_multiplier: float
+
+
 class Task(BaseModel):
     id: str
     namespace: str
@@ -27,6 +46,9 @@ class Task(BaseModel):
     input_payload: Optional[DataPayload] = None
     reducer_input_payload: Optional[DataPayload] = None
     output_payload_uri_prefix: Optional[str] = None
+    timeout: Optional[int] = None  # in seconds
+    resources: Optional[TaskResources] = None
+    retry_policy: Optional[TaskRetryPolicy] = None
 
 
 class FunctionURI(BaseModel):

indexify/executor/downloader.py

@@ -9,9 +9,7 @@ from tensorlake.function_executor.proto.message_validator import MessageValidato
 from tensorlake.utils.http_client import get_httpx_client
 
 from indexify.proto.executor_api_pb2 import DataPayload as DataPayloadProto
-from indexify.proto.executor_api_pb2 import (
-    DataPayloadEncoding,
-)
+from indexify.proto.executor_api_pb2 import DataPayloadEncoding
 
 from .api_objects import DataPayload
 from .blob_store.blob_store import BLOBStore

indexify/executor/function_executor/server/function_executor_server_factory.py

@@ -19,8 +19,15 @@ class FunctionExecutorServerConfiguration:
     executor_id: str
     function_executor_id: str
     namespace: str
+    graph_name: str
+    function_name: str
+    graph_version: str
     image_uri: Optional[str]
     secret_names: List[str]
+    cpu_ms_per_sec: Optional[int]
+    memory_bytes: Optional[int]
+    disk_bytes: Optional[int]
+    gpu_count: int
 
 
 class FunctionExecutorServerFactory:

indexify/executor/function_executor/single_task_runner.py

@@ -1,4 +1,5 @@
 from collections.abc import Awaitable, Callable
+from math import ceil
 from typing import Any, Optional
 
 import grpc
@@ -131,13 +132,33 @@ class SingleTaskRunner:
         self._function_executor_state.function_executor = FunctionExecutor(
             server_factory=self._function_executor_server_factory, logger=self._logger
         )
+        task: Task = self._task_input.task
         config: FunctionExecutorServerConfiguration = (
             FunctionExecutorServerConfiguration(
                 executor_id=self._executor_id,
                 function_executor_id=self._function_executor_state.id,
-                namespace=self._task_input.task.namespace,
-                image_uri=self._task_input.task.image_uri,
-                secret_names=self._task_input.task.secret_names or [],
+                namespace=task.namespace,
+                graph_name=task.compute_graph,
+                graph_version=task.graph_version,
+                function_name=task.compute_fn,
+                image_uri=task.image_uri,
+                secret_names=task.secret_names or [],
+                cpu_ms_per_sec=(
+                    None
+                    if task.resources.cpus is None
+                    else ceil(task.resources.cpus * 1000)
+                ),
+                memory_bytes=(
+                    None
+                    if task.resources.memory_mb is None
+                    else task.resources.memory_mb * 1024 * 1024
+                ),
+                disk_bytes=(
+                    None
+                    if task.resources.ephemeral_disk_mb is None
+                    else task.resources.ephemeral_disk_mb * 1024 * 1024
+                ),
+                gpu_count=0 if task.resources.gpu is None else task.resources.gpu.count,
             )
         )
         initialize_request: InitializeRequest = InitializeRequest(

indexify/executor/grpc/function_executor_controller.py

@@ -9,6 +9,7 @@ from tensorlake.function_executor.proto.message_validator import MessageValidato
 
 from indexify.proto.executor_api_pb2 import (
     FunctionExecutorDescription,
+    FunctionExecutorResources,
 )
 from indexify.proto.executor_api_pb2 import (
     FunctionExecutorStatus as FunctionExecutorStatusProto,
@@ -42,6 +43,13 @@ def validate_function_executor_description(
     # image_uri is optional.
     # secret_names can be empty.
     # resource_limits is optional.
+    # TODO: Make resources required after we migrate Server to them.
+    # validator.required_field("resources")
+    # validator = MessageValidator(function_executor_description.resources)
+    # validator.required_field("cpu_ms_per_sec")
+    # validator.required_field("memory_bytes")
+    # validator.required_field("disk_bytes")
+    # validator.required_field("gpu_count")
 
 
 def function_executor_logger(
@@ -333,11 +341,24 @@ async def _create_function_executor(
         executor_id=executor_id,
         function_executor_id=function_executor_description.id,
         namespace=function_executor_description.namespace,
+        graph_name=function_executor_description.graph_name,
+        graph_version=function_executor_description.graph_version,
+        function_name=function_executor_description.function_name,
         image_uri=None,
         secret_names=list(function_executor_description.secret_names),
+        cpu_ms_per_sec=None,
+        memory_bytes=None,
+        disk_bytes=None,
+        gpu_count=0,
     )
     if function_executor_description.HasField("image_uri"):
         config.image_uri = function_executor_description.image_uri
+    if function_executor_description.HasField("resources"):
+        resources: FunctionExecutorResources = function_executor_description.resources
+        config.cpu_ms_per_sec = resources.cpu_ms_per_sec
+        config.memory_bytes = resources.memory_bytes
+        config.disk_bytes = resources.disk_bytes
+        config.gpu_count = resources.gpu_count
 
     initialize_request: InitializeRequest = InitializeRequest(
         namespace=function_executor_description.namespace,

indexify/executor/grpc/state_reconciler.py

@@ -9,9 +9,7 @@ from indexify.proto.executor_api_pb2 import (
     GetDesiredExecutorStatesRequest,
     TaskAllocation,
 )
-from indexify.proto.executor_api_pb2_grpc import (
-    ExecutorAPIStub,
-)
+from indexify.proto.executor_api_pb2_grpc import ExecutorAPIStub
 
 from ..downloader import Downloader
 from ..function_executor.function_executor_state import FunctionExecutorState

indexify/executor/grpc/state_reporter.py

@@ -24,9 +24,7 @@ from indexify.proto.executor_api_pb2 import (
     HostResources,
     ReportExecutorStateRequest,
 )
-from indexify.proto.executor_api_pb2_grpc import (
-    ExecutorAPIStub,
-)
+from indexify.proto.executor_api_pb2_grpc import ExecutorAPIStub
 
 from ..api_objects import FunctionURI
 from ..executor_flavor import ExecutorFlavor

indexify/proto/executor_api.proto

@@ -83,6 +83,15 @@ enum FunctionExecutorStatus {
 }
 
 // Immutable information that identifies and describes a Function Executor.
+message FunctionExecutorResources {
+    // 1000 CPU ms per sec is one full CPU core.
+    // 2000 CPU ms per sec is two full CPU cores.
+    optional uint32 cpu_ms_per_sec = 1;
+    optional uint64 memory_bytes = 2;
+    optional uint64 disk_bytes = 3;
+    optional uint32 gpu_count = 4;
+}
+
 message FunctionExecutorDescription {
     optional string id = 1;
     optional string namespace = 2;
@@ -95,6 +104,7 @@ message FunctionExecutorDescription {
     // Timeout for customer code duration during FE creation.
     optional uint32 customer_code_timeout_ms = 9;
     optional DataPayload graph = 10;
+    optional FunctionExecutorResources resources = 11;
 }
 
 message FunctionExecutorState {
@@ -150,6 +160,14 @@ message ReportExecutorStateResponse {
 }
 
 // ===== get_desired_executor_states RPC =====
+message TaskRetryPolicy {
+    optional uint32 max_retries = 1;
+    optional uint32 initial_delay_ms = 2;
+    optional uint32 max_delay_ms = 3;
+    // The multiplier value is 1000x of the actual value to avoid working with floating point.
+    optional uint32 delay_multiplier = 4;
+}
+
 message Task {
     optional string id = 1;
     optional string namespace = 2;
@@ -166,6 +184,7 @@ message Task {
     // S3 URI if the data is stored in S3.
     // Starts with "file://"" prefix followed by an absolute directory path if the data is stored on a local file system.
     optional string output_payload_uri_prefix = 13;
+    optional TaskRetryPolicy retry_policy = 14;
 }
 
 message TaskAllocation {

indexify/proto/executor_api_pb2.py

@@ -19,7 +19,7 @@ _sym_db = _symbol_database.Default()
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"\x87\x02\n\x0b\x44\x61taPayload\x12\x11\n\x04path\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x11\n\x04size\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x18\n\x0bsha256_hash\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x10\n\x03uri\x18\x04 \x01(\tH\x03\x88\x01\x01\x12;\n\x08\x65ncoding\x18\x05 \x01(\x0e\x32$.executor_api_pb.DataPayloadEncodingH\x04\x88\x01\x01\x12\x1d\n\x10\x65ncoding_version\x18\x06 \x01(\x04H\x05\x88\x01\x01\x42\x07\n\x05_pathB\x07\n\x05_sizeB\x0e\n\x0c_sha256_hashB\x06\n\x04_uriB\x0b\n\t_encodingB\x13\n\x11_encoding_version"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xed\x03\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12<\n\x0fresource_limits\x18\x08 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12%\n\x18\x63ustomer_code_timeout_ms\x18\t \x01(\rH\x07\x88\x01\x01\x12\x30\n\x05graph\x18\n \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x08\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x12\n\x10_resource_limitsB\x1b\n\x19_customer_code_timeout_msB\x08\n\x06_graph"\xe8\x01\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x12\x1b\n\x0estatus_message\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_statusB\x11\n\x0f_status_message"\xcb\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1d\n\x10\x64\x65velopment_mode\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06\x66lavor\x18\x04 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorFlavorH\x03\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x05\x88\x01\x01\x12;\n\x0e\x66ree_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cserver_clock\x18\x0c \x01(\x04H\x08\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x13\n\x11_development_modeB\x0b\n\t_hostnameB\t\n\x07_flavorB\n\n\x08_versionB\t\n\x07_statusB\x11\n\x0f_free_resourcesB\r\n\x0b_state_hashB\x0f\n\r_server_clock"l\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x42\x11\n\x0f_executor_state"\x1d\n\x1bReportExecutorStateResponse"\xd6\x04\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x16\n\tinput_key\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x1f\n\x12reducer_output_key\x18\t \x01(\tH\x07\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\rH\x08\x88\x01\x01\x12\x30\n\x05input\x18\x0b \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\t\x88\x01\x01\x12\x38\n\rreducer_input\x18\x0c \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\r \x01(\tH\x0b\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\x0c\n\n_input_keyB\x15\n\x13_reducer_output_keyB\r\n\x0b_timeout_msB\x08\n\x06_inputB\x10\n\x0e_reducer_inputB\x1c\n\x1a_output_payload_uri_prefix"\x7f\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_task"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock"\x87\x06\n\x18ReportTaskOutcomeRequest\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x04\x88\x01\x01\x12\x32\n\x07outcome\x18\x07 \x01(\x0e\x32\x1c.executor_api_pb.TaskOutcomeH\x05\x88\x01\x01\x12\x1a\n\rinvocation_id\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0b\x65xecutor_id\x18\t \x01(\tH\x07\x88\x01\x01\x12\x14\n\x07reducer\x18\n \x01(\x08H\x08\x88\x01\x01\x12\x16\n\x0enext_functions\x18\x0b \x03(\t\x12\x30\n\nfn_outputs\x18\x0c \x03(\x0b\x32\x1c.executor_api_pb.DataPayload\x12\x31\n\x06stdout\x18\x0e \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\t\x88\x01\x01\x12\x31\n\x06stderr\x18\x0f \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12=\n\x0foutput_encoding\x18\r \x01(\x0e\x32\x1f.executor_api_pb.OutputEncodingH\x0b\x88\x01\x01\x12$\n\x17output_encoding_version\x18\x05 \x01(\x04H\x0c\x88\x01\x01\x42\n\n\x08_task_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_outcomeB\x10\n\x0e_invocation_idB\x0e\n\x0c_executor_idB\n\n\x08_reducerB\t\n\x07_stdoutB\t\n\x07_stderrB\x12\n\x10_output_encodingB\x1a\n\x18_output_encoding_version"\x1b\n\x19ReportTaskOutcomeResponse*\xab\x01\n\x13\x44\x61taPayloadEncoding\x12!\n\x1d\x44\x41TA_PAYLOAD_ENCODING_UNKNOWN\x10\x00\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_JSON\x10\x01\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_TEXT\x10\x02\x12\'\n#DATA_PAYLOAD_ENCODING_BINARY_PICKLE\x10\x03*\x86\x03\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12"\n\x1eGPU_MODEL_NVIDIA_TESLA_T4_16GB\x10\n\x12$\n GPU_MODEL_NVIDIA_TESLA_V100_16GB\x10\x14\x12\x1d\n\x19GPU_MODEL_NVIDIA_A10_24GB\x10\x1e\x12\x1f\n\x1bGPU_MODEL_NVIDIA_A6000_48GB\x10(\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_40GB\x10\x32\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_80GB\x10\x33\x12"\n\x1eGPU_MODEL_NVIDIA_A100_PCI_40GB\x10\x34\x12#\n\x1fGPU_MODEL_NVIDIA_H100_SXM5_80GB\x10<\x12"\n\x1eGPU_MODEL_NVIDIA_H100_PCI_80GB\x10=\x12"\n\x1eGPU_MODEL_NVIDIA_RTX_6000_24GB\x10>*\xca\x03\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12(\n$FUNCTION_EXECUTOR_STATUS_STARTING_UP\x10\x01\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR\x10\x02\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_PLATFORM_ERROR\x10\x03\x12!\n\x1d\x46UNCTION_EXECUTOR_STATUS_IDLE\x10\x04\x12)\n%FUNCTION_EXECUTOR_STATUS_RUNNING_TASK\x10\x05\x12&\n"FUNCTION_EXECUTOR_STATUS_UNHEALTHY\x10\x06\x12%\n!FUNCTION_EXECUTOR_STATUS_STOPPING\x10\x07\x12$\n FUNCTION_EXECUTOR_STATUS_STOPPED\x10\x08\x12%\n!FUNCTION_EXECUTOR_STATUS_SHUTDOWN\x10\t*\xc3\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1c\n\x18\x45XECUTOR_STATUS_STOPPING\x10\x04\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x05*d\n\x0e\x45xecutorFlavor\x12\x1b\n\x17\x45XECUTOR_FLAVOR_UNKNOWN\x10\x00\x12\x17\n\x13\x45XECUTOR_FLAVOR_OSS\x10\x01\x12\x1c\n\x18\x45XECUTOR_FLAVOR_PLATFORM\x10\x02*[\n\x0bTaskOutcome\x12\x18\n\x14TASK_OUTCOME_UNKNOWN\x10\x00\x12\x18\n\x14TASK_OUTCOME_SUCCESS\x10\x01\x12\x18\n\x14TASK_OUTCOME_FAILURE\x10\x02*\x7f\n\x0eOutputEncoding\x12\x1b\n\x17OUTPUT_ENCODING_UNKNOWN\x10\x00\x12\x18\n\x14OUTPUT_ENCODING_JSON\x10\x01\x12\x1a\n\x16OUTPUT_ENCODING_PICKLE\x10\x02\x12\x1a\n\x16OUTPUT_ENCODING_BINARY\x10\x03\x32\xef\x02\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x12n\n\x13report_task_outcome\x12).executor_api_pb.ReportTaskOutcomeRequest\x1a*.executor_api_pb.ReportTaskOutcomeResponse"\x00\x62\x06proto3'
+    b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"\x87\x02\n\x0b\x44\x61taPayload\x12\x11\n\x04path\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x11\n\x04size\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x18\n\x0bsha256_hash\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x10\n\x03uri\x18\x04 \x01(\tH\x03\x88\x01\x01\x12;\n\x08\x65ncoding\x18\x05 \x01(\x0e\x32$.executor_api_pb.DataPayloadEncodingH\x04\x88\x01\x01\x12\x1d\n\x10\x65ncoding_version\x18\x06 \x01(\x04H\x05\x88\x01\x01\x42\x07\n\x05_pathB\x07\n\x05_sizeB\x0e\n\x0c_sha256_hashB\x06\n\x04_uriB\x0b\n\t_encodingB\x13\n\x11_encoding_version"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xc5\x01\n\x19\x46unctionExecutorResources\x12\x1b\n\x0e\x63pu_ms_per_sec\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12\x16\n\tgpu_count\x18\x04 \x01(\rH\x03\x88\x01\x01\x42\x11\n\x0f_cpu_ms_per_secB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x0c\n\n_gpu_count"\xbf\x04\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12<\n\x0fresource_limits\x18\x08 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12%\n\x18\x63ustomer_code_timeout_ms\x18\t \x01(\rH\x07\x88\x01\x01\x12\x30\n\x05graph\x18\n \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x08\x88\x01\x01\x12\x42\n\tresources\x18\x0b \x01(\x0b\x32*.executor_api_pb.FunctionExecutorResourcesH\t\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x12\n\x10_resource_limitsB\x1b\n\x19_customer_code_timeout_msB\x08\n\x06_graphB\x0c\n\n_resources"\xe8\x01\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x12\x1b\n\x0estatus_message\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_statusB\x11\n\x0f_status_message"\xcb\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1d\n\x10\x64\x65velopment_mode\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06\x66lavor\x18\x04 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorFlavorH\x03\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x05\x88\x01\x01\x12;\n\x0e\x66ree_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x07\x88\x01\x01\x12\x19\n\x0cserver_clock\x18\x0c \x01(\x04H\x08\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x13\n\x11_development_modeB\x0b\n\t_hostnameB\t\n\x07_flavorB\n\n\x08_versionB\t\n\x07_statusB\x11\n\x0f_free_resourcesB\r\n\x0b_state_hashB\x0f\n\r_server_clock"l\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x42\x11\n\x0f_executor_state"\x1d\n\x1bReportExecutorStateResponse"\xcf\x01\n\x0fTaskRetryPolicy\x12\x18\n\x0bmax_retries\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x1d\n\x10initial_delay_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x19\n\x0cmax_delay_ms\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65lay_multiplier\x18\x04 \x01(\rH\x03\x88\x01\x01\x42\x0e\n\x0c_max_retriesB\x13\n\x11_initial_delay_msB\x0f\n\r_max_delay_msB\x13\n\x11_delay_multiplier"\xa4\x05\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x16\n\tinput_key\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x1f\n\x12reducer_output_key\x18\t \x01(\tH\x07\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\rH\x08\x88\x01\x01\x12\x30\n\x05input\x18\x0b \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\t\x88\x01\x01\x12\x38\n\rreducer_input\x18\x0c \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\r \x01(\tH\x0b\x88\x01\x01\x12;\n\x0cretry_policy\x18\x0e \x01(\x0b\x32 .executor_api_pb.TaskRetryPolicyH\x0c\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\x0c\n\n_input_keyB\x15\n\x13_reducer_output_keyB\r\n\x0b_timeout_msB\x08\n\x06_inputB\x10\n\x0e_reducer_inputB\x1c\n\x1a_output_payload_uri_prefixB\x0f\n\r_retry_policy"\x7f\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_task"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock"\x87\x06\n\x18ReportTaskOutcomeRequest\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x04\x88\x01\x01\x12\x32\n\x07outcome\x18\x07 \x01(\x0e\x32\x1c.executor_api_pb.TaskOutcomeH\x05\x88\x01\x01\x12\x1a\n\rinvocation_id\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x18\n\x0b\x65xecutor_id\x18\t \x01(\tH\x07\x88\x01\x01\x12\x14\n\x07reducer\x18\n \x01(\x08H\x08\x88\x01\x01\x12\x16\n\x0enext_functions\x18\x0b \x03(\t\x12\x30\n\nfn_outputs\x18\x0c \x03(\x0b\x32\x1c.executor_api_pb.DataPayload\x12\x31\n\x06stdout\x18\x0e \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\t\x88\x01\x01\x12\x31\n\x06stderr\x18\x0f \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12=\n\x0foutput_encoding\x18\r \x01(\x0e\x32\x1f.executor_api_pb.OutputEncodingH\x0b\x88\x01\x01\x12$\n\x17output_encoding_version\x18\x05 \x01(\x04H\x0c\x88\x01\x01\x42\n\n\x08_task_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_outcomeB\x10\n\x0e_invocation_idB\x0e\n\x0c_executor_idB\n\n\x08_reducerB\t\n\x07_stdoutB\t\n\x07_stderrB\x12\n\x10_output_encodingB\x1a\n\x18_output_encoding_version"\x1b\n\x19ReportTaskOutcomeResponse*\xab\x01\n\x13\x44\x61taPayloadEncoding\x12!\n\x1d\x44\x41TA_PAYLOAD_ENCODING_UNKNOWN\x10\x00\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_JSON\x10\x01\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_TEXT\x10\x02\x12\'\n#DATA_PAYLOAD_ENCODING_BINARY_PICKLE\x10\x03*\x86\x03\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12"\n\x1eGPU_MODEL_NVIDIA_TESLA_T4_16GB\x10\n\x12$\n GPU_MODEL_NVIDIA_TESLA_V100_16GB\x10\x14\x12\x1d\n\x19GPU_MODEL_NVIDIA_A10_24GB\x10\x1e\x12\x1f\n\x1bGPU_MODEL_NVIDIA_A6000_48GB\x10(\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_40GB\x10\x32\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_80GB\x10\x33\x12"\n\x1eGPU_MODEL_NVIDIA_A100_PCI_40GB\x10\x34\x12#\n\x1fGPU_MODEL_NVIDIA_H100_SXM5_80GB\x10<\x12"\n\x1eGPU_MODEL_NVIDIA_H100_PCI_80GB\x10=\x12"\n\x1eGPU_MODEL_NVIDIA_RTX_6000_24GB\x10>*\xca\x03\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12(\n$FUNCTION_EXECUTOR_STATUS_STARTING_UP\x10\x01\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR\x10\x02\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_PLATFORM_ERROR\x10\x03\x12!\n\x1d\x46UNCTION_EXECUTOR_STATUS_IDLE\x10\x04\x12)\n%FUNCTION_EXECUTOR_STATUS_RUNNING_TASK\x10\x05\x12&\n"FUNCTION_EXECUTOR_STATUS_UNHEALTHY\x10\x06\x12%\n!FUNCTION_EXECUTOR_STATUS_STOPPING\x10\x07\x12$\n FUNCTION_EXECUTOR_STATUS_STOPPED\x10\x08\x12%\n!FUNCTION_EXECUTOR_STATUS_SHUTDOWN\x10\t*\xc3\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1c\n\x18\x45XECUTOR_STATUS_STOPPING\x10\x04\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x05*d\n\x0e\x45xecutorFlavor\x12\x1b\n\x17\x45XECUTOR_FLAVOR_UNKNOWN\x10\x00\x12\x17\n\x13\x45XECUTOR_FLAVOR_OSS\x10\x01\x12\x1c\n\x18\x45XECUTOR_FLAVOR_PLATFORM\x10\x02*[\n\x0bTaskOutcome\x12\x18\n\x14TASK_OUTCOME_UNKNOWN\x10\x00\x12\x18\n\x14TASK_OUTCOME_SUCCESS\x10\x01\x12\x18\n\x14TASK_OUTCOME_FAILURE\x10\x02*\x7f\n\x0eOutputEncoding\x12\x1b\n\x17OUTPUT_ENCODING_UNKNOWN\x10\x00\x12\x18\n\x14OUTPUT_ENCODING_JSON\x10\x01\x12\x1a\n\x16OUTPUT_ENCODING_PICKLE\x10\x02\x12\x1a\n\x16OUTPUT_ENCODING_BINARY\x10\x03\x32\xef\x02\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x12n\n\x13report_task_outcome\x12).executor_api_pb.ReportTaskOutcomeRequest\x1a*.executor_api_pb.ReportTaskOutcomeResponse"\x00\x62\x06proto3'
 )
 
 _globals = globals()
@@ -31,20 +31,20 @@ if not _descriptor._USE_C_DESCRIPTORS:
     DESCRIPTOR._loaded_options = None
     _globals["_EXECUTORSTATE_LABELSENTRY"]._loaded_options = None
     _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_options = b"8\001"
-    _globals["_DATAPAYLOADENCODING"]._serialized_start = 4203
-    _globals["_DATAPAYLOADENCODING"]._serialized_end = 4374
-    _globals["_GPUMODEL"]._serialized_start = 4377
-    _globals["_GPUMODEL"]._serialized_end = 4767
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 4770
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 5228
-    _globals["_EXECUTORSTATUS"]._serialized_start = 5231
-    _globals["_EXECUTORSTATUS"]._serialized_end = 5426
-    _globals["_EXECUTORFLAVOR"]._serialized_start = 5428
-    _globals["_EXECUTORFLAVOR"]._serialized_end = 5528
-    _globals["_TASKOUTCOME"]._serialized_start = 5530
-    _globals["_TASKOUTCOME"]._serialized_end = 5621
-    _globals["_OUTPUTENCODING"]._serialized_start = 5623
-    _globals["_OUTPUTENCODING"]._serialized_end = 5750
+    _globals["_DATAPAYLOADENCODING"]._serialized_start = 4773
+    _globals["_DATAPAYLOADENCODING"]._serialized_end = 4944
+    _globals["_GPUMODEL"]._serialized_start = 4947
+    _globals["_GPUMODEL"]._serialized_end = 5337
+    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 5340
+    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 5798
+    _globals["_EXECUTORSTATUS"]._serialized_start = 5801
+    _globals["_EXECUTORSTATUS"]._serialized_end = 5996
+    _globals["_EXECUTORFLAVOR"]._serialized_start = 5998
+    _globals["_EXECUTORFLAVOR"]._serialized_end = 6098
+    _globals["_TASKOUTCOME"]._serialized_start = 6100
+    _globals["_TASKOUTCOME"]._serialized_end = 6191
+    _globals["_OUTPUTENCODING"]._serialized_start = 6193
+    _globals["_OUTPUTENCODING"]._serialized_end = 6320
     _globals["_DATAPAYLOAD"]._serialized_start = 55
     _globals["_DATAPAYLOAD"]._serialized_end = 318
     _globals["_GPURESOURCES"]._serialized_start = 320
@@ -53,30 +53,34 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_HOSTRESOURCES"]._serialized_end = 618
     _globals["_ALLOWEDFUNCTION"]._serialized_start = 621
     _globals["_ALLOWEDFUNCTION"]._serialized_end = 808
-    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_start = 811
-    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_end = 1304
-    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_start = 1307
-    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_end = 1539
-    _globals["_EXECUTORSTATE"]._serialized_start = 1542
-    _globals["_EXECUTORSTATE"]._serialized_end = 2257
-    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_start = 2077
-    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_end = 2122
-    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_start = 2259
-    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_end = 2367
-    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_start = 2369
-    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_end = 2398
-    _globals["_TASK"]._serialized_start = 2401
-    _globals["_TASK"]._serialized_end = 2999
-    _globals["_TASKALLOCATION"]._serialized_start = 3001
-    _globals["_TASKALLOCATION"]._serialized_end = 3128
-    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_start = 3130
-    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 3205
-    _globals["_DESIREDEXECUTORSTATE"]._serialized_start = 3208
-    _globals["_DESIREDEXECUTORSTATE"]._serialized_end = 3393
-    _globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_start = 3396
-    _globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_end = 4171
-    _globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_start = 4173
-    _globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_end = 4200
-    _globals["_EXECUTORAPI"]._serialized_start = 5753
-    _globals["_EXECUTORAPI"]._serialized_end = 6120
+    _globals["_FUNCTIONEXECUTORRESOURCES"]._serialized_start = 811
+    _globals["_FUNCTIONEXECUTORRESOURCES"]._serialized_end = 1008
+    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_start = 1011
+    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_end = 1586
+    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_start = 1589
+    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_end = 1821
+    _globals["_EXECUTORSTATE"]._serialized_start = 1824
+    _globals["_EXECUTORSTATE"]._serialized_end = 2539
+    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_start = 2359
+    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_end = 2404
+    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_start = 2541
+    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_end = 2649
+    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_start = 2651
+    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_end = 2680
+    _globals["_TASKRETRYPOLICY"]._serialized_start = 2683
+    _globals["_TASKRETRYPOLICY"]._serialized_end = 2890
+    _globals["_TASK"]._serialized_start = 2893
+    _globals["_TASK"]._serialized_end = 3569
+    _globals["_TASKALLOCATION"]._serialized_start = 3571
+    _globals["_TASKALLOCATION"]._serialized_end = 3698
+    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_start = 3700
+    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 3775
+    _globals["_DESIREDEXECUTORSTATE"]._serialized_start = 3778
+    _globals["_DESIREDEXECUTORSTATE"]._serialized_end = 3963
+    _globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_start = 3966
+    _globals["_REPORTTASKOUTCOMEREQUEST"]._serialized_end = 4741
+    _globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_start = 4743
+    _globals["_REPORTTASKOUTCOMERESPONSE"]._serialized_end = 4770
+    _globals["_EXECUTORAPI"]._serialized_start = 6323
+    _globals["_EXECUTORAPI"]._serialized_end = 6690
 # @@protoc_insertion_point(module_scope)

indexify/proto/executor_api_pb2.pyi

@@ -189,6 +189,24 @@ class AllowedFunction(_message.Message):
         graph_version: _Optional[str] = ...,
     ) -> None: ...
 
+class FunctionExecutorResources(_message.Message):
+    __slots__ = ("cpu_ms_per_sec", "memory_bytes", "disk_bytes", "gpu_count")
+    CPU_MS_PER_SEC_FIELD_NUMBER: _ClassVar[int]
+    MEMORY_BYTES_FIELD_NUMBER: _ClassVar[int]
+    DISK_BYTES_FIELD_NUMBER: _ClassVar[int]
+    GPU_COUNT_FIELD_NUMBER: _ClassVar[int]
+    cpu_ms_per_sec: int
+    memory_bytes: int
+    disk_bytes: int
+    gpu_count: int
+    def __init__(
+        self,
+        cpu_ms_per_sec: _Optional[int] = ...,
+        memory_bytes: _Optional[int] = ...,
+        disk_bytes: _Optional[int] = ...,
+        gpu_count: _Optional[int] = ...,
+    ) -> None: ...
+
 class FunctionExecutorDescription(_message.Message):
     __slots__ = (
         "id",
@@ -201,6 +219,7 @@ class FunctionExecutorDescription(_message.Message):
         "resource_limits",
         "customer_code_timeout_ms",
         "graph",
+        "resources",
     )
     ID_FIELD_NUMBER: _ClassVar[int]
     NAMESPACE_FIELD_NUMBER: _ClassVar[int]
@@ -212,6 +231,7 @@ class FunctionExecutorDescription(_message.Message):
     RESOURCE_LIMITS_FIELD_NUMBER: _ClassVar[int]
     CUSTOMER_CODE_TIMEOUT_MS_FIELD_NUMBER: _ClassVar[int]
     GRAPH_FIELD_NUMBER: _ClassVar[int]
+    RESOURCES_FIELD_NUMBER: _ClassVar[int]
     id: str
     namespace: str
     graph_name: str
@@ -222,6 +242,7 @@ class FunctionExecutorDescription(_message.Message):
     resource_limits: HostResources
     customer_code_timeout_ms: int
     graph: DataPayload
+    resources: FunctionExecutorResources
     def __init__(
         self,
         id: _Optional[str] = ...,
@@ -234,6 +255,7 @@ class FunctionExecutorDescription(_message.Message):
         resource_limits: _Optional[_Union[HostResources, _Mapping]] = ...,
         customer_code_timeout_ms: _Optional[int] = ...,
         graph: _Optional[_Union[DataPayload, _Mapping]] = ...,
+        resources: _Optional[_Union[FunctionExecutorResources, _Mapping]] = ...,
     ) -> None: ...
 
 class FunctionExecutorState(_message.Message):
@@ -335,6 +357,24 @@ class ReportExecutorStateResponse(_message.Message):
     __slots__ = ()
     def __init__(self) -> None: ...
 
+class TaskRetryPolicy(_message.Message):
+    __slots__ = ("max_retries", "initial_delay_ms", "max_delay_ms", "delay_multiplier")
+    MAX_RETRIES_FIELD_NUMBER: _ClassVar[int]
+    INITIAL_DELAY_MS_FIELD_NUMBER: _ClassVar[int]
+    MAX_DELAY_MS_FIELD_NUMBER: _ClassVar[int]
+    DELAY_MULTIPLIER_FIELD_NUMBER: _ClassVar[int]
+    max_retries: int
+    initial_delay_ms: int
+    max_delay_ms: int
+    delay_multiplier: int
+    def __init__(
+        self,
+        max_retries: _Optional[int] = ...,
+        initial_delay_ms: _Optional[int] = ...,
+        max_delay_ms: _Optional[int] = ...,
+        delay_multiplier: _Optional[int] = ...,
+    ) -> None: ...
+
 class Task(_message.Message):
     __slots__ = (
         "id",
@@ -349,6 +389,7 @@ class Task(_message.Message):
         "input",
         "reducer_input",
         "output_payload_uri_prefix",
+        "retry_policy",
     )
     ID_FIELD_NUMBER: _ClassVar[int]
     NAMESPACE_FIELD_NUMBER: _ClassVar[int]
@@ -362,6 +403,7 @@ class Task(_message.Message):
     INPUT_FIELD_NUMBER: _ClassVar[int]
     REDUCER_INPUT_FIELD_NUMBER: _ClassVar[int]
     OUTPUT_PAYLOAD_URI_PREFIX_FIELD_NUMBER: _ClassVar[int]
+    RETRY_POLICY_FIELD_NUMBER: _ClassVar[int]
     id: str
     namespace: str
     graph_name: str
@@ -374,6 +416,7 @@ class Task(_message.Message):
     input: DataPayload
     reducer_input: DataPayload
     output_payload_uri_prefix: str
+    retry_policy: TaskRetryPolicy
     def __init__(
         self,
         id: _Optional[str] = ...,
@@ -388,6 +431,7 @@ class Task(_message.Message):
         input: _Optional[_Union[DataPayload, _Mapping]] = ...,
         reducer_input: _Optional[_Union[DataPayload, _Mapping]] = ...,
         output_payload_uri_prefix: _Optional[str] = ...,
+        retry_policy: _Optional[_Union[TaskRetryPolicy, _Mapping]] = ...,
     ) -> None: ...
 
 class TaskAllocation(_message.Message):

{indexify-0.3.21.dist-info → indexify-0.3.23.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.21
+Version: 0.3.23
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.3.21.dist-info → indexify-0.3.23.dist-info}/RECORD

@@ -1,11 +1,11 @@
-indexify/cli/cli.py,sha256=L-QVyRO-nGazfrdIbhxYvxMwhEnNjJ4H32Lw9pz464I,9345
+indexify/cli/cli.py,sha256=k6vhgVKsHZvum2A2w7kbirpKusfqi_PyZGq29Q433_c,8950
 indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
-indexify/executor/api_objects.py,sha256=UEMpD_kKr6BNfu2JdYzq1CPNrEmVZYhwlZRKf3hB7KI,1750
+indexify/executor/api_objects.py,sha256=kHx5gKPwM0Rm64Ea__kPFwuarStX0u_9uaE7vV5M5z8,2222
 indexify/executor/blob_store/blob_store.py,sha256=XViw_KRfFSNqwcFYwMZixZF-EYCjXK2AQHdt0xh4UVo,2368
 indexify/executor/blob_store/local_fs_blob_store.py,sha256=6LexqMBGXp8f6Ka95R6xMIUyDutrZJABOMNcp-ssa98,1809
 indexify/executor/blob_store/metrics/blob_store.py,sha256=5_xiPREeHWFtxFh1NupDsF8zP4pmUPgLNNn-UE9Uzvc,1008
 indexify/executor/blob_store/s3_blob_store.py,sha256=G3B_V3gUE7XbUY42lDtBczUKuA7q8S7MD43tx1aHrJo,3445
-indexify/executor/downloader.py,sha256=7wMuOHNX0w3HM89RZaYFix_VpNfSmZl_O52Y7QB-ivU,15483
+indexify/executor/downloader.py,sha256=k9VbfOa-D6YH-cX8Sz-W-gWTsxmeVpSaIOq0xTC9KB0,15474
 indexify/executor/executor.py,sha256=K_xNJDImvIxgpE_ypw1ERyrijqOSe0EZIyVRVCqztVw,16697
 indexify/executor/executor_flavor.py,sha256=uilzDQVVYlQGR1MVnrUC4NevUActDWHdnJkr38M6kTk,118
 indexify/executor/function_executor/function_executor.py,sha256=agfUxzSQ-2TqkpMhW3OvOSMF_EhpemetaL3_dYp29Ro,11888
@@ -22,20 +22,20 @@ indexify/executor/function_executor/metrics/invocation_state_client.py,sha256=6F
 indexify/executor/function_executor/metrics/single_task_runner.py,sha256=7BJlGkdPGKeufMs3zWNO_1GRVzjINRY5rW3Mp4oWWec,805
 indexify/executor/function_executor/server/client_configuration.py,sha256=gOywMus0cotlX6NKIadEJwvOmBE-LbGE_wvoMi5-HzY,994
 indexify/executor/function_executor/server/function_executor_server.py,sha256=_DLivLDikupZusRk8gVWDk7fWPT9XjZ4un1yWSlOObs,883
-indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=cP93a3t1AfGx8qwageNLVdTwG52UOwzYNbbyrPq2TUQ,1692
+indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=xWEuDoxFqF-oC4RoiEer4S0Tk2tNbbJfA2kANZVShpM,1873
 indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=JekDOqF7oFD4J6zcN3xB0Dxd1cgpEXMOsb_rKZOeBlI,668
 indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=g1AUbhOoPsdhp_50Ayahdyv1Ix5-nEBE8orOQfkATpM,4470
-indexify/executor/function_executor/single_task_runner.py,sha256=wHllYigXy5R9ZiJREDf1EazqxUrhgxEJaSYGQwPUnZU,14140
+indexify/executor/function_executor/single_task_runner.py,sha256=6Fb9icnZ21pJcCq3mddFRoonPdNpUdSjTSQYh9nJXS0,14977
 indexify/executor/function_executor/task_input.py,sha256=wSrHR4m0juiGClQyeVdhRC37QzDt6Rrjq-ZXJkfBi9k,584
 indexify/executor/function_executor/task_output.py,sha256=vi0W75T8qbxjtAtI-812HksCv986lH_ijEqc3q0CVww,3424
 indexify/executor/grpc/channel_manager.py,sha256=ihDkLoiGBLfSmoA2szbntjCfL3E_NDf5LABRXE7YRec,6330
-indexify/executor/grpc/function_executor_controller.py,sha256=b_xqiXsA79yIMCHtRJv0i0px6QMToIj9nCTc_km9fHk,16070
+indexify/executor/grpc/function_executor_controller.py,sha256=3esPU9ILzYQ4gjfkRZ3IKHpqXWUQEmNit8XzQYKXoGs,17125
 indexify/executor/grpc/metrics/channel_manager.py,sha256=k-WArgklmP5WhjcmFmrgRblB7yc3XlaOXO8owRyV-mw,649
 indexify/executor/grpc/metrics/state_reconciler.py,sha256=0aI2IM4XztKxFa7NCxYSLafw_iiej3p07yEiKyewXIM,585
 indexify/executor/grpc/metrics/state_reporter.py,sha256=GggBEjMzQUYIG95LtTS4fUg1u9jYowkaXoUXppAXucs,543
 indexify/executor/grpc/metrics/task_controller.py,sha256=9Nm86nGxL2rZ3rAORB0_CBdO--Fe4MBrewVW4CqGyOU,222
-indexify/executor/grpc/state_reconciler.py,sha256=Yx_r1oUBGWjHWqUSUhap2ALZpz_o1Ue-SdL7BOgWQQU,19273
-indexify/executor/grpc/state_reporter.py,sha256=iixvl1nzJ_SfQhTUJpBEfiVlFPvOxQxvZyqT5Szk43Q,10503
+indexify/executor/grpc/state_reconciler.py,sha256=VYfKiy43XOA4OxbQUGVffIeqUaQQ37P4_aD9b_80Ls0,19264
+indexify/executor/grpc/state_reporter.py,sha256=uIouQMbk47pi90ceixKs7QEo_rjltQc0AfajryqbGzM,10494
 indexify/executor/grpc/task_controller.py,sha256=JrRUkVa8pGoYWBg_RVfn1ThvDhYCgJSMACFPUww7Lys,20851
 indexify/executor/metrics/downloader.py,sha256=lctPh8xjkXeLEFJnl1hNrD1yEhLhIl5sggsR4Yoe_Zc,2746
 indexify/executor/metrics/executor.py,sha256=ua-Vv_k1CB4juJdF7tEBQbBMksqWAA3iXKKMKXZUCLk,2369
@@ -55,11 +55,11 @@ indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3V
 indexify/executor/task_fetcher.py,sha256=p3iEsWyGi0ZMPAv0183smzOUD1KycQ_dXsyd9mpB9IU,3529
 indexify/executor/task_reporter.py,sha256=7X-IdLdwNBIfFbazG_4rtfR1A0ZFt03JGYpVJQUTKpE,16704
 indexify/executor/task_runner.py,sha256=UupZbGxU9BN4i1t6M8tH-5k3s4eUPEhMhar1YI0Aztk,7219
-indexify/proto/executor_api.proto,sha256=K1lwFmk042GA1tp8s633FZJVg6Fi8f8LtAuFj8Gz7XU,9930
-indexify/proto/executor_api_pb2.py,sha256=5y570_FIgc6WFhHVAKWFieMuUhyKBA7rPJJ4DJ5hcCM,14054
-indexify/proto/executor_api_pb2.pyi,sha256=5eJJJjPNdTMSttNUOtzGwADbASsCh7138de_Y3l8uq4,18612
+indexify/proto/executor_api.proto,sha256=e-RQGStJnr_QsT_0Glx16q33VVQ7jvO7u-v7sfbavgY,10603
+indexify/proto/executor_api_pb2.py,sha256=jR8kHjztV9TyVNErYHdtmWUzhvEbfLz4LuRMSznwvNI,15291
+indexify/proto/executor_api_pb2.pyi,sha256=N3cg87otIrr2pxFvf7NKRndDzQL1nmToSjHCfCyZX0I,20246
 indexify/proto/executor_api_pb2_grpc.py,sha256=GGiDtyQlA2382E_ZyKUBYcWNEJHH_RlulieStKfkJXI,9514
-indexify-0.3.21.dist-info/METADATA,sha256=FnyHAJo968nOsbZyCvFSL0rW-6bsaCAilmCZVOTL9lg,1198
-indexify-0.3.21.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.21.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
-indexify-0.3.21.dist-info/RECORD,,
+indexify-0.3.23.dist-info/METADATA,sha256=te3tex-vXTrDiQhM43N7u-cxLBmnJvpr00ftrl9YGt8,1198
+indexify-0.3.23.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.23.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
+indexify-0.3.23.dist-info/RECORD,,