indexify 0.4.8__py3-none-any.whl → 0.4.10__py3-none-any.whl

indexify/executor/executor.py

@@ -52,7 +52,7 @@ class Executor:
         blob_store: BLOBStore,
         host_resources_provider: HostResourcesProvider,
     ):
-        self._logger = structlog.get_logger(module=__name__)
+        self._logger = structlog.get_logger(module=__name__, executor_id=id)
         protocol: str = "http"
         if config_path:
             self._logger.info("running the extractor with TLS enabled")

indexify/executor/function_executor/function_executor.py

@@ -7,6 +7,8 @@ import grpc
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     InfoRequest,
     InfoResponse,
+    InitializationFailureReason,
+    InitializationOutcomeCode,
     InitializeRequest,
     InitializeResponse,
 )
@@ -315,16 +317,42 @@ async def _initialize_server(
                 initialize_request,
                 timeout=customer_code_timeout_sec,
             )
-            # TODO: set real stdout and stderr when their proper capturing on FE initialization is implemented.
-            if initialize_response.success:
-                return FunctionExecutorInitializationResult()
-            elif initialize_response.HasField("customer_error"):
+
+            if (
+                initialize_response.outcome_code
+                == InitializationOutcomeCode.INITIALIZE_OUTCOME_CODE_SUCCESS
+            ):
                 return FunctionExecutorInitializationResult(
-                    error=FunctionExecutorInitializationError.FUNCTION_ERROR,
-                    stderr=initialize_response.customer_error,
+                    stdout=initialize_response.stdout, stderr=initialize_response.stderr
                 )
+            elif (
+                initialize_response.outcome_code
+                == InitializationOutcomeCode.INITIALIZE_OUTCOME_CODE_FAILURE
+            ):
+                if (
+                    initialize_response.failure_reason
+                    == InitializationFailureReason.INITIALIZATION_FAILURE_REASON_FUNCTION_ERROR
+                ):
+                    return FunctionExecutorInitializationResult(
+                        error=FunctionExecutorInitializationError.FUNCTION_ERROR,
+                        stdout=initialize_response.stdout,
+                        stderr=initialize_response.stderr,
+                    )
+                elif (
+                    initialize_response.failure_reason
+                    == InitializationFailureReason.INITIALIZATION_FAILURE_REASON_INTERNAL_ERROR
+                ):
+                    # Don't add stdout/stderr because this is customer data.
+                    raise RuntimeError("initialize RPC failed with internal error")
+                else:
+                    raise ValueError(
+                        f"unexpected failure reason {InitializationFailureReason.Name(initialize_response.failure_reason)} in initialize RPC response"
+                    )
             else:
-                raise Exception("initialize RPC failed at function executor server")
+                raise ValueError(
+                    f"unexpected outcome code {InitializationOutcomeCode.Name(initialize_response.outcome_code)} in initialize RPC response"
+                )
+
         except grpc.aio.AioRpcError as e:
             if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
                 return FunctionExecutorInitializationResult(

indexify/executor/function_executor/invocation_state_client.py

@@ -8,6 +8,7 @@ from tensorlake.function_executor.proto.function_executor_pb2 import (
     InvocationStateRequest,
     InvocationStateResponse,
     SerializedObject,
+    SerializedObjectEncoding,
     SetInvocationStateResponse,
 )
 from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
@@ -25,6 +26,10 @@ from .metrics.invocation_state_client import (
     metric_server_set_state_requests,
 )
 
+# We're currently only supporting CloudPickle for invocation state values.
+# FIXME: if Function Executor sends us something else then we fail the calls.
+_VALUE_CONTENT_TYPE = "application/octet-stream"
+
 
 class InvocationStateClient:
     """InvocationStateClient is a client for the invocation state server of a Function Executor.
@@ -196,14 +201,21 @@ class InvocationStateClient:
         url: str = (
             f"{self._base_url}/internal/namespaces/{self._namespace}/compute_graphs/{self._graph}/invocations/{invocation_id}/ctx/{key}"
         )
-        payload = value.bytes if value.HasField("bytes") else value.string
+        if (
+            value.encoding
+            != SerializedObjectEncoding.SERIALIZED_OBJECT_ENCODING_BINARY_PICKLE
+        ):
+            raise ValueError(
+                f"Unsupported value encoding: {SerializedObjectEncoding.Name(value.encoding)}. "
+                "Only binary pickle is supported for invocation state values."
+            )
 
         response = await self._http_client.post(
             url=url,
             files=[
                 (
                     "value",
-                    ("value", payload, value.content_type),
+                    ("value", value.data, _VALUE_CONTENT_TYPE),
                 ),
             ],
         )
@@ -245,7 +257,7 @@ class InvocationStateClient:
             )
             raise
 
-        return serialized_object_from_http_response(response)
+        return _serialized_object_from_http_response(response)
 
     def _validate_request(self, request: InvocationStateRequest) -> None:
         (
@@ -265,17 +277,15 @@ class InvocationStateClient:
             raise ValueError("unknown request type")
 
 
-def serialized_object_from_http_response(response: httpx.Response) -> SerializedObject:
-    # We're hardcoding the content type currently used by Python SDK. It might change in the future.
-    # There's no other way for now to determine if the response is a bytes or string.
-    if response.headers["content-type"] in [
-        "application/octet-stream",
-        "application/pickle",
-    ]:
-        return SerializedObject(
-            bytes=response.content, content_type=response.headers["content-type"]
-        )
-    else:
-        return SerializedObject(
-            string=response.text, content_type=response.headers["content-type"]
+def _serialized_object_from_http_response(response: httpx.Response) -> SerializedObject:
+    if response.headers["content-type"] != _VALUE_CONTENT_TYPE:
+        raise ValueError(
+            f"Unexpected content type: {response.headers['content-type']}. "
+            f"Expected: {_VALUE_CONTENT_TYPE}."
         )
+
+    return SerializedObject(
+        data=response.content,
+        encoding=SerializedObjectEncoding.SERIALIZED_OBJECT_ENCODING_BINARY_PICKLE,
+        encoding_version=0,
+    )

indexify/executor/function_executor_controller/completed_task_metrics.py

@@ -66,6 +66,7 @@ def emit_completed_task_metrics(task_info: TaskInfo, logger: Any) -> None:
         elif task_failure_reason in [
             TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR,
             TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_TIMEOUT,
+            TaskFailureReason.TASK_FAILURE_REASON_INVOCATION_ERROR,
         ]:
             metric_tasks_completed.labels(
                 outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,

indexify/executor/function_executor_controller/create_function_executor.py

@@ -5,6 +5,7 @@ from typing import Any, Optional, Tuple
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
     SerializedObject,
+    SerializedObjectEncoding,
 )
 
 from indexify.executor.blob_store.blob_store import BLOBStore

indexify/executor/function_executor_controller/downloads.py

@@ -4,7 +4,10 @@ from pathlib import Path
 from typing import Any, Optional
 
 import nanoid
-from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
+from tensorlake.function_executor.proto.function_executor_pb2 import (
+    SerializedObject,
+    SerializedObjectEncoding,
+)
 
 from indexify.executor.blob_store.blob_store import BLOBStore
 from indexify.proto.executor_api_pb2 import (
@@ -179,20 +182,28 @@ def _serialized_object_from_data_payload_proto(
     """
     if data_payload.encoding == DataPayloadEncoding.DATA_PAYLOAD_ENCODING_BINARY_PICKLE:
         return SerializedObject(
-            bytes=data,
-            content_type="application/octet-stream",
+            data=data,
+            encoding=SerializedObjectEncoding.SERIALIZED_OBJECT_ENCODING_BINARY_PICKLE,
+            encoding_version=data_payload.encoding_version,
         )
     elif data_payload.encoding == DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_TEXT:
         return SerializedObject(
-            content_type="text/plain",
-            string=data.decode("utf-8"),
+            data=data,
+            encoding=SerializedObjectEncoding.SERIALIZED_OBJECT_ENCODING_UTF8_TEXT,
+            encoding_version=data_payload.encoding_version,
         )
     elif data_payload.encoding == DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_JSON:
-        result = SerializedObject(
-            content_type="application/json",
-            string=data.decode("utf-8"),
+        return SerializedObject(
+            data=data,
+            encoding=SerializedObjectEncoding.SERIALIZED_OBJECT_ENCODING_UTF8_JSON,
+            encoding_version=data_payload.encoding_version,
+        )
+    elif data_payload.encoding == DataPayloadEncoding.DATA_PAYLOAD_ENCODING_BINARY_ZIP:
+        return SerializedObject(
+            data=data,
+            encoding=SerializedObjectEncoding.SERIALIZED_OBJECT_ENCODING_BINARY_ZIP,
+            encoding_version=data_payload.encoding_version,
         )
-        return result
 
     raise ValueError(
         f"Can't convert data payload {data_payload} into serialized object"

indexify/executor/function_executor_controller/function_executor_controller.py

@@ -197,7 +197,6 @@ class FunctionExecutorController:
         task_info.is_cancelled = True
         logger.info(
             "cancelling task",
-            allocation_id=task_info.allocation_id,
         )
         if task_info.aio_task is not None:
             task_info.aio_task.cancel()
@@ -332,7 +331,7 @@ class FunctionExecutorController:
                     self._logger.error(
                         "unexpected exception in function executor controller control loop",
                         exc_info=e,
-                        fe_event=str(event),
+                        event_type=event.event_type.name,
                     )
 
     def _handle_event(self, event: BaseEvent) -> None:
@@ -786,16 +785,14 @@ def _to_task_result_proto(output: TaskOutput) -> TaskResult:
         graph_invocation_id=output.allocation.task.graph_invocation_id,
         reducer=output.reducer,
         outcome_code=output.outcome_code,
-        next_functions=(output.router_output.edges if output.router_output else []),
+        failure_reason=output.failure_reason,
+        next_functions=output.next_functions,
         function_outputs=output.uploaded_data_payloads,
+        invocation_error_output=output.uploaded_invocation_error_output,
     )
-    if output.failure_reason is not None:
-        task_result.failure_reason = output.failure_reason
     if output.uploaded_stdout is not None:
         task_result.stdout.CopyFrom(output.uploaded_stdout)
     if output.uploaded_stderr is not None:
         task_result.stderr.CopyFrom(output.uploaded_stderr)
-    if output.router_output is not None:
-        task_result.routing.next_functions[:] = output.router_output.edges
 
     return task_result

indexify/executor/function_executor_controller/run_task.py

@@ -6,6 +6,13 @@ import grpc
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     RunTaskRequest,
     RunTaskResponse,
+    SerializedObject,
+)
+from tensorlake.function_executor.proto.function_executor_pb2 import (
+    TaskFailureReason as FETaskFailureReason,
+)
+from tensorlake.function_executor.proto.function_executor_pb2 import (
+    TaskOutcomeCode as FETaskOutcomeCode,
 )
 from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
@@ -83,6 +90,7 @@ async def run_task_on_function_executor(
         task_info.output = _task_output_from_function_executor_response(
             allocation=task_info.allocation,
             response=response,
+            logger=logger,
         )
     except grpc.aio.AioRpcError as e:
         if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
@@ -140,13 +148,13 @@ async def run_task_on_function_executor(
 
 
 def _task_output_from_function_executor_response(
-    allocation: TaskAllocation, response: RunTaskResponse
+    allocation: TaskAllocation, response: RunTaskResponse, logger: Any
 ) -> TaskOutput:
     response_validator = MessageValidator(response)
     response_validator.required_field("stdout")
     response_validator.required_field("stderr")
     response_validator.required_field("is_reducer")
-    response_validator.required_field("success")
+    response_validator.required_field("outcome_code")
 
     metrics = TaskMetrics(counters={}, timers={})
     if response.HasField("metrics"):
@@ -154,31 +162,34 @@ def _task_output_from_function_executor_response(
         metrics.counters = dict(response.metrics.counters)
         metrics.timers = dict(response.metrics.timers)
 
-    output = TaskOutput(
+    outcome_code: TaskOutcomeCode = _to_task_outcome_code(
+        response.outcome_code, logger=logger
+    )
+    failure_reason: Optional[TaskFailureReason] = None
+    invocation_error_output: Optional[SerializedObject] = None
+
+    if outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE:
+        response_validator.required_field("failure_reason")
+        failure_reason: Optional[TaskFailureReason] = _to_task_failure_reason(
+            response.failure_reason, logger
+        )
+        if failure_reason == TaskFailureReason.TASK_FAILURE_REASON_INVOCATION_ERROR:
+            response_validator.required_field("invocation_error_output")
+            invocation_error_output = response.invocation_error_output
+
+    return TaskOutput(
         allocation=allocation,
-        outcome_code=(
-            TaskOutcomeCode.TASK_OUTCOME_CODE_SUCCESS
-            if response.success
-            else TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE
-        ),
-        failure_reason=(
-            None
-            if response.success
-            else TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR
-        ),
+        outcome_code=outcome_code,
+        failure_reason=failure_reason,
+        invocation_error_output=invocation_error_output,
+        function_outputs=response.function_outputs,
+        next_functions=response.next_functions,
         stdout=response.stdout,
         stderr=response.stderr,
         reducer=response.is_reducer,
         metrics=metrics,
     )
 
-    if response.HasField("function_output"):
-        output.function_output = response.function_output
-    if response.HasField("router_output"):
-        output.router_output = response.router_output
-
-    return output
-
 
 def _log_task_execution_finished(output: TaskOutput, logger: Any) -> None:
     logger.info(
@@ -191,3 +202,40 @@ def _log_task_execution_finished(output: TaskOutput, logger: Any) -> None:
             else None
         ),
     )
+
+
+def _to_task_outcome_code(
+    fe_task_outcome_code: FETaskOutcomeCode, logger
+) -> TaskOutcomeCode:
+    if fe_task_outcome_code == FETaskOutcomeCode.TASK_OUTCOME_CODE_SUCCESS:
+        return TaskOutcomeCode.TASK_OUTCOME_CODE_SUCCESS
+    elif fe_task_outcome_code == FETaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE:
+        return TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE
+    else:
+        logger.warning(
+            "Unknown TaskOutcomeCode received from Function Executor",
+            value=FETaskOutcomeCode.Name(fe_task_outcome_code),
+        )
+        return TaskOutcomeCode.TASK_OUTCOME_CODE_UNKNOWN
+
+
+def _to_task_failure_reason(
+    fe_task_failure_reason: FETaskFailureReason, logger: Any
+) -> TaskFailureReason:
+    if fe_task_failure_reason == FETaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR:
+        return TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR
+    elif (
+        fe_task_failure_reason
+        == FETaskFailureReason.TASK_FAILURE_REASON_INVOCATION_ERROR
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_INVOCATION_ERROR
+    elif (
+        fe_task_failure_reason == FETaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR
+    else:
+        logger.warning(
+            "Unknown TaskFailureReason received from Function Executor",
+            value=FETaskFailureReason.Name(fe_task_failure_reason),
+        )
+        return TaskFailureReason.TASK_FAILURE_REASON_UNKNOWN

indexify/executor/function_executor_controller/task_output.py

@@ -1,8 +1,7 @@
 from typing import Dict, List, Optional
 
 from tensorlake.function_executor.proto.function_executor_pb2 import (
-    FunctionOutput,
-    RouterOutput,
+    SerializedObject,
 )
 
 from indexify.proto.executor_api_pb2 import (
@@ -30,31 +29,29 @@ class TaskOutput:
         outcome_code: TaskOutcomeCode,
         # Optional[TaskFailureReason] is not supported in python 3.9
         failure_reason: TaskFailureReason = None,
-        output_encoding: Optional[str] = None,
-        function_output: Optional[FunctionOutput] = None,
-        router_output: Optional[RouterOutput] = None,
+        invocation_error_output: Optional[SerializedObject] = None,
+        function_outputs: List[SerializedObject] = [],
+        next_functions: List[str] = [],
         stdout: Optional[str] = None,
         stderr: Optional[str] = None,
         reducer: bool = False,
         metrics: Optional[TaskMetrics] = None,
-        uploaded_data_payloads: List[DataPayload] = [],
-        uploaded_stdout: Optional[DataPayload] = None,
-        uploaded_stderr: Optional[DataPayload] = None,
     ):
         self.task = allocation.task
         self.allocation = allocation
-        self.function_output = function_output
-        self.router_output = router_output
+        self.function_outputs = function_outputs
+        self.next_functions = next_functions
         self.stdout = stdout
         self.stderr = stderr
         self.reducer = reducer
         self.outcome_code = outcome_code
         self.failure_reason = failure_reason
+        self.invocation_error_output = invocation_error_output
         self.metrics = metrics
-        self.output_encoding = output_encoding
-        self.uploaded_data_payloads = uploaded_data_payloads
-        self.uploaded_stdout = uploaded_stdout
-        self.uploaded_stderr = uploaded_stderr
+        self.uploaded_data_payloads: List[DataPayload] = []
+        self.uploaded_stdout: Optional[DataPayload] = None
+        self.uploaded_stderr: Optional[DataPayload] = None
+        self.uploaded_invocation_error_output: Optional[DataPayload] = None
 
     @classmethod
     def internal_error(

indexify/executor/function_executor_controller/upload_task_output.py

@@ -1,7 +1,12 @@
 import asyncio
 import hashlib
 import time
-from typing import Any
+from typing import Any, List
+
+from tensorlake.function_executor.proto.function_executor_pb2 import (
+    SerializedObject,
+    SerializedObjectEncoding,
+)
 
 from indexify.executor.blob_store.blob_store import BLOBStore
 from indexify.proto.executor_api_pb2 import (
@@ -74,11 +79,13 @@ class _TaskOutputSummary:
     def __init__(self):
         self.output_count: int = 0
         self.output_total_bytes: int = 0
-        self.router_output_count: int = 0
+        self.next_functions_count: int = 0
         self.stdout_count: int = 0
         self.stdout_total_bytes: int = 0
         self.stderr_count: int = 0
         self.stderr_total_bytes: int = 0
+        self.invocation_error_output_count: int = 0
+        self.invocation_error_output_total_bytes: int = 0
         self.total_bytes: int = 0
 
 
@@ -95,12 +102,14 @@ async def _upload_task_output_once(
         total_bytes=output_summary.total_bytes,
         total_files=output_summary.output_count
         + output_summary.stdout_count
-        + output_summary.stderr_count,
+        + output_summary.stderr_count
+        + output_summary.invocation_error_output_count,
         output_files=output_summary.output_count,
         output_bytes=output_summary.total_bytes,
-        router_output_count=output_summary.router_output_count,
+        next_functions_count=output_summary.next_functions_count,
         stdout_bytes=output_summary.stdout_total_bytes,
         stderr_bytes=output_summary.stderr_total_bytes,
+        invocation_error_output_bytes=output_summary.invocation_error_output_total_bytes,
     )
 
     start_time = time.time()
@@ -109,7 +118,9 @@ async def _upload_task_output_once(
         metric_task_output_blob_store_upload_errors.count_exceptions(),
     ):
         metric_task_output_blob_store_uploads.inc()
-        await _upload_to_blob_store(output=output, blob_store=blob_store, logger=logger)
+        await _upload_to_blob_store(
+            task_output=output, blob_store=blob_store, logger=logger
+        )
 
     logger.info(
         "files uploaded to blob store",
@@ -118,86 +129,105 @@ async def _upload_task_output_once(
 
 
 async def _upload_to_blob_store(
-    output: TaskOutput, blob_store: BLOBStore, logger: Any
+    task_output: TaskOutput, blob_store: BLOBStore, logger: Any
 ) -> None:
-    if output.stdout is not None:
-        stdout_url = f"{output.allocation.task.output_payload_uri_prefix}.{output.allocation.task.id}.stdout"
-        stdout_bytes: bytes = output.stdout.encode()
+    if task_output.stdout is not None:
+        stdout_url = f"{task_output.allocation.task.output_payload_uri_prefix}.{task_output.allocation.task.id}.stdout"
+        stdout_bytes: bytes = task_output.stdout.encode()
         await blob_store.put(stdout_url, stdout_bytes, logger)
-        output.uploaded_stdout = DataPayload(
+        task_output.uploaded_stdout = DataPayload(
             uri=stdout_url,
             size=len(stdout_bytes),
             sha256_hash=compute_hash(stdout_bytes),
             encoding=DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_TEXT,
             encoding_version=0,
         )
-        # stdout is uploaded, free the memory used for it.
-        output.stdout = None
+        # stdout is uploaded, free the memory used for it and don't upload again if we retry overall output upload again.
+        task_output.stdout = None
 
-    if output.stderr is not None:
-        stderr_url = f"{output.allocation.task.output_payload_uri_prefix}.{output.allocation.task.id}.stderr"
-        stderr_bytes: bytes = output.stderr.encode()
+    if task_output.stderr is not None:
+        stderr_url = f"{task_output.allocation.task.output_payload_uri_prefix}.{task_output.allocation.task.id}.stderr"
+        stderr_bytes: bytes = task_output.stderr.encode()
         await blob_store.put(stderr_url, stderr_bytes, logger)
-        output.uploaded_stderr = DataPayload(
+        task_output.uploaded_stderr = DataPayload(
             uri=stderr_url,
             size=len(stderr_bytes),
             sha256_hash=compute_hash(stderr_bytes),
             encoding=DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_TEXT,
             encoding_version=0,
         )
-        # stderr is uploaded, free the memory used for it.
-        output.stderr = None
-
-    if output.function_output is not None:
-        # We can't use the default empty list output.uploaded_data_payloads because it's a singleton.
-        uploaded_data_payloads = []
-        for func_output_item in output.function_output.outputs:
-            node_output_sequence = len(uploaded_data_payloads)
-            output_url = f"{output.allocation.task.output_payload_uri_prefix}.{output.allocation.task.id}.{node_output_sequence}"
-            output_bytes: bytes = (
-                func_output_item.bytes
-                if func_output_item.HasField("bytes")
-                else func_output_item.string.encode()
-            )
-            await blob_store.put(output_url, output_bytes, logger)
-            uploaded_data_payloads.append(
-                DataPayload(
-                    uri=output_url,
-                    size=len(output_bytes),
-                    sha256_hash=compute_hash(output_bytes),
-                    encoding=_to_grpc_data_payload_encoding(output),
-                    encoding_version=0,
-                )
+        # stderr is uploaded, free the memory used for it and don't upload again if we retry overall output upload again.
+        task_output.stderr = None
+
+    if task_output.invocation_error_output is not None:
+        invocation_error_output_url = (
+            f"{task_output.allocation.task.output_payload_uri_prefix}.inverr."
+            f"{task_output.allocation.task.graph_invocation_id}"
+        )
+        invocation_error_output_bytes: bytes = task_output.invocation_error_output.data
+        await blob_store.put(
+            invocation_error_output_url, invocation_error_output_bytes, logger
+        )
+        task_output.uploaded_invocation_error_output = DataPayload(
+            uri=invocation_error_output_url,
+            size=len(invocation_error_output_bytes),
+            sha256_hash=compute_hash(invocation_error_output_bytes),
+            encoding=_to_grpc_data_payload_encoding(
+                task_output.invocation_error_output.encoding, logger
+            ),
+            encoding_version=0,
+        )
+        # Invocation error output is uploaded, free the memory used for it and don't upload again if we retry overall output upload again.
+        task_output.invocation_error_output = None
+
+    # We can't use the default empty list output.uploaded_data_payloads because it's a singleton.
+    uploaded_data_payloads: List[DataPayload] = []
+    for output in task_output.function_outputs:
+        output: SerializedObject
+        output_ix: int = len(uploaded_data_payloads)
+        output_url: str = (
+            f"{task_output.allocation.task.output_payload_uri_prefix}.{task_output.allocation.task.id}.{output_ix}"
+        )
+        await blob_store.put(output_url, output.data, logger)
+        uploaded_data_payloads.append(
+            DataPayload(
+                uri=output_url,
+                size=len(output.data),
+                sha256_hash=compute_hash(output.data),
+                encoding=_to_grpc_data_payload_encoding(output.encoding, logger),
+                encoding_version=0,
             )
+        )
 
-        output.uploaded_data_payloads = uploaded_data_payloads
-        # The output is uploaded, free the memory used for it.
-        output.function_output = None
+    task_output.uploaded_data_payloads = uploaded_data_payloads
+    # The output is uploaded, free the memory used for it and don't upload again if we retry overall output upload again.
+    task_output.function_outputs = []
 
 
-def _task_output_summary(output: TaskOutput) -> _TaskOutputSummary:
+def _task_output_summary(task_output: TaskOutput) -> _TaskOutputSummary:
     summary: _TaskOutputSummary = _TaskOutputSummary()
 
-    if output.stdout is not None:
+    if task_output.stdout is not None:
         summary.stdout_count += 1
-        summary.stdout_total_bytes += len(output.stdout)
+        summary.stdout_total_bytes += len(task_output.stdout)
 
-    if output.stderr is not None:
+    if task_output.stderr is not None:
         summary.stderr_count += 1
-        summary.stderr_total_bytes += len(output.stderr)
-
-    if output.function_output is not None:
-        for func_output_item in output.function_output.outputs:
-            output_len: bytes = len(
-                func_output_item.bytes
-                if func_output_item.HasField("bytes")
-                else func_output_item.string
-            )
-            summary.output_count += 1
-            summary.output_total_bytes += output_len
+        summary.stderr_total_bytes += len(task_output.stderr)
+
+    if task_output.invocation_error_output is not None:
+        summary.invocation_error_output_count += 1
+        summary.invocation_error_output_total_bytes += len(
+            task_output.invocation_error_output.data
+        )
 
-    if output.router_output is not None:
-        summary.router_output_count += 1
+    for output in task_output.function_outputs:
+        output: SerializedObject
+        output_len: bytes = len(output.data)
+        summary.output_count += 1
+        summary.output_total_bytes += output_len
+
+    summary.next_functions_count = len(task_output.next_functions)
 
     summary.total_bytes = (
         summary.output_total_bytes
@@ -207,11 +237,21 @@ def _task_output_summary(output: TaskOutput) -> _TaskOutputSummary:
     return summary
 
 
-def _to_grpc_data_payload_encoding(task_output: TaskOutput) -> DataPayloadEncoding:
-    if task_output.output_encoding == "json":
+def _to_grpc_data_payload_encoding(
+    encoding: SerializedObjectEncoding, logger: Any
+) -> DataPayloadEncoding:
+    if encoding == SerializedObjectEncoding.SERIALIZED_OBJECT_ENCODING_BINARY_PICKLE:
+        return DataPayloadEncoding.DATA_PAYLOAD_ENCODING_BINARY_PICKLE
+    elif encoding == SerializedObjectEncoding.SERIALIZED_OBJECT_ENCODING_UTF8_JSON:
         return DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_JSON
+    elif encoding == SerializedObjectEncoding.SERIALIZED_OBJECT_ENCODING_UTF8_TEXT:
+        return DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_TEXT
     else:
-        return DataPayloadEncoding.DATA_PAYLOAD_ENCODING_BINARY_PICKLE
+        logger.error(
+            "Unexpected encoding for SerializedObject",
+            encoding=SerializedObjectEncoding.Name(encoding),
+        )
+        return DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UNKNOWN
 
 
 def compute_hash(data: bytes) -> str:

indexify/proto/executor_api.proto

@@ -7,11 +7,10 @@ package executor_api_pb;
 // ===== DataPayload =====
 enum DataPayloadEncoding {
     DATA_PAYLOAD_ENCODING_UNKNOWN = 0;
-    // These encodings are currently mapping 1:1 to mime types.
-    // TODO: use SDK specific encodings becase 1:1 mapping might not work in the future.
     DATA_PAYLOAD_ENCODING_UTF8_JSON = 1;
     DATA_PAYLOAD_ENCODING_UTF8_TEXT = 2;
     DATA_PAYLOAD_ENCODING_BINARY_PICKLE = 3;
+    DATA_PAYLOAD_ENCODING_BINARY_ZIP = 4;
 }
 
 message DataPayload {
@@ -62,28 +61,38 @@ message AllowedFunction {
 
 enum FunctionExecutorStatus {
     FUNCTION_EXECUTOR_STATUS_UNKNOWN = 0;
-    // Function Executor is being created.
+    // FE is being created.
     FUNCTION_EXECUTOR_STATUS_PENDING = 1;
-    // Function Executor is running and ready to accept tasks.
+    // FE is running and ready to accept tasks.
     FUNCTION_EXECUTOR_STATUS_RUNNING = 2;
-    // Function Executor is terminated, all resources are freed.
+    // FE is terminated, all resources are freed.
     FUNCTION_EXECUTOR_STATUS_TERMINATED = 3;
 }
 
 enum FunctionExecutorTerminationReason {
     FUNCTION_EXECUTOR_TERMINATION_REASON_UNKNOWN = 0;
+    // Internal error aka platform error on FE startup.
     FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR = 1;
+    // A clear function constructor code error on FE startup.
+    // Typically an exception raised from the constructor.
     FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR = 2;
     // Timeout on FE startup while running the function constructor.
     FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT = 3;
 
+    // FE was terminated on Executor shutdown.
     FUNCTION_EXECUTOR_TERMINATION_REASON_EXECUTOR_SHUTDOWN = 10;
+    // FE was removed from the Executor desired state by Server.
     FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE = 11;
+    // FE was terminated because it failed a health check.
     FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY = 12;
+    // FE was terminated due to an unrecoverable internal error on Executor.
     FUNCTION_EXECUTOR_TERMINATION_REASON_INTERNAL_ERROR = 13;
-    // Timeout while running the function.
+    // FE was terminated because Function code exceeded its cofigured timeout.
+    // FE termination is the only way to reliably stop the function code execution.
     FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_TIMEOUT = 14;
-    // The running function allocation was removed from the desired state.
+    // FE was terminated because function allocation currently running on it was removed
+    // from Executor desired state.
+    // FE termination is the only way to reliably stop the function code execution.
     FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_CANCELLED = 15;
 }
 
@@ -129,9 +138,13 @@ message FunctionExecutorUpdate {
 
 enum ExecutorStatus {
     EXECUTOR_STATUS_UNKNOWN = 0;
+    // Executor is starting up, not ready to accept task allocations and FEs.
     EXECUTOR_STATUS_STARTING_UP = 1;
+    // Executor is ready to accept task allocations and FEs.
     EXECUTOR_STATUS_RUNNING = 2;
+    // No new task allocations or FEs should be places on the Executor.
     EXECUTOR_STATUS_DRAINED = 3;
+    // Executor is stopped, all resources are freed, all FEs are terminated.
     EXECUTOR_STATUS_STOPPED = 4;
 }
 
@@ -228,17 +241,19 @@ enum TaskOutcomeCode {
 
 enum TaskFailureReason {
     TASK_FAILURE_REASON_UNKNOWN = 0;
+    // Internal error on Executor aka platform error.
+    // Includes grey failures when we can't determine the exact cause.
     TASK_FAILURE_REASON_INTERNAL_ERROR = 1;
+    // Clear function code failure typically by raising an exception from the function code.
     TASK_FAILURE_REASON_FUNCTION_ERROR = 2;
+    // Function code run time exceeded its cofigured timeout.
     TASK_FAILURE_REASON_FUNCTION_TIMEOUT = 3;
-    TASK_FAILURE_REASON_TASK_CANCELLED = 4;
-    TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED = 5;
-}
-
-message ResultRouting {
-    // The list of next functions.
-    // NB: An empty list indicates that no routing should be performed.
-    repeated string next_functions = 1;
+    // Function code raised InvocationError to mark the invocation as permanently failed.
+    TASK_FAILURE_REASON_INVOCATION_ERROR = 4;
+    // Server removed the task allocation from Executor desired state. The task allocation didn't finish before the removal.
+    TASK_FAILURE_REASON_TASK_CANCELLED = 5;
+    // Function Executor terminated - can't run the task allocation on it anymore.
+    TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED = 6;
 }
 
 message TaskResult {
@@ -253,24 +268,13 @@ message TaskResult {
     optional TaskOutcomeCode outcome_code = 9;
     optional TaskFailureReason failure_reason = 10;
     // Edges that the function wants the invocation to be routed to.
-    // Previously called router_edges.
-    // NB: An empty list indicates that the graph's route definitions should be used,
-    // unless this field is overridden by the presence of the `routing` field.
     repeated string next_functions = 11;
     repeated DataPayload function_outputs = 12;
     // Standard output and error streams of the function.
     optional DataPayload stdout = 13;
     optional DataPayload stderr = 14;
-
-    // Indicates how the results should be routed.
-    // If this is present, it replaces `next_functions`.
-    //
-    // If absent, `next_functions` will be used; note that if no
-    // routes are defined in `next_functions`, this will use the
-    // graph's routing.  The long-term goal is to deprecate
-    // `next_functions`, so that if `routing` is not present, the
-    // graph's routing definitions will always be used.
-    ResultRouting routing = 15;
+    // User payload for invocation error if task failed with invocation error.
+    optional DataPayload invocation_error_output = 15;
 }
 
 // Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.

indexify/proto/executor_api_pb2.py

@@ -19,7 +19,7 @@ _sym_db = _symbol_database.Default()
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"\xeb\x01\n\x0b\x44\x61taPayload\x12\x11\n\x04size\x18\x02 \x01(\x04H\x00\x88\x01\x01\x12\x18\n\x0bsha256_hash\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03uri\x18\x04 \x01(\tH\x02\x88\x01\x01\x12;\n\x08\x65ncoding\x18\x05 \x01(\x0e\x32$.executor_api_pb.DataPayloadEncodingH\x03\x88\x01\x01\x12\x1d\n\x10\x65ncoding_version\x18\x06 \x01(\x04H\x04\x88\x01\x01\x42\x07\n\x05_sizeB\x0e\n\x0c_sha256_hashB\x06\n\x04_uriB\x0b\n\t_encodingB\x13\n\x11_encoding_version"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xd8\x01\n\x19\x46unctionExecutorResources\x12\x1b\n\x0e\x63pu_ms_per_sec\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x11\n\x0f_cpu_ms_per_secB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xb3\x04\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12%\n\x18\x63ustomer_code_timeout_ms\x18\t \x01(\rH\x06\x88\x01\x01\x12\x30\n\x05graph\x18\n \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x07\x88\x01\x01\x12\x42\n\tresources\x18\x0b \x01(\x0b\x32*.executor_api_pb.FunctionExecutorResourcesH\x08\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\x0c \x01(\tH\t\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x1b\n\x19_customer_code_timeout_msB\x08\n\x06_graphB\x0c\n\n_resourcesB\x1c\n\x1a_output_payload_uri_prefix"\xa4\x02\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x12S\n\x12termination_reason\x18\x03 \x01(\x0e\x32\x32.executor_api_pb.FunctionExecutorTerminationReasonH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_statusB\x15\n\x13_termination_reason"\x8c\x02\n\x16\x46unctionExecutorUpdate\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12\x39\n\x0estartup_stdout\x18\x02 \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x01\x88\x01\x01\x12\x39\n\x0estartup_stderr\x18\x03 \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\x11\n\x0f_startup_stdoutB\x11\n\x0f_startup_stderr"\xce\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x03\x88\x01\x01\x12<\n\x0ftotal_resources\x18\r \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x04\x88\x01\x01\x12N\n!total_function_executor_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x05\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x06\x88\x01\x01\x12\x19\n\x0cserver_clock\x18\x0c \x01(\x04H\x07\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x0b\n\t_hostnameB\n\n\x08_versionB\t\n\x07_statusB\x12\n\x10_total_resourcesB$\n"_total_function_executor_resourcesB\r\n\x0b_state_hashB\x0f\n\r_server_clock"\xb9\x01\n\x0e\x45xecutorUpdate\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x31\n\x0ctask_results\x18\x02 \x03(\x0b\x32\x1b.executor_api_pb.TaskResult\x12J\n\x19\x66unction_executor_updates\x18\x03 \x03(\x0b\x32\'.executor_api_pb.FunctionExecutorUpdateB\x0e\n\x0c_executor_id"\xbf\x01\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x12=\n\x0f\x65xecutor_update\x18\x02 \x01(\x0b\x32\x1f.executor_api_pb.ExecutorUpdateH\x01\x88\x01\x01\x42\x11\n\x0f_executor_stateB\x12\n\x10_executor_update"\x1d\n\x1bReportExecutorStateResponse"\xcf\x01\n\x0fTaskRetryPolicy\x12\x18\n\x0bmax_retries\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x1d\n\x10initial_delay_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x19\n\x0cmax_delay_ms\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65lay_multiplier\x18\x04 \x01(\rH\x03\x88\x01\x01\x42\x0e\n\x0c_max_retriesB\x13\n\x11_initial_delay_msB\x0f\n\r_max_delay_msB\x13\n\x11_delay_multiplier"\xc6\x04\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\rH\x06\x88\x01\x01\x12\x30\n\x05input\x18\x0b \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x07\x88\x01\x01\x12\x38\n\rreducer_input\x18\x0c \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x08\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\r \x01(\tH\t\x88\x01\x01\x12;\n\x0cretry_policy\x18\x0e \x01(\x0b\x32 .executor_api_pb.TaskRetryPolicyH\n\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\r\n\x0b_timeout_msB\x08\n\x06_inputB\x10\n\x0e_reducer_inputB\x1c\n\x1a_output_payload_uri_prefixB\x0f\n\r_retry_policy"\xad\x01\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x12\x1a\n\rallocation_id\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_taskB\x10\n\x0e_allocation_id"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock"\'\n\rResultRouting\x12\x16\n\x0enext_functions\x18\x01 \x03(\t"\x81\x06\n\nTaskResult\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rallocation_id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tnamespace\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\ngraph_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x06 \x01(\tH\x05\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x14\n\x07reducer\x18\x08 \x01(\x08H\x07\x88\x01\x01\x12;\n\x0coutcome_code\x18\t \x01(\x0e\x32 .executor_api_pb.TaskOutcomeCodeH\x08\x88\x01\x01\x12?\n\x0e\x66\x61ilure_reason\x18\n \x01(\x0e\x32".executor_api_pb.TaskFailureReasonH\t\x88\x01\x01\x12\x16\n\x0enext_functions\x18\x0b \x03(\t\x12\x36\n\x10\x66unction_outputs\x18\x0c \x03(\x0b\x32\x1c.executor_api_pb.DataPayload\x12\x31\n\x06stdout\x18\r \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12\x31\n\x06stderr\x18\x0e \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x0b\x88\x01\x01\x12/\n\x07routing\x18\x0f \x01(\x0b\x32\x1e.executor_api_pb.ResultRoutingB\n\n\x08_task_idB\x10\n\x0e_allocation_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_reducerB\x0f\n\r_outcome_codeB\x11\n\x0f_failure_reasonB\t\n\x07_stdoutB\t\n\x07_stderr*\xab\x01\n\x13\x44\x61taPayloadEncoding\x12!\n\x1d\x44\x41TA_PAYLOAD_ENCODING_UNKNOWN\x10\x00\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_JSON\x10\x01\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_TEXT\x10\x02\x12\'\n#DATA_PAYLOAD_ENCODING_BINARY_PICKLE\x10\x03*\xd6\x01\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12\x1e\n\x1aGPU_MODEL_NVIDIA_A100_40GB\x10\x01\x12\x1e\n\x1aGPU_MODEL_NVIDIA_A100_80GB\x10\x02\x12\x1e\n\x1aGPU_MODEL_NVIDIA_H100_80GB\x10\x03\x12\x1d\n\x19GPU_MODEL_NVIDIA_TESLA_T4\x10\x04\x12\x1a\n\x16GPU_MODEL_NVIDIA_A6000\x10\x05\x12\x18\n\x14GPU_MODEL_NVIDIA_A10\x10\x06*\xb3\x01\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12$\n FUNCTION_EXECUTOR_STATUS_PENDING\x10\x01\x12$\n FUNCTION_EXECUTOR_STATUS_RUNNING\x10\x02\x12\'\n#FUNCTION_EXECUTOR_STATUS_TERMINATED\x10\x03*\x95\x05\n!FunctionExecutorTerminationReason\x12\x30\n,FUNCTION_EXECUTOR_TERMINATION_REASON_UNKNOWN\x10\x00\x12\x46\nBFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR\x10\x01\x12\x46\nBFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR\x10\x02\x12H\nDFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT\x10\x03\x12:\n6FUNCTION_EXECUTOR_TERMINATION_REASON_EXECUTOR_SHUTDOWN\x10\n\x12\x43\n?FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE\x10\x0b\x12\x32\n.FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY\x10\x0c\x12\x37\n3FUNCTION_EXECUTOR_TERMINATION_REASON_INTERNAL_ERROR\x10\r\x12\x39\n5FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_TIMEOUT\x10\x0e\x12;\n7FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_CANCELLED\x10\x0f*\xa5\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x04*n\n\x0fTaskOutcomeCode\x12\x1d\n\x19TASK_OUTCOME_CODE_UNKNOWN\x10\x00\x12\x1d\n\x19TASK_OUTCOME_CODE_SUCCESS\x10\x01\x12\x1d\n\x19TASK_OUTCOME_CODE_FAILURE\x10\x02*\x8c\x02\n\x11TaskFailureReason\x12\x1f\n\x1bTASK_FAILURE_REASON_UNKNOWN\x10\x00\x12&\n"TASK_FAILURE_REASON_INTERNAL_ERROR\x10\x01\x12&\n"TASK_FAILURE_REASON_FUNCTION_ERROR\x10\x02\x12(\n$TASK_FAILURE_REASON_FUNCTION_TIMEOUT\x10\x03\x12&\n"TASK_FAILURE_REASON_TASK_CANCELLED\x10\x04\x12\x34\n0TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED\x10\x05\x32\xff\x01\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x62\x06proto3'
+    b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"\xeb\x01\n\x0b\x44\x61taPayload\x12\x11\n\x04size\x18\x02 \x01(\x04H\x00\x88\x01\x01\x12\x18\n\x0bsha256_hash\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03uri\x18\x04 \x01(\tH\x02\x88\x01\x01\x12;\n\x08\x65ncoding\x18\x05 \x01(\x0e\x32$.executor_api_pb.DataPayloadEncodingH\x03\x88\x01\x01\x12\x1d\n\x10\x65ncoding_version\x18\x06 \x01(\x04H\x04\x88\x01\x01\x42\x07\n\x05_sizeB\x0e\n\x0c_sha256_hashB\x06\n\x04_uriB\x0b\n\t_encodingB\x13\n\x11_encoding_version"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xd8\x01\n\x19\x46unctionExecutorResources\x12\x1b\n\x0e\x63pu_ms_per_sec\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x11\n\x0f_cpu_ms_per_secB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xb3\x04\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12%\n\x18\x63ustomer_code_timeout_ms\x18\t \x01(\rH\x06\x88\x01\x01\x12\x30\n\x05graph\x18\n \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x07\x88\x01\x01\x12\x42\n\tresources\x18\x0b \x01(\x0b\x32*.executor_api_pb.FunctionExecutorResourcesH\x08\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\x0c \x01(\tH\t\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x1b\n\x19_customer_code_timeout_msB\x08\n\x06_graphB\x0c\n\n_resourcesB\x1c\n\x1a_output_payload_uri_prefix"\xa4\x02\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x12S\n\x12termination_reason\x18\x03 \x01(\x0e\x32\x32.executor_api_pb.FunctionExecutorTerminationReasonH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_statusB\x15\n\x13_termination_reason"\x8c\x02\n\x16\x46unctionExecutorUpdate\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12\x39\n\x0estartup_stdout\x18\x02 \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x01\x88\x01\x01\x12\x39\n\x0estartup_stderr\x18\x03 \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\x11\n\x0f_startup_stdoutB\x11\n\x0f_startup_stderr"\xce\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x03\x88\x01\x01\x12<\n\x0ftotal_resources\x18\r \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x04\x88\x01\x01\x12N\n!total_function_executor_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x05\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x06\x88\x01\x01\x12\x19\n\x0cserver_clock\x18\x0c \x01(\x04H\x07\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x0b\n\t_hostnameB\n\n\x08_versionB\t\n\x07_statusB\x12\n\x10_total_resourcesB$\n"_total_function_executor_resourcesB\r\n\x0b_state_hashB\x0f\n\r_server_clock"\xb9\x01\n\x0e\x45xecutorUpdate\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x31\n\x0ctask_results\x18\x02 \x03(\x0b\x32\x1b.executor_api_pb.TaskResult\x12J\n\x19\x66unction_executor_updates\x18\x03 \x03(\x0b\x32\'.executor_api_pb.FunctionExecutorUpdateB\x0e\n\x0c_executor_id"\xbf\x01\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x12=\n\x0f\x65xecutor_update\x18\x02 \x01(\x0b\x32\x1f.executor_api_pb.ExecutorUpdateH\x01\x88\x01\x01\x42\x11\n\x0f_executor_stateB\x12\n\x10_executor_update"\x1d\n\x1bReportExecutorStateResponse"\xcf\x01\n\x0fTaskRetryPolicy\x12\x18\n\x0bmax_retries\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x1d\n\x10initial_delay_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x19\n\x0cmax_delay_ms\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65lay_multiplier\x18\x04 \x01(\rH\x03\x88\x01\x01\x42\x0e\n\x0c_max_retriesB\x13\n\x11_initial_delay_msB\x0f\n\r_max_delay_msB\x13\n\x11_delay_multiplier"\xc6\x04\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\rH\x06\x88\x01\x01\x12\x30\n\x05input\x18\x0b \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x07\x88\x01\x01\x12\x38\n\rreducer_input\x18\x0c \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x08\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\r \x01(\tH\t\x88\x01\x01\x12;\n\x0cretry_policy\x18\x0e \x01(\x0b\x32 .executor_api_pb.TaskRetryPolicyH\n\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\r\n\x0b_timeout_msB\x08\n\x06_inputB\x10\n\x0e_reducer_inputB\x1c\n\x1a_output_payload_uri_prefixB\x0f\n\r_retry_policy"\xad\x01\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x12\x1a\n\rallocation_id\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_taskB\x10\n\x0e_allocation_id"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock"\xb0\x06\n\nTaskResult\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rallocation_id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tnamespace\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\ngraph_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x06 \x01(\tH\x05\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x14\n\x07reducer\x18\x08 \x01(\x08H\x07\x88\x01\x01\x12;\n\x0coutcome_code\x18\t \x01(\x0e\x32 .executor_api_pb.TaskOutcomeCodeH\x08\x88\x01\x01\x12?\n\x0e\x66\x61ilure_reason\x18\n \x01(\x0e\x32".executor_api_pb.TaskFailureReasonH\t\x88\x01\x01\x12\x16\n\x0enext_functions\x18\x0b \x03(\t\x12\x36\n\x10\x66unction_outputs\x18\x0c \x03(\x0b\x32\x1c.executor_api_pb.DataPayload\x12\x31\n\x06stdout\x18\r \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12\x31\n\x06stderr\x18\x0e \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x0b\x88\x01\x01\x12\x42\n\x17invocation_error_output\x18\x0f \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x0c\x88\x01\x01\x42\n\n\x08_task_idB\x10\n\x0e_allocation_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_reducerB\x0f\n\r_outcome_codeB\x11\n\x0f_failure_reasonB\t\n\x07_stdoutB\t\n\x07_stderrB\x1a\n\x18_invocation_error_output*\xd1\x01\n\x13\x44\x61taPayloadEncoding\x12!\n\x1d\x44\x41TA_PAYLOAD_ENCODING_UNKNOWN\x10\x00\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_JSON\x10\x01\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_TEXT\x10\x02\x12\'\n#DATA_PAYLOAD_ENCODING_BINARY_PICKLE\x10\x03\x12$\n DATA_PAYLOAD_ENCODING_BINARY_ZIP\x10\x04*\xd6\x01\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12\x1e\n\x1aGPU_MODEL_NVIDIA_A100_40GB\x10\x01\x12\x1e\n\x1aGPU_MODEL_NVIDIA_A100_80GB\x10\x02\x12\x1e\n\x1aGPU_MODEL_NVIDIA_H100_80GB\x10\x03\x12\x1d\n\x19GPU_MODEL_NVIDIA_TESLA_T4\x10\x04\x12\x1a\n\x16GPU_MODEL_NVIDIA_A6000\x10\x05\x12\x18\n\x14GPU_MODEL_NVIDIA_A10\x10\x06*\xb3\x01\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12$\n FUNCTION_EXECUTOR_STATUS_PENDING\x10\x01\x12$\n FUNCTION_EXECUTOR_STATUS_RUNNING\x10\x02\x12\'\n#FUNCTION_EXECUTOR_STATUS_TERMINATED\x10\x03*\x95\x05\n!FunctionExecutorTerminationReason\x12\x30\n,FUNCTION_EXECUTOR_TERMINATION_REASON_UNKNOWN\x10\x00\x12\x46\nBFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR\x10\x01\x12\x46\nBFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR\x10\x02\x12H\nDFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT\x10\x03\x12:\n6FUNCTION_EXECUTOR_TERMINATION_REASON_EXECUTOR_SHUTDOWN\x10\n\x12\x43\n?FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE\x10\x0b\x12\x32\n.FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY\x10\x0c\x12\x37\n3FUNCTION_EXECUTOR_TERMINATION_REASON_INTERNAL_ERROR\x10\r\x12\x39\n5FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_TIMEOUT\x10\x0e\x12;\n7FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_CANCELLED\x10\x0f*\xa5\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x04*n\n\x0fTaskOutcomeCode\x12\x1d\n\x19TASK_OUTCOME_CODE_UNKNOWN\x10\x00\x12\x1d\n\x19TASK_OUTCOME_CODE_SUCCESS\x10\x01\x12\x1d\n\x19TASK_OUTCOME_CODE_FAILURE\x10\x02*\xb6\x02\n\x11TaskFailureReason\x12\x1f\n\x1bTASK_FAILURE_REASON_UNKNOWN\x10\x00\x12&\n"TASK_FAILURE_REASON_INTERNAL_ERROR\x10\x01\x12&\n"TASK_FAILURE_REASON_FUNCTION_ERROR\x10\x02\x12(\n$TASK_FAILURE_REASON_FUNCTION_TIMEOUT\x10\x03\x12(\n$TASK_FAILURE_REASON_INVOCATION_ERROR\x10\x04\x12&\n"TASK_FAILURE_REASON_TASK_CANCELLED\x10\x05\x12\x34\n0TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED\x10\x06\x32\xff\x01\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x62\x06proto3'
 )
 
 _globals = globals()
@@ -31,20 +31,20 @@ if not _descriptor._USE_C_DESCRIPTORS:
     DESCRIPTOR._loaded_options = None
     _globals["_EXECUTORSTATE_LABELSENTRY"]._loaded_options = None
     _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_options = b"8\001"
-    _globals["_DATAPAYLOADENCODING"]._serialized_start = 5317
-    _globals["_DATAPAYLOADENCODING"]._serialized_end = 5488
-    _globals["_GPUMODEL"]._serialized_start = 5491
-    _globals["_GPUMODEL"]._serialized_end = 5705
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 5708
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 5887
-    _globals["_FUNCTIONEXECUTORTERMINATIONREASON"]._serialized_start = 5890
-    _globals["_FUNCTIONEXECUTORTERMINATIONREASON"]._serialized_end = 6551
-    _globals["_EXECUTORSTATUS"]._serialized_start = 6554
-    _globals["_EXECUTORSTATUS"]._serialized_end = 6719
-    _globals["_TASKOUTCOMECODE"]._serialized_start = 6721
-    _globals["_TASKOUTCOMECODE"]._serialized_end = 6831
-    _globals["_TASKFAILUREREASON"]._serialized_start = 6834
-    _globals["_TASKFAILUREREASON"]._serialized_end = 7102
+    _globals["_DATAPAYLOADENCODING"]._serialized_start = 5323
+    _globals["_DATAPAYLOADENCODING"]._serialized_end = 5532
+    _globals["_GPUMODEL"]._serialized_start = 5535
+    _globals["_GPUMODEL"]._serialized_end = 5749
+    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 5752
+    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 5931
+    _globals["_FUNCTIONEXECUTORTERMINATIONREASON"]._serialized_start = 5934
+    _globals["_FUNCTIONEXECUTORTERMINATIONREASON"]._serialized_end = 6595
+    _globals["_EXECUTORSTATUS"]._serialized_start = 6598
+    _globals["_EXECUTORSTATUS"]._serialized_end = 6763
+    _globals["_TASKOUTCOMECODE"]._serialized_start = 6765
+    _globals["_TASKOUTCOMECODE"]._serialized_end = 6875
+    _globals["_TASKFAILUREREASON"]._serialized_start = 6878
+    _globals["_TASKFAILUREREASON"]._serialized_end = 7188
     _globals["_DATAPAYLOAD"]._serialized_start = 55
     _globals["_DATAPAYLOAD"]._serialized_end = 290
     _globals["_GPURESOURCES"]._serialized_start = 292
@@ -81,10 +81,8 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 4313
     _globals["_DESIREDEXECUTORSTATE"]._serialized_start = 4316
     _globals["_DESIREDEXECUTORSTATE"]._serialized_end = 4501
-    _globals["_RESULTROUTING"]._serialized_start = 4503
-    _globals["_RESULTROUTING"]._serialized_end = 4542
-    _globals["_TASKRESULT"]._serialized_start = 4545
-    _globals["_TASKRESULT"]._serialized_end = 5314
-    _globals["_EXECUTORAPI"]._serialized_start = 7105
-    _globals["_EXECUTORAPI"]._serialized_end = 7360
+    _globals["_TASKRESULT"]._serialized_start = 4504
+    _globals["_TASKRESULT"]._serialized_end = 5320
+    _globals["_EXECUTORAPI"]._serialized_start = 7191
+    _globals["_EXECUTORAPI"]._serialized_end = 7446
 # @@protoc_insertion_point(module_scope)

indexify/proto/executor_api_pb2.pyi

@@ -17,6 +17,7 @@ class DataPayloadEncoding(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     DATA_PAYLOAD_ENCODING_UTF8_JSON: _ClassVar[DataPayloadEncoding]
     DATA_PAYLOAD_ENCODING_UTF8_TEXT: _ClassVar[DataPayloadEncoding]
     DATA_PAYLOAD_ENCODING_BINARY_PICKLE: _ClassVar[DataPayloadEncoding]
+    DATA_PAYLOAD_ENCODING_BINARY_ZIP: _ClassVar[DataPayloadEncoding]
 
 class GPUModel(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     __slots__ = ()
@@ -90,6 +91,7 @@ class TaskFailureReason(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     TASK_FAILURE_REASON_INTERNAL_ERROR: _ClassVar[TaskFailureReason]
     TASK_FAILURE_REASON_FUNCTION_ERROR: _ClassVar[TaskFailureReason]
     TASK_FAILURE_REASON_FUNCTION_TIMEOUT: _ClassVar[TaskFailureReason]
+    TASK_FAILURE_REASON_INVOCATION_ERROR: _ClassVar[TaskFailureReason]
     TASK_FAILURE_REASON_TASK_CANCELLED: _ClassVar[TaskFailureReason]
     TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED: _ClassVar[TaskFailureReason]
 
@@ -97,6 +99,7 @@ DATA_PAYLOAD_ENCODING_UNKNOWN: DataPayloadEncoding
 DATA_PAYLOAD_ENCODING_UTF8_JSON: DataPayloadEncoding
 DATA_PAYLOAD_ENCODING_UTF8_TEXT: DataPayloadEncoding
 DATA_PAYLOAD_ENCODING_BINARY_PICKLE: DataPayloadEncoding
+DATA_PAYLOAD_ENCODING_BINARY_ZIP: DataPayloadEncoding
 GPU_MODEL_UNKNOWN: GPUModel
 GPU_MODEL_NVIDIA_A100_40GB: GPUModel
 GPU_MODEL_NVIDIA_A100_80GB: GPUModel
@@ -142,6 +145,7 @@ TASK_FAILURE_REASON_UNKNOWN: TaskFailureReason
 TASK_FAILURE_REASON_INTERNAL_ERROR: TaskFailureReason
 TASK_FAILURE_REASON_FUNCTION_ERROR: TaskFailureReason
 TASK_FAILURE_REASON_FUNCTION_TIMEOUT: TaskFailureReason
+TASK_FAILURE_REASON_INVOCATION_ERROR: TaskFailureReason
 TASK_FAILURE_REASON_TASK_CANCELLED: TaskFailureReason
 TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED: TaskFailureReason
 
@@ -527,12 +531,6 @@ class DesiredExecutorState(_message.Message):
         clock: _Optional[int] = ...,
     ) -> None: ...
 
-class ResultRouting(_message.Message):
-    __slots__ = ("next_functions",)
-    NEXT_FUNCTIONS_FIELD_NUMBER: _ClassVar[int]
-    next_functions: _containers.RepeatedScalarFieldContainer[str]
-    def __init__(self, next_functions: _Optional[_Iterable[str]] = ...) -> None: ...
-
 class TaskResult(_message.Message):
     __slots__ = (
         "task_id",
@@ -549,7 +547,7 @@ class TaskResult(_message.Message):
         "function_outputs",
         "stdout",
         "stderr",
-        "routing",
+        "invocation_error_output",
     )
     TASK_ID_FIELD_NUMBER: _ClassVar[int]
     ALLOCATION_ID_FIELD_NUMBER: _ClassVar[int]
@@ -565,7 +563,7 @@ class TaskResult(_message.Message):
     FUNCTION_OUTPUTS_FIELD_NUMBER: _ClassVar[int]
     STDOUT_FIELD_NUMBER: _ClassVar[int]
     STDERR_FIELD_NUMBER: _ClassVar[int]
-    ROUTING_FIELD_NUMBER: _ClassVar[int]
+    INVOCATION_ERROR_OUTPUT_FIELD_NUMBER: _ClassVar[int]
     task_id: str
     allocation_id: str
     namespace: str
@@ -580,7 +578,7 @@ class TaskResult(_message.Message):
     function_outputs: _containers.RepeatedCompositeFieldContainer[DataPayload]
     stdout: DataPayload
     stderr: DataPayload
-    routing: ResultRouting
+    invocation_error_output: DataPayload
     def __init__(
         self,
         task_id: _Optional[str] = ...,
@@ -597,5 +595,5 @@ class TaskResult(_message.Message):
         function_outputs: _Optional[_Iterable[_Union[DataPayload, _Mapping]]] = ...,
         stdout: _Optional[_Union[DataPayload, _Mapping]] = ...,
         stderr: _Optional[_Union[DataPayload, _Mapping]] = ...,
-        routing: _Optional[_Union[ResultRouting, _Mapping]] = ...,
+        invocation_error_output: _Optional[_Union[DataPayload, _Mapping]] = ...,
     ) -> None: ...

{indexify-0.4.8.dist-info → indexify-0.4.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.8
+Version: 0.4.10
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -17,7 +17,7 @@ Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
 Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: tensorlake (==0.2.6)
+Requires-Dist: tensorlake (==0.2.7)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.4.8.dist-info → indexify-0.4.10.dist-info}/RECORD

@@ -8,11 +8,11 @@ indexify/executor/blob_store/local_fs_blob_store.py,sha256=6LexqMBGXp8f6Ka95R6xM
 indexify/executor/blob_store/metrics/blob_store.py,sha256=5_xiPREeHWFtxFh1NupDsF8zP4pmUPgLNNn-UE9Uzvc,1008
 indexify/executor/blob_store/s3_blob_store.py,sha256=G3B_V3gUE7XbUY42lDtBczUKuA7q8S7MD43tx1aHrJo,3445
 indexify/executor/channel_manager.py,sha256=SeBaOYJXJ6KNw59c4C4n54qhcCusNda2756Rbu8pFFM,8642
-indexify/executor/executor.py,sha256=AGtOHbznjGLQMjJHie10gxq1fLq3h23uXvk4tVkxxEw,6294
+indexify/executor/executor.py,sha256=hR49lBFGaqSRj8RtLlzd4cd8hRUwhMrwUHwPx53EHDU,6310
 indexify/executor/function_allowlist.py,sha256=PCelCW6qIe_2sH11BCKr7LDqarRV5kwNsrfB2EV7Zwo,1772
-indexify/executor/function_executor/function_executor.py,sha256=buY2JvIVPbwEwl_CUuSGkn4Tb6gJIUxGwcMnT_Y3kNI,12376
+indexify/executor/function_executor/function_executor.py,sha256=Hz_dT_2i1m9akUGfULWQpDlMsn0CI1AX4Mdt7-oOknI,13598
 indexify/executor/function_executor/health_checker.py,sha256=IxE0jnC99K_lvnizFLjXqS1942H8-FNAN4AlhLIjg2Y,6373
-indexify/executor/function_executor/invocation_state_client.py,sha256=tARfeEEOZJoWd38wA5e7ihKMbR3llk90eD_KQ1QD6cs,10582
+indexify/executor/function_executor/invocation_state_client.py,sha256=YYJElvpkCH-OmBXplLJX59dwMZELMHXSDR4VLLn1u7k,10939
 indexify/executor/function_executor/metrics/function_executor.py,sha256=TDksxLRJr-P9ZKhF2Orsaxzzb4lVIBxFEjd_9Zv53Ng,6313
 indexify/executor/function_executor/metrics/health_checker.py,sha256=EaeIYJPrQ-qqNMGZVGkvjPoeQSCl4FzPKXEv3Cly1NE,456
 indexify/executor/function_executor/metrics/invocation_state_client.py,sha256=6FCW6rXHVZZSmwLquZdpjgQPSmE_99naDLke5rZiwMI,1867
@@ -22,13 +22,13 @@ indexify/executor/function_executor/server/function_executor_server_factory.py,s
 indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=JekDOqF7oFD4J6zcN3xB0Dxd1cgpEXMOsb_rKZOeBlI,668
 indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=w5aGQPHWLpixlP9-BbZu6oL_muMA95-hr7WKVxiEL7Q,4303
 indexify/executor/function_executor_controller/__init__.py,sha256=VPuuBEYOKf7OWyPPjy-jGOv-d5xJqHvkJfFT_oj-AsE,492
-indexify/executor/function_executor_controller/completed_task_metrics.py,sha256=_awcdfAd04YjddV-X1W5_2o-odwuu4BTuH1E0QHjgzI,3596
-indexify/executor/function_executor_controller/create_function_executor.py,sha256=wfogvQlwLnYo2EfJyHM7gLtESc2tZNP90EnNsh0_0jM,9966
+indexify/executor/function_executor_controller/completed_task_metrics.py,sha256=MhnC-ddgmTK4yTsuZxgTKnqZ-YSVeWn2EhbbiggsSKk,3664
+indexify/executor/function_executor_controller/create_function_executor.py,sha256=qQ_xBQRYUwz8SbNH7-H6D9GP-NJM3OlxPaIuLnPpbRM,9996
 indexify/executor/function_executor_controller/debug_event_loop.py,sha256=VJOKe_c9HjIDVCjhMY3Yqyeq1tAM1eVa2chZa6CMf-U,1016
 indexify/executor/function_executor_controller/destroy_function_executor.py,sha256=D2idVC-q4etkt65C0qH7TENHDVidSLsSalr6e3HwZWU,843
-indexify/executor/function_executor_controller/downloads.py,sha256=iVXuDl9mUOOMurpd6wrtIexUHubgvM9GdKxwgwQA44E,6487
+indexify/executor/function_executor_controller/downloads.py,sha256=XjCUmLY_jrI3AlnXC7aDwwKWTvsQjV7I9AXzrbIeY6c,7063
 indexify/executor/function_executor_controller/events.py,sha256=KiK0-fNGHyeLfDiw8dKwl_qIMkJPjiUPdLY0FZYaMf8,5467
-indexify/executor/function_executor_controller/function_executor_controller.py,sha256=ZwFA10rJbt34Z_TXYvc2Au9BpokjA-O83-AhuBev7RQ,33566
+indexify/executor/function_executor_controller/function_executor_controller.py,sha256=aKyRdxwTeCL9yJpgsJAoqVpYApBcdhvJHod92633oOQ,33391
 indexify/executor/function_executor_controller/function_executor_startup_output.py,sha256=PXg2r440kqHI3oHGZbb58ehuAuW_fmEdxLTAa-0V3p4,715
 indexify/executor/function_executor_controller/loggers.py,sha256=zEY2nt15gboX3SX6Kh1xjeCljZJZSE4lp27qNrg8yPY,3637
 indexify/executor/function_executor_controller/message_validators.py,sha256=aNiZhYA87pnxUJtZKvKGDt40rfox-TYH2J6mW7o-Pkw,2981
@@ -38,10 +38,10 @@ indexify/executor/function_executor_controller/metrics/function_executor_control
 indexify/executor/function_executor_controller/metrics/run_task.py,sha256=kdjw-Bi788B8Pq4eL08k8HVtcjIA3y8pLLSNWRBI8YA,1006
 indexify/executor/function_executor_controller/metrics/upload_task_output.py,sha256=Ppf8NujCNbQzFelJiuh8Sutcjty7hnkFz1dWQLYIgQI,1464
 indexify/executor/function_executor_controller/prepare_task.py,sha256=AKbo_H_5pOKdxFKKkzdOb1WhQ0XT-4Qm9D3iIsukyMU,1247
-indexify/executor/function_executor_controller/run_task.py,sha256=aCeVwtbwz0T5Qw-vkMw9j5keiC76zgyvJa-4ZC_RsNg,7745
+indexify/executor/function_executor_controller/run_task.py,sha256=Sbg22yo07FQPyicxKmzPHhWRtELhTR51sF5fZ_y5lIY,9927
 indexify/executor/function_executor_controller/task_info.py,sha256=ZEdypd8QVmYbrLt1186Ed9YEQwrO0Sx_hKH0QLg1DVY,1181
-indexify/executor/function_executor_controller/task_output.py,sha256=Me6s2FLqqDHH6QLmfOTDs2F40PpZG0kfRJvbeBEfdtk,3823
-indexify/executor/function_executor_controller/upload_task_output.py,sha256=CTZ9iyA178gkV9JvfRDlowIHK9ff8Fx_j2_6EGaiyqk,8314
+indexify/executor/function_executor_controller/task_output.py,sha256=A5KJh60OI1FdHw6ABX3KxL9OMrC6ZADJHeo0W5iZpz8,3764
+indexify/executor/function_executor_controller/upload_task_output.py,sha256=fEZm5eodx5rNLQYFhmdkMDD9qjX3_wKo64x4aUKTu34,10403
 indexify/executor/host_resources/host_resources.py,sha256=ZSfox24jaz1IIaQWUmciOoev0l35rk8LHbnb_koJWno,3810
 indexify/executor/host_resources/nvidia_gpu.py,sha256=BIxBcWenyhZe0fuPQT9I0g6zAWMDPcm_oZEfgOoYsFU,3306
 indexify/executor/host_resources/nvidia_gpu_allocator.py,sha256=AOcXKglLyRD-GrZzyCoi_oDRJoaOhFKWBSlUOxHeAP8,2114
@@ -59,11 +59,11 @@ indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFs
 indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpxWDfed1VHtKKnwvLQ8-frpG98,425
 indexify/executor/state_reconciler.py,sha256=yCDLRfPmfU71aqUNnRLxDt6y4AWdSVkrEwG2HZq7nso,18125
 indexify/executor/state_reporter.py,sha256=0qs9n2n8bWn77hdW8qsuUqHS7G8iyW7-9_-s8l6aJSU,14195
-indexify/proto/executor_api.proto,sha256=2rLo6BBeVOkU8rEBSgwmXc7qccLS-Of0CEJKwF0oMIY,11628
-indexify/proto/executor_api_pb2.py,sha256=h_1nT-4L7dPR9_ZQ0-igtA97il4wG03dVnQlM8jyD6U,16334
-indexify/proto/executor_api_pb2.pyi,sha256=RR4J8IotdGYOEqC4iZ5ene6j-sOQRuS8xO6j-pk_lok,22867
+indexify/proto/executor_api.proto,sha256=x52XYq4VJLhFGAQ3EaH_hr-wWUTspsKL5XTaPZyi87k,12359
+indexify/proto/executor_api_pb2.py,sha256=OjLV2DJZI9zOXcppNniWQOc0wnI9A3Fp52ZEr6TuYOc,16332
+indexify/proto/executor_api_pb2.pyi,sha256=CEkNPPyIZK4CAJt7_z-bHiftuiSEjuf3-_p4cMrQP4I,22902
 indexify/proto/executor_api_pb2_grpc.py,sha256=JpT5K6jiS0NJVNyTt1mAPpyJMXuEGeNN2V6R3KmLHZ4,7607
-indexify-0.4.8.dist-info/METADATA,sha256=2vCZHQqVcPDwbvN8ezUVRH-GFO7DWq3Nw9hpINMyX5I,1114
-indexify-0.4.8.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.4.8.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
-indexify-0.4.8.dist-info/RECORD,,
+indexify-0.4.10.dist-info/METADATA,sha256=o7zdxoOttCT8_rSI_bgRjW8LX4TIMWzV4-kQhjc0Ux4,1115
+indexify-0.4.10.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.4.10.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
+indexify-0.4.10.dist-info/RECORD,,