indexify 0.4.22__py3-none-any.whl → 0.4.24__py3-none-any.whl

indexify/executor/function_executor_controller/metrics/finalize_task.py

@@ -0,0 +1,20 @@
+import prometheus_client
+
+from indexify.executor.monitoring.metrics import latency_metric_for_fast_operation
+
+# Task finalization metrics.
+metric_task_finalizations: prometheus_client.Counter = prometheus_client.Counter(
+    "task_finalizations",
+    "Number of task finalizations",
+)
+metric_task_finalization_errors: prometheus_client.Counter = prometheus_client.Counter(
+    "task_finalization_errors",
+    "Number of task finalization errors",
+)
+metric_tasks_finalizing: prometheus_client.Gauge = prometheus_client.Gauge(
+    "tasks_finalizing",
+    "Number of tasks currently finalizing",
+)
+metric_task_finalization_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation("task_finalization", "task finalization")
+)

indexify/executor/function_executor_controller/metrics/prepare_task.py

@@ -0,0 +1,18 @@
+import prometheus_client
+
+from indexify.executor.monitoring.metrics import latency_metric_for_fast_operation
+
+metric_task_preparations: prometheus_client.Counter = prometheus_client.Counter(
+    "task_preparations", "Number of task preparations for execution"
+)
+metric_task_preparation_errors: prometheus_client.Counter = prometheus_client.Counter(
+    "task_preparation_errors", "Number of task preparation errors"
+)
+metric_task_preparation_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "task_preparation", "task preparation for execution"
+    )
+)
+metric_tasks_getting_prepared: prometheus_client.Gauge = prometheus_client.Gauge(
+    "tasks_getting_prepared", "Number of tasks currently getting prepared for execution"
+)

indexify/executor/function_executor_controller/prepare_task.py

@@ -1,38 +1,256 @@
-from typing import Any
+import asyncio
+import time
+from typing import Any, List, Optional
+
+from tensorlake.function_executor.proto.function_executor_pb2 import (
+    BLOB,
+    BLOBChunk,
+    FunctionInputs,
+    SerializedObjectInsideBLOB,
+)
 
 from indexify.executor.blob_store.blob_store import BLOBStore
+from indexify.proto.executor_api_pb2 import DataPayload, Task
 
-from .downloads import download_init_value, download_input
+from .downloads import serialized_object_manifest_from_data_payload_proto
 from .events import TaskPreparationFinished
+from .metrics.prepare_task import (
+    metric_task_preparation_errors,
+    metric_task_preparation_latency,
+    metric_task_preparations,
+    metric_tasks_getting_prepared,
+)
 from .task_info import TaskInfo
+from .task_input import TaskInput
+
+# The following constants are subject to S3 limits,
+# see https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html.
+#
+# 7 days - max presigned URL validity duration and limit on max function duration
+_MAX_PRESIGNED_URI_EXPIRATION_SEC: int = 7 * 24 * 60 * 60
+# This chunk size gives the best performance with S3. Based on our benchmarking.
+_BLOB_OPTIMAL_CHUNK_SIZE_BYTES: int = 100 * 1024 * 1024  # 100 MB
+# Max output size with optimal chunks is 100 * 100 MB = 10 GB.
+# Each chunk requires a separate S3 presign operation, so we limit the number of optimal chunks to 100.
+# S3 presign operations are local, it typically takes 30 ms per 100 URLs.
+_OUTPUT_BLOB_OPTIMAL_CHUNKS_COUNT: int = 100
+# This chunk size gives ~20% slower performance with S3 compared to optimal. Based on our benchmarking.
+_OUTPUT_BLOB_SLOWER_CHUNK_SIZE_BYTES: int = 1 * 1024 * 1024 * 1024  # 1 GB
+# Max output size with slower chunks is 100 * 1 GB = 100 GB.
+_OUTPUT_BLOB_SLOWER_CHUNKS_COUNT: int = 100
+# Invocation error output is using a single chunk.
+_INVOCATION_ERROR_MAX_SIZE_BYTES: int = 10 * 1024 * 1024  # 10 MB
 
 
 async def prepare_task(
     task_info: TaskInfo, blob_store: BLOBStore, logger: Any
 ) -> TaskPreparationFinished:
-    """Prepares the task by downloading the input and init value if available.
+    """Prepares the task for execution.
 
+    If successful then the task is runnable.
     Doesn't raise any exceptions.
     """
     logger = logger.bind(module=__name__)
+    start_time = time.monotonic()
     try:
-        task_info.input = await download_input(
-            data_payload=task_info.allocation.task.input,
+        with (
+            metric_task_preparation_errors.count_exceptions(),
+            metric_tasks_getting_prepared.track_inprogress(),
+            metric_task_preparation_latency.time(),
+        ):
+            metric_task_preparations.inc()
+            task_info.input = await _prepare_task_input(
+                task_info=task_info,
+                blob_store=blob_store,
+                logger=logger,
+            )
+            logger.info(
+                "Task was prepared for execution",
+                duration=time.monotonic() - start_time,
+            )
+            return TaskPreparationFinished(
+                task_info=task_info,
+                is_success=True,
+            )
+    except asyncio.CancelledError:
+        return TaskPreparationFinished(task_info=task_info, is_success=False)
+    except BaseException as e:
+        logger.error(
+            "Failed to prepare task for execution",
+            exc_info=e,
+            duration=time.monotonic() - start_time,
+        )
+        return TaskPreparationFinished(task_info=task_info, is_success=False)
+
+
+async def _prepare_task_input(
+    task_info: TaskInfo, blob_store: BLOBStore, logger: Any
+) -> TaskInput:
+    """Prepares the task for execution.
+
+    Raises an exception on error.
+    """
+    task: Task = task_info.allocation.task
+    function_init_value_blob: Optional[BLOB] = None
+    function_init_value: Optional[SerializedObjectInsideBLOB] = None
+    if task.HasField("reducer_input"):
+        function_init_value_blob = await _presign_function_input_blob(
+            data_payload=task.reducer_input,
             blob_store=blob_store,
             logger=logger,
         )
+        function_init_value = _to_serialized_object_inside_blob(task.reducer_input)
+
+    function_outputs_blob_uri: str = (
+        f"{task.output_payload_uri_prefix}.{task_info.allocation.allocation_id}.output"
+    )
+    invocation_error_blob_uri: str = (
+        f"{task.invocation_error_payload_uri_prefix}.{task.graph_invocation_id}.inverr"
+    )
+
+    # The uploads are completed when finalizing the task.
+    function_outputs_blob_upload_id: Optional[str] = None
+    invocation_error_blob_upload_id: Optional[str] = None
+
+    try:
+        function_outputs_blob_upload_id = await blob_store.create_multipart_upload(
+            uri=function_outputs_blob_uri,
+            logger=logger,
+        )
+        invocation_error_blob_upload_id = await blob_store.create_multipart_upload(
+            uri=invocation_error_blob_uri,
+            logger=logger,
+        )
+    except BaseException:
+        if function_outputs_blob_upload_id is not None:
+            await blob_store.abort_multipart_upload(
+                uri=function_outputs_blob_uri,
+                upload_id=function_outputs_blob_upload_id,
+                logger=logger,
+            )
+        if invocation_error_blob_upload_id is not None:
+            await blob_store.abort_multipart_upload(
+                uri=invocation_error_blob_uri,
+                upload_id=invocation_error_blob_upload_id,
+                logger=logger,
+            )
+        raise
 
-        if task_info.allocation.task.HasField("reducer_input"):
-            task_info.init_value = await download_init_value(
-                data_payload=task_info.allocation.task.reducer_input,
+    return TaskInput(
+        function_inputs=FunctionInputs(
+            function_input_blob=await _presign_function_input_blob(
+                data_payload=task.input,
+                blob_store=blob_store,
+                logger=logger,
+            ),
+            function_input=_to_serialized_object_inside_blob(task.input),
+            function_init_value_blob=function_init_value_blob,
+            function_init_value=function_init_value,
+            function_outputs_blob=await _presign_function_outputs_blob(
+                uri=function_outputs_blob_uri,
+                upload_id=function_outputs_blob_upload_id,
                 blob_store=blob_store,
                 logger=logger,
+            ),
+            invocation_error_blob=await _presign_invocation_error_blob(
+                uri=invocation_error_blob_uri,
+                upload_id=invocation_error_blob_upload_id,
+                blob_store=blob_store,
+                logger=logger,
+            ),
+        ),
+        function_outputs_blob_uri=function_outputs_blob_uri,
+        function_outputs_blob_upload_id=function_outputs_blob_upload_id,
+        invocation_error_blob_uri=invocation_error_blob_uri,
+        invocation_error_blob_upload_id=invocation_error_blob_upload_id,
+    )
+
+
+async def _presign_function_input_blob(
+    data_payload: DataPayload, blob_store: BLOBStore, logger: Any
+) -> BLOB:
+    get_blob_uri: str = await blob_store.presign_get_uri(
+        uri=data_payload.uri,
+        expires_in_sec=_MAX_PRESIGNED_URI_EXPIRATION_SEC,
+        logger=logger,
+    )
+    chunks: List[BLOBChunk] = []
+
+    while len(chunks) * _BLOB_OPTIMAL_CHUNK_SIZE_BYTES < data_payload.size:
+        chunks.append(
+            BLOBChunk(
+                uri=get_blob_uri,  # The URI allows to read any byte range in the BLOB.
+                size=_BLOB_OPTIMAL_CHUNK_SIZE_BYTES,
+                # ETag is only set by FE when returning BLOBs to us
             )
+        )
 
-        return TaskPreparationFinished(task_info=task_info, is_success=True)
-    except Exception as e:
-        logger.error(
-            "Failed to prepare task",
-            exc_info=e,
+    return BLOB(
+        chunks=chunks,
+    )
+
+
+async def _presign_function_outputs_blob(
+    uri: str, upload_id: str, blob_store: BLOBStore, logger: Any
+) -> BLOB:
+    """Presigns the output blob for the task."""
+    chunks: List[BLOBChunk] = []
+
+    while len(chunks) != (
+        _OUTPUT_BLOB_OPTIMAL_CHUNKS_COUNT + _OUTPUT_BLOB_SLOWER_CHUNKS_COUNT
+    ):
+        upload_chunk_uri: str = await blob_store.presign_upload_part_uri(
+            uri=uri,
+            part_number=len(chunks) + 1,
+            upload_id=upload_id,
+            expires_in_sec=_MAX_PRESIGNED_URI_EXPIRATION_SEC,
+            logger=logger,
         )
-        return TaskPreparationFinished(task_info=task_info, is_success=False)
+
+        chunk_size: int = (
+            _BLOB_OPTIMAL_CHUNK_SIZE_BYTES
+            if len(chunks) < _OUTPUT_BLOB_OPTIMAL_CHUNKS_COUNT
+            else _OUTPUT_BLOB_SLOWER_CHUNK_SIZE_BYTES
+        )
+        chunks.append(
+            BLOBChunk(
+                uri=upload_chunk_uri,
+                size=chunk_size,
+                # ETag is only set by FE when returning BLOBs to us
+            )
+        )
+
+    return BLOB(
+        chunks=chunks,
+    )
+
+
+async def _presign_invocation_error_blob(
+    uri: str, upload_id: str, blob_store: BLOBStore, logger: Any
+) -> BLOB:
+    """Presigns the output blob for the invocation error."""
+    upload_chunk_uri: str = await blob_store.presign_upload_part_uri(
+        uri=uri,
+        part_number=1,
+        upload_id=upload_id,
+        expires_in_sec=_MAX_PRESIGNED_URI_EXPIRATION_SEC,
+        logger=logger,
+    )
+    return BLOB(
+        chunks=[
+            BLOBChunk(
+                uri=upload_chunk_uri,
+                size=_INVOCATION_ERROR_MAX_SIZE_BYTES,
+                # ETag is only set by FE when returning BLOBs to us
+            )
+        ]
+    )
+
+
+def _to_serialized_object_inside_blob(
+    data_payload: DataPayload,
+) -> SerializedObjectInsideBLOB:
+    return SerializedObjectInsideBLOB(
+        manifest=serialized_object_manifest_from_data_payload_proto(data_payload),
+        offset=data_payload.offset,
+    )

indexify/executor/function_executor_controller/run_task.py

@@ -1,18 +1,17 @@
 import asyncio
-import os
-import random
 import time
 from typing import Any, Optional
 
 import grpc
 from tensorlake.function_executor.proto.function_executor_pb2 import (
+    BLOB,
     AwaitTaskProgress,
     AwaitTaskRequest,
     CreateTaskRequest,
     DeleteTaskRequest,
-    FunctionInputs,
-    SerializedObject,
+    SerializedObjectInsideBLOB,
     Task,
+    TaskDiagnostics,
 )
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     TaskFailureReason as FETaskFailureReason,
@@ -47,10 +46,6 @@ from .metrics.run_task import (
 from .task_info import TaskInfo
 from .task_output import TaskMetrics, TaskOutput
 
-_ENABLE_INJECT_TASK_CANCELLATIONS = (
-    os.getenv("INDEXIFY_INJECT_TASK_CANCELLATIONS", "0") == "1"
-)
-
 _CREATE_TASK_TIMEOUT_SECS = 5
 _DELETE_TASK_TIMEOUT_SECS = 5
 
@@ -63,23 +58,31 @@ async def run_task_on_function_executor(
     Doesn't raise any exceptions.
     """
     logger = logger.bind(module=__name__)
+
+    if task_info.input is None:
+        logger.error(
+            "task input is None, this should never happen",
+        )
+        task_info.output = TaskOutput.internal_error(
+            allocation=task_info.allocation,
+            execution_start_time=None,
+            execution_end_time=None,
+        )
+        return TaskExecutionFinished(
+            task_info=task_info,
+            function_executor_termination_reason=None,
+        )
+
     task = Task(
-        task_id=task_info.allocation.task.id,
         namespace=task_info.allocation.task.namespace,
         graph_name=task_info.allocation.task.graph_name,
         graph_version=task_info.allocation.task.graph_version,
         function_name=task_info.allocation.task.function_name,
         graph_invocation_id=task_info.allocation.task.graph_invocation_id,
+        task_id=task_info.allocation.task.id,
         allocation_id=task_info.allocation.allocation_id,
-        request=FunctionInputs(function_input=task_info.input),
+        request=task_info.input.function_inputs,
     )
-    # Don't keep the input in memory after we started running the task.
-    task_info.input = None
-
-    if task_info.init_value is not None:
-        task.request.function_init_value.CopyFrom(task_info.init_value)
-        # Don't keep the init value in memory after we started running the task.
-        task_info.init_value = None
 
     function_executor.invocation_state_client().add_task_to_invocation_id_entry(
         task_id=task_info.allocation.task.id,
@@ -104,10 +107,13 @@ async def run_task_on_function_executor(
     # If this RPC failed due to customer code crashing the server we won't be
     # able to detect this. We'll treat this as our own error for now and thus
     # let the AioRpcError to be raised here.
-    timeout_sec = task_info.allocation.task.timeout_ms / 1000.0
+    timeout_sec: float = task_info.allocation.task.timeout_ms / 1000.0
     try:
+        # This aio task can only be cancelled during this await call.
         task_result = await _run_task_rpcs(task, function_executor, timeout_sec)
 
+        _process_task_diagnostics(task_result.diagnostics, logger)
+
         task_info.output = _task_output_from_function_executor_result(
             allocation=task_info.allocation,
             result=task_result,
@@ -123,7 +129,6 @@ async def run_task_on_function_executor(
         )
         task_info.output = TaskOutput.function_timeout(
             allocation=task_info.allocation,
-            timeout_sec=timeout_sec,
             execution_start_time=execution_start_time,
             execution_end_time=time.monotonic(),
         )
@@ -156,8 +161,8 @@ async def run_task_on_function_executor(
             execution_start_time=execution_start_time,
             execution_end_time=time.monotonic(),
         )
-
     except asyncio.CancelledError:
+        # Handle aio task cancellation during `await _run_task_rpcs`.
         # The task is still running in FE, we only cancelled the client-side RPC.
         function_executor_termination_reason = (
             FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_CANCELLED
@@ -192,16 +197,21 @@ async def run_task_on_function_executor(
         task_info.output.outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE
         and function_executor_termination_reason is None
     ):
-        # Check if the task failed because the FE is unhealthy to prevent more tasks failing.
-        result: HealthCheckResult = await function_executor.health_checker().check()
-        if not result.is_healthy:
-            function_executor_termination_reason = (
-                FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY
-            )
-            logger.error(
-                "Function Executor health check failed after running task, shutting down Function Executor",
-                health_check_fail_reason=result.reason,
-            )
+        try:
+            # Check if the task failed because the FE is unhealthy to prevent more tasks failing.
+            result: HealthCheckResult = await function_executor.health_checker().check()
+            if not result.is_healthy:
+                function_executor_termination_reason = (
+                    FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY
+                )
+                logger.error(
+                    "Function Executor health check failed after running task, shutting down Function Executor",
+                    health_check_fail_reason=result.reason,
+                )
+        except asyncio.CancelledError:
+            # The aio task was cancelled during the health check await.
+            # We can't conclude anything about the health of the FE here.
+            pass
 
     _log_task_execution_finished(output=task_info.output, logger=logger)
 
@@ -215,8 +225,7 @@ async def _run_task_rpcs(
     task: Task, function_executor: FunctionExecutor, timeout_sec: float
 ) -> TaskResult:
     """Runs the task, returning the result, reporting errors via exceptions."""
-
-    response: AwaitTaskProgress
+    task_result: Optional[TaskResult] = None
     channel: grpc.aio.Channel = function_executor.channel()
     fe_stub = FunctionExecutorStub(channel)
 
@@ -231,24 +240,19 @@ async def _run_task_rpcs(
     try:
         while True:
             # Wait for next response with fresh timeout each time
-            response = await asyncio.wait_for(await_rpc.read(), timeout=timeout_sec)
-            if response.WhichOneof("response") == "task_result":
-                # We're done waiting.
+            response: AwaitTaskProgress = await asyncio.wait_for(
+                await_rpc.read(), timeout=timeout_sec
+            )
+
+            if response == grpc.aio.EOF:
+                break
+            elif response.WhichOneof("response") == "task_result":
+                task_result = response.task_result
                 break
 
             # NB: We don't actually check for other message types
             # here; any message from the FE is treated as an
             # indication that it's making forward progress.
-
-            if response == grpc.aio.EOF:
-                # Protocol error: we should get a task_result before
-                # we see the RPC complete.
-                raise grpc.aio.AioRpcError(
-                    grpc.StatusCode.CANCELLED,
-                    None,
-                    None,
-                    "Function Executor didn't return function/task alloc response",
-                )
     finally:
         # Cancel the outstanding RPC to ensure any resources in use
         # are cleaned up; note that this is idempotent (in case the
@@ -260,7 +264,15 @@ async def _run_task_rpcs(
         DeleteTaskRequest(task_id=task.task_id), timeout=_DELETE_TASK_TIMEOUT_SECS
     )
 
-    return response.task_result
+    if task_result is None:
+        raise grpc.aio.AioRpcError(
+            grpc.StatusCode.CANCELLED,
+            None,
+            None,
+            "Function Executor didn't return function/task alloc result",
+        )
+
+    return task_result
 
 
 def _task_output_from_function_executor_result(
@@ -271,8 +283,6 @@ def _task_output_from_function_executor_result(
     logger: Any,
 ) -> TaskOutput:
     response_validator = MessageValidator(result)
-    response_validator.required_field("stdout")
-    response_validator.required_field("stderr")
     response_validator.required_field("outcome_code")
 
     metrics = TaskMetrics(counters={}, timers={})
@@ -285,7 +295,8 @@ def _task_output_from_function_executor_result(
         result.outcome_code, logger=logger
     )
     failure_reason: Optional[TaskFailureReason] = None
-    invocation_error_output: Optional[SerializedObject] = None
+    invocation_error_output: Optional[SerializedObjectInsideBLOB] = None
+    uploaded_invocation_error_blob: Optional[BLOB] = None
 
     if outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE:
         response_validator.required_field("failure_reason")
@@ -294,25 +305,22 @@ def _task_output_from_function_executor_result(
         )
         if failure_reason == TaskFailureReason.TASK_FAILURE_REASON_INVOCATION_ERROR:
             response_validator.required_field("invocation_error_output")
+            response_validator.required_field("uploaded_invocation_error_blob")
             invocation_error_output = result.invocation_error_output
-
-    if _ENABLE_INJECT_TASK_CANCELLATIONS:
-        logger.warning("injecting cancellation failure for the task allocation")
-        if (
-            random.random() < 0.5
-        ):  # 50% chance to get stable reproduction in manual testing
-            outcome_code = TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE
-            failure_reason = TaskFailureReason.TASK_FAILURE_REASON_TASK_CANCELLED
+            uploaded_invocation_error_blob = result.uploaded_invocation_error_blob
+    elif outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_SUCCESS:
+        # function_outputs can have no items, this happens when the function returns None.
+        response_validator.required_field("uploaded_function_outputs_blob")
 
     return TaskOutput(
         allocation=allocation,
         outcome_code=outcome_code,
         failure_reason=failure_reason,
+        function_outputs=list(result.function_outputs),
+        uploaded_function_outputs_blob=result.uploaded_function_outputs_blob,
         invocation_error_output=invocation_error_output,
-        function_outputs=result.function_outputs,
-        next_functions=result.next_functions,
-        stdout=result.stdout,
-        stderr=result.stderr,
+        uploaded_invocation_error_blob=uploaded_invocation_error_blob,
+        next_functions=list(result.next_functions),
         metrics=metrics,
         execution_start_time=execution_start_time,
         execution_end_time=execution_end_time,
@@ -332,6 +340,14 @@ def _log_task_execution_finished(output: TaskOutput, logger: Any) -> None:
     )
 
 
+def _process_task_diagnostics(task_diagnostics: TaskDiagnostics, logger: Any) -> None:
+    MessageValidator(task_diagnostics).required_field("function_executor_log")
+    # Uncomment these lines once we stop printing FE logs to stdout/stderr.
+    # Print FE logs directly to Executor logs so operators can see them.
+    # logger.info("Function Executor logs during task execution:")
+    # print(task_diagnostics.function_executor_log)
+
+
 def _to_task_outcome_code(
     fe_task_outcome_code: FETaskOutcomeCode, logger
 ) -> TaskOutcomeCode:

indexify/executor/function_executor_controller/task_info.py

@@ -2,10 +2,9 @@ import asyncio
 from dataclasses import dataclass
 from typing import Optional
 
-from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
-
 from indexify.proto.executor_api_pb2 import TaskAllocation
 
+from .task_input import TaskInput
 from .task_output import TaskOutput
 
 
@@ -22,11 +21,9 @@ class TaskInfo:
     is_cancelled: bool = False
     # aio task that is currently executing a lifecycle step of this task.
     aio_task: Optional[asyncio.Task] = None
-    # Downloaded input if function was prepared successfully.
-    input: Optional[SerializedObject] = None
-    # Downloaded init value if function was prepared successfully and is a reducer.
-    init_value: Optional[SerializedObject] = None
-    # Output of the task.
+    # Input if function was prepared successfully.
+    input: Optional[TaskInput] = None
+    # Output of the task, always set when the task is completed.
     output: Optional[TaskOutput] = None
     # True if the task is fully completed and was added to state reporter.
     is_completed: bool = False

indexify/executor/function_executor_controller/task_input.py

@@ -0,0 +1,21 @@
+from tensorlake.function_executor.proto.function_executor_pb2 import FunctionInputs
+
+
+class TaskInput:
+    """Represents the input for a task in the function executor controller."""
+
+    def __init__(
+        self,
+        function_inputs: FunctionInputs,
+        function_outputs_blob_uri: str,
+        function_outputs_blob_upload_id: str,
+        invocation_error_blob_uri: str,
+        invocation_error_blob_upload_id: str,
+    ):
+        # Actual input object sent to FE.
+        self.function_inputs = function_inputs
+        # Executor side function input related bookkeeping.
+        self.function_outputs_blob_uri = function_outputs_blob_uri
+        self.function_outputs_blob_upload_id = function_outputs_blob_upload_id
+        self.invocation_error_blob_uri = invocation_error_blob_uri
+        self.invocation_error_blob_upload_id = invocation_error_blob_upload_id