indexify 0.4.10__tar.gz → 0.4.11__tar.gz

{indexify-0.4.10 → indexify-0.4.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.10
+Version: 0.4.11
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -17,7 +17,7 @@ Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
 Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: tensorlake (==0.2.7)
+Requires-Dist: tensorlake (==0.2.8)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.4.10 → indexify-0.4.11}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.4.10"
+version = "0.4.11"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -25,7 +25,7 @@ prometheus-client = "^0.21.1"
 psutil = "^7.0.0"
 # Adds function-executor binary, utils lib, sdk used in indexify-cli commands.
 # We need to specify the tensorlake version exactly because pip install doesn't respect poetry.lock files.
-tensorlake = "0.2.7"
+tensorlake = "0.2.8"
 # Uncomment the next line to use local tensorlake package (only for development!)
 # tensorlake = { path = "../tensorlake", develop = true }
 # pydantic is provided by tensorlake

indexify-0.4.11/src/indexify/executor/function_executor_controller/destroy_function_executor.py

@@ -0,0 +1,31 @@
+import asyncio
+from typing import Any, Optional
+
+from indexify.executor.function_executor.function_executor import FunctionExecutor
+
+from .events import FunctionExecutorDestroyed
+
+
+async def destroy_function_executor(
+    function_executor: Optional[FunctionExecutor],
+    lock: asyncio.Lock,
+    logger: Any,
+) -> FunctionExecutorDestroyed:
+    """Destroys the function executor if it's not None.
+
+    The supplied lock is used to ensure that if a destroy operation is in progress,
+    then another caller won't return immediately assuming that the destroy is complete
+    due to its idempotency.
+
+    Doesn't raise any exceptions.
+    """
+    logger = logger.bind(module=__name__)
+
+    if function_executor is not None:
+        async with lock:
+            logger.info(
+                "destroying function executor",
+            )
+            await function_executor.destroy()
+
+    return FunctionExecutorDestroyed(is_success=True)

{indexify-0.4.10 → indexify-0.4.11}/src/indexify/executor/function_executor_controller/events.py

@@ -55,19 +55,12 @@ class FunctionExecutorDestroyed(BaseEvent):
     Event indicating that Function Executor has been destroyed.
     """
 
-    def __init__(
-        self, is_success: bool, termination_reason: FunctionExecutorTerminationReason
-    ):
+    def __init__(self, is_success: bool):
         super().__init__(EventType.FUNCTION_EXECUTOR_DESTROYED)
         self.is_success: bool = is_success
-        self.termination_reason: FunctionExecutorTerminationReason = termination_reason
 
     def __str__(self) -> str:
-        return (
-            f"Event(type={self.event_type.name}, "
-            f"is_success={self.is_success}, "
-            f"termination_reason={FunctionExecutorTerminationReason.Name(self.termination_reason)})"
-        )
+        return f"Event(type={self.event_type.name}, " f"is_success={self.is_success})"
 
 
 class ShutdownInitiated(BaseEvent):
@@ -75,15 +68,8 @@ class ShutdownInitiated(BaseEvent):
     Event indicating that Function Executor shutdown has been initiated.
     """
 
-    def __init__(self, termination_reason: FunctionExecutorTerminationReason):
+    def __init__(self):
         super().__init__(EventType.SHUTDOWN_INITIATED)
-        self.termination_reason: FunctionExecutorTerminationReason = termination_reason
-
-    def __str__(self) -> str:
-        return (
-            f"Event(type={self.event_type.name}, "
-            f"termination_reason={FunctionExecutorTerminationReason.Name(self.termination_reason)})"
-        )
 
 
 class TaskPreparationFinished(BaseEvent):

{indexify-0.4.10 → indexify-0.4.11}/src/indexify/executor/function_executor_controller/function_executor_controller.py

@@ -1,6 +1,7 @@
 import asyncio
 import time
 from collections.abc import Coroutine
+from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
@@ -43,12 +44,14 @@ from .events import (
 from .function_executor_startup_output import FunctionExecutorStartupOutput
 from .loggers import function_executor_logger, task_allocation_logger
 from .metrics.function_executor_controller import (
-    METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_PENDING,
-    METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_RUNNING,
-    METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_TERMINATED,
-    METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_UNKNOWN,
+    METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_NOT_STARTED,
+    METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_RUNNING,
+    METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_STARTING_UP,
+    METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_TERMINATED,
+    METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_TERMINATING,
+    METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_UNKNOWN,
     metric_control_loop_handle_event_latency,
-    metric_function_executors_with_status,
+    metric_function_executors_with_state,
     metric_runnable_tasks,
     metric_runnable_tasks_per_function_name,
     metric_schedule_task_latency,
@@ -61,6 +64,16 @@ from .task_output import TaskOutput
 from .upload_task_output import upload_task_output
 
 
+# Actual FE controller states, they are a bit different from statuses reported to the Server.
+# All the valid state transitions are forward only (can skip multiple states in a row).
+class _FE_CONTROLLER_STATE(Enum):
+    NOT_STARTED = 1
+    STARTING_UP = 2
+    RUNNING = 3
+    TERMINATING = 4
+    TERMINATED = 5
+
+
 class FunctionExecutorController:
     def __init__(
         self,
@@ -94,19 +107,18 @@ class FunctionExecutorController:
         self._logger: Any = function_executor_logger(
             function_executor_description, logger.bind(module=__name__)
         )
-        # Mutable state. No lock needed as it's modified by async tasks running in
-        # the same event loop.
+        self._destroy_lock: asyncio.Lock = asyncio.Lock()
+        # Mutable state. No lock needed as it's modified by async tasks running in the same event loop.
         self._fe: Optional[FunctionExecutor] = None
-        self._fe_termination_reason: FunctionExecutorTerminationReason = (
-            None  # Optional
-        )
-        # FE Status reported to Server.
-        self._status: FunctionExecutorStatus = (
-            FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_UNKNOWN
-        )
-        metric_function_executors_with_status.labels(
-            status=_to_fe_status_metric_label(self._status, self._logger)
+        self._fe_termination_reason: Optional[FunctionExecutorTerminationReason] = None
+        self._internal_state = _FE_CONTROLLER_STATE.NOT_STARTED
+        metric_function_executors_with_state.labels(
+            state=_to_fe_state_metric_label(self._internal_state, self._logger)
         ).inc()
+        self._reported_state: FunctionExecutorState = FunctionExecutorState(
+            description=function_executor_description,
+            status=FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_UNKNOWN,
+        )
         # Ordered list of events to be processed by the control loop.
         self._events: List[BaseEvent] = []
         # Asyncio event used to notify the control loop that there are new events to process.
@@ -124,13 +136,6 @@ class FunctionExecutorController:
     def function_executor_id(self) -> str:
         return self._fe_description.id
 
-    def status(self) -> FunctionExecutorStatus:
-        """Returns the current status of the Function Executor.
-
-        Not blocking.
-        """
-        return self._status
-
     def add_task_allocation(self, task_allocation: TaskAllocation) -> None:
         """Adds a task to the Function Executor.
 
@@ -205,9 +210,10 @@ class FunctionExecutorController:
         """Starts up the Function Executor and prepares it to run tasks.
 
         Not blocking. Never raises exceptions."""
-        if self._control_loop_aio_task is not None:
+        if self._internal_state != _FE_CONTROLLER_STATE.NOT_STARTED:
             self._logger.warning(
-                "ignoring startup call as the Function Executor is already started"
+                "function executor state is not NOT_STARTED, ignoring startup call",
+                internal_state=self._internal_state.name,
             )
             return
 
@@ -215,7 +221,13 @@ class FunctionExecutorController:
             self._control_loop(),
             name="function executor control loop",
         )
-        self._set_status(FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_PENDING)
+        self._update_internal_state(_FE_CONTROLLER_STATE.STARTING_UP)
+        self._update_reported_state(
+            FunctionExecutorState(
+                description=self._fe_description,
+                status=FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_PENDING,
+            )
+        )
         next_aio = create_function_executor(
             function_executor_description=self._fe_description,
             function_executor_server_factory=self._fe_server_factory,
@@ -237,17 +249,13 @@ class FunctionExecutorController:
             ),
         )
 
-    async def shutdown(
-        self, termination_reason: FunctionExecutorTerminationReason
-    ) -> None:
+    async def shutdown(self) -> None:
         """Shutsdown the Function Executor and frees all of its resources.
 
-        All the tasks are reported as failed with FE Terminated failure code.
+        No task outcomes and outputs are getting reported to Server after this call.
         Doesn't raise any exceptions. Blocks until the shutdown is complete.
         """
-        self._add_event(
-            ShutdownInitiated(termination_reason=termination_reason), source="shutdown"
-        )
+        self._add_event(ShutdownInitiated(), source="shutdown")
         try:
             await self._control_loop_aio_task
         except asyncio.CancelledError:
@@ -259,51 +267,49 @@ class FunctionExecutorController:
             )
         self._logger.info("function executor controller shutdown finished")
 
-    def _set_status(
+    def _update_internal_state(self, new_state: _FE_CONTROLLER_STATE) -> None:
+        """Updates the internal state of the Function Executor Controller.
+
+        Not blocking. Never raises exceptions."""
+        old_state: _FE_CONTROLLER_STATE = self._internal_state
+        self._internal_state = new_state
+
+        self._logger.info(
+            "function executor internal state changed",
+            old_state=old_state.name,
+            new_state=new_state.name,
+        )
+
+        metric_function_executors_with_state.labels(
+            state=_to_fe_state_metric_label(old_state, self._logger)
+        ).dec()
+        metric_function_executors_with_state.labels(
+            state=_to_fe_state_metric_label(new_state, self._logger)
+        ).inc()
+
+    def _update_reported_state(
         self,
-        status: FunctionExecutorStatus,
+        new_state: FunctionExecutorState,
     ) -> None:
-        """Sets Function Executor status and reports it to the Server.
+        """Sets new Function Executor state and reports it to the Server.
 
         Not blocking. Never raises exceptions."""
-        old_status: FunctionExecutorStatus = self._status
-        new_status: FunctionExecutorStatus = status
-        self._status: FunctionExecutorStatus = new_status
+        old_state: FunctionExecutorState = self._reported_state
+        self._reported_state = new_state
 
         self._logger.info(
-            "function executor status changed",
-            old_status=FunctionExecutorStatus.Name(old_status),
-            new_status=FunctionExecutorStatus.Name(new_status),
+            "function executor grpc status changed",
+            old_status=FunctionExecutorStatus.Name(old_state.status),
+            new_status=FunctionExecutorStatus.Name(new_state.status),
             termination_reason=_termination_reason_to_short_name(
-                self._fe_termination_reason
+                new_state.termination_reason
             ),
         )
-        metric_function_executors_with_status.labels(
-            status=_to_fe_status_metric_label(old_status, self._logger)
-        ).dec()
-        metric_function_executors_with_status.labels(
-            status=_to_fe_status_metric_label(new_status, self._logger)
-        ).inc()
 
-        self._state_reporter.update_function_executor_state(self._current_state())
+        self._state_reporter.update_function_executor_state(new_state)
         # Report the status change to the Server asap to reduce latency in the system.
         self._state_reporter.schedule_state_report()
 
-    def _current_state(self) -> FunctionExecutorState:
-        """Returns the current state of the Function Executor.
-
-        Not blocking. Never raises exceptions.
-        """
-        termination_reason: Optional[FunctionExecutorTerminationReason] = None
-        if self._fe_termination_reason is not None:
-            termination_reason = self._fe_termination_reason
-
-        return FunctionExecutorState(
-            description=self._fe_description,
-            status=self._status,
-            termination_reason=termination_reason,
-        )
-
     async def _control_loop(self) -> None:
         """Runs control loop that coordinates all the work done by the Function Executor.
 
@@ -454,13 +460,17 @@ class FunctionExecutorController:
         self._state_reporter.schedule_state_report()
 
         if event.function_executor is None:
-            self._destroy_function_executor_before_termination(
-                event.output.termination_reason
-            )
+            self._start_termination(termination_reason=event.output.termination_reason)
             return
 
         self._fe = event.function_executor
-        self._set_status(FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_RUNNING)
+        self._update_internal_state(_FE_CONTROLLER_STATE.RUNNING)
+        self._update_reported_state(
+            FunctionExecutorState(
+                description=self._fe_description,
+                status=FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_RUNNING,
+            )
+        )
         # Health checker starts after FE creation and gets automatically stopped on FE destroy.
         self._fe.health_checker().start(self._health_check_failed_callback)
         self._add_event(
@@ -479,9 +489,18 @@ class FunctionExecutorController:
             self._logger.error(
                 "Function Executor destroy failed unexpectedly, this should never happen",
             )
-        # Set the status only after the FE got destroyed because Server assumes that all FE resources are freed when the status changes.
-        self._fe_termination_reason = event.termination_reason
-        self._set_status(FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_TERMINATED)
+
+        self._fe = None
+        # Set reported status only after the FE got destroyed because Server assumes that all FE resources are freed when the status changes.
+        self._update_reported_state(
+            FunctionExecutorState(
+                description=self._fe_description,
+                status=FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_TERMINATED,
+                termination_reason=self._fe_termination_reason,
+            )
+        )
+        self._update_internal_state(_FE_CONTROLLER_STATE.TERMINATED)
+
         # Invoke the scheduler so it can fail runnable tasks with FE Terminated error.
         self._add_event(
             ScheduleTaskExecution(),
@@ -493,7 +512,7 @@ class FunctionExecutorController:
             "Function Executor health check failed, terminating Function Executor",
             reason=result.reason,
         )
-        self._destroy_function_executor_before_termination(
+        self._start_termination(
             termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY
         )
 
@@ -532,14 +551,15 @@ class FunctionExecutorController:
         if len(self._runnable_tasks) == 0:
             return
 
-        if self._status not in [
-            FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_RUNNING,
-            FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_TERMINATED,
+        if self._internal_state not in [
+            _FE_CONTROLLER_STATE.RUNNING,
+            _FE_CONTROLLER_STATE.TERMINATING,
+            _FE_CONTROLLER_STATE.TERMINATED,
         ]:
-            return  # Can't progress pending task with the current status.
+            return  # Can't progress runnable tasks in the current state.
 
         if (
-            self._status == FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_RUNNING
+            self._internal_state == _FE_CONTROLLER_STATE.RUNNING
             and self._running_task is not None
         ):
             return
@@ -555,12 +575,15 @@ class FunctionExecutorController:
         if task_info.is_cancelled:
             task_info.output = TaskOutput.task_cancelled(task_info.allocation)
             self._start_task_output_upload(task_info)
-        elif self._status == FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_TERMINATED:
+        elif self._internal_state in [
+            _FE_CONTROLLER_STATE.TERMINATING,
+            _FE_CONTROLLER_STATE.TERMINATED,
+        ]:
             task_info.output = TaskOutput.function_executor_terminated(
                 task_info.allocation
             )
             self._start_task_output_upload(task_info)
-        elif self._status == FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_RUNNING:
+        elif self._internal_state == _FE_CONTROLLER_STATE.RUNNING:
             self._running_task = task_info
             next_aio = run_task_on_function_executor(
                 task_info=task_info,
@@ -603,7 +626,7 @@ class FunctionExecutorController:
                 ScheduleTaskExecution(), source="_handle_event_task_execution_finished"
             )
         else:
-            self._destroy_function_executor_before_termination(
+            self._start_termination(
                 termination_reason=event.function_executor_termination_reason
             )
 
@@ -660,24 +683,31 @@ class FunctionExecutorController:
         )
         self._state_reporter.schedule_state_report()
 
-    def _destroy_function_executor_before_termination(
+    def _start_termination(
         self, termination_reason: FunctionExecutorTerminationReason
     ) -> None:
-        """Destroys the Function Executor and frees all its resources to prepare for transitioning to the TERMINATED state.
+        """Starts termination of the Function Executor if it's not started yet.
 
         Doesn't raise any exceptions. Doesn't block.
         """
+        if self._internal_state in [
+            _FE_CONTROLLER_STATE.TERMINATING,
+            _FE_CONTROLLER_STATE.TERMINATED,
+        ]:
+            # _start_termination() can be called multiple times, e.g. by each failed task alloc
+            # when the FE is unhealthy. Dedup the calls to keep state machine consistent.
+            return
+
+        self._fe_termination_reason = termination_reason
+        self._update_internal_state(_FE_CONTROLLER_STATE.TERMINATING)
         next_aio = destroy_function_executor(
             function_executor=self._fe,
-            termination_reason=termination_reason,
+            lock=self._destroy_lock,
             logger=self._logger,
         )
-        self._fe = None
         self._spawn_aio_for_fe(
             aio=next_aio,
-            on_exception=FunctionExecutorDestroyed(
-                is_success=False, termination_reason=termination_reason
-            ),
+            on_exception=FunctionExecutorDestroyed(is_success=False),
         )
 
     async def _shutdown_no_exceptions(self, event: ShutdownInitiated) -> None:
@@ -716,16 +746,15 @@ class FunctionExecutorController:
                 # BaseException includes asyncio.CancelledError which is always raised here.
                 pass
 
-        if self._status != FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_TERMINATED:
-            self._handle_event_function_executor_destroyed(
-                await destroy_function_executor(
-                    function_executor=self._fe,
-                    termination_reason=event.termination_reason,
-                    logger=self._logger,
-                )
-            )
-        metric_function_executors_with_status.labels(
-            status=_to_fe_status_metric_label(self._status, self._logger)
+        await destroy_function_executor(
+            function_executor=self._fe,
+            lock=self._destroy_lock,
+            logger=self._logger,
+        )
+
+        # Cleanup the metric from this FE.
+        metric_function_executors_with_state.labels(
+            state=_to_fe_state_metric_label(self._internal_state, self._logger)
         ).dec()
 
         self._state_reporter.remove_function_executor_state(self.function_executor_id())
@@ -735,21 +764,23 @@ class FunctionExecutorController:
         debug_print_events(events=self._events, logger=self._logger)
 
 
-def _to_fe_status_metric_label(status: FunctionExecutorStatus, logger: Any) -> str:
-    if status == FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_UNKNOWN:
-        return METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_UNKNOWN
-    elif status == FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_PENDING:
-        return METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_PENDING
-    elif status == FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_RUNNING:
-        return METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_RUNNING
-    elif status == FunctionExecutorStatus.FUNCTION_EXECUTOR_STATUS_TERMINATED:
-        return METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_TERMINATED
+def _to_fe_state_metric_label(state: _FE_CONTROLLER_STATE, logger: Any) -> str:
+    if state == _FE_CONTROLLER_STATE.NOT_STARTED:
+        return METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_NOT_STARTED
+    elif state == _FE_CONTROLLER_STATE.STARTING_UP:
+        return METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_STARTING_UP
+    elif state == _FE_CONTROLLER_STATE.RUNNING:
+        return METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_RUNNING
+    elif state == _FE_CONTROLLER_STATE.TERMINATING:
+        return METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_TERMINATING
+    elif state == _FE_CONTROLLER_STATE.TERMINATED:
+        return METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_TERMINATED
     else:
         logger.error(
-            "unexpected Function Executor status",
-            status=FunctionExecutorStatus.Name(status),
+            "unexpected Function Executor internal state",
+            state=state.name,
         )
-        return METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_UNKNOWN
+        return METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_UNKNOWN
 
 
 _termination_reason_to_short_name_map = {
@@ -757,8 +788,6 @@ _termination_reason_to_short_name_map = {
     FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR: "STARTUP_FAILED_INTERNAL_ERROR",
     FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR: "STARTUP_FAILED_FUNCTION_ERROR",
     FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT: "STARTUP_FAILED_FUNCTION_TIMEOUT",
-    FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_EXECUTOR_SHUTDOWN: "EXECUTOR_SHUTDOWN",
-    FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE: "REMOVED_FROM_DESIRED_STATE",
     FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY: "UNHEALTHY",
     FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_INTERNAL_ERROR: "INTERNAL_ERROR",
     FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_TIMEOUT: "FUNCTION_TIMEOUT",

{indexify-0.4.10 → indexify-0.4.11}/src/indexify/executor/function_executor_controller/metrics/function_executor_controller.py

@@ -34,27 +34,34 @@ metric_runnable_tasks_per_function_name: prometheus_client.Gauge = (
     )
 )
 
-metric_function_executors_with_status: prometheus_client.Gauge = (
-    prometheus_client.Gauge(
-        "function_executors_with_status",
-        "Number of Function Executors with a particular status",
-        ["status"],
-    )
+metric_function_executors_with_state: prometheus_client.Gauge = prometheus_client.Gauge(
+    "function_executors_with_state",
+    "Number of Function Executors with a particular internal state",
+    ["state"],
 )
-METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_UNKNOWN = "unknown"
-METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_PENDING = "pending"
-METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_RUNNING = "running"
-METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_TERMINATED = "terminated"
+METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_UNKNOWN = "unknown"
+METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_NOT_STARTED = "not_started"
+METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_STARTING_UP = "starting_up"
+METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_RUNNING = "running"
+METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_TERMINATING = "terminating"
+METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_TERMINATED = "terminated"
+
 
-metric_function_executors_with_status.labels(
-    status=METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_UNKNOWN
+metric_function_executors_with_state.labels(
+    state=METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_UNKNOWN
+)
+metric_function_executors_with_state.labels(
+    state=METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_NOT_STARTED
+)
+metric_function_executors_with_state.labels(
+    state=METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_STARTING_UP
 )
-metric_function_executors_with_status.labels(
-    status=METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_PENDING
+metric_function_executors_with_state.labels(
+    state=METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_RUNNING
 )
-metric_function_executors_with_status.labels(
-    status=METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_RUNNING
+metric_function_executors_with_state.labels(
+    state=METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_TERMINATING
 )
-metric_function_executors_with_status.labels(
-    status=METRIC_FUNCTION_EXECUTORS_WITH_STATUS_LABEL_TERMINATED
+metric_function_executors_with_state.labels(
+    state=METRIC_FUNCTION_EXECUTORS_WITH_STATE_LABEL_TERMINATED
 )

{indexify-0.4.10 → indexify-0.4.11}/src/indexify/executor/function_executor_controller/run_task.py

@@ -1,4 +1,6 @@
 import asyncio
+import os
+import random
 import time
 from typing import Any, Optional
 
@@ -38,6 +40,10 @@ from .metrics.run_task import (
 from .task_info import TaskInfo
 from .task_output import TaskMetrics, TaskOutput
 
+_ENABLE_INJECT_TASK_CANCELLATIONS = (
+    os.getenv("INDEXIFY_INJECT_TASK_CANCELLATIONS", "0") == "1"
+)
+
 
 async def run_task_on_function_executor(
     task_info: TaskInfo, function_executor: FunctionExecutor, logger: Any
@@ -177,6 +183,14 @@ def _task_output_from_function_executor_response(
             response_validator.required_field("invocation_error_output")
             invocation_error_output = response.invocation_error_output
 
+    if _ENABLE_INJECT_TASK_CANCELLATIONS:
+        logger.warning("injecting cancellation failure for the task allocation")
+        if (
+            random.random() < 0.5
+        ):  # 50% chance to get stable reproduction in manual testing
+            outcome_code = TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE
+            failure_reason = TaskFailureReason.TASK_FAILURE_REASON_TASK_CANCELLED
+
     return TaskOutput(
         allocation=allocation,
         outcome_code=outcome_code,

{indexify-0.4.10 → indexify-0.4.11}/src/indexify/executor/state_reconciler.py

@@ -7,7 +7,6 @@ from tensorlake.function_executor.proto.message_validator import MessageValidato
 from indexify.proto.executor_api_pb2 import (
     DesiredExecutorState,
     FunctionExecutorDescription,
-    FunctionExecutorTerminationReason,
     GetDesiredExecutorStatesRequest,
     TaskAllocation,
 )
@@ -123,9 +122,7 @@ class ExecutorStateReconciler:
         for fe_controller in self._function_executor_controllers.values():
             fe_shutdown_tasks.append(
                 asyncio.create_task(
-                    fe_controller.shutdown(
-                        termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_EXECUTOR_SHUTDOWN
-                    ),
+                    fe_controller.shutdown(),
                     name=f"Shutdown Function Executor {fe_controller.function_executor_id()}",
                 )
             )
@@ -327,9 +324,7 @@ class ExecutorStateReconciler:
             self._function_executor_controllers.pop(function_executor_id)
         )
         asyncio.create_task(
-            fe_controller.shutdown(
-                termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE
-            ),
+            fe_controller.shutdown(),
             name=f"Shutdown Function Executor {function_executor_id}",
         )
 

{indexify-0.4.10 → indexify-0.4.11}/src/indexify/proto/executor_api.proto

@@ -69,6 +69,7 @@ enum FunctionExecutorStatus {
     FUNCTION_EXECUTOR_STATUS_TERMINATED = 3;
 }
 
+// The reasons why an Executor decided to terminate a Function Executor.
 enum FunctionExecutorTerminationReason {
     FUNCTION_EXECUTOR_TERMINATION_REASON_UNKNOWN = 0;
     // Internal error aka platform error on FE startup.
@@ -79,10 +80,6 @@ enum FunctionExecutorTerminationReason {
     // Timeout on FE startup while running the function constructor.
     FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT = 3;
 
-    // FE was terminated on Executor shutdown.
-    FUNCTION_EXECUTOR_TERMINATION_REASON_EXECUTOR_SHUTDOWN = 10;
-    // FE was removed from the Executor desired state by Server.
-    FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE = 11;
     // FE was terminated because it failed a health check.
     FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY = 12;
     // FE was terminated due to an unrecoverable internal error on Executor.

{indexify-0.4.10 → indexify-0.4.11}/src/indexify/proto/executor_api_pb2.py

@@ -19,7 +19,7 @@ _sym_db = _symbol_database.Default()
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"\xeb\x01\n\x0b\x44\x61taPayload\x12\x11\n\x04size\x18\x02 \x01(\x04H\x00\x88\x01\x01\x12\x18\n\x0bsha256_hash\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03uri\x18\x04 \x01(\tH\x02\x88\x01\x01\x12;\n\x08\x65ncoding\x18\x05 \x01(\x0e\x32$.executor_api_pb.DataPayloadEncodingH\x03\x88\x01\x01\x12\x1d\n\x10\x65ncoding_version\x18\x06 \x01(\x04H\x04\x88\x01\x01\x42\x07\n\x05_sizeB\x0e\n\x0c_sha256_hashB\x06\n\x04_uriB\x0b\n\t_encodingB\x13\n\x11_encoding_version"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xd8\x01\n\x19\x46unctionExecutorResources\x12\x1b\n\x0e\x63pu_ms_per_sec\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x11\n\x0f_cpu_ms_per_secB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xb3\x04\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12%\n\x18\x63ustomer_code_timeout_ms\x18\t \x01(\rH\x06\x88\x01\x01\x12\x30\n\x05graph\x18\n \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x07\x88\x01\x01\x12\x42\n\tresources\x18\x0b \x01(\x0b\x32*.executor_api_pb.FunctionExecutorResourcesH\x08\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\x0c \x01(\tH\t\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x1b\n\x19_customer_code_timeout_msB\x08\n\x06_graphB\x0c\n\n_resourcesB\x1c\n\x1a_output_payload_uri_prefix"\xa4\x02\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x12S\n\x12termination_reason\x18\x03 \x01(\x0e\x32\x32.executor_api_pb.FunctionExecutorTerminationReasonH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_statusB\x15\n\x13_termination_reason"\x8c\x02\n\x16\x46unctionExecutorUpdate\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12\x39\n\x0estartup_stdout\x18\x02 \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x01\x88\x01\x01\x12\x39\n\x0estartup_stderr\x18\x03 \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\x11\n\x0f_startup_stdoutB\x11\n\x0f_startup_stderr"\xce\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x03\x88\x01\x01\x12<\n\x0ftotal_resources\x18\r \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x04\x88\x01\x01\x12N\n!total_function_executor_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x05\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x06\x88\x01\x01\x12\x19\n\x0cserver_clock\x18\x0c \x01(\x04H\x07\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x0b\n\t_hostnameB\n\n\x08_versionB\t\n\x07_statusB\x12\n\x10_total_resourcesB$\n"_total_function_executor_resourcesB\r\n\x0b_state_hashB\x0f\n\r_server_clock"\xb9\x01\n\x0e\x45xecutorUpdate\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x31\n\x0ctask_results\x18\x02 \x03(\x0b\x32\x1b.executor_api_pb.TaskResult\x12J\n\x19\x66unction_executor_updates\x18\x03 \x03(\x0b\x32\'.executor_api_pb.FunctionExecutorUpdateB\x0e\n\x0c_executor_id"\xbf\x01\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x12=\n\x0f\x65xecutor_update\x18\x02 \x01(\x0b\x32\x1f.executor_api_pb.ExecutorUpdateH\x01\x88\x01\x01\x42\x11\n\x0f_executor_stateB\x12\n\x10_executor_update"\x1d\n\x1bReportExecutorStateResponse"\xcf\x01\n\x0fTaskRetryPolicy\x12\x18\n\x0bmax_retries\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x1d\n\x10initial_delay_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x19\n\x0cmax_delay_ms\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65lay_multiplier\x18\x04 \x01(\rH\x03\x88\x01\x01\x42\x0e\n\x0c_max_retriesB\x13\n\x11_initial_delay_msB\x0f\n\r_max_delay_msB\x13\n\x11_delay_multiplier"\xc6\x04\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\rH\x06\x88\x01\x01\x12\x30\n\x05input\x18\x0b \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x07\x88\x01\x01\x12\x38\n\rreducer_input\x18\x0c \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x08\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\r \x01(\tH\t\x88\x01\x01\x12;\n\x0cretry_policy\x18\x0e \x01(\x0b\x32 .executor_api_pb.TaskRetryPolicyH\n\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\r\n\x0b_timeout_msB\x08\n\x06_inputB\x10\n\x0e_reducer_inputB\x1c\n\x1a_output_payload_uri_prefixB\x0f\n\r_retry_policy"\xad\x01\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x12\x1a\n\rallocation_id\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_taskB\x10\n\x0e_allocation_id"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock"\xb0\x06\n\nTaskResult\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rallocation_id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tnamespace\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\ngraph_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x06 \x01(\tH\x05\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x14\n\x07reducer\x18\x08 \x01(\x08H\x07\x88\x01\x01\x12;\n\x0coutcome_code\x18\t \x01(\x0e\x32 .executor_api_pb.TaskOutcomeCodeH\x08\x88\x01\x01\x12?\n\x0e\x66\x61ilure_reason\x18\n \x01(\x0e\x32".executor_api_pb.TaskFailureReasonH\t\x88\x01\x01\x12\x16\n\x0enext_functions\x18\x0b \x03(\t\x12\x36\n\x10\x66unction_outputs\x18\x0c \x03(\x0b\x32\x1c.executor_api_pb.DataPayload\x12\x31\n\x06stdout\x18\r \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12\x31\n\x06stderr\x18\x0e \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x0b\x88\x01\x01\x12\x42\n\x17invocation_error_output\x18\x0f \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x0c\x88\x01\x01\x42\n\n\x08_task_idB\x10\n\x0e_allocation_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_reducerB\x0f\n\r_outcome_codeB\x11\n\x0f_failure_reasonB\t\n\x07_stdoutB\t\n\x07_stderrB\x1a\n\x18_invocation_error_output*\xd1\x01\n\x13\x44\x61taPayloadEncoding\x12!\n\x1d\x44\x41TA_PAYLOAD_ENCODING_UNKNOWN\x10\x00\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_JSON\x10\x01\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_TEXT\x10\x02\x12\'\n#DATA_PAYLOAD_ENCODING_BINARY_PICKLE\x10\x03\x12$\n DATA_PAYLOAD_ENCODING_BINARY_ZIP\x10\x04*\xd6\x01\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12\x1e\n\x1aGPU_MODEL_NVIDIA_A100_40GB\x10\x01\x12\x1e\n\x1aGPU_MODEL_NVIDIA_A100_80GB\x10\x02\x12\x1e\n\x1aGPU_MODEL_NVIDIA_H100_80GB\x10\x03\x12\x1d\n\x19GPU_MODEL_NVIDIA_TESLA_T4\x10\x04\x12\x1a\n\x16GPU_MODEL_NVIDIA_A6000\x10\x05\x12\x18\n\x14GPU_MODEL_NVIDIA_A10\x10\x06*\xb3\x01\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12$\n FUNCTION_EXECUTOR_STATUS_PENDING\x10\x01\x12$\n FUNCTION_EXECUTOR_STATUS_RUNNING\x10\x02\x12\'\n#FUNCTION_EXECUTOR_STATUS_TERMINATED\x10\x03*\x95\x05\n!FunctionExecutorTerminationReason\x12\x30\n,FUNCTION_EXECUTOR_TERMINATION_REASON_UNKNOWN\x10\x00\x12\x46\nBFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR\x10\x01\x12\x46\nBFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR\x10\x02\x12H\nDFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT\x10\x03\x12:\n6FUNCTION_EXECUTOR_TERMINATION_REASON_EXECUTOR_SHUTDOWN\x10\n\x12\x43\n?FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE\x10\x0b\x12\x32\n.FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY\x10\x0c\x12\x37\n3FUNCTION_EXECUTOR_TERMINATION_REASON_INTERNAL_ERROR\x10\r\x12\x39\n5FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_TIMEOUT\x10\x0e\x12;\n7FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_CANCELLED\x10\x0f*\xa5\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x04*n\n\x0fTaskOutcomeCode\x12\x1d\n\x19TASK_OUTCOME_CODE_UNKNOWN\x10\x00\x12\x1d\n\x19TASK_OUTCOME_CODE_SUCCESS\x10\x01\x12\x1d\n\x19TASK_OUTCOME_CODE_FAILURE\x10\x02*\xb6\x02\n\x11TaskFailureReason\x12\x1f\n\x1bTASK_FAILURE_REASON_UNKNOWN\x10\x00\x12&\n"TASK_FAILURE_REASON_INTERNAL_ERROR\x10\x01\x12&\n"TASK_FAILURE_REASON_FUNCTION_ERROR\x10\x02\x12(\n$TASK_FAILURE_REASON_FUNCTION_TIMEOUT\x10\x03\x12(\n$TASK_FAILURE_REASON_INVOCATION_ERROR\x10\x04\x12&\n"TASK_FAILURE_REASON_TASK_CANCELLED\x10\x05\x12\x34\n0TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED\x10\x06\x32\xff\x01\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x62\x06proto3'
+    b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"\xeb\x01\n\x0b\x44\x61taPayload\x12\x11\n\x04size\x18\x02 \x01(\x04H\x00\x88\x01\x01\x12\x18\n\x0bsha256_hash\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03uri\x18\x04 \x01(\tH\x02\x88\x01\x01\x12;\n\x08\x65ncoding\x18\x05 \x01(\x0e\x32$.executor_api_pb.DataPayloadEncodingH\x03\x88\x01\x01\x12\x1d\n\x10\x65ncoding_version\x18\x06 \x01(\x04H\x04\x88\x01\x01\x42\x07\n\x05_sizeB\x0e\n\x0c_sha256_hashB\x06\n\x04_uriB\x0b\n\t_encodingB\x13\n\x11_encoding_version"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xd8\x01\n\x19\x46unctionExecutorResources\x12\x1b\n\x0e\x63pu_ms_per_sec\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x11\n\x0f_cpu_ms_per_secB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xb3\x04\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12%\n\x18\x63ustomer_code_timeout_ms\x18\t \x01(\rH\x06\x88\x01\x01\x12\x30\n\x05graph\x18\n \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x07\x88\x01\x01\x12\x42\n\tresources\x18\x0b \x01(\x0b\x32*.executor_api_pb.FunctionExecutorResourcesH\x08\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\x0c \x01(\tH\t\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x1b\n\x19_customer_code_timeout_msB\x08\n\x06_graphB\x0c\n\n_resourcesB\x1c\n\x1a_output_payload_uri_prefix"\xa4\x02\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x12S\n\x12termination_reason\x18\x03 \x01(\x0e\x32\x32.executor_api_pb.FunctionExecutorTerminationReasonH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_statusB\x15\n\x13_termination_reason"\x8c\x02\n\x16\x46unctionExecutorUpdate\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12\x39\n\x0estartup_stdout\x18\x02 \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x01\x88\x01\x01\x12\x39\n\x0estartup_stderr\x18\x03 \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x02\x88\x01\x01\x42\x0e\n\x0c_descriptionB\x11\n\x0f_startup_stdoutB\x11\n\x0f_startup_stderr"\xce\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x03\x88\x01\x01\x12<\n\x0ftotal_resources\x18\r \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x04\x88\x01\x01\x12N\n!total_function_executor_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x05\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x06\x88\x01\x01\x12\x19\n\x0cserver_clock\x18\x0c \x01(\x04H\x07\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x0b\n\t_hostnameB\n\n\x08_versionB\t\n\x07_statusB\x12\n\x10_total_resourcesB$\n"_total_function_executor_resourcesB\r\n\x0b_state_hashB\x0f\n\r_server_clock"\xb9\x01\n\x0e\x45xecutorUpdate\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x31\n\x0ctask_results\x18\x02 \x03(\x0b\x32\x1b.executor_api_pb.TaskResult\x12J\n\x19\x66unction_executor_updates\x18\x03 \x03(\x0b\x32\'.executor_api_pb.FunctionExecutorUpdateB\x0e\n\x0c_executor_id"\xbf\x01\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x12=\n\x0f\x65xecutor_update\x18\x02 \x01(\x0b\x32\x1f.executor_api_pb.ExecutorUpdateH\x01\x88\x01\x01\x42\x11\n\x0f_executor_stateB\x12\n\x10_executor_update"\x1d\n\x1bReportExecutorStateResponse"\xcf\x01\n\x0fTaskRetryPolicy\x12\x18\n\x0bmax_retries\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x1d\n\x10initial_delay_ms\x18\x02 \x01(\rH\x01\x88\x01\x01\x12\x19\n\x0cmax_delay_ms\x18\x03 \x01(\rH\x02\x88\x01\x01\x12\x1d\n\x10\x64\x65lay_multiplier\x18\x04 \x01(\rH\x03\x88\x01\x01\x42\x0e\n\x0c_max_retriesB\x13\n\x11_initial_delay_msB\x0f\n\r_max_delay_msB\x13\n\x11_delay_multiplier"\xc6\x04\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\rH\x06\x88\x01\x01\x12\x30\n\x05input\x18\x0b \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x07\x88\x01\x01\x12\x38\n\rreducer_input\x18\x0c \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x08\x88\x01\x01\x12&\n\x19output_payload_uri_prefix\x18\r \x01(\tH\t\x88\x01\x01\x12;\n\x0cretry_policy\x18\x0e \x01(\x0b\x32 .executor_api_pb.TaskRetryPolicyH\n\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\r\n\x0b_timeout_msB\x08\n\x06_inputB\x10\n\x0e_reducer_inputB\x1c\n\x1a_output_payload_uri_prefixB\x0f\n\r_retry_policy"\xad\x01\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x12\x1a\n\rallocation_id\x18\x03 \x01(\tH\x02\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_taskB\x10\n\x0e_allocation_id"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock"\xb0\x06\n\nTaskResult\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1a\n\rallocation_id\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tnamespace\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x17\n\ngraph_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x06 \x01(\tH\x05\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x07 \x01(\tH\x06\x88\x01\x01\x12\x14\n\x07reducer\x18\x08 \x01(\x08H\x07\x88\x01\x01\x12;\n\x0coutcome_code\x18\t \x01(\x0e\x32 .executor_api_pb.TaskOutcomeCodeH\x08\x88\x01\x01\x12?\n\x0e\x66\x61ilure_reason\x18\n \x01(\x0e\x32".executor_api_pb.TaskFailureReasonH\t\x88\x01\x01\x12\x16\n\x0enext_functions\x18\x0b \x03(\t\x12\x36\n\x10\x66unction_outputs\x18\x0c \x03(\x0b\x32\x1c.executor_api_pb.DataPayload\x12\x31\n\x06stdout\x18\r \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\n\x88\x01\x01\x12\x31\n\x06stderr\x18\x0e \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x0b\x88\x01\x01\x12\x42\n\x17invocation_error_output\x18\x0f \x01(\x0b\x32\x1c.executor_api_pb.DataPayloadH\x0c\x88\x01\x01\x42\n\n\x08_task_idB\x10\n\x0e_allocation_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_reducerB\x0f\n\r_outcome_codeB\x11\n\x0f_failure_reasonB\t\n\x07_stdoutB\t\n\x07_stderrB\x1a\n\x18_invocation_error_output*\xd1\x01\n\x13\x44\x61taPayloadEncoding\x12!\n\x1d\x44\x41TA_PAYLOAD_ENCODING_UNKNOWN\x10\x00\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_JSON\x10\x01\x12#\n\x1f\x44\x41TA_PAYLOAD_ENCODING_UTF8_TEXT\x10\x02\x12\'\n#DATA_PAYLOAD_ENCODING_BINARY_PICKLE\x10\x03\x12$\n DATA_PAYLOAD_ENCODING_BINARY_ZIP\x10\x04*\xd6\x01\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12\x1e\n\x1aGPU_MODEL_NVIDIA_A100_40GB\x10\x01\x12\x1e\n\x1aGPU_MODEL_NVIDIA_A100_80GB\x10\x02\x12\x1e\n\x1aGPU_MODEL_NVIDIA_H100_80GB\x10\x03\x12\x1d\n\x19GPU_MODEL_NVIDIA_TESLA_T4\x10\x04\x12\x1a\n\x16GPU_MODEL_NVIDIA_A6000\x10\x05\x12\x18\n\x14GPU_MODEL_NVIDIA_A10\x10\x06*\xb3\x01\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12$\n FUNCTION_EXECUTOR_STATUS_PENDING\x10\x01\x12$\n FUNCTION_EXECUTOR_STATUS_RUNNING\x10\x02\x12\'\n#FUNCTION_EXECUTOR_STATUS_TERMINATED\x10\x03*\x94\x04\n!FunctionExecutorTerminationReason\x12\x30\n,FUNCTION_EXECUTOR_TERMINATION_REASON_UNKNOWN\x10\x00\x12\x46\nBFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR\x10\x01\x12\x46\nBFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR\x10\x02\x12H\nDFUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT\x10\x03\x12\x32\n.FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY\x10\x0c\x12\x37\n3FUNCTION_EXECUTOR_TERMINATION_REASON_INTERNAL_ERROR\x10\r\x12\x39\n5FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_TIMEOUT\x10\x0e\x12;\n7FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_CANCELLED\x10\x0f*\xa5\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x04*n\n\x0fTaskOutcomeCode\x12\x1d\n\x19TASK_OUTCOME_CODE_UNKNOWN\x10\x00\x12\x1d\n\x19TASK_OUTCOME_CODE_SUCCESS\x10\x01\x12\x1d\n\x19TASK_OUTCOME_CODE_FAILURE\x10\x02*\xb6\x02\n\x11TaskFailureReason\x12\x1f\n\x1bTASK_FAILURE_REASON_UNKNOWN\x10\x00\x12&\n"TASK_FAILURE_REASON_INTERNAL_ERROR\x10\x01\x12&\n"TASK_FAILURE_REASON_FUNCTION_ERROR\x10\x02\x12(\n$TASK_FAILURE_REASON_FUNCTION_TIMEOUT\x10\x03\x12(\n$TASK_FAILURE_REASON_INVOCATION_ERROR\x10\x04\x12&\n"TASK_FAILURE_REASON_TASK_CANCELLED\x10\x05\x12\x34\n0TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED\x10\x06\x32\xff\x01\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x62\x06proto3'
 )
 
 _globals = globals()
@@ -38,13 +38,13 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 5752
     _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 5931
     _globals["_FUNCTIONEXECUTORTERMINATIONREASON"]._serialized_start = 5934
-    _globals["_FUNCTIONEXECUTORTERMINATIONREASON"]._serialized_end = 6595
-    _globals["_EXECUTORSTATUS"]._serialized_start = 6598
-    _globals["_EXECUTORSTATUS"]._serialized_end = 6763
-    _globals["_TASKOUTCOMECODE"]._serialized_start = 6765
-    _globals["_TASKOUTCOMECODE"]._serialized_end = 6875
-    _globals["_TASKFAILUREREASON"]._serialized_start = 6878
-    _globals["_TASKFAILUREREASON"]._serialized_end = 7188
+    _globals["_FUNCTIONEXECUTORTERMINATIONREASON"]._serialized_end = 6466
+    _globals["_EXECUTORSTATUS"]._serialized_start = 6469
+    _globals["_EXECUTORSTATUS"]._serialized_end = 6634
+    _globals["_TASKOUTCOMECODE"]._serialized_start = 6636
+    _globals["_TASKOUTCOMECODE"]._serialized_end = 6746
+    _globals["_TASKFAILUREREASON"]._serialized_start = 6749
+    _globals["_TASKFAILUREREASON"]._serialized_end = 7059
     _globals["_DATAPAYLOAD"]._serialized_start = 55
     _globals["_DATAPAYLOAD"]._serialized_end = 290
     _globals["_GPURESOURCES"]._serialized_start = 292
@@ -83,6 +83,6 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_DESIREDEXECUTORSTATE"]._serialized_end = 4501
     _globals["_TASKRESULT"]._serialized_start = 4504
     _globals["_TASKRESULT"]._serialized_end = 5320
-    _globals["_EXECUTORAPI"]._serialized_start = 7191
-    _globals["_EXECUTORAPI"]._serialized_end = 7446
+    _globals["_EXECUTORAPI"]._serialized_start = 7062
+    _globals["_EXECUTORAPI"]._serialized_end = 7317
 # @@protoc_insertion_point(module_scope)

{indexify-0.4.10 → indexify-0.4.11}/src/indexify/proto/executor_api_pb2.pyi

@@ -52,12 +52,6 @@ class FunctionExecutorTerminationReason(
     FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT: _ClassVar[
         FunctionExecutorTerminationReason
     ]
-    FUNCTION_EXECUTOR_TERMINATION_REASON_EXECUTOR_SHUTDOWN: _ClassVar[
-        FunctionExecutorTerminationReason
-    ]
-    FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE: _ClassVar[
-        FunctionExecutorTerminationReason
-    ]
     FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY: _ClassVar[
         FunctionExecutorTerminationReason
     ]
@@ -121,12 +115,6 @@ FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR: (
 FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT: (
     FunctionExecutorTerminationReason
 )
-FUNCTION_EXECUTOR_TERMINATION_REASON_EXECUTOR_SHUTDOWN: (
-    FunctionExecutorTerminationReason
-)
-FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE: (
-    FunctionExecutorTerminationReason
-)
 FUNCTION_EXECUTOR_TERMINATION_REASON_UNHEALTHY: FunctionExecutorTerminationReason
 FUNCTION_EXECUTOR_TERMINATION_REASON_INTERNAL_ERROR: FunctionExecutorTerminationReason
 FUNCTION_EXECUTOR_TERMINATION_REASON_FUNCTION_TIMEOUT: FunctionExecutorTerminationReason

indexify-0.4.10/src/indexify/executor/function_executor_controller/destroy_function_executor.py

@@ -1,28 +0,0 @@
-from typing import Any, Optional
-
-from indexify.executor.function_executor.function_executor import FunctionExecutor
-from indexify.proto.executor_api_pb2 import FunctionExecutorTerminationReason
-
-from .events import FunctionExecutorDestroyed
-
-
-async def destroy_function_executor(
-    function_executor: Optional[FunctionExecutor],
-    termination_reason: FunctionExecutorTerminationReason,
-    logger: Any,
-) -> FunctionExecutorDestroyed:
-    """Destroys a function executor if it's not None.
-
-    Doesn't raise any exceptions.
-    """
-    logger = logger.bind(module=__name__)
-
-    if function_executor is not None:
-        logger.info(
-            "destroying function executor",
-        )
-        await function_executor.destroy()
-
-    return FunctionExecutorDestroyed(
-        is_success=True, termination_reason=termination_reason
-    )