indexify 0.4.12__tar.gz → 0.4.14__tar.gz

{indexify-0.4.12 → indexify-0.4.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.12
+Version: 0.4.14
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -17,7 +17,7 @@ Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
 Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: tensorlake (==0.2.8)
+Requires-Dist: tensorlake (==0.2.15)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.4.12 → indexify-0.4.14}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.4.12"
+version = "0.4.14"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -25,7 +25,7 @@ prometheus-client = "^0.21.1"
 psutil = "^7.0.0"
 # Adds function-executor binary, utils lib, sdk used in indexify-cli commands.
 # We need to specify the tensorlake version exactly because pip install doesn't respect poetry.lock files.
-tensorlake = "0.2.8"
+tensorlake = "0.2.15"
 # Uncomment the next line to use local tensorlake package (only for development!)
 # tensorlake = { path = "../tensorlake", develop = true }
 # pydantic is provided by tensorlake

{indexify-0.4.12 → indexify-0.4.14}/src/indexify/executor/function_executor_controller/create_function_executor.py

@@ -5,7 +5,6 @@ from typing import Any, Optional, Tuple
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
     SerializedObject,
-    SerializedObjectEncoding,
 )
 
 from indexify.executor.blob_store.blob_store import BLOBStore
@@ -73,25 +72,21 @@ async def create_function_executor(
     except BaseException as e:
         if isinstance(e, asyncio.CancelledError):
             logger.info("function executor startup was cancelled")
-            return FunctionExecutorCreated(
-                function_executor=None,
-                output=FunctionExecutorStartupOutput(
-                    function_executor_description=function_executor_description,
-                    termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE,
-                ),
-            )
         else:
             logger.error(
                 "failed to create function executor due to platform error",
                 exc_info=e,
             )
-            return FunctionExecutorCreated(
-                function_executor=None,
-                output=FunctionExecutorStartupOutput(
-                    function_executor_description=function_executor_description,
-                    termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR,
-                ),
-            )
+
+        # Cancelled FE startup means that Server removed it from desired state so it doesn't matter what termination_reason we return
+        # in this case cause this FE will be removed from Executor reported state.
+        return FunctionExecutorCreated(
+            function_executor=None,
+            output=FunctionExecutorStartupOutput(
+                function_executor_description=function_executor_description,
+                termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR,
+            ),
+        )
 
 
 async def _initialization_result_to_fe_creation_output(

{indexify-0.4.12 → indexify-0.4.14}/src/indexify/executor/function_executor_controller/function_executor_controller.py

@@ -252,7 +252,7 @@ class FunctionExecutorController:
         """Shutsdown the Function Executor and frees all of its resources.
 
         No task outcomes and outputs are getting reported to Server after this call.
-        Doesn't raise any exceptions. Blocks until the shutdown is complete.
+        Doesn't raise any exceptions. Blocks until the shutdown is complete. Idempotent.
         """
         self._add_event(ShutdownInitiated(), source="shutdown")
         try:
@@ -261,10 +261,9 @@ class FunctionExecutorController:
             pass  # Expected exception on shutdown
         except Exception as e:
             self._logger.error(
-                "function executor controller control loop raised unexpected exception",
+                "function executor controller control loop task raised unexpected exception",
                 exc_info=e,
             )
-        self._logger.info("function executor controller shutdown finished")
 
     def _update_internal_state(self, new_state: _FE_CONTROLLER_STATE) -> None:
         """Updates the internal state of the Function Executor Controller.
@@ -461,12 +460,25 @@ class FunctionExecutorController:
         if event.function_executor is None:
             # Server needs to increment attempts counter for all the tasks that were pending while FE was starting up.
             # This prevents infinite retries if FEs consistently fail to start up.
+            # The allocations we marked here also need to not used FE terminated failure reason in their outputs
+            # because FE terminated means that the allocation wasn't the cause of the FE termination.
+            allocation_ids_caused_termination: List[str] = []
+            for task_info in self._tasks.values():
+                task_logger = task_allocation_logger(task_info.allocation, self._logger)
+                task_logger.info(
+                    "marking allocation failed on function executor startup failure"
+                )
+                allocation_ids_caused_termination.append(
+                    task_info.allocation.allocation_id
+                )
+                task_info.output = TaskOutput.function_executor_startup_failed(
+                    allocation=task_info.allocation,
+                    fe_startup_output=event.output,
+                    logger=task_logger,
+                )
             self._start_termination(
                 fe_termination_reason=event.output.termination_reason,
-                allocation_ids_caused_termination=[
-                    task_info.allocation.allocation_id
-                    for task_info in self._tasks.values()
-                ],
+                allocation_ids_caused_termination=allocation_ids_caused_termination,
             )
             return
 
@@ -593,9 +605,11 @@ class FunctionExecutorController:
             _FE_CONTROLLER_STATE.TERMINATING,
             _FE_CONTROLLER_STATE.TERMINATED,
         ]:
-            task_info.output = TaskOutput.function_executor_terminated(
-                task_info.allocation
-            )
+            if task_info.output is None:
+                # The output can be set already by FE startup failure handler.
+                task_info.output = TaskOutput.function_executor_terminated(
+                    task_info.allocation
+                )
             self._start_task_output_upload(task_info)
         elif self._internal_state == _FE_CONTROLLER_STATE.RUNNING:
             self._running_task = task_info
@@ -787,7 +801,7 @@ class FunctionExecutorController:
         self._state_reporter.remove_function_executor_state(self.function_executor_id())
         self._state_reporter.schedule_state_report()
 
-        self._logger.info("function executor controller control loop finished")
+        self._logger.info("function executor controller shutdown finished")
         debug_print_events(events=self._events, logger=self._logger)
 
 

{indexify-0.4.12 → indexify-0.4.14}/src/indexify/executor/function_executor_controller/task_output.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     SerializedObject,
@@ -6,11 +6,14 @@ from tensorlake.function_executor.proto.function_executor_pb2 import (
 
 from indexify.proto.executor_api_pb2 import (
     DataPayload,
+    FunctionExecutorTerminationReason,
     TaskAllocation,
     TaskFailureReason,
     TaskOutcomeCode,
 )
 
+from .function_executor_startup_output import FunctionExecutorStartupOutput
+
 
 class TaskMetrics:
     """Metrics for a task."""
@@ -105,3 +108,52 @@ class TaskOutput:
             outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
             failure_reason=TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
         )
+
+    @classmethod
+    def function_executor_startup_failed(
+        cls,
+        allocation: TaskAllocation,
+        fe_startup_output: FunctionExecutorStartupOutput,
+        logger: Any,
+    ) -> "TaskOutput":
+        """Creates a TaskOutput for the case when we fail a task because its FE startup failed."""
+        output = TaskOutput(
+            allocation=allocation,
+            outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
+            failure_reason=_fe_startup_failure_reason_to_task_failure_reason(
+                fe_startup_output.termination_reason, logger
+            ),
+        )
+        # Use FE startup stdout, stderr for allocations that we failed because FE startup failed.
+        output.uploaded_stdout = fe_startup_output.stdout
+        output.uploaded_stderr = fe_startup_output.stderr
+        return output
+
+
+def _fe_startup_failure_reason_to_task_failure_reason(
+    fe_termination_reason: FunctionExecutorTerminationReason, logger: Any
+) -> TaskFailureReason:
+    # Only need to check FE termination reasons happening on FE startup.
+    if (
+        fe_termination_reason
+        == FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR
+    elif (
+        fe_termination_reason
+        == FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_TIMEOUT
+    elif (
+        fe_termination_reason
+        == FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR
+    else:
+        logger.error(
+            "unexpected function executor startup failure reason",
+            fe_termination_reason=FunctionExecutorTerminationReason.Name(
+                fe_termination_reason
+            ),
+        )
+        return TaskFailureReason.TASK_FAILURE_REASON_UNKNOWN

{indexify-0.4.12 → indexify-0.4.14}/src/indexify/executor/state_reconciler.py

@@ -66,6 +66,7 @@ class ExecutorStateReconciler:
         self._desired_states_reader_task: Optional[asyncio.Task] = None
         self._reconciliation_loop_task: Optional[asyncio.Task] = None
         self._function_executor_controllers: Dict[str, FunctionExecutorController] = {}
+        self._shutting_down_fe_ids: Set[str] = set()
         self._last_server_clock: Optional[int] = None
 
         self._last_desired_state_lock = asyncio.Lock()
@@ -320,14 +321,35 @@ class ExecutorStateReconciler:
             logger.error("failed adding Function Executor", exc_info=e)
 
     def _remove_function_executor_controller(self, function_executor_id: str) -> None:
-        fe_controller: FunctionExecutorController = (
-            self._function_executor_controllers.pop(function_executor_id)
-        )
+        # Don't remove the FE controller from self._function_executor_controllers until
+        # its shutdown is complete. Otherwise, if Server re-adds the FE to desired state
+        # before FE shutdown completes then we'll have two FE controllers for the same
+        # FE ID which results in many bugs.
+        if function_executor_id in self._shutting_down_fe_ids:
+            return
+
+        self._shutting_down_fe_ids.add(function_executor_id)
         asyncio.create_task(
-            fe_controller.shutdown(),
+            self._shutdown_function_executor_controller(function_executor_id),
             name=f"Shutdown Function Executor {function_executor_id}",
         )
 
+    async def _shutdown_function_executor_controller(
+        self, function_executor_id: str
+    ) -> None:
+        # We are not cancelling this aio task in self.shutdown(). Because of this the code here should
+        # not fail if the FE controller is not found in internal data structures. It can be removed
+        # by self.shutdown() at any time while we're running this aio task.
+        fe_controller: Optional[FunctionExecutorController] = (
+            self._function_executor_controllers.get(function_executor_id)
+        )
+        if fe_controller is None:
+            return
+
+        await fe_controller.shutdown()
+        self._function_executor_controllers.pop(function_executor_id, None)
+        self._shutting_down_fe_ids.discard(function_executor_id)
+
     def _reconcile_tasks(self, task_allocations: Iterable[TaskAllocation]):
         valid_task_allocations: List[TaskAllocation] = self._valid_task_allocations(
             task_allocations
@@ -393,8 +415,6 @@ class ExecutorStateReconciler:
                 task_allocation.function_executor_id
                 not in self._function_executor_controllers
             ):
-                # Current policy: don't report task outcomes for tasks that didn't run.
-                # This is required to simplify the protocol so Server doesn't need to care about task states.
                 logger.error(
                     "received TaskAllocation for a Function Executor that doesn't exist, dropping it from desired state"
                 )