indexify 0.4.12__tar.gz → 0.4.13__tar.gz

{indexify-0.4.12 → indexify-0.4.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.12
+Version: 0.4.13
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -17,7 +17,7 @@ Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
 Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: tensorlake (==0.2.8)
+Requires-Dist: tensorlake (==0.2.12)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.4.12 → indexify-0.4.13}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.4.12"
+version = "0.4.13"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -25,7 +25,7 @@ prometheus-client = "^0.21.1"
 psutil = "^7.0.0"
 # Adds function-executor binary, utils lib, sdk used in indexify-cli commands.
 # We need to specify the tensorlake version exactly because pip install doesn't respect poetry.lock files.
-tensorlake = "0.2.8"
+tensorlake = "0.2.12"
 # Uncomment the next line to use local tensorlake package (only for development!)
 # tensorlake = { path = "../tensorlake", develop = true }
 # pydantic is provided by tensorlake

{indexify-0.4.12 → indexify-0.4.13}/src/indexify/executor/function_executor_controller/create_function_executor.py

@@ -5,7 +5,6 @@ from typing import Any, Optional, Tuple
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
     SerializedObject,
-    SerializedObjectEncoding,
 )
 
 from indexify.executor.blob_store.blob_store import BLOBStore
@@ -73,25 +72,21 @@ async def create_function_executor(
     except BaseException as e:
         if isinstance(e, asyncio.CancelledError):
             logger.info("function executor startup was cancelled")
-            return FunctionExecutorCreated(
-                function_executor=None,
-                output=FunctionExecutorStartupOutput(
-                    function_executor_description=function_executor_description,
-                    termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE,
-                ),
-            )
         else:
             logger.error(
                 "failed to create function executor due to platform error",
                 exc_info=e,
             )
-            return FunctionExecutorCreated(
-                function_executor=None,
-                output=FunctionExecutorStartupOutput(
-                    function_executor_description=function_executor_description,
-                    termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR,
-                ),
-            )
+
+        # Cancelled FE startup means that Server removed it from desired state so it doesn't matter what termination_reason we return
+        # in this case cause this FE will be removed from Executor reported state.
+        return FunctionExecutorCreated(
+            function_executor=None,
+            output=FunctionExecutorStartupOutput(
+                function_executor_description=function_executor_description,
+                termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR,
+            ),
+        )
 
 
 async def _initialization_result_to_fe_creation_output(

{indexify-0.4.12 → indexify-0.4.13}/src/indexify/executor/function_executor_controller/function_executor_controller.py

@@ -461,12 +461,25 @@ class FunctionExecutorController:
         if event.function_executor is None:
             # Server needs to increment attempts counter for all the tasks that were pending while FE was starting up.
             # This prevents infinite retries if FEs consistently fail to start up.
+            # The allocations we marked here also need to not used FE terminated failure reason in their outputs
+            # because FE terminated means that the allocation wasn't the cause of the FE termination.
+            allocation_ids_caused_termination: List[str] = []
+            for task_info in self._tasks.values():
+                task_logger = task_allocation_logger(task_info.allocation, self._logger)
+                task_logger.info(
+                    "marking allocation failed on function executor startup failure"
+                )
+                allocation_ids_caused_termination.append(
+                    task_info.allocation.allocation_id
+                )
+                task_info.output = TaskOutput.function_executor_startup_failed(
+                    allocation=task_info.allocation,
+                    fe_startup_output=event.output,
+                    logger=task_logger,
+                )
             self._start_termination(
                 fe_termination_reason=event.output.termination_reason,
-                allocation_ids_caused_termination=[
-                    task_info.allocation.allocation_id
-                    for task_info in self._tasks.values()
-                ],
+                allocation_ids_caused_termination=allocation_ids_caused_termination,
             )
             return
 
@@ -593,9 +606,11 @@ class FunctionExecutorController:
             _FE_CONTROLLER_STATE.TERMINATING,
             _FE_CONTROLLER_STATE.TERMINATED,
         ]:
-            task_info.output = TaskOutput.function_executor_terminated(
-                task_info.allocation
-            )
+            if task_info.output is None:
+                # The output can be set already by FE startup failure handler.
+                task_info.output = TaskOutput.function_executor_terminated(
+                    task_info.allocation
+                )
             self._start_task_output_upload(task_info)
         elif self._internal_state == _FE_CONTROLLER_STATE.RUNNING:
             self._running_task = task_info

{indexify-0.4.12 → indexify-0.4.13}/src/indexify/executor/function_executor_controller/task_output.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     SerializedObject,
@@ -6,11 +6,14 @@ from tensorlake.function_executor.proto.function_executor_pb2 import (
 
 from indexify.proto.executor_api_pb2 import (
     DataPayload,
+    FunctionExecutorTerminationReason,
     TaskAllocation,
     TaskFailureReason,
     TaskOutcomeCode,
 )
 
+from .function_executor_startup_output import FunctionExecutorStartupOutput
+
 
 class TaskMetrics:
     """Metrics for a task."""
@@ -105,3 +108,52 @@ class TaskOutput:
             outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
             failure_reason=TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
         )
+
+    @classmethod
+    def function_executor_startup_failed(
+        cls,
+        allocation: TaskAllocation,
+        fe_startup_output: FunctionExecutorStartupOutput,
+        logger: Any,
+    ) -> "TaskOutput":
+        """Creates a TaskOutput for the case when we fail a task because its FE startup failed."""
+        output = TaskOutput(
+            allocation=allocation,
+            outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
+            failure_reason=_fe_startup_failure_reason_to_task_failure_reason(
+                fe_startup_output.termination_reason, logger
+            ),
+        )
+        # Use FE startup stdout, stderr for allocations that we failed because FE startup failed.
+        output.uploaded_stdout = fe_startup_output.stdout
+        output.uploaded_stderr = fe_startup_output.stderr
+        return output
+
+
+def _fe_startup_failure_reason_to_task_failure_reason(
+    fe_termination_reason: FunctionExecutorTerminationReason, logger: Any
+) -> TaskFailureReason:
+    # Only need to check FE termination reasons happening on FE startup.
+    if (
+        fe_termination_reason
+        == FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR
+    elif (
+        fe_termination_reason
+        == FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_TIMEOUT
+    elif (
+        fe_termination_reason
+        == FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR
+    else:
+        logger.error(
+            "unexpected function executor startup failure reason",
+            fe_termination_reason=FunctionExecutorTerminationReason.Name(
+                fe_termination_reason
+            ),
+        )
+        return TaskFailureReason.TASK_FAILURE_REASON_UNKNOWN