indexify 0.4.12__py3-none-any.whl → 0.4.13__py3-none-any.whl

indexify/executor/function_executor_controller/create_function_executor.py

@@ -5,7 +5,6 @@ from typing import Any, Optional, Tuple
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
     SerializedObject,
-    SerializedObjectEncoding,
 )
 
 from indexify.executor.blob_store.blob_store import BLOBStore
@@ -73,25 +72,21 @@ async def create_function_executor(
     except BaseException as e:
         if isinstance(e, asyncio.CancelledError):
             logger.info("function executor startup was cancelled")
-            return FunctionExecutorCreated(
-                function_executor=None,
-                output=FunctionExecutorStartupOutput(
-                    function_executor_description=function_executor_description,
-                    termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE,
-                ),
-            )
         else:
             logger.error(
                 "failed to create function executor due to platform error",
                 exc_info=e,
             )
-            return FunctionExecutorCreated(
-                function_executor=None,
-                output=FunctionExecutorStartupOutput(
-                    function_executor_description=function_executor_description,
-                    termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR,
-                ),
-            )
+
+        # Cancelled FE startup means that Server removed it from desired state so it doesn't matter what termination_reason we return
+        # in this case cause this FE will be removed from Executor reported state.
+        return FunctionExecutorCreated(
+            function_executor=None,
+            output=FunctionExecutorStartupOutput(
+                function_executor_description=function_executor_description,
+                termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR,
+            ),
+        )
 
 
 async def _initialization_result_to_fe_creation_output(

indexify/executor/function_executor_controller/function_executor_controller.py

@@ -461,12 +461,25 @@ class FunctionExecutorController:
         if event.function_executor is None:
             # Server needs to increment attempts counter for all the tasks that were pending while FE was starting up.
             # This prevents infinite retries if FEs consistently fail to start up.
+            # The allocations we marked here also need to not used FE terminated failure reason in their outputs
+            # because FE terminated means that the allocation wasn't the cause of the FE termination.
+            allocation_ids_caused_termination: List[str] = []
+            for task_info in self._tasks.values():
+                task_logger = task_allocation_logger(task_info.allocation, self._logger)
+                task_logger.info(
+                    "marking allocation failed on function executor startup failure"
+                )
+                allocation_ids_caused_termination.append(
+                    task_info.allocation.allocation_id
+                )
+                task_info.output = TaskOutput.function_executor_startup_failed(
+                    allocation=task_info.allocation,
+                    fe_startup_output=event.output,
+                    logger=task_logger,
+                )
             self._start_termination(
                 fe_termination_reason=event.output.termination_reason,
-                allocation_ids_caused_termination=[
-                    task_info.allocation.allocation_id
-                    for task_info in self._tasks.values()
-                ],
+                allocation_ids_caused_termination=allocation_ids_caused_termination,
             )
             return
 
@@ -593,9 +606,11 @@ class FunctionExecutorController:
             _FE_CONTROLLER_STATE.TERMINATING,
             _FE_CONTROLLER_STATE.TERMINATED,
         ]:
-            task_info.output = TaskOutput.function_executor_terminated(
-                task_info.allocation
-            )
+            if task_info.output is None:
+                # The output can be set already by FE startup failure handler.
+                task_info.output = TaskOutput.function_executor_terminated(
+                    task_info.allocation
+                )
             self._start_task_output_upload(task_info)
         elif self._internal_state == _FE_CONTROLLER_STATE.RUNNING:
             self._running_task = task_info

indexify/executor/function_executor_controller/task_output.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     SerializedObject,
@@ -6,11 +6,14 @@ from tensorlake.function_executor.proto.function_executor_pb2 import (
 
 from indexify.proto.executor_api_pb2 import (
     DataPayload,
+    FunctionExecutorTerminationReason,
     TaskAllocation,
     TaskFailureReason,
     TaskOutcomeCode,
 )
 
+from .function_executor_startup_output import FunctionExecutorStartupOutput
+
 
 class TaskMetrics:
     """Metrics for a task."""
@@ -105,3 +108,52 @@ class TaskOutput:
             outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
             failure_reason=TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
         )
+
+    @classmethod
+    def function_executor_startup_failed(
+        cls,
+        allocation: TaskAllocation,
+        fe_startup_output: FunctionExecutorStartupOutput,
+        logger: Any,
+    ) -> "TaskOutput":
+        """Creates a TaskOutput for the case when we fail a task because its FE startup failed."""
+        output = TaskOutput(
+            allocation=allocation,
+            outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
+            failure_reason=_fe_startup_failure_reason_to_task_failure_reason(
+                fe_startup_output.termination_reason, logger
+            ),
+        )
+        # Use FE startup stdout, stderr for allocations that we failed because FE startup failed.
+        output.uploaded_stdout = fe_startup_output.stdout
+        output.uploaded_stderr = fe_startup_output.stderr
+        return output
+
+
+def _fe_startup_failure_reason_to_task_failure_reason(
+    fe_termination_reason: FunctionExecutorTerminationReason, logger: Any
+) -> TaskFailureReason:
+    # Only need to check FE termination reasons happening on FE startup.
+    if (
+        fe_termination_reason
+        == FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR
+    elif (
+        fe_termination_reason
+        == FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_TIMEOUT
+    elif (
+        fe_termination_reason
+        == FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR
+    ):
+        return TaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR
+    else:
+        logger.error(
+            "unexpected function executor startup failure reason",
+            fe_termination_reason=FunctionExecutorTerminationReason.Name(
+                fe_termination_reason
+            ),
+        )
+        return TaskFailureReason.TASK_FAILURE_REASON_UNKNOWN

{indexify-0.4.12.dist-info → indexify-0.4.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.12
+Version: 0.4.13
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -17,7 +17,7 @@ Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
 Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: tensorlake (==0.2.8)
+Requires-Dist: tensorlake (==0.2.12)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.4.12.dist-info → indexify-0.4.13.dist-info}/RECORD

@@ -23,11 +23,11 @@ indexify/executor/function_executor/server/subprocess_function_executor_server.p
 indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=w5aGQPHWLpixlP9-BbZu6oL_muMA95-hr7WKVxiEL7Q,4303
 indexify/executor/function_executor_controller/__init__.py,sha256=VPuuBEYOKf7OWyPPjy-jGOv-d5xJqHvkJfFT_oj-AsE,492
 indexify/executor/function_executor_controller/completed_task_metrics.py,sha256=MhnC-ddgmTK4yTsuZxgTKnqZ-YSVeWn2EhbbiggsSKk,3664
-indexify/executor/function_executor_controller/create_function_executor.py,sha256=qQ_xBQRYUwz8SbNH7-H6D9GP-NJM3OlxPaIuLnPpbRM,9996
+indexify/executor/function_executor_controller/create_function_executor.py,sha256=DA8niVgftxSE_OYARw7nLSJujVu8HDzHh1EW_cXquC0,9766
 indexify/executor/function_executor_controller/debug_event_loop.py,sha256=VJOKe_c9HjIDVCjhMY3Yqyeq1tAM1eVa2chZa6CMf-U,1016
 indexify/executor/function_executor_controller/downloads.py,sha256=XjCUmLY_jrI3AlnXC7aDwwKWTvsQjV7I9AXzrbIeY6c,7063
 indexify/executor/function_executor_controller/events.py,sha256=r2K3k9Nnkzh0j6HHZC0DxOdQ3HtCmzt4eN2DIwTa7NM,5456
-indexify/executor/function_executor_controller/function_executor_controller.py,sha256=OgPlcYELVehuDVd1mzo2-LMzvIKBsYMZDGUaGDwkigc,35454
+indexify/executor/function_executor_controller/function_executor_controller.py,sha256=CpIpZ0l8s5O22RLq_rQOBXRYrDq3YAWFXGQF1BfCbxQ,36415
 indexify/executor/function_executor_controller/function_executor_startup_output.py,sha256=PXg2r440kqHI3oHGZbb58ehuAuW_fmEdxLTAa-0V3p4,715
 indexify/executor/function_executor_controller/loggers.py,sha256=zEY2nt15gboX3SX6Kh1xjeCljZJZSE4lp27qNrg8yPY,3637
 indexify/executor/function_executor_controller/message_validators.py,sha256=aNiZhYA87pnxUJtZKvKGDt40rfox-TYH2J6mW7o-Pkw,2981
@@ -39,7 +39,7 @@ indexify/executor/function_executor_controller/metrics/upload_task_output.py,sha
 indexify/executor/function_executor_controller/prepare_task.py,sha256=AKbo_H_5pOKdxFKKkzdOb1WhQ0XT-4Qm9D3iIsukyMU,1247
 indexify/executor/function_executor_controller/run_task.py,sha256=ARa-OgqgJ1o_LmO1FIhkXBvqLWXCzYLiSiG2jGw_NR8,10448
 indexify/executor/function_executor_controller/task_info.py,sha256=ZEdypd8QVmYbrLt1186Ed9YEQwrO0Sx_hKH0QLg1DVY,1181
-indexify/executor/function_executor_controller/task_output.py,sha256=A5KJh60OI1FdHw6ABX3KxL9OMrC6ZADJHeo0W5iZpz8,3764
+indexify/executor/function_executor_controller/task_output.py,sha256=xj2QhYsw2QEX13B43i3HJS6kJyR9WreoQth4hIVXIqk,5938
 indexify/executor/function_executor_controller/terminate_function_executor.py,sha256=YLDlKoanfUBcy7A9ydCYdUsDwApjcTTn1o4tjNVN_QA,1281
 indexify/executor/function_executor_controller/upload_task_output.py,sha256=fEZm5eodx5rNLQYFhmdkMDD9qjX3_wKo64x4aUKTu34,10403
 indexify/executor/host_resources/host_resources.py,sha256=ZSfox24jaz1IIaQWUmciOoev0l35rk8LHbnb_koJWno,3810
@@ -63,7 +63,7 @@ indexify/proto/executor_api.proto,sha256=MEsBzpL5tlBd6Yc3ZfheQ60DbXAZWQWGM1dlldh
 indexify/proto/executor_api_pb2.py,sha256=w2Wj1MEtTihNZ4TvPKPYDjMryO52fPa7A4dKv7vb__s,16236
 indexify/proto/executor_api_pb2.pyi,sha256=AEHW4YFm5FT8f66qKGkPH36szlWgCgXrH3CQ8rgxSps,22754
 indexify/proto/executor_api_pb2_grpc.py,sha256=JpT5K6jiS0NJVNyTt1mAPpyJMXuEGeNN2V6R3KmLHZ4,7607
-indexify-0.4.12.dist-info/METADATA,sha256=r21ToO5MZ3_RK7w-iSaXuwXjHSCwx_8-JdTF8_a9lTc,1115
-indexify-0.4.12.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.4.12.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
-indexify-0.4.12.dist-info/RECORD,,
+indexify-0.4.13.dist-info/METADATA,sha256=ZN16aCArggFFTIlaWelbhaF36t2CEtTAm6dbJxVbFU0,1116
+indexify-0.4.13.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.4.13.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
+indexify-0.4.13.dist-info/RECORD,,