indexify 0.3.16__tar.gz → 0.3.17__tar.gz

{indexify-0.3.16 → indexify-0.3.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.16
+Version: 0.3.17
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.3.16 → indexify-0.3.17}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.16"
+version = "0.3.17"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"

{indexify-0.3.16 → indexify-0.3.17}/src/indexify/executor/function_executor/function_executor.py

@@ -110,7 +110,7 @@ class FunctionExecutor:
                     config_path=config_path,
                     initialize_request=initialize_request,
                 )
-                await self._create_health_checker(stub)
+                await self._create_health_checker(self._channel, stub)
                 self._initialized = True
         except Exception:
             await self.destroy()
@@ -243,12 +243,15 @@ class FunctionExecutor:
         finally:
             self._invocation_state_client = None
 
-    async def _create_health_checker(self, stub: FunctionExecutorStub) -> None:
+    async def _create_health_checker(
+        self, channel: grpc.aio.Channel, stub: FunctionExecutorStub
+    ) -> None:
         with (
             metric_create_health_checker_errors.count_exceptions(),
             metric_create_health_checker_latency.time(),
         ):
             self._health_checker = HealthChecker(
+                channel=channel,
                 stub=stub,
                 logger=self._logger,
             )

{indexify-0.3.16 → indexify-0.3.17}/src/indexify/executor/function_executor/health_checker.py

@@ -1,8 +1,10 @@
 import asyncio
+import os
 from collections.abc import Awaitable, Callable
 from typing import Any, Optional
 
-from grpc.aio import AioRpcError
+import grpc
+import grpc.aio
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     HealthCheckRequest,
     HealthCheckResponse,
@@ -27,7 +29,10 @@ class HealthCheckResult:
 
 
 class HealthChecker:
-    def __init__(self, stub: FunctionExecutorStub, logger: Any):
+    def __init__(
+        self, channel: grpc.aio.Channel, stub: FunctionExecutorStub, logger: Any
+    ):
+        self._channel: grpc.aio.Channel = channel
         self._stub: FunctionExecutorStub = stub
         self._logger: Any = logger.bind(module=__name__)
         self._health_check_loop_task: Optional[asyncio.Task] = None
@@ -39,6 +44,12 @@ class HealthChecker:
         """Runs the health check once and returns the result.
 
         Does not raise any exceptions."""
+        if os.getenv("INDEXIFY_DISABLE_FUNCTION_EXECUTOR_HEALTH_CHECKS", "0") == "1":
+            return HealthCheckResult(
+                is_healthy=True,
+                reason="Function Executor health checks are disabled using INDEXIFY_DISABLE_FUNCTION_EXECUTOR_HEALTH_CHECKS env var.",
+            )
+
         with metric_health_check_latency.time():
             try:
                 response: HealthCheckResponse = await self._stub.check_health(
@@ -49,19 +60,32 @@ class HealthChecker:
                 return HealthCheckResult(
                     is_healthy=response.healthy, reason=response.status_message
                 )
-            except AioRpcError as e:
-                metric_failed_health_checks.inc()
-                # Expected exception when there are problems with communication because e.g. the server is unhealthy.
-                return HealthCheckResult(
-                    is_healthy=False,
-                    reason=f"Executor side RPC channel error: {str(e)}",
-                )
+            except grpc.aio.AioRpcError as e:
+                # Due to the customer code running in Function Executor we can't reliably conclude
+                # that the FE is unhealthy when RPC status code is not OK. E.g. customer code can
+                # hold Python GIL and prevent the health check RPC from being processed by FE Python code.
+                #
+                # The only unhealthy condition we can be sure about is when the channel can't re-establish
+                # the TCP connection within HEALTH_CHECK_TIMEOUT_SEC deadline. This is because FE Python
+                # code is not involved when TCP connections are established to FE. Problems reestablishing
+                # the TCP connection are usually due to the FE process crashing and its gRPC server socket
+                # not being available anymore or due to prolonged local networking failures on Executor.
+                channel_connectivity = self._channel.get_state()
+                if channel_connectivity == grpc.ChannelConnectivity.TRANSIENT_FAILURE:
+                    return HealthCheckResult(
+                        is_healthy=False,
+                        reason="Channel is in TRANSIENT_FAILURE state, assuming Function Executor crashed.",
+                    )
+                else:
+                    return HealthCheckResult(
+                        is_healthy=True,
+                        reason=f"Health check RPC failed with status code: {e.code().name}. Assuming Function Executor is healthy.",
+                    )
             except Exception as e:
-                metric_failed_health_checks.inc()
-                self._logger.warning("Got unexpected exception, ignoring", exc_info=e)
+                self._logger.error("Got unexpected exception, ignoring", exc_info=e)
                 return HealthCheckResult(
-                    is_healthy=False,
-                    reason=f"Unexpected exception in Executor: {str(e)}",
+                    is_healthy=True,
+                    reason=f"Unexpected exception in Executor: {str(e)}. Assuming Function Executor is healthy.",
                 )
 
     def start(self, callback: Callable[[HealthCheckResult], Awaitable[None]]) -> None:

{indexify-0.3.16 → indexify-0.3.17}/src/indexify/executor/metrics/task_runner.py

@@ -23,6 +23,13 @@ metric_tasks_blocked_by_policy: prometheus_client.Gauge = prometheus_client.Gaug
     "tasks_blocked_by_policy",
     "Number of tasks that are ready for execution but are blocked according to the current policy (typically waiting for a free Function Executor)",
 )
+metric_tasks_blocked_by_policy_per_function_name: prometheus_client.Gauge = (
+    prometheus_client.Gauge(
+        "tasks_blocked_by_policy_per_function_name",
+        "Number of tasks that are ready for execution but are blocked according to the current policy (typically waiting for a free Function Executor)",
+        ["function_name"],
+    )
+)
 
 # Metrics for the stage when task is running.
 metric_task_runs: prometheus_client.Counter = prometheus_client.Counter(

{indexify-0.3.16 → indexify-0.3.17}/src/indexify/executor/task_runner.py

@@ -22,6 +22,7 @@ from .metrics.task_runner import (
     metric_task_run_platform_errors,
     metric_task_runs,
     metric_tasks_blocked_by_policy,
+    metric_tasks_blocked_by_policy_per_function_name,
     metric_tasks_running,
 )
 
@@ -55,6 +56,9 @@ class TaskRunner:
             with (
                 metric_task_policy_errors.count_exceptions(),
                 metric_tasks_blocked_by_policy.track_inprogress(),
+                metric_tasks_blocked_by_policy_per_function_name.labels(
+                    function_name=task_input.task.compute_fn
+                ).track_inprogress(),
                 metric_task_policy_latency.time(),
             ):
                 metric_task_policy_runs.inc()