indexify 0.3.16__py3-none-any.whl → 0.3.17__py3-none-any.whl

indexify/executor/function_executor/function_executor.py

@@ -110,7 +110,7 @@ class FunctionExecutor:
                     config_path=config_path,
                     initialize_request=initialize_request,
                 )
-                await self._create_health_checker(stub)
+                await self._create_health_checker(self._channel, stub)
                 self._initialized = True
         except Exception:
             await self.destroy()
@@ -243,12 +243,15 @@ class FunctionExecutor:
         finally:
             self._invocation_state_client = None
 
-    async def _create_health_checker(self, stub: FunctionExecutorStub) -> None:
+    async def _create_health_checker(
+        self, channel: grpc.aio.Channel, stub: FunctionExecutorStub
+    ) -> None:
         with (
             metric_create_health_checker_errors.count_exceptions(),
             metric_create_health_checker_latency.time(),
         ):
             self._health_checker = HealthChecker(
+                channel=channel,
                 stub=stub,
                 logger=self._logger,
             )

indexify/executor/function_executor/health_checker.py

@@ -1,8 +1,10 @@
 import asyncio
+import os
 from collections.abc import Awaitable, Callable
 from typing import Any, Optional
 
-from grpc.aio import AioRpcError
+import grpc
+import grpc.aio
 from tensorlake.function_executor.proto.function_executor_pb2 import (
     HealthCheckRequest,
     HealthCheckResponse,
@@ -27,7 +29,10 @@ class HealthCheckResult:
 
 
 class HealthChecker:
-    def __init__(self, stub: FunctionExecutorStub, logger: Any):
+    def __init__(
+        self, channel: grpc.aio.Channel, stub: FunctionExecutorStub, logger: Any
+    ):
+        self._channel: grpc.aio.Channel = channel
         self._stub: FunctionExecutorStub = stub
         self._logger: Any = logger.bind(module=__name__)
         self._health_check_loop_task: Optional[asyncio.Task] = None
@@ -39,6 +44,12 @@ class HealthChecker:
         """Runs the health check once and returns the result.
 
         Does not raise any exceptions."""
+        if os.getenv("INDEXIFY_DISABLE_FUNCTION_EXECUTOR_HEALTH_CHECKS", "0") == "1":
+            return HealthCheckResult(
+                is_healthy=True,
+                reason="Function Executor health checks are disabled using INDEXIFY_DISABLE_FUNCTION_EXECUTOR_HEALTH_CHECKS env var.",
+            )
+
         with metric_health_check_latency.time():
             try:
                 response: HealthCheckResponse = await self._stub.check_health(
@@ -49,19 +60,32 @@ class HealthChecker:
                 return HealthCheckResult(
                     is_healthy=response.healthy, reason=response.status_message
                 )
-            except AioRpcError as e:
-                metric_failed_health_checks.inc()
-                # Expected exception when there are problems with communication because e.g. the server is unhealthy.
-                return HealthCheckResult(
-                    is_healthy=False,
-                    reason=f"Executor side RPC channel error: {str(e)}",
-                )
+            except grpc.aio.AioRpcError as e:
+                # Due to the customer code running in Function Executor we can't reliably conclude
+                # that the FE is unhealthy when RPC status code is not OK. E.g. customer code can
+                # hold Python GIL and prevent the health check RPC from being processed by FE Python code.
+                #
+                # The only unhealthy condition we can be sure about is when the channel can't re-establish
+                # the TCP connection within HEALTH_CHECK_TIMEOUT_SEC deadline. This is because FE Python
+                # code is not involved when TCP connections are established to FE. Problems reestablishing
+                # the TCP connection are usually due to the FE process crashing and its gRPC server socket
+                # not being available anymore or due to prolonged local networking failures on Executor.
+                channel_connectivity = self._channel.get_state()
+                if channel_connectivity == grpc.ChannelConnectivity.TRANSIENT_FAILURE:
+                    return HealthCheckResult(
+                        is_healthy=False,
+                        reason="Channel is in TRANSIENT_FAILURE state, assuming Function Executor crashed.",
+                    )
+                else:
+                    return HealthCheckResult(
+                        is_healthy=True,
+                        reason=f"Health check RPC failed with status code: {e.code().name}. Assuming Function Executor is healthy.",
+                    )
             except Exception as e:
-                metric_failed_health_checks.inc()
-                self._logger.warning("Got unexpected exception, ignoring", exc_info=e)
+                self._logger.error("Got unexpected exception, ignoring", exc_info=e)
                 return HealthCheckResult(
-                    is_healthy=False,
-                    reason=f"Unexpected exception in Executor: {str(e)}",
+                    is_healthy=True,
+                    reason=f"Unexpected exception in Executor: {str(e)}. Assuming Function Executor is healthy.",
                 )
 
     def start(self, callback: Callable[[HealthCheckResult], Awaitable[None]]) -> None:

indexify/executor/metrics/task_runner.py

@@ -23,6 +23,13 @@ metric_tasks_blocked_by_policy: prometheus_client.Gauge = prometheus_client.Gaug
     "tasks_blocked_by_policy",
     "Number of tasks that are ready for execution but are blocked according to the current policy (typically waiting for a free Function Executor)",
 )
+metric_tasks_blocked_by_policy_per_function_name: prometheus_client.Gauge = (
+    prometheus_client.Gauge(
+        "tasks_blocked_by_policy_per_function_name",
+        "Number of tasks that are ready for execution but are blocked according to the current policy (typically waiting for a free Function Executor)",
+        ["function_name"],
+    )
+)
 
 # Metrics for the stage when task is running.
 metric_task_runs: prometheus_client.Counter = prometheus_client.Counter(

indexify/executor/task_runner.py

@@ -22,6 +22,7 @@ from .metrics.task_runner import (
     metric_task_run_platform_errors,
     metric_task_runs,
     metric_tasks_blocked_by_policy,
+    metric_tasks_blocked_by_policy_per_function_name,
     metric_tasks_running,
 )
 
@@ -55,6 +56,9 @@ class TaskRunner:
             with (
                 metric_task_policy_errors.count_exceptions(),
                 metric_tasks_blocked_by_policy.track_inprogress(),
+                metric_tasks_blocked_by_policy_per_function_name.labels(
+                    function_name=task_input.task.compute_fn
+                ).track_inprogress(),
                 metric_task_policy_latency.time(),
             ):
                 metric_task_policy_runs.inc()

{indexify-0.3.16.dist-info → indexify-0.3.17.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.16
+Version: 0.3.17
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.3.16.dist-info → indexify-0.3.17.dist-info}/RECORD

@@ -3,11 +3,11 @@ indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2
 indexify/executor/api_objects.py,sha256=oUlH-GQPuPmwgcBzMpI2HehXeElBTCULECk-oHiBHwU,1263
 indexify/executor/downloader.py,sha256=LkvAXfKxddnDzgfmwHcpDB_n795-eVKzn-hLjq4nUEM,9412
 indexify/executor/executor.py,sha256=FTp05YxuKlMUbI99OV7NkL3KuFD12evKcqxzb-fXWBk,14641
-indexify/executor/function_executor/function_executor.py,sha256=sQ5FOdrjybDDsjagghlfjV06IXTpWWDBSTHqQXI-w9M,11245
+indexify/executor/function_executor/function_executor.py,sha256=s1mc7g6b8ilc98Fp7RFElEBSLJl0UGNQY0iZzCpuR2A,11334
 indexify/executor/function_executor/function_executor_state.py,sha256=b2taGClg0BUnlD_rYGkpom6syXBMUp7UWWrjLrUCwyo,3966
 indexify/executor/function_executor/function_executor_states_container.py,sha256=RclJDJqIr8ywKipPBC6_idnPAqYi0dPa1d4QUAaXqbw,3460
 indexify/executor/function_executor/function_executor_status.py,sha256=U4p1fcdVWlHr7uPY7e7ZSb2_WelUmPeH-WgboQQ9mw4,3336
-indexify/executor/function_executor/health_checker.py,sha256=qUUpG4oeVsPLibiCspAiRm-2Ldg46ulnnpj9EBXr1NQ,3916
+indexify/executor/function_executor/health_checker.py,sha256=CPUWvvtJtJCwbEsfr_BOhSKkRki4qOoSk1oeyBosWz0,5464
 indexify/executor/function_executor/invocation_state_client.py,sha256=p-xgM4__cHR1ApvMV9hShrGWee_Je0VDhICZUGjpQY4,9644
 indexify/executor/function_executor/metrics/function_executor.py,sha256=TDksxLRJr-P9ZKhF2Orsaxzzb4lVIBxFEjd_9Zv53Ng,6313
 indexify/executor/function_executor/metrics/function_executor_state.py,sha256=qheMhnoiYLiZB7ky5EyegfDy4Mr0Zh83bOE0gJ38YmU,1607
@@ -32,7 +32,7 @@ indexify/executor/metrics/downloader.py,sha256=lctPh8xjkXeLEFJnl1hNrD1yEhLhIl5sg
 indexify/executor/metrics/executor.py,sha256=ua-Vv_k1CB4juJdF7tEBQbBMksqWAA3iXKKMKXZUCLk,2369
 indexify/executor/metrics/task_fetcher.py,sha256=iJEwCLzYr2cuz7hRvNiqaa2nvQP4OrA0hm0iJY0YKG0,736
 indexify/executor/metrics/task_reporter.py,sha256=zUA9RpkSgx5lG_ZqDDuela5VuhtsnC0IKoQcEvHND0Y,730
-indexify/executor/metrics/task_runner.py,sha256=o5ERNePKPmVKknFoSZUr-r597dEOOWvWn3ocbiL2jxI,1699
+indexify/executor/metrics/task_runner.py,sha256=ZGFrl7zzfUdgPZnklxRIbnv9wVcHIQRhOGNqn9V2hSk,2047
 indexify/executor/monitoring/function_allowlist.py,sha256=wUGeiv3aAGWMlQXzHXq9O6MVHby6Tu-zY4U0MyWiQu0,683
 indexify/executor/monitoring/handler.py,sha256=Cj1cu_LcsAP0tdviqNhoEtGm4h0OJAxxzW9C2YdNXYU,240
 indexify/executor/monitoring/health_check_handler.py,sha256=e1pEtWFKaVs6H57Z4YLejNECrJtC38PweZc7xTJeqVw,695
@@ -45,12 +45,12 @@ indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpx
 indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
 indexify/executor/task_fetcher.py,sha256=NpFfHgaY99bSL-K2D5kcDAMNUG2FArq0-qF_mgF-LBQ,3375
 indexify/executor/task_reporter.py,sha256=mYgwozUO95PEwYMmeeIS0-HfMrO4z3Nhy6IduMsMahM,7367
-indexify/executor/task_runner.py,sha256=TGiTNE68HCm38HcpPRN5-hwNiTFw_gwBRkCcrw7sz2Q,6847
+indexify/executor/task_runner.py,sha256=1zYH03yS_FaFk9xXBl-ioM74-L2xdW3vHJt522mseds,7073
 indexify/proto/task_scheduler.proto,sha256=kxMIJCj1pXG-fHeJGHXlthZTsB1dy_yvshQLt0UJRTM,5672
 indexify/proto/task_scheduler_pb2.py,sha256=X97JBJZ2n6ToDtUlDjPFV66_vZ05-vO8wPATrpzAonA,9085
 indexify/proto/task_scheduler_pb2.pyi,sha256=aXrB7-eNwgchy2OVlvEfPXtr9EyYoU-sgbdSRVNEI8s,11357
 indexify/proto/task_scheduler_pb2_grpc.py,sha256=STtk9XrBzLbmWdLwpL55Obyf9ehUesfxxysxER32SEE,6854
-indexify-0.3.16.dist-info/METADATA,sha256=3jBagszOve3WAI9dTFN7ZtT1x_0hBFhLPOFYs-KQ6AU,1158
-indexify-0.3.16.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.16.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
-indexify-0.3.16.dist-info/RECORD,,
+indexify-0.3.17.dist-info/METADATA,sha256=hREMWJfSrd4Vcclp2w8fcUnjtvkiXHw6jMCTWECKAtw,1158
+indexify-0.3.17.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.17.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
+indexify-0.3.17.dist-info/RECORD,,