indexify 0.4.15__tar.gz → 0.4.17__tar.gz

{indexify-0.4.15 → indexify-0.4.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.15
+Version: 0.4.17
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -14,10 +14,10 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.12.14,<4.0.0)
-Requires-Dist: boto3 (>=1.39.4,<2.0.0)
+Requires-Dist: boto3 (>=1.39.7,<2.0.0)
 Requires-Dist: prometheus-client (>=0.22.1,<0.23.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: tensorlake (==0.2.24)
+Requires-Dist: tensorlake (==0.2.25)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.4.15 → indexify-0.4.17}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.4.15"
+version = "0.4.17"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -23,10 +23,10 @@ python = "^3.10"
 aiohttp = "^3.12.14"
 prometheus-client = "^0.22.1"
 psutil = "^7.0.0"
-boto3 = "^1.39.4"
+boto3 = "^1.39.7"
 # Adds function-executor binary, utils lib, sdk used in indexify-cli commands.
 # We need to specify the tensorlake version exactly because pip install doesn't respect poetry.lock files.
-tensorlake = "0.2.24"
+tensorlake = "0.2.25"
 # Uncomment the next line to use local tensorlake package (only for development!)
 # tensorlake = { path = "../tensorlake", develop = true }
 # pydantic is provided by tensorlake

{indexify-0.4.15 → indexify-0.4.17}/src/indexify/executor/channel_manager.py

@@ -1,5 +1,5 @@
 import asyncio
-import os
+import time
 from typing import Any, Dict, Optional
 
 import grpc.aio
@@ -10,16 +10,23 @@ from .metrics.channel_manager import (
     metric_grpc_server_channel_creation_retries,
     metric_grpc_server_channel_creations,
 )
+from .monitoring.health_checker.health_checker import HealthChecker
 
 _RETRY_INTERVAL_SEC = 5
 _CONNECT_TIMEOUT_SEC = 5
 
 
 class ChannelManager:
-    def __init__(self, server_address: str, config_path: Optional[str], logger: Any):
+    def __init__(
+        self,
+        server_address: str,
+        config_path: Optional[str],
+        health_checker: HealthChecker,
+        logger: Any,
+    ):
         self._logger: Any = logger.bind(module=__name__, server_address=server_address)
-        self._keep_alive_period_sec: int = _keep_alive_period_sec_from_env(logger)
         self._server_address: str = server_address
+        self._health_checker: HealthChecker = health_checker
         self._channel_credentials: Optional[grpc.ChannelCredentials] = None
         # This lock protects the fields below.
         self._lock = asyncio.Lock()
@@ -86,31 +93,33 @@ class ChannelManager:
         # Use the lock to ensure that we only create one channel without race conditions.
         async with self._lock:
             if self._channel is None:
+                # Only called on Executor startup when we establish the channel for the first time.
                 self._channel = await self._create_ready_channel()
             elif not await self._locked_channel_is_healthy():
                 self._logger.info("grpc channel to server is unhealthy")
+                self._health_checker.server_connection_state_changed(
+                    is_healthy=False,
+                    status_message="grpc channel to server is unhealthy",
+                )
                 await self._destroy_locked_channel()
                 self._channel = await self._create_ready_channel()
+                self._health_checker.server_connection_state_changed(
+                    is_healthy=True, status_message="grpc channel to server is healthy"
+                )
 
             return self._channel
 
     def create_channel(self) -> grpc.aio.Channel:
         """Creates a new channel to the gRPC server.
 
-        The channel is not be ready to use. Raises an exception on failure.
+        The channel is not ready to use. Raises an exception on failure.
         """
-        channel_options: list[tuple[str, int]] = _channel_options(
-            self._keep_alive_period_sec
-        )
         if self._channel_credentials is None:
-            return grpc.aio.insecure_channel(
-                target=self._server_address, options=channel_options
-            )
+            return grpc.aio.insecure_channel(target=self._server_address)
         else:
             return grpc.aio.secure_channel(
                 target=self._server_address,
                 credentials=self._channel_credentials,
-                options=channel_options,
             )
 
     async def _create_ready_channel(self) -> grpc.aio.Channel:
@@ -119,25 +128,41 @@ class ChannelManager:
         Returns a ready to use channel. Blocks until the channel
         is ready, never raises any exceptions.
         """
-        self._logger.info("creating new grpc server channel")
-
         with metric_grpc_server_channel_creation_latency.time():
             metric_grpc_server_channel_creations.inc()
             while True:
                 try:
-                    channel = self.create_channel()
+                    self._logger.info("creating new grpc server channel")
+                    create_channel_start = time.monotonic()
+                    channel: grpc.Channel = self.create_channel()
+                    self._logger.info(
+                        "grpc server channel created",
+                        duration_sec=time.monotonic() - create_channel_start,
+                    )
+
+                    self._logger.info(
+                        "waiting for grpc server channel to get established (ready)"
+                    )
+                    channel_ready_start = time.monotonic()
                     await asyncio.wait_for(
                         channel.channel_ready(),
                         timeout=_CONNECT_TIMEOUT_SEC,
                     )
+                    self._logger.info(
+                        "grpc server channel is established (ready)",
+                        duration_sec=time.monotonic() - channel_ready_start,
+                    )
+
                     return channel
-                except Exception:
+                except BaseException as e:
                     self._logger.error(
-                        f"failed establishing grpc server channel in {_CONNECT_TIMEOUT_SEC} sec, retrying in {_RETRY_INTERVAL_SEC} sec"
+                        f"failed establishing grpc server channel in {_CONNECT_TIMEOUT_SEC} sec, retrying in {_RETRY_INTERVAL_SEC} sec",
+                        exc_info=e,
                     )
+
                     try:
                         await channel.close()
-                    except Exception as e:
+                    except BaseException as e:
                         self._logger.error(
                             "failed closing not established channel", exc_info=e
                         )
@@ -173,45 +198,3 @@ class ChannelManager:
         except Exception as e:
             self._logger.error("failed closing channel", exc_info=e)
         self._channel = None
-
-
-def _channel_options(keep_alive_period_sec: int) -> list[tuple[str, int]]:
-    """Returns the gRPC channel options."""
-    # See https://grpc.io/docs/guides/keepalive/.
-    #
-    # NB: Rust Tonic framework that we're using in Server is not using gRPC core and doesn't support
-    # these options. From https://github.com/hyperium/tonic/issues/258 it supports gRPC PINGs when
-    # there are in-flight RPCs (and streams) without any extra configuration.
-    return [
-        ("grpc.keepalive_time_ms", keep_alive_period_sec * 1000),
-        (
-            "grpc.http2.max_pings_without_data",
-            -1,
-        ),  # Allow any number of empty PING messages
-        (
-            "grpc.keepalive_permit_without_calls",
-            0,
-        ),  # Don't send PINGs when there are no in-flight RPCs (and streams)
-    ]
-
-
-def _keep_alive_period_sec_from_env(logger: Any) -> int:
-    """Returns the keep alive period in seconds."""
-    # We have to use gRPC keep alive (PING) to prevent proxies/load-balancers from closing underlying HTTP/2
-    # (TCP) connections due to periods of idleness in gRPC streams that we use between Executor and Server.
-    # If a proxy/load-balancer closes the connection, then we see it as gRPC stream errors which results in
-    # a lot of error logs noise.
-    #
-    # The default period of 50 sec is used for one of the standard proxy/load-balancer timeouts of 1 minute.
-    DEFAULT_KEEP_ALIVE_PERIOD_SEC = "50"
-    keep_alive_period_sec = int(
-        os.getenv(
-            "INDEXIFY_EXECUTOR_GRPC_KEEP_ALIVE_PERIOD_SEC",
-            DEFAULT_KEEP_ALIVE_PERIOD_SEC,
-        )
-    )
-    if keep_alive_period_sec != int(DEFAULT_KEEP_ALIVE_PERIOD_SEC):
-        logger.info(
-            f"gRPC keep alive (PING) period is set to {keep_alive_period_sec} sec"
-        )
-    return keep_alive_period_sec

{indexify-0.4.15 → indexify-0.4.17}/src/indexify/executor/executor.py

@@ -69,6 +69,7 @@ class Executor:
         self._channel_manager = ChannelManager(
             server_address=grpc_server_addr,
             config_path=config_path,
+            health_checker=health_checker,
             logger=self._logger,
         )
         function_allowlist: List[FunctionURI] = parse_function_uris(function_uris)

indexify-0.4.17/src/indexify/executor/monitoring/health_checker/generic_health_checker.py

@@ -0,0 +1,40 @@
+from typing import Optional
+
+from .health_checker import HealthChecker, HealthCheckResult
+from .metrics.health_checker import metric_healthy
+
+_HEALTH_CHECKER_NAME = "GenericHealthChecker"
+
+
+class GenericHealthChecker(HealthChecker):
+    """A generic health checker that doesn't depend on machine type and other features of the environment.
+
+    The health checker uses software signals available in all environments like Function Executor failure rates.
+    """
+
+    def __init__(self):
+        self._server_connection_unhealthy_status_message: Optional[str] = None
+        metric_healthy.set(1)
+
+    def server_connection_state_changed(self, is_healthy: bool, status_message: str):
+        """Handle changes in server connection state."""
+        if is_healthy:
+            self._server_connection_unhealthy_status_message = None
+            metric_healthy.set(1)
+        else:
+            self._server_connection_unhealthy_status_message = status_message
+            metric_healthy.set(0)
+
+    async def check(self) -> HealthCheckResult:
+        if self._server_connection_unhealthy_status_message is not None:
+            return HealthCheckResult(
+                is_success=False,
+                status_message=self._server_connection_unhealthy_status_message,
+                checker_name=_HEALTH_CHECKER_NAME,
+            )
+
+        return HealthCheckResult(
+            is_success=True,
+            status_message="Successful",
+            checker_name=_HEALTH_CHECKER_NAME,
+        )

indexify-0.4.17/src/indexify/executor/monitoring/health_checker/health_checker.py

@@ -0,0 +1,16 @@
+class HealthCheckResult:
+    def __init__(self, checker_name: str, is_success: bool, status_message: str):
+        self.checker_name: str = checker_name
+        self.is_success: bool = is_success
+        self.status_message: str = status_message
+
+
+class HealthChecker:
+    """Abstract base class for health checkers."""
+
+    def server_connection_state_changed(self, is_healthy: bool, status_message: str):
+        """Handle changes in server connection state."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    async def check(self) -> HealthCheckResult:
+        raise NotImplementedError("Subclasses must implement this method.")

indexify-0.4.17/src/indexify/executor/monitoring/health_checker/metrics/health_checker.py

@@ -0,0 +1,5 @@
+import prometheus_client
+
+metric_healthy = prometheus_client.Gauge(
+    "healthy", "1 if the executor is healthy, 0 otherwise"
+)

indexify-0.4.15/src/indexify/executor/monitoring/health_checker/generic_health_checker.py

@@ -1,20 +0,0 @@
-from .health_checker import HealthChecker, HealthCheckResult
-
-HEALTH_CHECKER_NAME = "GenericHealthChecker"
-
-
-class GenericHealthChecker(HealthChecker):
-    """A generic health checker that doesn't depend on machine type and other features of the environment.
-
-    The health checker uses software signals available in all environments like Function Executor failure rates.
-    """
-
-    def __init__(self):
-        pass
-
-    async def check(self) -> HealthCheckResult:
-        return HealthCheckResult(
-            is_success=True,
-            status_message="The health check is always successful",
-            checker_name=HEALTH_CHECKER_NAME,
-        )

indexify-0.4.15/src/indexify/executor/monitoring/health_checker/health_checker.py

@@ -1,12 +0,0 @@
-class HealthCheckResult:
-    def __init__(self, checker_name: str, is_success: bool, status_message: str):
-        self.checker_name = checker_name
-        self.is_success = is_success
-        self.status_message = status_message
-
-
-class HealthChecker:
-    """Abstract base class for health checkers."""
-
-    async def check(self) -> HealthCheckResult:
-        raise NotImplementedError("Subclasses must implement this method.")