indexify 0.4.15__py3-none-any.whl → 0.4.16__py3-none-any.whl

indexify/executor/channel_manager.py

@@ -1,5 +1,5 @@
 import asyncio
-import os
+import time
 from typing import Any, Dict, Optional
 
 import grpc.aio
@@ -10,16 +10,23 @@ from .metrics.channel_manager import (
     metric_grpc_server_channel_creation_retries,
     metric_grpc_server_channel_creations,
 )
+from .monitoring.health_checker.health_checker import HealthChecker
 
 _RETRY_INTERVAL_SEC = 5
 _CONNECT_TIMEOUT_SEC = 5
 
 
 class ChannelManager:
-    def __init__(self, server_address: str, config_path: Optional[str], logger: Any):
+    def __init__(
+        self,
+        server_address: str,
+        config_path: Optional[str],
+        health_checker: HealthChecker,
+        logger: Any,
+    ):
         self._logger: Any = logger.bind(module=__name__, server_address=server_address)
-        self._keep_alive_period_sec: int = _keep_alive_period_sec_from_env(logger)
         self._server_address: str = server_address
+        self._health_checker: HealthChecker = health_checker
         self._channel_credentials: Optional[grpc.ChannelCredentials] = None
         # This lock protects the fields below.
         self._lock = asyncio.Lock()
@@ -86,31 +93,33 @@ class ChannelManager:
         # Use the lock to ensure that we only create one channel without race conditions.
         async with self._lock:
             if self._channel is None:
+                # Only called on Executor startup when we establish the channel for the first time.
                 self._channel = await self._create_ready_channel()
             elif not await self._locked_channel_is_healthy():
                 self._logger.info("grpc channel to server is unhealthy")
+                self._health_checker.server_connection_state_changed(
+                    is_healthy=False,
+                    status_message="grpc channel to server is unhealthy",
+                )
                 await self._destroy_locked_channel()
                 self._channel = await self._create_ready_channel()
+                self._health_checker.server_connection_state_changed(
+                    is_healthy=True, status_message="grpc channel to server is healthy"
+                )
 
             return self._channel
 
     def create_channel(self) -> grpc.aio.Channel:
         """Creates a new channel to the gRPC server.
 
-        The channel is not be ready to use. Raises an exception on failure.
+        The channel is not ready to use. Raises an exception on failure.
         """
-        channel_options: list[tuple[str, int]] = _channel_options(
-            self._keep_alive_period_sec
-        )
         if self._channel_credentials is None:
-            return grpc.aio.insecure_channel(
-                target=self._server_address, options=channel_options
-            )
+            return grpc.aio.insecure_channel(target=self._server_address)
         else:
             return grpc.aio.secure_channel(
                 target=self._server_address,
                 credentials=self._channel_credentials,
-                options=channel_options,
             )
 
     async def _create_ready_channel(self) -> grpc.aio.Channel:
@@ -119,25 +128,36 @@ class ChannelManager:
         Returns a ready to use channel. Blocks until the channel
         is ready, never raises any exceptions.
         """
-        self._logger.info("creating new grpc server channel")
-
         with metric_grpc_server_channel_creation_latency.time():
             metric_grpc_server_channel_creations.inc()
             while True:
                 try:
-                    channel = self.create_channel()
+                    self._logger.info("creating new grpc server channel")
+                    create_channel_start = time.monotonic()
+                    channel: grpc.Channel = self.create_channel()
+                    self._logger.info(
+                        "grpc server channel created",
+                        duration_sec=time.monotonic() - create_channel_start,
+                    )
+
+                    channel_ready_start = time.monotonic()
                     await asyncio.wait_for(
                         channel.channel_ready(),
                         timeout=_CONNECT_TIMEOUT_SEC,
                     )
+                    self._logger.info(
+                        "grpc server channel is established (ready)",
+                        duration_sec=time.monotonic() - channel_ready_start,
+                    )
+
                     return channel
-                except Exception:
+                except BaseException:
                     self._logger.error(
                         f"failed establishing grpc server channel in {_CONNECT_TIMEOUT_SEC} sec, retrying in {_RETRY_INTERVAL_SEC} sec"
                     )
                     try:
                         await channel.close()
-                    except Exception as e:
+                    except BaseException as e:
                         self._logger.error(
                             "failed closing not established channel", exc_info=e
                         )
@@ -173,45 +193,3 @@ class ChannelManager:
         except Exception as e:
             self._logger.error("failed closing channel", exc_info=e)
         self._channel = None
-
-
-def _channel_options(keep_alive_period_sec: int) -> list[tuple[str, int]]:
-    """Returns the gRPC channel options."""
-    # See https://grpc.io/docs/guides/keepalive/.
-    #
-    # NB: Rust Tonic framework that we're using in Server is not using gRPC core and doesn't support
-    # these options. From https://github.com/hyperium/tonic/issues/258 it supports gRPC PINGs when
-    # there are in-flight RPCs (and streams) without any extra configuration.
-    return [
-        ("grpc.keepalive_time_ms", keep_alive_period_sec * 1000),
-        (
-            "grpc.http2.max_pings_without_data",
-            -1,
-        ),  # Allow any number of empty PING messages
-        (
-            "grpc.keepalive_permit_without_calls",
-            0,
-        ),  # Don't send PINGs when there are no in-flight RPCs (and streams)
-    ]
-
-
-def _keep_alive_period_sec_from_env(logger: Any) -> int:
-    """Returns the keep alive period in seconds."""
-    # We have to use gRPC keep alive (PING) to prevent proxies/load-balancers from closing underlying HTTP/2
-    # (TCP) connections due to periods of idleness in gRPC streams that we use between Executor and Server.
-    # If a proxy/load-balancer closes the connection, then we see it as gRPC stream errors which results in
-    # a lot of error logs noise.
-    #
-    # The default period of 50 sec is used for one of the standard proxy/load-balancer timeouts of 1 minute.
-    DEFAULT_KEEP_ALIVE_PERIOD_SEC = "50"
-    keep_alive_period_sec = int(
-        os.getenv(
-            "INDEXIFY_EXECUTOR_GRPC_KEEP_ALIVE_PERIOD_SEC",
-            DEFAULT_KEEP_ALIVE_PERIOD_SEC,
-        )
-    )
-    if keep_alive_period_sec != int(DEFAULT_KEEP_ALIVE_PERIOD_SEC):
-        logger.info(
-            f"gRPC keep alive (PING) period is set to {keep_alive_period_sec} sec"
-        )
-    return keep_alive_period_sec

indexify/executor/executor.py

@@ -69,6 +69,7 @@ class Executor:
         self._channel_manager = ChannelManager(
             server_address=grpc_server_addr,
             config_path=config_path,
+            health_checker=health_checker,
             logger=self._logger,
         )
         function_allowlist: List[FunctionURI] = parse_function_uris(function_uris)

indexify/executor/monitoring/health_checker/generic_health_checker.py

@@ -1,6 +1,8 @@
+from typing import Optional
+
 from .health_checker import HealthChecker, HealthCheckResult
 
-HEALTH_CHECKER_NAME = "GenericHealthChecker"
+_HEALTH_CHECKER_NAME = "GenericHealthChecker"
 
 
 class GenericHealthChecker(HealthChecker):
@@ -10,11 +12,25 @@ class GenericHealthChecker(HealthChecker):
     """
 
     def __init__(self):
-        pass
+        self._server_connection_unhealthy_status_message: Optional[str] = None
+
+    def server_connection_state_changed(self, is_healthy: bool, status_message: str):
+        """Handle changes in server connection state."""
+        if is_healthy:
+            self._server_connection_unhealthy_status_message = None
+        else:
+            self._server_connection_unhealthy_status_message = status_message
 
     async def check(self) -> HealthCheckResult:
+        if self._server_connection_unhealthy_status_message is not None:
+            return HealthCheckResult(
+                is_success=False,
+                status_message=self._server_connection_unhealthy_status_message,
+                checker_name=_HEALTH_CHECKER_NAME,
+            )
+
         return HealthCheckResult(
             is_success=True,
-            status_message="The health check is always successful",
-            checker_name=HEALTH_CHECKER_NAME,
+            status_message="Successful",
+            checker_name=_HEALTH_CHECKER_NAME,
         )

indexify/executor/monitoring/health_checker/health_checker.py

@@ -1,12 +1,16 @@
 class HealthCheckResult:
     def __init__(self, checker_name: str, is_success: bool, status_message: str):
-        self.checker_name = checker_name
-        self.is_success = is_success
-        self.status_message = status_message
+        self.checker_name: str = checker_name
+        self.is_success: bool = is_success
+        self.status_message: str = status_message
 
 
 class HealthChecker:
     """Abstract base class for health checkers."""
 
+    def server_connection_state_changed(self, is_healthy: bool, status_message: str):
+        """Handle changes in server connection state."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
     async def check(self) -> HealthCheckResult:
         raise NotImplementedError("Subclasses must implement this method.")

{indexify-0.4.15.dist-info → indexify-0.4.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.15
+Version: 0.4.16
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -14,10 +14,10 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.12.14,<4.0.0)
-Requires-Dist: boto3 (>=1.39.4,<2.0.0)
+Requires-Dist: boto3 (>=1.39.6,<2.0.0)
 Requires-Dist: prometheus-client (>=0.22.1,<0.23.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: tensorlake (==0.2.24)
+Requires-Dist: tensorlake (==0.2.25)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.4.15.dist-info → indexify-0.4.16.dist-info}/RECORD

@@ -7,8 +7,8 @@ indexify/executor/blob_store/blob_store.py,sha256=XViw_KRfFSNqwcFYwMZixZF-EYCjXK
 indexify/executor/blob_store/local_fs_blob_store.py,sha256=6LexqMBGXp8f6Ka95R6xMIUyDutrZJABOMNcp-ssa98,1809
 indexify/executor/blob_store/metrics/blob_store.py,sha256=5_xiPREeHWFtxFh1NupDsF8zP4pmUPgLNNn-UE9Uzvc,1008
 indexify/executor/blob_store/s3_blob_store.py,sha256=G3B_V3gUE7XbUY42lDtBczUKuA7q8S7MD43tx1aHrJo,3445
-indexify/executor/channel_manager.py,sha256=SeBaOYJXJ6KNw59c4C4n54qhcCusNda2756Rbu8pFFM,8642
-indexify/executor/executor.py,sha256=hR49lBFGaqSRj8RtLlzd4cd8hRUwhMrwUHwPx53EHDU,6310
+indexify/executor/channel_manager.py,sha256=rulAmCvgEupKZ0esuO4COfosuhm1iVZ27IIqeqFu0Vg,7750
+indexify/executor/executor.py,sha256=iMmysK_6jZkczcVqjzbQdXzrALEig_Qtb51MhQUfRqc,6353
 indexify/executor/function_allowlist.py,sha256=PCelCW6qIe_2sH11BCKr7LDqarRV5kwNsrfB2EV7Zwo,1772
 indexify/executor/function_executor/function_executor.py,sha256=Hz_dT_2i1m9akUGfULWQpDlMsn0CI1AX4Mdt7-oOknI,13598
 indexify/executor/function_executor/health_checker.py,sha256=IxE0jnC99K_lvnizFLjXqS1942H8-FNAN4AlhLIjg2Y,6373
@@ -51,8 +51,8 @@ indexify/executor/metrics/state_reconciler.py,sha256=BSlRgvgtwih6QcYrsFU5P2ylaXA
 indexify/executor/metrics/state_reporter.py,sha256=_dssgz335UyZ67OoKNyI50gn4kzPZyFN4VpFGMNw4qE,542
 indexify/executor/monitoring/handler.py,sha256=Cj1cu_LcsAP0tdviqNhoEtGm4h0OJAxxzW9C2YdNXYU,240
 indexify/executor/monitoring/health_check_handler.py,sha256=e1pEtWFKaVs6H57Z4YLejNECrJtC38PweZc7xTJeqVw,695
-indexify/executor/monitoring/health_checker/generic_health_checker.py,sha256=NQ1NTcaOJKd7xL9IaAtA8-VW4PZWIpW6N97MtTxmWVc,655
-indexify/executor/monitoring/health_checker/health_checker.py,sha256=SiWJ_bKNLsxHCzwBMjS-DnFmXmN8CuPA61aTvv_iyTI,429
+indexify/executor/monitoring/health_checker/generic_health_checker.py,sha256=8vvny1DOOaZMxPwcpeTuuVh7l42YCaOd6IkIxzNUURg,1344
+indexify/executor/monitoring/health_checker/health_checker.py,sha256=B-Q4KM1iEUSMA2fr9PBhBLdA7sYII_NuTRmPuRILGSo,665
 indexify/executor/monitoring/metrics.py,sha256=Dx2wPcTKvbd5Y5rGOfeyscFtAQ2DZ16_s5BX6d4nhI8,6660
 indexify/executor/monitoring/prometheus_metrics_handler.py,sha256=KiGqSf7rkXTfbDwThyXFpFe2jnuZD5q-5SBP_0GDo8Y,591
 indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFseOmHyQpTo,1254
@@ -63,7 +63,7 @@ indexify/proto/executor_api.proto,sha256=vP14TJCGO2BSwZ6piqaltiLp5YNPT5jCH2yaehy
 indexify/proto/executor_api_pb2.py,sha256=zdl00UOqgOB1KeRIAceh_43RpAOVLEs9RSbzxQ0hmKY,16163
 indexify/proto/executor_api_pb2.pyi,sha256=adD5mqqJhmTgRCa_4v1cR6GcOY-VOLOBV9k8T5iaqPc,22647
 indexify/proto/executor_api_pb2_grpc.py,sha256=gPtP9GscW2D9yUMXl4uEIO3cwe1B3bKprJjFfNrWou4,7607
-indexify-0.4.15.dist-info/METADATA,sha256=IgeB_Sdi6bQk8Va3dMbkblVkewzV_FSH49F6bLS0MgI,1116
-indexify-0.4.15.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.4.15.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
-indexify-0.4.15.dist-info/RECORD,,
+indexify-0.4.16.dist-info/METADATA,sha256=k9tCSJJYV9imbC2kJKhKYQz9_preBM7vd2gipq4Fhlw,1116
+indexify-0.4.16.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.4.16.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
+indexify-0.4.16.dist-info/RECORD,,