indexify 0.4.28__py3-none-any.whl → 0.4.29__py3-none-any.whl

indexify/executor/executor.py

@@ -23,9 +23,11 @@ from .metrics.executor import (
     metric_executor_info,
     metric_executor_state,
 )
+from .monitoring.desired_state_handler import DesiredStateHandler
 from .monitoring.health_check_handler import HealthCheckHandler
 from .monitoring.health_checker.health_checker import HealthChecker
 from .monitoring.prometheus_metrics_handler import PrometheusMetricsHandler
+from .monitoring.reported_state_handler import ReportedStateHandler
 from .monitoring.server import MonitoringServer
 from .monitoring.startup_probe_handler import StartupProbeHandler
 from .state_reconciler import ExecutorStateReconciler
@@ -59,13 +61,6 @@ class Executor:
             protocol = "https"
 
         self._startup_probe_handler = StartupProbeHandler()
-        self._monitoring_server = MonitoringServer(
-            host=monitoring_server_host,
-            port=monitoring_server_port,
-            startup_probe_handler=self._startup_probe_handler,
-            health_probe_handler=HealthCheckHandler(health_checker),
-            metrics_handler=PrometheusMetricsHandler(),
-        )
         self._channel_manager = ChannelManager(
             server_address=grpc_server_addr,
             config_path=config_path,
@@ -96,6 +91,15 @@ class Executor:
             state_reporter=self._state_reporter,
             logger=self._logger,
         )
+        self._monitoring_server = MonitoringServer(
+            host=monitoring_server_host,
+            port=monitoring_server_port,
+            startup_probe_handler=self._startup_probe_handler,
+            health_probe_handler=HealthCheckHandler(health_checker),
+            metrics_handler=PrometheusMetricsHandler(),
+            reported_state_handler=ReportedStateHandler(self._state_reporter),
+            desired_state_handler=DesiredStateHandler(self._state_reconciler),
+        )
         self._run_aio_task: Optional[asyncio.Task] = None
         self._shutdown_aio_task: Optional[asyncio.Task] = None
 

indexify/executor/monitoring/desired_state_handler.py

@@ -0,0 +1,24 @@
+from aiohttp import web
+
+from indexify.proto.executor_api_pb2 import (
+    DesiredExecutorState,
+)
+
+from ..state_reconciler import ExecutorStateReconciler
+from .handler import Handler
+
+
+class DesiredStateHandler(Handler):
+    def __init__(self, state_reconciler: ExecutorStateReconciler):
+        self._state_reconciler = state_reconciler
+
+    async def handle(self, request: web.Request) -> web.Response:
+        desired_state: DesiredExecutorState | None = (
+            self._state_reconciler.get_desired_state()
+        )
+        if desired_state is None:
+            return web.Response(
+                status=200, text="No desired state received from Server yet"
+            )
+        else:
+            return web.Response(text=str(desired_state))

indexify/executor/monitoring/reported_state_handler.py

@@ -0,0 +1,22 @@
+from aiohttp import web
+
+from indexify.proto.executor_api_pb2 import (
+    ReportExecutorStateRequest,
+)
+
+from ..state_reporter import ExecutorStateReporter
+from .handler import Handler
+
+
+class ReportedStateHandler(Handler):
+    def __init__(self, state_reporter: ExecutorStateReporter):
+        self._state_reporter = state_reporter
+
+    async def handle(self, request: web.Request) -> web.Response:
+        request: ReportExecutorStateRequest | None = (
+            self._state_reporter.last_state_report_request()
+        )
+        if request is None:
+            return web.Response(status=200, text="No state reported so far")
+        else:
+            return web.Response(text=str(request))

indexify/executor/monitoring/server.py

@@ -11,6 +11,8 @@ class MonitoringServer:
         startup_probe_handler: Handler,
         health_probe_handler: Handler,
         metrics_handler: Handler,
+        reported_state_handler: Handler,
+        desired_state_handler: Handler,
     ):
         self._host = host
         self._port = port
@@ -20,6 +22,8 @@ class MonitoringServer:
                 web.get("/monitoring/startup", startup_probe_handler.handle),
                 web.get("/monitoring/health", health_probe_handler.handle),
                 web.get("/monitoring/metrics", metrics_handler.handle),
+                web.get("/state/reported", reported_state_handler.handle),
+                web.get("/state/desired", desired_state_handler.handle),
             ]
         )
         self._app_runner: web.AppRunner = web.AppRunner(self._app)

indexify/executor/state_reconciler.py

@@ -88,6 +88,9 @@ class ExecutorStateReconciler:
         )
         self._last_desired_state: Optional[DesiredExecutorState] = None
 
+    def get_desired_state(self) -> Optional[DesiredExecutorState]:
+        return self._last_desired_state
+
     def run(self):
         """Runs the state reconciler.
 

indexify/executor/state_reporter.py

@@ -89,6 +89,10 @@ class ExecutorStateReporter:
         self._pending_task_results: List[TaskResult] = []
         self._pending_fe_updates: List[FunctionExecutorUpdate] = []
         self._function_executor_states: Dict[str, FunctionExecutorState] = {}
+        self._last_state_report_request: Optional[ReportExecutorStateRequest] = None
+
+    def last_state_report_request(self) -> Optional[ReportExecutorStateRequest]:
+        return self._last_state_report_request
 
     def update_executor_status(self, value: ExecutorStatus) -> None:
         self._executor_status = value
@@ -203,7 +207,11 @@ class ExecutorStateReporter:
                 try:
                     state: ExecutorState = self._current_executor_state()
                     update: ExecutorUpdate = self._remove_pending_update()
+                    request: ReportExecutorStateRequest = ReportExecutorStateRequest(
+                        executor_state=state, executor_update=update
+                    )
                     _log_reported_executor_update(update, self._logger)
+                    self._last_state_report_request = request
 
                     with (
                         metric_state_report_rpc_errors.count_exceptions(),
@@ -211,10 +219,7 @@ class ExecutorStateReporter:
                     ):
                         metric_state_report_rpcs.inc()
                         await stub.report_executor_state(
-                            ReportExecutorStateRequest(
-                                executor_state=state, executor_update=update
-                            ),
-                            timeout=_REPORT_RPC_TIMEOUT_SEC,
+                            request, timeout=_REPORT_RPC_TIMEOUT_SEC
                         )
                     self._state_reported_event.set()
                     self._health_checker.server_connection_state_changed(

{indexify-0.4.28.dist-info → indexify-0.4.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.28
+Version: 0.4.29
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.4.28.dist-info → indexify-0.4.29.dist-info}/RECORD

@@ -8,7 +8,7 @@ indexify/executor/blob_store/local_fs_blob_store.py,sha256=nRFawLMbOCCFlCIx2ccmh
 indexify/executor/blob_store/metrics/blob_store.py,sha256=3lmLU8q4Yx87RIYcy56nmFiNQTPY94pB12ht7X6MyhA,3811
 indexify/executor/blob_store/s3_blob_store.py,sha256=wJlDBTTaq48Vp1I0LvP2958b1Xe8esvarkr5PVRawU0,7609
 indexify/executor/channel_manager.py,sha256=ihKfWJmUqQvh4UKXewZLzyJWW_f50P4fnwPqPonrozw,6651
-indexify/executor/executor.py,sha256=rM7BmJDqC_YwdwPfDGFGiFO2WxOW3Nj8Z7rwRw8UcFk,6353
+indexify/executor/executor.py,sha256=--qzHfQnbP70hsV7Y1L6dlgsORAi-Ugwu0W795NgYyc,6645
 indexify/executor/function_allowlist.py,sha256=PCelCW6qIe_2sH11BCKr7LDqarRV5kwNsrfB2EV7Zwo,1772
 indexify/executor/function_executor/function_executor.py,sha256=dTZ8y15ifu7GKmNLU-SQH5M3COa1_8ec_2439h67Pd8,12381
 indexify/executor/function_executor/health_checker.py,sha256=IxE0jnC99K_lvnizFLjXqS1942H8-FNAN4AlhLIjg2Y,6373
@@ -52,6 +52,7 @@ indexify/executor/metrics/channel_manager.py,sha256=1dU9bzF3xqBy1nY9Sc66GfQQWnWZ
 indexify/executor/metrics/executor.py,sha256=8dJXmyGqKlBSrPuyWXW7O2I21uxQ687l-2dYTvz4fmk,398
 indexify/executor/metrics/state_reconciler.py,sha256=BSlRgvgtwih6QcYrsFU5P2ylaXAsC_X70DbzDuv9NsU,584
 indexify/executor/metrics/state_reporter.py,sha256=JvyP_IUfJQetEjzmoWe9q6HCA4Ao1GLocaa7Od_jl2g,550
+indexify/executor/monitoring/desired_state_handler.py,sha256=jmpTSQY6VyDbPFGKMpj-dAv0un5dtQmYc1tonQDBWL8,755
 indexify/executor/monitoring/handler.py,sha256=Cj1cu_LcsAP0tdviqNhoEtGm4h0OJAxxzW9C2YdNXYU,240
 indexify/executor/monitoring/health_check_handler.py,sha256=e1pEtWFKaVs6H57Z4YLejNECrJtC38PweZc7xTJeqVw,695
 indexify/executor/monitoring/health_checker/generic_health_checker.py,sha256=vJRV879GrdZFqwTnM9pRLA97LRMutGz2sWRy-KS-tfg,1493
@@ -59,15 +60,16 @@ indexify/executor/monitoring/health_checker/health_checker.py,sha256=B-Q4KM1iEUS
 indexify/executor/monitoring/health_checker/metrics/health_checker.py,sha256=50JS4JaOdAgSk7iYaBV4J3tGXkRTzmIVR_jVOV66YOc,129
 indexify/executor/monitoring/metrics.py,sha256=5BpNqDBDQiL2K962WDPQU2eSo5zD6I9vF2flGyBejts,7388
 indexify/executor/monitoring/prometheus_metrics_handler.py,sha256=KiGqSf7rkXTfbDwThyXFpFe2jnuZD5q-5SBP_0GDo8Y,591
-indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFseOmHyQpTo,1254
+indexify/executor/monitoring/reported_state_handler.py,sha256=R1C3tk8CF2xh7pbBgKzM1ADReDMEV9CyIRlAZ9NFado,697
+indexify/executor/monitoring/server.py,sha256=aAKzL9J243Q9_41JY-4tSBdFKXR_ZOMz-DEJNtxfYC4,1483
 indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpxWDfed1VHtKKnwvLQ8-frpG98,425
-indexify/executor/state_reconciler.py,sha256=8l4O0IjovnQNI39AQsst4qPb2qFdEncZiEvVl8nLjYI,20248
-indexify/executor/state_reporter.py,sha256=zf5UBhBZVv9SQ1Ju_bY8w6D_t1hBZ5YVXhjeFMEgRms,15208
+indexify/executor/state_reconciler.py,sha256=SUhzcazNmVoJMLYTFWcP911irN3zVzugNx11afiUyws,20356
+indexify/executor/state_reporter.py,sha256=zXb6SvD1yA4tMDWxT_p995y8l490hifXRHX4LjN6WOA,15505
 indexify/proto/executor_api.proto,sha256=YwLeLjyLHhs5qoWLA50uHY2KdKRGfBQBKZwE8VXmzeo,12871
 indexify/proto/executor_api_pb2.py,sha256=vTG1-2Pp4OnTWFD4GYphgJ3cUbTbDjCOKstKrLBXB-E,16472
 indexify/proto/executor_api_pb2.pyi,sha256=-6P-ef-fBJF0CTc4UucIzrDLCBVZpIEhEz2qhexvwjk,23175
 indexify/proto/executor_api_pb2_grpc.py,sha256=u9GEQV4nm_GvApRxjVo806CkgBMBVReb5IVrcaDaliY,7520
-indexify-0.4.28.dist-info/METADATA,sha256=gtwcTBq-IRRXsCSEXv4hO2SJuFWQyC_bTJ5MeJEV-f8,1390
-indexify-0.4.28.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.4.28.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
-indexify-0.4.28.dist-info/RECORD,,
+indexify-0.4.29.dist-info/METADATA,sha256=kF8bbOI0OHcO7tHtlyUAsEvFfrlfymHKqlNR9lt6yuM,1390
+indexify-0.4.29.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.4.29.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
+indexify-0.4.29.dist-info/RECORD,,