indexify 0.4.28__py3-none-any.whl → 0.4.30__py3-none-any.whl

indexify/executor/function_executor_controller/{task_output.py → task_allocation_output.py}

@@ -13,16 +13,16 @@ from indexify.proto.executor_api_pb2 import (
 )
 
 
-class TaskMetrics:
-    """Metrics for a task."""
+class TaskAllocationMetrics:
+    """Metrics for a task allocation."""
 
     def __init__(self, counters: Dict[str, int], timers: Dict[str, float]):
         self.counters = counters
         self.timers = timers
 
 
-class TaskOutput:
-    """Result of running a task."""
+class TaskAllocationOutput:
+    """Result of running a task allocation."""
 
     def __init__(
         self,
@@ -34,7 +34,7 @@ class TaskOutput:
         invocation_error_output: Optional[SerializedObjectInsideBLOB] = None,
         uploaded_invocation_error_blob: Optional[BLOB] = None,
         next_functions: List[str] = [],
-        metrics: Optional[TaskMetrics] = None,
+        metrics: Optional[TaskAllocationMetrics] = None,
         execution_start_time: Optional[float] = None,
         execution_end_time: Optional[float] = None,
     ):
@@ -56,9 +56,9 @@ class TaskOutput:
         allocation: TaskAllocation,
         execution_start_time: Optional[float],
         execution_end_time: Optional[float],
-    ) -> "TaskOutput":
-        """Creates a TaskOutput for an internal error."""
-        return TaskOutput(
+    ) -> "TaskAllocationOutput":
+        """Creates a TaskAllocationOutput for an internal error."""
+        return TaskAllocationOutput(
             allocation=allocation,
             outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
             failure_reason=TaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR,
@@ -72,9 +72,9 @@ class TaskOutput:
         allocation: TaskAllocation,
         execution_start_time: Optional[float],
         execution_end_time: Optional[float],
-    ) -> "TaskOutput":
-        """Creates a TaskOutput for a function timeout error."""
-        return TaskOutput(
+    ) -> "TaskAllocationOutput":
+        """Creates a TaskAllocationOutput for a function timeout error."""
+        return TaskAllocationOutput(
             allocation=allocation,
             outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
             failure_reason=TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_TIMEOUT,
@@ -88,10 +88,10 @@ class TaskOutput:
         allocation: TaskAllocation,
         execution_start_time: Optional[float],
         execution_end_time: Optional[float],
-    ) -> "TaskOutput":
-        """Creates a TaskOutput for an unresponsive FE aka grey failure."""
+    ) -> "TaskAllocationOutput":
+        """Creates a TaskAllocationOutput for an unresponsive FE aka grey failure."""
         # When FE is unresponsive we don't know exact cause of the failure.
-        return TaskOutput(
+        return TaskAllocationOutput(
             allocation=allocation,
             outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
             # Treat the grey failure as a function error and thus charge the customer.
@@ -102,14 +102,14 @@ class TaskOutput:
         )
 
     @classmethod
-    def task_cancelled(
+    def task_allocation_cancelled(
         cls,
         allocation: TaskAllocation,
         execution_start_time: Optional[float],
         execution_end_time: Optional[float],
-    ) -> "TaskOutput":
-        """Creates a TaskOutput for the case when task didn't finish because its allocation was removed by Server."""
-        return TaskOutput(
+    ) -> "TaskAllocationOutput":
+        """Creates a TaskAllocationOutput for the case when task allocation didn't finish because its allocation was removed by Server."""
+        return TaskAllocationOutput(
             allocation=allocation,
             outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
             failure_reason=TaskFailureReason.TASK_FAILURE_REASON_TASK_CANCELLED,
@@ -121,9 +121,9 @@ class TaskOutput:
     def function_executor_terminated(
         cls,
         allocation: TaskAllocation,
-    ) -> "TaskOutput":
-        """Creates a TaskOutput for the case when task didn't run because its FE terminated."""
-        return TaskOutput(
+    ) -> "TaskAllocationOutput":
+        """Creates a TaskAllocationOutput for the case when task allocation didn't run because its FE terminated."""
+        return TaskAllocationOutput(
             allocation=allocation,
             outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
             failure_reason=TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
@@ -135,9 +135,9 @@ class TaskOutput:
         allocation: TaskAllocation,
         fe_termination_reason: FunctionExecutorTerminationReason,
         logger: Any,
-    ) -> "TaskOutput":
-        """Creates a TaskOutput for the case when we fail a task that didn't run because its FE startup failed."""
-        return TaskOutput(
+    ) -> "TaskAllocationOutput":
+        """Creates a TaskAllocationOutput for the case when we fail a task allocation that didn't run because its FE startup failed."""
+        return TaskAllocationOutput(
             allocation=allocation,
             outcome_code=TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE,
             failure_reason=_fe_startup_failure_reason_to_task_failure_reason(

indexify/executor/monitoring/desired_state_handler.py

@@ -0,0 +1,24 @@
+from aiohttp import web
+
+from indexify.proto.executor_api_pb2 import (
+    DesiredExecutorState,
+)
+
+from ..state_reconciler import ExecutorStateReconciler
+from .handler import Handler
+
+
+class DesiredStateHandler(Handler):
+    def __init__(self, state_reconciler: ExecutorStateReconciler):
+        self._state_reconciler = state_reconciler
+
+    async def handle(self, request: web.Request) -> web.Response:
+        desired_state: DesiredExecutorState | None = (
+            self._state_reconciler.get_desired_state()
+        )
+        if desired_state is None:
+            return web.Response(
+                status=200, text="No desired state received from Server yet"
+            )
+        else:
+            return web.Response(text=str(desired_state))

indexify/executor/monitoring/reported_state_handler.py

@@ -0,0 +1,22 @@
+from aiohttp import web
+
+from indexify.proto.executor_api_pb2 import (
+    ReportExecutorStateRequest,
+)
+
+from ..state_reporter import ExecutorStateReporter
+from .handler import Handler
+
+
+class ReportedStateHandler(Handler):
+    def __init__(self, state_reporter: ExecutorStateReporter):
+        self._state_reporter = state_reporter
+
+    async def handle(self, request: web.Request) -> web.Response:
+        request: ReportExecutorStateRequest | None = (
+            self._state_reporter.last_state_report_request()
+        )
+        if request is None:
+            return web.Response(status=200, text="No state reported so far")
+        else:
+            return web.Response(text=str(request))

indexify/executor/monitoring/server.py

@@ -11,6 +11,8 @@ class MonitoringServer:
         startup_probe_handler: Handler,
         health_probe_handler: Handler,
         metrics_handler: Handler,
+        reported_state_handler: Handler,
+        desired_state_handler: Handler,
     ):
         self._host = host
         self._port = port
@@ -20,6 +22,8 @@ class MonitoringServer:
                 web.get("/monitoring/startup", startup_probe_handler.handle),
                 web.get("/monitoring/health", health_probe_handler.handle),
                 web.get("/monitoring/metrics", metrics_handler.handle),
+                web.get("/state/reported", reported_state_handler.handle),
+                web.get("/state/desired", desired_state_handler.handle),
             ]
         )
         self._app_runner: web.AppRunner = web.AppRunner(self._app)

indexify/executor/state_reconciler.py

@@ -88,6 +88,9 @@ class ExecutorStateReconciler:
         )
         self._last_desired_state: Optional[DesiredExecutorState] = None
 
+    def get_desired_state(self) -> Optional[DesiredExecutorState]:
+        return self._last_desired_state
+
     def run(self):
         """Runs the state reconciler.
 
@@ -244,6 +247,8 @@ class ExecutorStateReconciler:
             with metric_state_reconciliation_latency.time():
                 metric_state_reconciliations.inc()
                 await self._reconcile_state(last_reconciled_state)
+                # Update the clock regardless of success or failure.
+                # This is to show Server that we actually processed the message.
                 self._state_reporter.update_last_server_clock(
                     last_reconciled_state.clock
                 )
@@ -258,7 +263,7 @@ class ExecutorStateReconciler:
             try:
                 # Reconcile FEs first because Tasks depend on them.
                 self._reconcile_function_executors(desired_state.function_executors)
-                self._reconcile_tasks(desired_state.task_allocations)
+                self._reconcile_task_allocations(desired_state.task_allocations)
                 return
             except Exception as e:
                 self._logger.error(
@@ -380,38 +385,38 @@ class ExecutorStateReconciler:
         self._function_executor_controllers.pop(function_executor_id, None)
         self._shutting_down_fe_ids.discard(function_executor_id)
 
-    def _reconcile_tasks(self, task_allocations: Iterable[TaskAllocation]):
+    def _reconcile_task_allocations(self, task_allocations: Iterable[TaskAllocation]):
         valid_task_allocations: List[TaskAllocation] = self._valid_task_allocations(
             task_allocations
         )
         for task_allocation in valid_task_allocations:
-            self._reconcile_task(task_allocation)
+            self._reconcile_task_allocation(task_allocation)
 
         # Cancel tasks that are no longer in the desired state.
-        # FE ID => [Task ID]
-        desired_task_ids_per_fe: Dict[str, List[str]] = {}
+        # FE ID => [Allocation ID]
+        desired_alloc_ids_per_fe: Dict[str, List[str]] = {}
         for task_allocation in valid_task_allocations:
-            if task_allocation.function_executor_id not in desired_task_ids_per_fe:
-                desired_task_ids_per_fe[task_allocation.function_executor_id] = []
-            desired_task_ids_per_fe[task_allocation.function_executor_id].append(
-                task_allocation.task.id
+            if task_allocation.function_executor_id not in desired_alloc_ids_per_fe:
+                desired_alloc_ids_per_fe[task_allocation.function_executor_id] = []
+            desired_alloc_ids_per_fe[task_allocation.function_executor_id].append(
+                task_allocation.allocation_id
             )
 
         for fe_controller in self._function_executor_controllers.values():
             fe_controller: FunctionExecutorController
-            if fe_controller.function_executor_id() in desired_task_ids_per_fe:
-                desired_fe_task_ids: Set[str] = set(
-                    desired_task_ids_per_fe[fe_controller.function_executor_id()]
+            if fe_controller.function_executor_id() in desired_alloc_ids_per_fe:
+                desired_fe_alloc_ids: Set[str] = set(
+                    desired_alloc_ids_per_fe[fe_controller.function_executor_id()]
                 )
             else:
                 # No tasks desired for this FE, so cancel all its tasks.
-                desired_fe_task_ids: Set[str] = set()
-            actual_fe_task_ids: Set[str] = set(fe_controller.task_ids())
-            task_ids_to_remove: Set[str] = actual_fe_task_ids - desired_fe_task_ids
-            for task_id in task_ids_to_remove:
-                fe_controller.remove_task(task_id)
+                desired_fe_alloc_ids: Set[str] = set()
+            actual_fe_alloc_ids: Set[str] = set(fe_controller.task_allocation_ids())
+            alloc_ids_to_remove: Set[str] = actual_fe_alloc_ids - desired_fe_alloc_ids
+            for alloc_id in alloc_ids_to_remove:
+                fe_controller.remove_task_allocation(alloc_id)
 
-    def _reconcile_task(self, task_allocation: TaskAllocation):
+    def _reconcile_task_allocation(self, task_allocation: TaskAllocation):
         """Reconciles a single TaskAllocation with the desired state.
 
         Doesn't raise any exceptions.
@@ -419,7 +424,9 @@ class ExecutorStateReconciler:
         function_executor_controller: FunctionExecutorController = (
             self._function_executor_controllers[task_allocation.function_executor_id]
         )
-        if function_executor_controller.has_task(task_allocation.task.id):
+        if function_executor_controller.has_task_allocation(
+            task_allocation.allocation_id
+        ):
             # Nothing to do, task already exists and it's immutable.
             return
 

indexify/executor/state_reporter.py

@@ -89,6 +89,10 @@ class ExecutorStateReporter:
         self._pending_task_results: List[TaskResult] = []
         self._pending_fe_updates: List[FunctionExecutorUpdate] = []
         self._function_executor_states: Dict[str, FunctionExecutorState] = {}
+        self._last_state_report_request: Optional[ReportExecutorStateRequest] = None
+
+    def last_state_report_request(self) -> Optional[ReportExecutorStateRequest]:
+        return self._last_state_report_request
 
     def update_executor_status(self, value: ExecutorStatus) -> None:
         self._executor_status = value
@@ -203,7 +207,11 @@ class ExecutorStateReporter:
                 try:
                     state: ExecutorState = self._current_executor_state()
                     update: ExecutorUpdate = self._remove_pending_update()
+                    request: ReportExecutorStateRequest = ReportExecutorStateRequest(
+                        executor_state=state, executor_update=update
+                    )
                     _log_reported_executor_update(update, self._logger)
+                    self._last_state_report_request = request
 
                     with (
                         metric_state_report_rpc_errors.count_exceptions(),
@@ -211,10 +219,7 @@ class ExecutorStateReporter:
                     ):
                         metric_state_report_rpcs.inc()
                         await stub.report_executor_state(
-                            ReportExecutorStateRequest(
-                                executor_state=state, executor_update=update
-                            ),
-                            timeout=_REPORT_RPC_TIMEOUT_SEC,
+                            request, timeout=_REPORT_RPC_TIMEOUT_SEC
                         )
                     self._state_reported_event.set()
                     self._health_checker.server_connection_state_changed(

{indexify-0.4.28.dist-info → indexify-0.4.30.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.28
+Version: 0.4.30
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.12.15,<4.0.0)
-Requires-Dist: boto3 (>=1.40.12,<2.0.0)
+Requires-Dist: boto3 (>=1.40.15,<2.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: httpx[http2] (==0.27.2)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)

{indexify-0.4.28.dist-info → indexify-0.4.30.dist-info}/RECORD

@@ -8,7 +8,7 @@ indexify/executor/blob_store/local_fs_blob_store.py,sha256=nRFawLMbOCCFlCIx2ccmh
 indexify/executor/blob_store/metrics/blob_store.py,sha256=3lmLU8q4Yx87RIYcy56nmFiNQTPY94pB12ht7X6MyhA,3811
 indexify/executor/blob_store/s3_blob_store.py,sha256=wJlDBTTaq48Vp1I0LvP2958b1Xe8esvarkr5PVRawU0,7609
 indexify/executor/channel_manager.py,sha256=ihKfWJmUqQvh4UKXewZLzyJWW_f50P4fnwPqPonrozw,6651
-indexify/executor/executor.py,sha256=rM7BmJDqC_YwdwPfDGFGiFO2WxOW3Nj8Z7rwRw8UcFk,6353
+indexify/executor/executor.py,sha256=--qzHfQnbP70hsV7Y1L6dlgsORAi-Ugwu0W795NgYyc,6645
 indexify/executor/function_allowlist.py,sha256=PCelCW6qIe_2sH11BCKr7LDqarRV5kwNsrfB2EV7Zwo,1772
 indexify/executor/function_executor/function_executor.py,sha256=dTZ8y15ifu7GKmNLU-SQH5M3COa1_8ec_2439h67Pd8,12381
 indexify/executor/function_executor/health_checker.py,sha256=IxE0jnC99K_lvnizFLjXqS1942H8-FNAN4AlhLIjg2Y,6373
@@ -21,28 +21,28 @@ indexify/executor/function_executor/server/function_executor_server.py,sha256=_D
 indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=pZ3tQoaeWP2NDaR-A0PUYmzrBz768U2b9ENBFQG1INg,1814
 indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=JekDOqF7oFD4J6zcN3xB0Dxd1cgpEXMOsb_rKZOeBlI,668
 indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=w5aGQPHWLpixlP9-BbZu6oL_muMA95-hr7WKVxiEL7Q,4303
-indexify/executor/function_executor_controller/__init__.py,sha256=VPuuBEYOKf7OWyPPjy-jGOv-d5xJqHvkJfFT_oj-AsE,492
+indexify/executor/function_executor_controller/__init__.py,sha256=jJQa09kQw05ycJhIWGv5-RoWe8WIymoCki5mo5P3Nyo,523
 indexify/executor/function_executor_controller/aio_utils.py,sha256=nohPk9k38FpZ87y5jgbb-UhUNvf-GRETkyyRBp7WnVw,804
-indexify/executor/function_executor_controller/completed_task_metrics.py,sha256=MhnC-ddgmTK4yTsuZxgTKnqZ-YSVeWn2EhbbiggsSKk,3664
+indexify/executor/function_executor_controller/completed_task_allocation_metrics.py,sha256=eYtV0--vwd_xwdRBTr_FRsbQ3D4bjOuIQ94TjEy5kR8,4137
 indexify/executor/function_executor_controller/create_function_executor.py,sha256=_VLmT9zmo0Hvt4K4WkC8PCB9qNgTv8k9QkwTSAOQRDU,11158
 indexify/executor/function_executor_controller/debug_event_loop.py,sha256=VJOKe_c9HjIDVCjhMY3Yqyeq1tAM1eVa2chZa6CMf-U,1016
 indexify/executor/function_executor_controller/downloads.py,sha256=B2dbaa6osp1_vCQ6WY_9znAca3Z2qqVzQAF2av3v8Pg,5304
-indexify/executor/function_executor_controller/events.py,sha256=M3taTBSxHG5CYWXfvk-BPtcV9bX-VDmSQDaNdGKK7Hk,5633
-indexify/executor/function_executor_controller/finalize_task.py,sha256=letfBqGXPTubvOfbRg7cvdgtvrwkSnSezx4XRknYvKM,6624
-indexify/executor/function_executor_controller/function_executor_controller.py,sha256=euQWkm3JFKaZFE28hwr-3vnbBnlSQuQbd3R7zApkvIU,39810
-indexify/executor/function_executor_controller/loggers.py,sha256=zEY2nt15gboX3SX6Kh1xjeCljZJZSE4lp27qNrg8yPY,3637
-indexify/executor/function_executor_controller/message_validators.py,sha256=24T3Nm8tkyo2XXlcIg6b2e0P5_M26O62jcI8o7nsnVo,3176
-indexify/executor/function_executor_controller/metrics/completed_task_metrics.py,sha256=53EGBCLwCEV-RBBeyLPTElrtcveaEM0Fwxs9NmC1Hn8,2724
+indexify/executor/function_executor_controller/events.py,sha256=Ly8ypjgyPdOThyj9TXN15RZTJ6ipP_nDNiYND9Ghyr4,5795
+indexify/executor/function_executor_controller/finalize_task_allocation.py,sha256=hYCULPN0LWpPJXwHtyLUYSUQOGC3Lp47SQuLLJAO90A,7135
+indexify/executor/function_executor_controller/function_executor_controller.py,sha256=jq4UV_6op9GbjmcjEmSDxVbb_DtUj1gVKF24rmZU4dU,41698
+indexify/executor/function_executor_controller/loggers.py,sha256=KNXlb7n3CssMbYmt5DrH-cUgfYEf38JM1W82uzkYsZY,3556
+indexify/executor/function_executor_controller/message_validators.py,sha256=fxF-sNC1Pf84NmbDb0-Yg6I6OccgjQeWjSkaPkIcip8,3055
+indexify/executor/function_executor_controller/metrics/completed_task_allocation_metrics.py,sha256=iW-is4V4mv9umIQQar_2k_rlwnaTLs5uzCo-TuCylpg,3155
 indexify/executor/function_executor_controller/metrics/downloads.py,sha256=G8UUDfnzmiK_26OvZYTqH0KgNb3kI-0TgzGLFEuSEFc,892
-indexify/executor/function_executor_controller/metrics/finalize_task.py,sha256=KlJ9o3DQ8VSWNBpMrugr0CT7sSZm2J1LH6lvZhzsQ6E,743
-indexify/executor/function_executor_controller/metrics/function_executor_controller.py,sha256=gyIZHbdsNSnrA6z4WDaRAunNolFrbbg1pa8JaL_ODNE,2666
-indexify/executor/function_executor_controller/metrics/prepare_task.py,sha256=7nHuerFWGqRCRqtgpL3vJVs-DHwxwFhBZaGjrtfOlys,764
-indexify/executor/function_executor_controller/metrics/run_task.py,sha256=ZFv_nw5_pKUJoTaavSyzdglQKW4uvC2XyK8S6xi9xLQ,1064
-indexify/executor/function_executor_controller/prepare_task.py,sha256=EPdqidd4MpCvpvgTZKQsJJL3iwfhpwaz37_EY3z_XS0,9170
-indexify/executor/function_executor_controller/run_task.py,sha256=eaESmOz7IQWFT6Q6VbZV_uSB3mjy47s4ihw2BBD3Oa4,15483
-indexify/executor/function_executor_controller/task_info.py,sha256=ufhb4PvQuXyY4JUlddNyN2bJQdUeGlMTMIRlKz_WzXc,1015
-indexify/executor/function_executor_controller/task_input.py,sha256=PHCzqpjzTzw4TJTn6wncon3P08EiTVRJazEYRbTqDu8,876
-indexify/executor/function_executor_controller/task_output.py,sha256=_uf0Wi1K-kaKPXED1RmKQZ9rpmjXFA4ONLn6ZOj2-UE,7127
+indexify/executor/function_executor_controller/metrics/finalize_task_allocation.py,sha256=o2T3j9mTJ1Zjfje1Zuiw2BMOTLoSLyF-wSVhmNEUcbQ,940
+indexify/executor/function_executor_controller/metrics/function_executor_controller.py,sha256=3nMsgtRbPHIleU8FQZqrdcraJd-7rAjqF7i1PcQRyq8,2803
+indexify/executor/function_executor_controller/metrics/prepare_task_allocation.py,sha256=re07otwicIbPBFN43kihKEzuxN-4ZNt74LyrXKtI68I,971
+indexify/executor/function_executor_controller/metrics/run_task_allocation.py,sha256=ZFv_nw5_pKUJoTaavSyzdglQKW4uvC2XyK8S6xi9xLQ,1064
+indexify/executor/function_executor_controller/prepare_task_allocation.py,sha256=uv6_4Qd8DhJUhmp6eaCDrua4j6LLFqOjr6IwYmiOIHQ,9503
+indexify/executor/function_executor_controller/run_task_allocation.py,sha256=5J4BhxBhmvqm9nd1i5YNN6Z_49BGdPafXZw4VWoX71Q,15842
+indexify/executor/function_executor_controller/task_allocation_info.py,sha256=7gqSp90SuoGxpV5K80YrK2YhrLteeh5xSH--eStkX8s,1098
+indexify/executor/function_executor_controller/task_allocation_input.py,sha256=lST2UgjzsDbMAh6G9vsdnb_lRq9s0BObbz4RX3ggXag,897
+indexify/executor/function_executor_controller/task_allocation_output.py,sha256=pJC3FYxFyVMnbv2ci8KR8ONAWtjO48cE39dpM9sdSCA,7403
 indexify/executor/function_executor_controller/terminate_function_executor.py,sha256=GHkMEidd4zbkulFWAeLGX1HsXtZvPJXh4dEusgy2ioA,1731
 indexify/executor/host_resources/host_resources.py,sha256=eUyP05EX7QdOtQ5vbX_KCpvnBS2B7fl06UWeF9Oigns,3813
 indexify/executor/host_resources/nvidia_gpu.py,sha256=uTCkLXnozZSpax8VApt0QMMM9YcBUK9eggYpwmLz09I,3308
@@ -52,6 +52,7 @@ indexify/executor/metrics/channel_manager.py,sha256=1dU9bzF3xqBy1nY9Sc66GfQQWnWZ
 indexify/executor/metrics/executor.py,sha256=8dJXmyGqKlBSrPuyWXW7O2I21uxQ687l-2dYTvz4fmk,398
 indexify/executor/metrics/state_reconciler.py,sha256=BSlRgvgtwih6QcYrsFU5P2ylaXAsC_X70DbzDuv9NsU,584
 indexify/executor/metrics/state_reporter.py,sha256=JvyP_IUfJQetEjzmoWe9q6HCA4Ao1GLocaa7Od_jl2g,550
+indexify/executor/monitoring/desired_state_handler.py,sha256=jmpTSQY6VyDbPFGKMpj-dAv0un5dtQmYc1tonQDBWL8,755
 indexify/executor/monitoring/handler.py,sha256=Cj1cu_LcsAP0tdviqNhoEtGm4h0OJAxxzW9C2YdNXYU,240
 indexify/executor/monitoring/health_check_handler.py,sha256=e1pEtWFKaVs6H57Z4YLejNECrJtC38PweZc7xTJeqVw,695
 indexify/executor/monitoring/health_checker/generic_health_checker.py,sha256=vJRV879GrdZFqwTnM9pRLA97LRMutGz2sWRy-KS-tfg,1493
@@ -59,15 +60,16 @@ indexify/executor/monitoring/health_checker/health_checker.py,sha256=B-Q4KM1iEUS
 indexify/executor/monitoring/health_checker/metrics/health_checker.py,sha256=50JS4JaOdAgSk7iYaBV4J3tGXkRTzmIVR_jVOV66YOc,129
 indexify/executor/monitoring/metrics.py,sha256=5BpNqDBDQiL2K962WDPQU2eSo5zD6I9vF2flGyBejts,7388
 indexify/executor/monitoring/prometheus_metrics_handler.py,sha256=KiGqSf7rkXTfbDwThyXFpFe2jnuZD5q-5SBP_0GDo8Y,591
-indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFseOmHyQpTo,1254
+indexify/executor/monitoring/reported_state_handler.py,sha256=R1C3tk8CF2xh7pbBgKzM1ADReDMEV9CyIRlAZ9NFado,697
+indexify/executor/monitoring/server.py,sha256=aAKzL9J243Q9_41JY-4tSBdFKXR_ZOMz-DEJNtxfYC4,1483
 indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpxWDfed1VHtKKnwvLQ8-frpG98,425
-indexify/executor/state_reconciler.py,sha256=8l4O0IjovnQNI39AQsst4qPb2qFdEncZiEvVl8nLjYI,20248
-indexify/executor/state_reporter.py,sha256=zf5UBhBZVv9SQ1Ju_bY8w6D_t1hBZ5YVXhjeFMEgRms,15208
+indexify/executor/state_reconciler.py,sha256=fA-2hgPR2YFI8wOwLXzURn-OhsZEqNhMzFlksDESclc,20638
+indexify/executor/state_reporter.py,sha256=zXb6SvD1yA4tMDWxT_p995y8l490hifXRHX4LjN6WOA,15505
 indexify/proto/executor_api.proto,sha256=YwLeLjyLHhs5qoWLA50uHY2KdKRGfBQBKZwE8VXmzeo,12871
 indexify/proto/executor_api_pb2.py,sha256=vTG1-2Pp4OnTWFD4GYphgJ3cUbTbDjCOKstKrLBXB-E,16472
 indexify/proto/executor_api_pb2.pyi,sha256=-6P-ef-fBJF0CTc4UucIzrDLCBVZpIEhEz2qhexvwjk,23175
 indexify/proto/executor_api_pb2_grpc.py,sha256=u9GEQV4nm_GvApRxjVo806CkgBMBVReb5IVrcaDaliY,7520
-indexify-0.4.28.dist-info/METADATA,sha256=gtwcTBq-IRRXsCSEXv4hO2SJuFWQyC_bTJ5MeJEV-f8,1390
-indexify-0.4.28.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.4.28.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
-indexify-0.4.28.dist-info/RECORD,,
+indexify-0.4.30.dist-info/METADATA,sha256=rtJwA6hJc1w8kSC0HTsesyum276C4gH1iXKg_jK45rY,1390
+indexify-0.4.30.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.4.30.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
+indexify-0.4.30.dist-info/RECORD,,

indexify/executor/function_executor_controller/completed_task_metrics.py

@@ -1,83 +0,0 @@
-import time
-from typing import Any
-
-from indexify.proto.executor_api_pb2 import (
-    TaskFailureReason,
-    TaskOutcomeCode,
-)
-
-from .metrics.completed_task_metrics import (
-    METRIC_TASKS_COMPLETED_FAILURE_REASON_ALL,
-    METRIC_TASKS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR,
-    METRIC_TASKS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
-    METRIC_TASKS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR,
-    METRIC_TASKS_COMPLETED_FAILURE_REASON_NONE,
-    METRIC_TASKS_COMPLETED_FAILURE_REASON_TASK_CANCELLED,
-    METRIC_TASKS_COMPLETED_FAILURE_REASON_UNKNOWN,
-    METRIC_TASKS_COMPLETED_OUTCOME_CODE_ALL,
-    METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-    METRIC_TASKS_COMPLETED_OUTCOME_CODE_SUCCESS,
-    metric_task_completion_latency,
-    metric_tasks_completed,
-)
-from .task_info import TaskInfo
-
-
-def emit_completed_task_metrics(task_info: TaskInfo, logger: Any) -> None:
-    """Emits Prometheus metrics for a completed task.
-
-    Doesn't raise any exceptions.
-    """
-    logger = logger.bind(module=__name__)
-    metric_task_completion_latency.observe(time.monotonic() - task_info.start_time)
-
-    task_outcome_code: TaskOutcomeCode = task_info.output.outcome_code
-    task_failure_reason: TaskFailureReason = task_info.output.failure_reason
-    metric_tasks_completed.labels(
-        outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_ALL,
-        failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_ALL,
-    ).inc()
-    if task_outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_SUCCESS:
-        metric_tasks_completed.labels(
-            outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_SUCCESS,
-            failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_NONE,
-        ).inc()
-    elif task_outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE:
-        if task_failure_reason == TaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR:
-            metric_tasks_completed.labels(
-                outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-                failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR,
-            ).inc()
-        elif (
-            task_failure_reason
-            == TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED
-        ):
-            metric_tasks_completed.labels(
-                outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-                failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
-            ).inc()
-        elif (
-            task_failure_reason == TaskFailureReason.TASK_FAILURE_REASON_TASK_CANCELLED
-        ):
-            metric_tasks_completed.labels(
-                outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-                failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_TASK_CANCELLED,
-            ).inc()
-        elif task_failure_reason in [
-            TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR,
-            TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_TIMEOUT,
-            TaskFailureReason.TASK_FAILURE_REASON_INVOCATION_ERROR,
-        ]:
-            metric_tasks_completed.labels(
-                outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-                failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR,
-            ).inc()
-        else:
-            metric_tasks_completed.labels(
-                outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-                failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_UNKNOWN,
-            ).inc()
-            logger.warning(
-                "unexpected task failure reason",
-                failure_reason=TaskFailureReason.Name(task_failure_reason),
-            )

indexify/executor/function_executor_controller/metrics/completed_task_metrics.py

@@ -1,68 +0,0 @@
-import prometheus_client
-
-from indexify.executor.monitoring.metrics import (
-    latency_metric_for_customer_controlled_operation,
-)
-
-metric_tasks_completed: prometheus_client.Counter = prometheus_client.Counter(
-    "tasks_completed",
-    "Number of tasks that were completed",
-    ["outcome_code", "failure_reason"],
-)
-METRIC_TASKS_COMPLETED_OUTCOME_CODE_ALL = "all"
-METRIC_TASKS_COMPLETED_OUTCOME_CODE_SUCCESS = "success"
-METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE = "failure"
-
-METRIC_TASKS_COMPLETED_FAILURE_REASON_ALL = "all"
-# Used when the task is successfull.
-METRIC_TASKS_COMPLETED_FAILURE_REASON_NONE = "none"
-# Matches TASK_FAILURE_REASON_UNKNOWN
-METRIC_TASKS_COMPLETED_FAILURE_REASON_UNKNOWN = "unknown"
-# Includes all function errors including timeouts to reduce cardinality.
-METRIC_TASKS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR = "function_error"
-# Includes all internal errors to reduce cardinality.
-METRIC_TASKS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR = "internal_error"
-# Matches TASK_FAILURE_REASON_TASK_CANCELLED
-METRIC_TASKS_COMPLETED_FAILURE_REASON_TASK_CANCELLED = "task_cancelled"
-# Matches TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED
-METRIC_TASKS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED = (
-    "function_executor_terminated"
-)
-
-# Valid combinations of the labels:
-metric_tasks_completed.labels(
-    outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_ALL,
-    failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_ALL,
-)
-metric_tasks_completed.labels(
-    outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_SUCCESS,
-    failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_NONE,
-)
-
-metric_tasks_completed.labels(
-    outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-    failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_UNKNOWN,
-)
-metric_tasks_completed.labels(
-    outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-    failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR,
-)
-metric_tasks_completed.labels(
-    outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-    failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR,
-)
-metric_tasks_completed.labels(
-    outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-    failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_TASK_CANCELLED,
-)
-metric_tasks_completed.labels(
-    outcome_code=METRIC_TASKS_COMPLETED_OUTCOME_CODE_FAILURE,
-    failure_reason=METRIC_TASKS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
-)
-
-metric_task_completion_latency: prometheus_client.Histogram = (
-    latency_metric_for_customer_controlled_operation(
-        "task_completion",
-        "task completion from the moment it got fetched until its output got uploaded to blob store",
-    )
-)

indexify/executor/function_executor_controller/metrics/finalize_task.py

@@ -1,20 +0,0 @@
-import prometheus_client
-
-from indexify.executor.monitoring.metrics import latency_metric_for_fast_operation
-
-# Task finalization metrics.
-metric_task_finalizations: prometheus_client.Counter = prometheus_client.Counter(
-    "task_finalizations",
-    "Number of task finalizations",
-)
-metric_task_finalization_errors: prometheus_client.Counter = prometheus_client.Counter(
-    "task_finalization_errors",
-    "Number of task finalization errors",
-)
-metric_tasks_finalizing: prometheus_client.Gauge = prometheus_client.Gauge(
-    "tasks_finalizing",
-    "Number of tasks currently finalizing",
-)
-metric_task_finalization_latency: prometheus_client.Histogram = (
-    latency_metric_for_fast_operation("task_finalization", "task finalization")
-)

indexify/executor/function_executor_controller/metrics/prepare_task.py

@@ -1,18 +0,0 @@
-import prometheus_client
-
-from indexify.executor.monitoring.metrics import latency_metric_for_fast_operation
-
-metric_task_preparations: prometheus_client.Counter = prometheus_client.Counter(
-    "task_preparations", "Number of task preparations for execution"
-)
-metric_task_preparation_errors: prometheus_client.Counter = prometheus_client.Counter(
-    "task_preparation_errors", "Number of task preparation errors"
-)
-metric_task_preparation_latency: prometheus_client.Histogram = (
-    latency_metric_for_fast_operation(
-        "task_preparation", "task preparation for execution"
-    )
-)
-metric_tasks_getting_prepared: prometheus_client.Gauge = prometheus_client.Gauge(
-    "tasks_getting_prepared", "Number of tasks currently getting prepared for execution"
-)