indexify 0.4.13__py3-none-any.whl → 0.4.14__py3-none-any.whl

indexify/executor/function_executor_controller/function_executor_controller.py

@@ -252,7 +252,7 @@ class FunctionExecutorController:
         """Shutsdown the Function Executor and frees all of its resources.
 
         No task outcomes and outputs are getting reported to Server after this call.
-        Doesn't raise any exceptions. Blocks until the shutdown is complete.
+        Doesn't raise any exceptions. Blocks until the shutdown is complete. Idempotent.
         """
         self._add_event(ShutdownInitiated(), source="shutdown")
         try:
@@ -261,10 +261,9 @@ class FunctionExecutorController:
             pass  # Expected exception on shutdown
         except Exception as e:
             self._logger.error(
-                "function executor controller control loop raised unexpected exception",
+                "function executor controller control loop task raised unexpected exception",
                 exc_info=e,
             )
-        self._logger.info("function executor controller shutdown finished")
 
     def _update_internal_state(self, new_state: _FE_CONTROLLER_STATE) -> None:
         """Updates the internal state of the Function Executor Controller.
@@ -802,7 +801,7 @@ class FunctionExecutorController:
         self._state_reporter.remove_function_executor_state(self.function_executor_id())
         self._state_reporter.schedule_state_report()
 
-        self._logger.info("function executor controller control loop finished")
+        self._logger.info("function executor controller shutdown finished")
         debug_print_events(events=self._events, logger=self._logger)
 
 

indexify/executor/state_reconciler.py

@@ -66,6 +66,7 @@ class ExecutorStateReconciler:
         self._desired_states_reader_task: Optional[asyncio.Task] = None
         self._reconciliation_loop_task: Optional[asyncio.Task] = None
         self._function_executor_controllers: Dict[str, FunctionExecutorController] = {}
+        self._shutting_down_fe_ids: Set[str] = set()
         self._last_server_clock: Optional[int] = None
 
         self._last_desired_state_lock = asyncio.Lock()
@@ -320,14 +321,35 @@ class ExecutorStateReconciler:
             logger.error("failed adding Function Executor", exc_info=e)
 
     def _remove_function_executor_controller(self, function_executor_id: str) -> None:
-        fe_controller: FunctionExecutorController = (
-            self._function_executor_controllers.pop(function_executor_id)
-        )
+        # Don't remove the FE controller from self._function_executor_controllers until
+        # its shutdown is complete. Otherwise, if Server re-adds the FE to desired state
+        # before FE shutdown completes then we'll have two FE controllers for the same
+        # FE ID which results in many bugs.
+        if function_executor_id in self._shutting_down_fe_ids:
+            return
+
+        self._shutting_down_fe_ids.add(function_executor_id)
         asyncio.create_task(
-            fe_controller.shutdown(),
+            self._shutdown_function_executor_controller(function_executor_id),
             name=f"Shutdown Function Executor {function_executor_id}",
         )
 
+    async def _shutdown_function_executor_controller(
+        self, function_executor_id: str
+    ) -> None:
+        # We are not cancelling this aio task in self.shutdown(). Because of this the code here should
+        # not fail if the FE controller is not found in internal data structures. It can be removed
+        # by self.shutdown() at any time while we're running this aio task.
+        fe_controller: Optional[FunctionExecutorController] = (
+            self._function_executor_controllers.get(function_executor_id)
+        )
+        if fe_controller is None:
+            return
+
+        await fe_controller.shutdown()
+        self._function_executor_controllers.pop(function_executor_id, None)
+        self._shutting_down_fe_ids.discard(function_executor_id)
+
     def _reconcile_tasks(self, task_allocations: Iterable[TaskAllocation]):
         valid_task_allocations: List[TaskAllocation] = self._valid_task_allocations(
             task_allocations
@@ -393,8 +415,6 @@ class ExecutorStateReconciler:
                 task_allocation.function_executor_id
                 not in self._function_executor_controllers
             ):
-                # Current policy: don't report task outcomes for tasks that didn't run.
-                # This is required to simplify the protocol so Server doesn't need to care about task states.
                 logger.error(
                     "received TaskAllocation for a Function Executor that doesn't exist, dropping it from desired state"
                 )

{indexify-0.4.13.dist-info → indexify-0.4.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.13
+Version: 0.4.14
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -17,7 +17,7 @@ Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
 Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: tensorlake (==0.2.12)
+Requires-Dist: tensorlake (==0.2.15)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.4.13.dist-info → indexify-0.4.14.dist-info}/RECORD

@@ -27,7 +27,7 @@ indexify/executor/function_executor_controller/create_function_executor.py,sha25
 indexify/executor/function_executor_controller/debug_event_loop.py,sha256=VJOKe_c9HjIDVCjhMY3Yqyeq1tAM1eVa2chZa6CMf-U,1016
 indexify/executor/function_executor_controller/downloads.py,sha256=XjCUmLY_jrI3AlnXC7aDwwKWTvsQjV7I9AXzrbIeY6c,7063
 indexify/executor/function_executor_controller/events.py,sha256=r2K3k9Nnkzh0j6HHZC0DxOdQ3HtCmzt4eN2DIwTa7NM,5456
-indexify/executor/function_executor_controller/function_executor_controller.py,sha256=CpIpZ0l8s5O22RLq_rQOBXRYrDq3YAWFXGQF1BfCbxQ,36415
+indexify/executor/function_executor_controller/function_executor_controller.py,sha256=Ff_1O5m03xdw-Ut04Bi8GmCjmk_VkCVyZ8nBHgl-7CE,36352
 indexify/executor/function_executor_controller/function_executor_startup_output.py,sha256=PXg2r440kqHI3oHGZbb58ehuAuW_fmEdxLTAa-0V3p4,715
 indexify/executor/function_executor_controller/loggers.py,sha256=zEY2nt15gboX3SX6Kh1xjeCljZJZSE4lp27qNrg8yPY,3637
 indexify/executor/function_executor_controller/message_validators.py,sha256=aNiZhYA87pnxUJtZKvKGDt40rfox-TYH2J6mW7o-Pkw,2981
@@ -57,13 +57,13 @@ indexify/executor/monitoring/metrics.py,sha256=Dx2wPcTKvbd5Y5rGOfeyscFtAQ2DZ16_s
 indexify/executor/monitoring/prometheus_metrics_handler.py,sha256=KiGqSf7rkXTfbDwThyXFpFe2jnuZD5q-5SBP_0GDo8Y,591
 indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFseOmHyQpTo,1254
 indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpxWDfed1VHtKKnwvLQ8-frpG98,425
-indexify/executor/state_reconciler.py,sha256=pZ4a0OHeLkPHTuqiwh1G8YXvh23kERVUWZemol6F5E8,17787
+indexify/executor/state_reconciler.py,sha256=tibVENdOTCU-cDoi7v6WTx223wKeBt3LAg2Fc0vwkNo,18768
 indexify/executor/state_reporter.py,sha256=0qs9n2n8bWn77hdW8qsuUqHS7G8iyW7-9_-s8l6aJSU,14195
 indexify/proto/executor_api.proto,sha256=MEsBzpL5tlBd6Yc3ZfheQ60DbXAZWQWGM1dlldhH4i8,12240
 indexify/proto/executor_api_pb2.py,sha256=w2Wj1MEtTihNZ4TvPKPYDjMryO52fPa7A4dKv7vb__s,16236
 indexify/proto/executor_api_pb2.pyi,sha256=AEHW4YFm5FT8f66qKGkPH36szlWgCgXrH3CQ8rgxSps,22754
 indexify/proto/executor_api_pb2_grpc.py,sha256=JpT5K6jiS0NJVNyTt1mAPpyJMXuEGeNN2V6R3KmLHZ4,7607
-indexify-0.4.13.dist-info/METADATA,sha256=ZN16aCArggFFTIlaWelbhaF36t2CEtTAm6dbJxVbFU0,1116
-indexify-0.4.13.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.4.13.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
-indexify-0.4.13.dist-info/RECORD,,
+indexify-0.4.14.dist-info/METADATA,sha256=gjWx_i1QygT4oZTyzDfDVSesR0AxEif3j-iB9LGT0CQ,1116
+indexify-0.4.14.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.4.14.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
+indexify-0.4.14.dist-info/RECORD,,