indexify 0.4.26__py3-none-any.whl → 0.4.27__py3-none-any.whl

indexify/executor/function_executor/function_executor.py

@@ -308,10 +308,7 @@ async def _initialize_server(
     customer_code_timeout_sec: float,
     logger: Any,
 ) -> FunctionExecutorInitializationResult:
-    with (
-        metric_initialize_rpc_errors.count_exceptions(),
-        metric_initialize_rpc_latency.time(),
-    ):
+    with metric_initialize_rpc_latency.time():
         try:
             initialize_response: InitializeResponse = await stub.initialize(
                 initialize_request,
@@ -322,6 +319,9 @@ async def _initialize_server(
                 response=initialize_response,
             )
         except grpc.aio.AioRpcError as e:
+            # Increment the metric manually as we're not raising this exception.
+            metric_initialize_rpc_errors.inc()
+            metric_create_errors.inc()
             if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
                 return FunctionExecutorInitializationResult(
                     is_timeout=True,

indexify/executor/function_executor_controller/aio_utils.py

@@ -0,0 +1,26 @@
+import asyncio
+from typing import Any, Optional
+
+
+async def shielded_await(task: asyncio.Task, logger: Any) -> Any:
+    """Awaits the supplied task and ignores cancellations until it's done.
+
+    Cancels itself if the task is cancelled once the task is done.
+    """
+    cancelled_error: Optional[asyncio.CancelledError] = None
+
+    while not task.done():
+        try:
+            # Shield to make sure that task is not cancelled.
+            await asyncio.shield(task)
+        except asyncio.CancelledError as e:
+            logger.info(
+                "ignoring aio task cancellation until it's finished",
+                task_name=task.get_name(),
+            )
+            cancelled_error = e
+
+    if cancelled_error is not None:
+        raise cancelled_error
+    else:
+        return task.result()

indexify/executor/function_executor_controller/create_function_executor.py

@@ -25,6 +25,7 @@ from indexify.proto.executor_api_pb2 import (
     FunctionExecutorTerminationReason,
 )
 
+from .aio_utils import shielded_await
 from .downloads import download_graph
 from .events import FunctionExecutorCreated
 
@@ -82,8 +83,13 @@ async def create_function_executor(
         logger=logger,
     )
     if fe_created_event.function_executor is None:
+        # _to_fe_created_event doesn't like the FE, destroy it.
+        fe_destroy_task: asyncio.Task = asyncio.create_task(
+            function_executor.destroy(),
+            name=f"destroy function executor {function_executor_description.id}",
+        )
         try:
-            await asyncio.shield(function_executor.destroy())
+            await shielded_await(fe_destroy_task, logger)
         except asyncio.CancelledError:
             # destroy() finished due to the shield, return fe_created_event.
             pass
@@ -230,9 +236,16 @@ async def _create_function_executor(
             )
         )
         return (function_executor, result)
-    except BaseException:  # includes asyncio.CancelledError and anything else
+    except BaseException:
+        fe_destroy_task: asyncio.Task = asyncio.create_task(
+            function_executor.destroy(),
+            name=f"destroy function executor {function_executor_description.id}",
+        )
         # This await is a cancellation point, need to shield to ensure we destroyed the FE.
-        await asyncio.shield(function_executor.destroy())
+        await shielded_await(
+            fe_destroy_task,
+            logger,
+        )
         raise
 
 

indexify/executor/function_executor_controller/function_executor_controller.py

@@ -407,7 +407,7 @@ class FunctionExecutorController:
         """Spawns an aio task for the supplied coroutine.
 
         The coroutine should return an event that will be added to the FE controller events.
-        The coroutine should not raise any exceptions including BaseException and asyncio.CancelledError.
+        The coroutine should not raise any exceptions including BaseException.
         on_exception event will be added to the FE controller events if the aio task raises an unexpected exception.
         on_exception is required to not silently stall the task processing due to an unexpected exception.
         If task_info is not None, the aio task will be associated with the task_info while the aio task is running.
@@ -422,6 +422,14 @@ class FunctionExecutorController:
         async def coroutine_wrapper() -> None:
             try:
                 self._add_event(await aio, source=aio_task_name)
+            except asyncio.CancelledError:
+                # Workaround for scenario when coroutine_wrapper gets cancelled at `await aio` before aio starts.
+                # In this case aio doesn't handle the cancellation and doesn't return the right event.
+                # A fix for this is to cancel aio instead of coroutine_wrapper. We'll need to keep
+                # references to both coroutine_wrapper and aio, cause event loop uses weak references to
+                # tasks. Not doing this for now. Using on_exception is good enough because not started aios don't
+                # need to do anything special on cancellation.
+                self._add_event(on_exception, source=aio_task_name)
             except BaseException as e:
                 logger.error(
                     "unexpected exception in aio task",

indexify/executor/function_executor_controller/terminate_function_executor.py

@@ -4,6 +4,7 @@ from typing import Any, List, Optional
 from indexify.executor.function_executor.function_executor import FunctionExecutor
 from indexify.proto.executor_api_pb2 import FunctionExecutorTerminationReason
 
+from .aio_utils import shielded_await
 from .events import FunctionExecutorTerminated
 
 
@@ -18,7 +19,7 @@ async def terminate_function_executor(
 
     The supplied lock is used to ensure that if a destroy operation is in progress,
     then another caller won't return immediately assuming that the destroy is complete
-    due to its idempotency.
+    due to its idempotency. Ignores cancellations while destroying the function executor.
 
     Doesn't raise any exceptions.
     """
@@ -29,9 +30,12 @@ async def terminate_function_executor(
             logger.info(
                 "destroying function executor",
             )
+            fe_destroy_task: asyncio.Task = asyncio.create_task(
+                function_executor.destroy(),
+                name="destroy function executor",
+            )
             try:
-                # This await is a cancellation point, need to shield to ensure we destroyed the FE.
-                await asyncio.shield(function_executor.destroy())
+                await shielded_await(fe_destroy_task, logger)
             except asyncio.CancelledError:
                 # We actually destroyed the FE so we can return without error.
                 pass

indexify/executor/state_reconciler.py

@@ -193,6 +193,8 @@ class ExecutorStateReconciler:
                     timeout=_DESIRED_EXECUTOR_STATES_TIMEOUT_SEC,
                 )
             except asyncio.TimeoutError:
+                # These log lines really help to debug networking issues. When there are
+                # no networking issues and the fleet is not idle we don't get excessive logging here.
                 self._logger.info(
                     f"No desired state received from Server within {_DESIRED_EXECUTOR_STATES_TIMEOUT_SEC} sec, recreating the stream to ensure it is healthy"
                 )

{indexify-0.4.26.dist-info → indexify-0.4.27.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.26
+Version: 0.4.27
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.4.26.dist-info → indexify-0.4.27.dist-info}/RECORD

@@ -10,7 +10,7 @@ indexify/executor/blob_store/s3_blob_store.py,sha256=wJlDBTTaq48Vp1I0LvP2958b1Xe
 indexify/executor/channel_manager.py,sha256=ihKfWJmUqQvh4UKXewZLzyJWW_f50P4fnwPqPonrozw,6651
 indexify/executor/executor.py,sha256=rM7BmJDqC_YwdwPfDGFGiFO2WxOW3Nj8Z7rwRw8UcFk,6353
 indexify/executor/function_allowlist.py,sha256=PCelCW6qIe_2sH11BCKr7LDqarRV5kwNsrfB2EV7Zwo,1772
-indexify/executor/function_executor/function_executor.py,sha256=MrbGEqsxpzLVOrEAH_Kmm3cW3q_5h-GYUIkenuPvyT4,12288
+indexify/executor/function_executor/function_executor.py,sha256=dTZ8y15ifu7GKmNLU-SQH5M3COa1_8ec_2439h67Pd8,12381
 indexify/executor/function_executor/health_checker.py,sha256=IxE0jnC99K_lvnizFLjXqS1942H8-FNAN4AlhLIjg2Y,6373
 indexify/executor/function_executor/invocation_state_client.py,sha256=q3YWnoTCYCtBrCOBlLd3b13_vwzibFOJJBHHami28Yc,11269
 indexify/executor/function_executor/metrics/function_executor.py,sha256=TDksxLRJr-P9ZKhF2Orsaxzzb4lVIBxFEjd_9Zv53Ng,6313
@@ -22,13 +22,14 @@ indexify/executor/function_executor/server/function_executor_server_factory.py,s
 indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=JekDOqF7oFD4J6zcN3xB0Dxd1cgpEXMOsb_rKZOeBlI,668
 indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=w5aGQPHWLpixlP9-BbZu6oL_muMA95-hr7WKVxiEL7Q,4303
 indexify/executor/function_executor_controller/__init__.py,sha256=VPuuBEYOKf7OWyPPjy-jGOv-d5xJqHvkJfFT_oj-AsE,492
+indexify/executor/function_executor_controller/aio_utils.py,sha256=nohPk9k38FpZ87y5jgbb-UhUNvf-GRETkyyRBp7WnVw,804
 indexify/executor/function_executor_controller/completed_task_metrics.py,sha256=MhnC-ddgmTK4yTsuZxgTKnqZ-YSVeWn2EhbbiggsSKk,3664
-indexify/executor/function_executor_controller/create_function_executor.py,sha256=wCMlECcXZ9FMF0NlXqww6lbHRwy71w204q_Rh6nFoOE,10694
+indexify/executor/function_executor_controller/create_function_executor.py,sha256=_VLmT9zmo0Hvt4K4WkC8PCB9qNgTv8k9QkwTSAOQRDU,11158
 indexify/executor/function_executor_controller/debug_event_loop.py,sha256=VJOKe_c9HjIDVCjhMY3Yqyeq1tAM1eVa2chZa6CMf-U,1016
 indexify/executor/function_executor_controller/downloads.py,sha256=B2dbaa6osp1_vCQ6WY_9znAca3Z2qqVzQAF2av3v8Pg,5304
 indexify/executor/function_executor_controller/events.py,sha256=M3taTBSxHG5CYWXfvk-BPtcV9bX-VDmSQDaNdGKK7Hk,5633
 indexify/executor/function_executor_controller/finalize_task.py,sha256=letfBqGXPTubvOfbRg7cvdgtvrwkSnSezx4XRknYvKM,6624
-indexify/executor/function_executor_controller/function_executor_controller.py,sha256=uBlQgSGIuVlxAQnT7Qs-T-8c5vABeITFCu_oY9-vNmU,39132
+indexify/executor/function_executor_controller/function_executor_controller.py,sha256=WeetrN34i6YPA3z1VQvd_zw0bpodwiSArCb7-rYlm7M,39814
 indexify/executor/function_executor_controller/loggers.py,sha256=zEY2nt15gboX3SX6Kh1xjeCljZJZSE4lp27qNrg8yPY,3637
 indexify/executor/function_executor_controller/message_validators.py,sha256=7Hgu6GItKU5_zz_YHtHT5NlWIHnvof2P-o_21pto9vU,3128
 indexify/executor/function_executor_controller/metrics/completed_task_metrics.py,sha256=53EGBCLwCEV-RBBeyLPTElrtcveaEM0Fwxs9NmC1Hn8,2724
@@ -42,7 +43,7 @@ indexify/executor/function_executor_controller/run_task.py,sha256=kbQGhopCzbARcz
 indexify/executor/function_executor_controller/task_info.py,sha256=ufhb4PvQuXyY4JUlddNyN2bJQdUeGlMTMIRlKz_WzXc,1015
 indexify/executor/function_executor_controller/task_input.py,sha256=PHCzqpjzTzw4TJTn6wncon3P08EiTVRJazEYRbTqDu8,876
 indexify/executor/function_executor_controller/task_output.py,sha256=_uf0Wi1K-kaKPXED1RmKQZ9rpmjXFA4ONLn6ZOj2-UE,7127
-indexify/executor/function_executor_controller/terminate_function_executor.py,sha256=2eqcpAo5brfMPFtYCpTlI3FJ6ESRU9xUzbXDFU0lcP4,1561
+indexify/executor/function_executor_controller/terminate_function_executor.py,sha256=GHkMEidd4zbkulFWAeLGX1HsXtZvPJXh4dEusgy2ioA,1731
 indexify/executor/host_resources/host_resources.py,sha256=eUyP05EX7QdOtQ5vbX_KCpvnBS2B7fl06UWeF9Oigns,3813
 indexify/executor/host_resources/nvidia_gpu.py,sha256=uTCkLXnozZSpax8VApt0QMMM9YcBUK9eggYpwmLz09I,3308
 indexify/executor/host_resources/nvidia_gpu_allocator.py,sha256=AOcXKglLyRD-GrZzyCoi_oDRJoaOhFKWBSlUOxHeAP8,2114
@@ -60,13 +61,13 @@ indexify/executor/monitoring/metrics.py,sha256=5BpNqDBDQiL2K962WDPQU2eSo5zD6I9vF
 indexify/executor/monitoring/prometheus_metrics_handler.py,sha256=KiGqSf7rkXTfbDwThyXFpFe2jnuZD5q-5SBP_0GDo8Y,591
 indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFseOmHyQpTo,1254
 indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpxWDfed1VHtKKnwvLQ8-frpG98,425
-indexify/executor/state_reconciler.py,sha256=hPTjCUkXQV0HIwa5JczYpb5gvTGonQpkxqOvQXf-QU4,20057
+indexify/executor/state_reconciler.py,sha256=8l4O0IjovnQNI39AQsst4qPb2qFdEncZiEvVl8nLjYI,20248
 indexify/executor/state_reporter.py,sha256=zf5UBhBZVv9SQ1Ju_bY8w6D_t1hBZ5YVXhjeFMEgRms,15208
 indexify/proto/executor_api.proto,sha256=yYLx3LTx2hGDFJX8myCosS3_eAWUMWdsWt59PH7O-QQ,12864
 indexify/proto/executor_api_pb2.py,sha256=ZrJg5ivsv-Kn8QRRsFWupdKCo6Eru6UNR8iYRqo5AhE,16458
 indexify/proto/executor_api_pb2.pyi,sha256=YfKsaIYaePZkpLTk_toAsWfH50xOOYfjjU3aWsci9w0,23151
 indexify/proto/executor_api_pb2_grpc.py,sha256=u9GEQV4nm_GvApRxjVo806CkgBMBVReb5IVrcaDaliY,7520
-indexify-0.4.26.dist-info/METADATA,sha256=ICKghkdJs_FKfudkQCaej0E3aQtwFy6gxauxyuIMXoE,1389
-indexify-0.4.26.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.4.26.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
-indexify-0.4.26.dist-info/RECORD,,
+indexify-0.4.27.dist-info/METADATA,sha256=oBpQeVTH9cqACfgnpRxemwOHt5Y70tqbw-5bPNVB_NQ,1389
+indexify-0.4.27.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.4.27.dist-info/entry_points.txt,sha256=rMJqbE5KPZIXTPIfAtVIM4zpUElqYVgEYd6i7N23zzg,49
+indexify-0.4.27.dist-info/RECORD,,