indexify 0.3.19__py3-none-any.whl → 0.3.21__py3-none-any.whl

indexify/executor/executor.py

@@ -12,6 +12,7 @@ from tensorlake.utils.logging import suppress as suppress_logging
 from indexify.proto.executor_api_pb2 import ExecutorStatus
 
 from .api_objects import FunctionURI, Task
+from .blob_store.blob_store import BLOBStore
 from .downloader import Downloader
 from .executor_flavor import ExecutorFlavor
 from .function_executor.function_executor_states_container import (
@@ -69,6 +70,7 @@ class Executor:
         monitoring_server_host: str,
         monitoring_server_port: int,
         enable_grpc_state_reconciler: bool,
+        blob_store: BLOBStore,
     ):
         self._logger = structlog.get_logger(module=__name__)
         self._is_shutdown: bool = False
@@ -95,7 +97,10 @@ class Executor:
             self._function_executor_states
         )
         self._downloader = Downloader(
-            code_path=code_path, base_url=self._base_url, config_path=config_path
+            code_path=code_path,
+            base_url=self._base_url,
+            blob_store=blob_store,
+            config_path=config_path,
         )
         self._function_allowlist: Optional[List[FunctionURI]] = function_allowlist
         self._function_executor_server_factory = function_executor_server_factory
@@ -123,6 +128,7 @@ class Executor:
             executor_id=id,
             config_path=config_path,
             channel_manager=self._channel_manager,
+            blob_store=blob_store,
         )
 
         # HTTP mode task runner
@@ -189,12 +195,15 @@ class Executor:
                 signum, self.shutdown, asyncio.get_event_loop()
             )
 
-        asyncio.get_event_loop().create_task(self._monitoring_server.run())
-        if self._state_reporter is not None:
-            self._state_reporter.update_executor_status(
-                ExecutorStatus.EXECUTOR_STATUS_RUNNING
-            )
-            asyncio.get_event_loop().create_task(self._state_reporter.run())
+        asyncio.get_event_loop().create_task(
+            self._monitoring_server.run(), name="monitoring server runner"
+        )
+        self._state_reporter.update_executor_status(
+            ExecutorStatus.EXECUTOR_STATUS_RUNNING
+        )
+        asyncio.get_event_loop().create_task(
+            self._state_reporter.run(), name="state reporter runner"
+        )
 
         metric_executor_state.state("running")
         self._startup_probe_handler.set_ready()
@@ -215,7 +224,6 @@ class Executor:
         """Runs the gRPC state reconciler and state reporter.
 
         Never raises any exceptions."""
-        asyncio.create_task(self._state_reporter.run())
         await self._state_reconciler.run()
 
     async def _http_task_runner_loop(self):
@@ -224,11 +232,15 @@ class Executor:
                 async for task in self._task_fetcher.run():
                     metric_tasks_fetched.inc()
                     if not self._is_shutdown:
-                        asyncio.create_task(self._run_task(task))
+                        asyncio.create_task(
+                            self._run_task(task), name="task runner (http mode)"
+                        )
+                self._logger.info("fetching tasks finished, reconnecting in 5 seconds")
             except Exception as e:
                 self._logger.error(
                     "failed fetching tasks, retrying in 5 seconds", exc_info=e
                 )
+            if not self._is_shutdown:
                 await asyncio.sleep(5)
 
     async def _run_task(self, task: Task) -> None:
@@ -250,6 +262,7 @@ class Executor:
                 function_name=task.compute_fn,
                 graph_version=task.graph_version,
                 graph_invocation_id=task.invocation_id,
+                output_payload_uri_prefix=task.output_payload_uri_prefix,
             )
             logger.error("task execution failed", exc_info=e)
 
@@ -293,17 +306,19 @@ class Executor:
             graph_name=task.compute_graph,
             graph_version=task.graph_version,
             logger=logger,
+            data_payload=task.graph_payload,
         )
         input: SerializedObject = await self._downloader.download_input(
             namespace=task.namespace,
             graph_name=task.compute_graph,
             graph_invocation_id=task.invocation_id,
             input_key=task.input_key,
+            data_payload=task.input_payload,
             logger=logger,
         )
         init_value: Optional[SerializedObject] = (
             None
-            if task.reducer_output_id is None
+            if task.reducer_output_id is None and task.reducer_input_payload is None
             else (
                 await self._downloader.download_init_value(
                     namespace=task.namespace,
@@ -311,6 +326,7 @@ class Executor:
                     function_name=task.compute_fn,
                     graph_invocation_id=task.invocation_id,
                     reducer_output_key=task.reducer_output_id,
+                    data_payload=task.reducer_input_payload,
                     logger=logger,
                 )
             )
@@ -380,12 +396,12 @@ class Executor:
         if self._task_runner is not None:
             await self._task_runner.shutdown()
 
-        if self._channel_manager is not None:
-            await self._channel_manager.shutdown()
         if self._state_reporter is not None:
             await self._state_reporter.shutdown()
         if self._state_reconciler is not None:
             await self._state_reconciler.shutdown()
+        if self._channel_manager is not None:
+            await self._channel_manager.destroy()
 
         # We need to shutdown all users of FE states first,
         # otherwise states might disappear unexpectedly and we might
@@ -397,7 +413,7 @@ class Executor:
         # The current task is cancelled, the code after this line will not run.
 
     def shutdown(self, loop):
-        loop.create_task(self._shutdown(loop))
+        loop.create_task(self._shutdown(loop), name="executor shutdown")
 
     def _task_logger(self, task: Task) -> Any:
         return self._logger.bind(

indexify/executor/function_executor/function_executor.py

@@ -318,6 +318,6 @@ async def _initialize_server(
         except grpc.aio.AioRpcError as e:
             if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
                 raise CustomerError(
-                    f"Customer code timeout {customer_code_timeout_sec} sec expired"
+                    f"Customer code timeout of {customer_code_timeout_sec:.3f} sec expired"
                 ) from e
             raise

indexify/executor/function_executor/function_executor_states_container.py

@@ -54,6 +54,11 @@ class FunctionExecutorStatesContainer:
 
             return self._states[id]
 
+    async def get(self, id: str) -> FunctionExecutorState:
+        """Get the state with the given ID. Raises Exception if the state does not exist."""
+        async with self._lock:
+            return self._states[id]
+
     async def __aiter__(self) -> AsyncGenerator[FunctionExecutorState, None]:
         async with self._lock:
             for state in self._states.values():

indexify/executor/function_executor/function_executor_status.py

@@ -34,6 +34,8 @@ class FunctionExecutorStatus(Enum):
     SHUTDOWN = "Shutdown"  # Permanent stop state
 
 
+# TODO: After removing HTTP code simplify state transitions by not allowing to
+# startup an FE after it was destroyed. grpc protocol treats FEs as ephimeral and never revives them.
 def is_status_change_allowed(
     current_status: FunctionExecutorStatus, new_status: FunctionExecutorStatus
 ) -> bool:

indexify/executor/function_executor/health_checker.py

@@ -107,7 +107,9 @@ class HealthChecker:
             return
 
         self._health_check_failed_callback = callback
-        self._health_check_loop_task = asyncio.create_task(self._health_check_loop())
+        self._health_check_loop_task = asyncio.create_task(
+            self._health_check_loop(), name="function executor health checker loop"
+        )
 
     def stop(self) -> None:
         """Stops the periodic health checks.
@@ -126,7 +128,10 @@ class HealthChecker:
                 break
             await asyncio.sleep(HEALTH_CHECK_POLL_PERIOD_SEC)
 
-        asyncio.create_task(self._health_check_failed_callback(result))
+        asyncio.create_task(
+            self._health_check_failed_callback(result),
+            name="function executor health check failure callback",
+        )
         self._health_check_loop_task = None
 
 

indexify/executor/function_executor/invocation_state_client.py

@@ -67,7 +67,8 @@ class InvocationStateClient:
             self._response_generator()
         )
         self._request_loop_task = asyncio.create_task(
-            self._request_loop(server_requests)
+            self._request_loop(server_requests),
+            name="graph invocation state client request processing loop",
         )
 
     def add_task_to_invocation_id_entry(self, task_id: str, invocation_id: str) -> None:
@@ -100,7 +101,8 @@ class InvocationStateClient:
             pass
         except asyncio.CancelledError:
             # This async task was cancelled by destroy(). Normal situation too.
-            pass
+            # This exception should not be suppressed, see Python asyncio docs.
+            raise
         except Exception as e:
             metric_request_read_errors.inc()
             self._logger.error(

indexify/executor/function_executor/single_task_runner.py

@@ -96,6 +96,7 @@ class SingleTaskRunner:
                     graph_invocation_id=self._task_input.task.invocation_id,
                     stderr=str(e),
                     success=False,
+                    output_payload_uri_prefix=self._task_input.task.output_payload_uri_prefix,
                 )
 
         try:
@@ -311,6 +312,7 @@ def _task_output(task: Task, response: RunTaskResponse) -> TaskOutput:
         reducer=response.is_reducer,
         success=response.success,
         metrics=metrics,
+        output_payload_uri_prefix=task.output_payload_uri_prefix,
     )
 
     if response.HasField("function_output"):

indexify/executor/function_executor/task_output.py

@@ -25,6 +25,7 @@ class TaskOutput:
         function_name: str,
         graph_version: str,
         graph_invocation_id: str,
+        output_payload_uri_prefix: Optional[str],
         output_encoding: Optional[str] = None,
         function_output: Optional[FunctionOutput] = None,
         router_output: Optional[RouterOutput] = None,
@@ -50,6 +51,7 @@ class TaskOutput:
         self.is_internal_error = is_internal_error
         self.metrics = metrics
         self.output_encoding = output_encoding
+        self.output_payload_uri_prefix = output_payload_uri_prefix
 
     @classmethod
     def internal_error(
@@ -60,6 +62,7 @@ class TaskOutput:
         function_name: str,
         graph_version: str,
         graph_invocation_id: str,
+        output_payload_uri_prefix: Optional[str],
     ) -> "TaskOutput":
         """Creates a TaskOutput for an internal error."""
         # We are not sharing internal error messages with the customer.
@@ -72,6 +75,7 @@ class TaskOutput:
             graph_invocation_id=graph_invocation_id,
             stderr="Platform failed to execute the function.",
             is_internal_error=True,
+            output_payload_uri_prefix=output_payload_uri_prefix,
         )
 
     @classmethod
@@ -83,6 +87,8 @@ class TaskOutput:
         function_name: str,
         graph_version: str,
         graph_invocation_id: str,
+        timeout_sec: float,
+        output_payload_uri_prefix: Optional[str],
     ) -> "TaskOutput":
         """Creates a TaskOutput for an function timeout error."""
         # Task stdout, stderr is not available.
@@ -93,6 +99,7 @@ class TaskOutput:
             function_name=function_name,
             graph_version=graph_version,
             graph_invocation_id=graph_invocation_id,
-            stderr="Function execution timed out.",
+            stderr=f"Function exceeded its configured timeout of {timeout_sec:.3f} sec.",
             is_internal_error=False,
+            output_payload_uri_prefix=output_payload_uri_prefix,
         )

indexify/executor/grpc/channel_manager.py

@@ -69,6 +69,10 @@ class ChannelManager:
             certificate_chain=certificate_chain,
         )
 
+    async def destroy(self):
+        if self._channel is not None:
+            await self._destroy_locked_channel()
+
     async def get_channel(self) -> grpc.aio.Channel:
         """Returns a channel to the gRPC server.
 
@@ -155,6 +159,3 @@ class ChannelManager:
         except Exception as e:
             self._logger.error("failed closing channel", exc_info=e)
         self._channel = None
-
-    async def shutdown(self):
-        pass