indexify 0.3.3__tar.gz → 0.3.5__tar.gz

{indexify-0.3.3 → indexify-0.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.3
+Version: 0.3.5
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -17,13 +17,13 @@ Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: grpcio (==1.68.1)
 Requires-Dist: grpcio-tools (==1.68.1)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
-Requires-Dist: httpx[http2] (>=0.28.1,<0.29.0)
+Requires-Dist: httpx[http2] (>=0.27,<0.28)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: pydantic (==2.10.4)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.7,<0.2.0)
+Requires-Dist: tensorlake (>=0.1.11)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.3 → indexify-0.3.5}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.3"
+version = "0.3.5"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -18,12 +18,12 @@ function-executor = "indexify.function_executor.main:main"
 python = "^3.9"
 structlog = "^24.4.0"
 pyyaml = "^6"
-httpx = { version = "^0.28.1", extras = ["http2"] }
+httpx = { version = "^0.27", extras = ["http2"] }
 grpcio = "1.68.1"
 
 # Function Executor only
 grpcio-tools = "1.68.1"
-tensorlake = "~=0.1.7"
+tensorlake = ">=0.1.11"
 
 # Executor only
 pydantic = "2.10.4"
@@ -36,7 +36,7 @@ nanoid = "^2.0.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.10.0"
-pylint = "^2.4.0"
+pylint = "^3.3.0"
 parameterized = "^0.9.0"
 
 [build-system]

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/cli/cli.py

@@ -219,12 +219,13 @@ def executor(
             )
 
     id = nanoid.generate()
+    executor_version = version("indexify")
     logger.info(
         "starting executor",
         server_addr=server_addr,
         config_path=config_path,
         executor_id=id,
-        executor_version=version("indexify"),
+        executor_version=executor_version,
         executor_cache=executor_cache,
         ports=ports,
         functions=function_uris,
@@ -247,7 +248,8 @@ def executor(
         exit(1)
 
     Executor(
-        id,
+        id=id,
+        version=executor_version,
         server_addr=server_addr,
         config_path=config_path,
         code_path=executor_cache,

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/executor/executor.py

@@ -21,7 +21,8 @@ from .task_runner import TaskInput, TaskOutput, TaskRunner
 class Executor:
     def __init__(
         self,
-        executor_id: str,
+        id: str,
+        version: str,
         code_path: Path,
         function_allowlist: Optional[List[FunctionURI]],
         function_executor_server_factory: FunctionExecutorServerFactory,
@@ -48,15 +49,16 @@ class Executor:
             code_path=code_path, base_url=self._base_url, config_path=config_path
         )
         self._task_fetcher = TaskFetcher(
+            executor_id=id,
+            executor_version=version,
+            function_allowlist=function_allowlist,
             protocol=protocol,
             indexify_server_addr=self._server_addr,
-            executor_id=executor_id,
-            function_allowlist=function_allowlist,
             config_path=config_path,
         )
         self._task_reporter = TaskReporter(
             base_url=self._base_url,
-            executor_id=executor_id,
+            executor_id=id,
             config_path=self._config_path,
         )
 

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/executor/function_executor/function_executor.py

@@ -12,6 +12,7 @@ from indexify.function_executor.proto.function_executor_pb2_grpc import (
 )
 from indexify.utils.http_client import get_httpx_client
 
+from .health_checker import HealthChecker
 from .invocation_state_client import InvocationStateClient
 from .server.function_executor_server import (
     FUNCTION_EXECUTOR_SERVER_READY_TIMEOUT_SEC,
@@ -45,6 +46,7 @@ class FunctionExecutor:
         self._server: Optional[FunctionExecutorServer] = None
         self._channel: Optional[grpc.aio.Channel] = None
         self._invocation_state_client: Optional[InvocationStateClient] = None
+        self._health_checker: Optional[HealthChecker] = None
         self._initialized = False
 
     async def initialize(
@@ -78,6 +80,11 @@ class FunctionExecutor:
             )
             await self._invocation_state_client.start()
 
+            self._health_checker = HealthChecker(
+                stub=stub,
+                logger=self._logger,
+            )
+
             self._initialized = True
         except Exception:
             await self.destroy()
@@ -91,10 +98,21 @@ class FunctionExecutor:
         self._check_initialized()
         return self._invocation_state_client
 
+    def health_checker(self) -> HealthChecker:
+        self._check_initialized()
+        return self._health_checker
+
     async def destroy(self):
         """Destroys all resources owned by this FunctionExecutor.
 
         Never raises any exceptions but logs them."""
+        try:
+            if self._health_checker is not None:
+                self._health_checker.stop()
+                self._health_checker = None
+        except Exception as e:
+            self._logger.error("failed to stop HealthChecker", exc_info=e)
+
         try:
             if self._invocation_state_client is not None:
                 await self._invocation_state_client.destroy()

indexify-0.3.5/src/indexify/executor/function_executor/health_checker.py

@@ -0,0 +1,80 @@
+import asyncio
+from collections.abc import Awaitable, Callable
+from typing import Any, Optional
+
+from grpc.aio import AioRpcError
+
+from indexify.function_executor.proto.function_executor_pb2 import (
+    HealthCheckRequest,
+    HealthCheckResponse,
+)
+from indexify.function_executor.proto.function_executor_pb2_grpc import (
+    FunctionExecutorStub,
+)
+
+from .server.client_configuration import HEALTH_CHECK_TIMEOUT_SEC
+
+HEALTH_CHECK_POLL_PERIOD_SEC = 10
+
+
+class HealthChecker:
+    def __init__(self, stub: FunctionExecutorStub, logger: Any):
+        self._stub: FunctionExecutorStub = stub
+        self._logger: Any = logger.bind(module=__name__)
+        self._health_check_loop_task: Optional[asyncio.Task] = None
+        self._health_check_failed_callback: Optional[Callable[[], Awaitable[None]]] = (
+            None
+        )
+
+    async def check(self) -> bool:
+        """Runs the health check once and returns the result.
+
+        Does not raise any exceptions."""
+        try:
+            response: HealthCheckResponse = await self._stub.check_health(
+                HealthCheckRequest(), timeout=HEALTH_CHECK_TIMEOUT_SEC
+            )
+            return response.healthy
+        except AioRpcError:
+            return False
+        except Exception as e:
+            self._logger.warning("Got unexpected exception, ignoring", exc_info=e)
+            return False
+
+    def start(self, callback: Callable[[], Awaitable[None]]) -> None:
+        """Starts periodic health checks.
+
+        The supplied callback is an async function called in the calling thread's
+        event loop when the health check fails. The callback is called only once
+        and then health checks are stopped.
+
+        Without a periodic health check a TCP client socket won't detect a server
+        socket problem (e.g. it's closed due to server crash) because there are no
+        TCP packets floating between them.
+
+        Does not raise any exceptions.
+        """
+        if self._health_check_loop_task is not None:
+            return
+
+        self._health_check_failed_callback = callback
+        self._health_check_loop_task = asyncio.create_task(self._health_check_loop())
+
+    def stop(self) -> None:
+        """Stops the periodic health checks.
+
+        Does not raise any exceptions."""
+        if self._health_check_loop_task is None:
+            return
+
+        self._health_check_loop_task.cancel()
+        self._health_check_loop_task = None
+
+    async def _health_check_loop(self) -> None:
+        while True:
+            if not await self.check():
+                break
+            await asyncio.sleep(HEALTH_CHECK_POLL_PERIOD_SEC)
+
+        asyncio.create_task(self._health_check_failed_callback())
+        self._health_check_loop_task = None

indexify-0.3.5/src/indexify/executor/function_executor/server/client_configuration.py

@@ -0,0 +1,20 @@
+# We send function inputs and outputs over gRPC.
+# -1 means unlimited. We don't want to limit the size of data customers are using.
+# The effective max message size in this case is about 1.9 GB, see the max payload test.
+# This is due to internal hard gRPC limits. When we want to increase the message sizes
+# we'll have to implement chunking for large messages.
+_MAX_GRPC_MESSAGE_LENGTH = -1
+
+# Optimize the channels for low latency connection establishement as we are running on the same host.
+_RECONNECT_BACKOFF_MS = 100
+
+GRPC_CHANNEL_OPTIONS = [
+    ("grpc.max_receive_message_length", _MAX_GRPC_MESSAGE_LENGTH),
+    ("grpc.max_send_message_length", _MAX_GRPC_MESSAGE_LENGTH),
+    ("grpc.min_reconnect_backoff_ms", _RECONNECT_BACKOFF_MS),
+    ("grpc.max_reconnect_backoff_ms", _RECONNECT_BACKOFF_MS),
+    ("grpc.initial_reconnect_backoff_ms", _RECONNECT_BACKOFF_MS),
+]
+
+# If a health check takes more than this duration then the server is considered unhealthy.
+HEALTH_CHECK_TIMEOUT_SEC = 5

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/executor/function_executor/server/subprocess_function_executor_server.py

@@ -3,8 +3,7 @@ from typing import Any
 
 import grpc
 
-from indexify.function_executor.proto.configuration import GRPC_CHANNEL_OPTIONS
-
+from .client_configuration import GRPC_CHANNEL_OPTIONS
 from .function_executor_server import FunctionExecutorServer
 
 

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py

@@ -22,11 +22,6 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
     async def create(
         self, config: FunctionExecutorServerConfiguration, logger: Any
     ) -> SubprocessFunctionExecutorServer:
-        if config.image_uri is not None:
-            raise ValueError(
-                "SubprocessFunctionExecutorServerFactory doesn't support container images"
-            )
-
         logger = logger.bind(module=__name__)
         port: Optional[int] = None
 

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/executor/function_executor/single_task_runner.py

@@ -1,6 +1,8 @@
+from collections.abc import Awaitable, Callable
 from typing import Any, Optional
 
 import grpc
+from grpc.aio import AioRpcError
 
 from indexify.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
@@ -52,6 +54,12 @@ class SingleTaskRunner:
         if self._state.is_shutdown:
             raise RuntimeError("Function Executor state is shutting down.")
 
+        # If Function Executor is not healthy then recreate it.
+        if self._state.function_executor is not None:
+            if not await self._state.function_executor.health_checker().check():
+                self._logger.error("Health check failed, destroying FunctionExecutor.")
+                await self._state.destroy_function_executor()
+
         if self._state.function_executor is None:
             try:
                 await self._create_function_executor()
@@ -110,13 +118,23 @@ class SingleTaskRunner:
         async with _RunningTaskContextManager(
             invocation_id=self._task_input.task.invocation_id,
             task_id=self._task_input.task.id,
+            health_check_failed_callback=self._health_check_failed_callback,
             function_executor_state=self._state,
         ):
+            # If this RPC failed due to customer code crashing the server we won't be
+            # able to detect this. We'll treat this as our own error for now and thus
+            # let the AioRpcError to be raised here.
             response: RunTaskResponse = await FunctionExecutorStub(channel).run_task(
                 request
             )
             return _task_output(task=self._task_input.task, response=response)
 
+    async def _health_check_failed_callback(self):
+        # The Function Executor needs to get recreated on next task run.
+        self._logger.error("Health check failed, destroying FunctionExecutor.")
+        async with self._state.lock:
+            await self._state.destroy_function_executor()
+
 
 class _RunningTaskContextManager:
     """Performs all the actions required before and after running a task."""
@@ -125,10 +143,14 @@ class _RunningTaskContextManager:
         self,
         invocation_id: str,
         task_id: str,
+        health_check_failed_callback: Callable[[], Awaitable[None]],
         function_executor_state: FunctionExecutorState,
     ):
         self._invocation_id: str = invocation_id
         self._task_id: str = task_id
+        self._health_check_failed_callback: Callable[[], Awaitable[None]] = (
+            health_check_failed_callback
+        )
         self._state: FunctionExecutorState = function_executor_state
 
     async def __aenter__(self):
@@ -137,6 +159,9 @@ class _RunningTaskContextManager:
             task_id=self._task_id,
             invocation_id=self._invocation_id,
         )
+        self._state.function_executor.health_checker().start(
+            self._health_check_failed_callback
+        )
         # Unlock the state so other tasks can act depending on it.
         self._state.lock.release()
         return self
@@ -144,9 +169,12 @@ class _RunningTaskContextManager:
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         await self._state.lock.acquire()
         self._state.decrement_running_tasks()
-        self._state.function_executor.invocation_state_client().remove_task_to_invocation_id_entry(
-            task_id=self._task_id
-        )
+        # Health check callback could destroy the FunctionExecutor.
+        if self._state.function_executor is not None:
+            self._state.function_executor.invocation_state_client().remove_task_to_invocation_id_entry(
+                task_id=self._task_id
+            )
+            self._state.function_executor.health_checker().stop()
 
 
 def _task_output(task: Task, response: RunTaskResponse) -> TaskOutput:

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/executor/task_fetcher.py

@@ -16,10 +16,11 @@ class TaskFetcher:
 
     def __init__(
         self,
-        protocol: str,
-        indexify_server_addr: str,
         executor_id: str,
+        executor_version: str,
         function_allowlist: Optional[List[FunctionURI]],
+        protocol: str,
+        indexify_server_addr: str,
         config_path: Optional[str] = None,
     ):
         self._protocol: str = protocol
@@ -30,7 +31,7 @@ class TaskFetcher:
         probe_info: ProbeInfo = RuntimeProbes().probe()
         self._executor_metadata: ExecutorMetadata = ExecutorMetadata(
             id=executor_id,
-            executor_version=version("indexify"),
+            executor_version=executor_version,
             addr="",
             function_allowlist=function_allowlist,
             labels=probe_info.labels,

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/function_executor/handlers/run_function/handler.py

@@ -2,13 +2,12 @@ import io
 import sys
 import traceback
 from contextlib import redirect_stderr, redirect_stdout
-from typing import Any, Union
+from typing import Any
 
 from tensorlake.functions_sdk.functions import (
     FunctionCallResult,
     GraphInvocationContext,
     RouterCallResult,
-    TensorlakeCompute,
     TensorlakeFunctionWrapper,
     TensorlakeRouter,
 )
@@ -26,15 +25,20 @@ class Handler:
         graph_name: str,
         graph_version: str,
         function_name: str,
-        function: Union[TensorlakeCompute, TensorlakeCompute],
         invocation_state: InvocationState,
+        function_wrapper: TensorlakeFunctionWrapper,
         logger: Any,
     ):
+        self._invocation_id: str = request.graph_invocation_id
+        self._graph_name: str = graph_name
+        self._graph_version: str = graph_version
         self._function_name: str = function_name
+        self._invocation_state: InvocationState = invocation_state
         self._logger = logger.bind(
             graph_invocation_id=request.graph_invocation_id,
             task_id=request.task_id,
         )
+        self._function_wrapper = function_wrapper
         self._input_loader = FunctionInputsLoader(request)
         self._response_helper = ResponseHelper(task_id=request.task_id)
         # TODO: use files for stdout, stderr capturing. This puts a natural and thus reasonable
@@ -42,16 +46,6 @@ class Handler:
         self._func_stdout: io.StringIO = io.StringIO()
         self._func_stderr: io.StringIO = io.StringIO()
 
-        self._function_wrapper: TensorlakeFunctionWrapper = TensorlakeFunctionWrapper(
-            indexify_function=function,
-            context=GraphInvocationContext(
-                invocation_id=request.graph_invocation_id,
-                graph_name=graph_name,
-                graph_version=graph_version,
-                invocation_state=invocation_state,
-            ),
-        )
-
     def run(self) -> RunTaskResponse:
         """Runs the task.
 
@@ -81,9 +75,15 @@ class Handler:
             )
 
     def _run_func(self, inputs: FunctionInputs) -> RunTaskResponse:
+        ctx: GraphInvocationContext = GraphInvocationContext(
+            invocation_id=self._invocation_id,
+            graph_name=self._graph_name,
+            graph_version=self._graph_version,
+            invocation_state=self._invocation_state,
+        )
         if _is_router(self._function_wrapper):
             result: RouterCallResult = self._function_wrapper.invoke_router(
-                self._function_name, inputs.input
+                ctx, self._function_name, inputs.input
             )
             return self._response_helper.router_response(
                 result=result,
@@ -92,11 +92,11 @@ class Handler:
             )
         else:
             result: FunctionCallResult = self._function_wrapper.invoke_fn_ser(
-                self._function_name, inputs.input, inputs.init_value
+                ctx, self._function_name, inputs.input, inputs.init_value
             )
             return self._response_helper.function_response(
                 result=result,
-                is_reducer=_func_is_reducer(self._function_wrapper),
+                is_reducer=_function_is_reducer(self._function_wrapper),
                 stdout=self._func_stdout.getvalue(),
                 stderr=self._func_stderr.getvalue(),
             )
@@ -122,5 +122,5 @@ def _is_router(func_wrapper: TensorlakeFunctionWrapper) -> bool:
     )
 
 
-def _func_is_reducer(func_wrapper: TensorlakeFunctionWrapper) -> bool:
+def _function_is_reducer(func_wrapper: TensorlakeFunctionWrapper) -> bool:
     return func_wrapper.indexify_function.accumulate is not None

indexify-0.3.5/src/indexify/function_executor/info.py

@@ -0,0 +1,16 @@
+import importlib.metadata
+import sys
+from typing import Any, Dict
+
+
+def info_response_kv_args() -> Dict[str, Any]:
+    sdk_version = importlib.metadata.version("tensorlake")
+    python_version = (
+        f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
+    )
+    return {
+        "version": "0.1.0",
+        "sdk_version": sdk_version,
+        "sdk_language": "python",
+        "sdk_language_version": python_version,
+    }

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/function_executor/main.py

@@ -10,10 +10,11 @@ import argparse
 
 import structlog
 
+from .info import info_response_kv_args
 from .server import Server
 from .service import Service
 
-logger = structlog.get_logger(module=__name__)
+logger = structlog.get_logger(module=__name__).bind(**info_response_kv_args())
 
 
 def validate_args(args):

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/function_executor/proto/function_executor.proto

@@ -106,6 +106,28 @@ message RunTaskResponse {
     optional bool success = 7;
 }
 
+message HealthCheckRequest {}
+
+message HealthCheckResponse {
+    optional bool healthy = 1;
+}
+
+message InfoRequest {}
+
+message InfoResponse {
+    // Internal version of this Function Executor.
+    // Semantic versioning schema is used with format 0.0.0.
+    // Used to support migrations.
+    optional string version = 1;
+    // The version of the SDK used in this Function Executor to run customer code.
+    optional string sdk_version = 2;
+    // The language of the SDK. Currently supported values:
+    // - "python"
+    optional string sdk_language = 3;
+    // The version of the SDK language. The language's versioning format is used.
+    optional string sdk_language_version = 4;
+}
+
 service FunctionExecutor {
     // Initializes the Function Executor to run tasks
     // for a particular function. This method is called only
@@ -119,4 +141,8 @@ service FunctionExecutor {
     // Executes the task defined in the request.
     // Multiple tasks can be running in parallel.
     rpc run_task(RunTaskRequest) returns (RunTaskResponse);
+    // Health check method to check if the Function Executor is healthy.
+    rpc check_health(HealthCheckRequest) returns (HealthCheckResponse);
+    // Information about this Function Executor.
+    rpc get_info(InfoRequest) returns (InfoResponse);
 }

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/function_executor/proto/function_executor_pb2.py

@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n8indexify/function_executor/proto/function_executor.proto\x12\x19\x66unction_executor_service"i\n\x10SerializedObject\x12\x0f\n\x05\x62ytes\x18\x01 \x01(\x0cH\x00\x12\x10\n\x06string\x18\x02 \x01(\tH\x00\x12\x19\n\x0c\x63ontent_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x06\n\x04\x64\x61taB\x0f\n\r_content_type"\x88\x02\n\x11InitializeRequest\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x03\x88\x01\x01\x12?\n\x05graph\x18\x07 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x04\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x08\n\x06_graph"f\n\x12InitializeResponse\x12\x14\n\x07success\x18\x01 \x01(\x08H\x00\x88\x01\x01\x12\x1b\n\x0e\x63ustomer_error\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_successB\x11\n\x0f_customer_error"\x80\x01\n\x19SetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\x1c\n\x1aSetInvocationStateResponse"5\n\x19GetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_key"\x81\x01\n\x1aGetInvocationStateResponse\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\xf7\x01\n\x16InvocationStateRequest\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07task_id\x18\x02 \x01(\tH\x02\x88\x01\x01\x12\x43\n\x03set\x18\x03 \x01(\x0b\x32\x34.function_executor_service.SetInvocationStateRequestH\x00\x12\x43\n\x03get\x18\x04 \x01(\x0b\x32\x34.function_executor_service.GetInvocationStateRequestH\x00\x42\t\n\x07requestB\r\n\x0b_request_idB\n\n\x08_task_id"\xfb\x01\n\x17InvocationStateResponse\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07success\x18\x02 \x01(\x08H\x02\x88\x01\x01\x12\x44\n\x03set\x18\x03 \x01(\x0b\x32\x35.function_executor_service.SetInvocationStateResponseH\x00\x12\x44\n\x03get\x18\x04 \x01(\x0b\x32\x35.function_executor_service.GetInvocationStateResponseH\x00\x42\n\n\x08responseB\r\n\x0b_request_idB\n\n\x08_success"N\n\x0e\x46unctionOutput\x12<\n\x07outputs\x18\x01 \x03(\x0b\x32+.function_executor_service.SerializedObject"\x1d\n\x0cRouterOutput\x12\r\n\x05\x65\x64ges\x18\x01 \x03(\t"\xda\x03\n\x0eRunTaskRequest\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x14\n\x07task_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12H\n\x0e\x66unction_input\x18\x07 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x06\x88\x01\x01\x12M\n\x13\x66unction_init_value\x18\x08 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x07\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_task_idB\x11\n\x0f_function_inputB\x16\n\x14_function_init_value"\xf1\x02\n\x0fRunTaskResponse\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12G\n\x0f\x66unction_output\x18\x02 \x01(\x0b\x32).function_executor_service.FunctionOutputH\x01\x88\x01\x01\x12\x43\n\rrouter_output\x18\x03 \x01(\x0b\x32\'.function_executor_service.RouterOutputH\x02\x88\x01\x01\x12\x13\n\x06stdout\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06stderr\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x17\n\nis_reducer\x18\x06 \x01(\x08H\x05\x88\x01\x01\x12\x14\n\x07success\x18\x07 \x01(\x08H\x06\x88\x01\x01\x42\n\n\x08_task_idB\x12\n\x10_function_outputB\x10\n\x0e_router_outputB\t\n\x07_stdoutB\t\n\x07_stderrB\r\n\x0b_is_reducerB\n\n\x08_success2\xf2\x02\n\x10\x46unctionExecutor\x12i\n\ninitialize\x12,.function_executor_service.InitializeRequest\x1a-.function_executor_service.InitializeResponse\x12\x8f\x01\n"initialize_invocation_state_server\x12\x32.function_executor_service.InvocationStateResponse\x1a\x31.function_executor_service.InvocationStateRequest(\x01\x30\x01\x12\x61\n\x08run_task\x12).function_executor_service.RunTaskRequest\x1a*.function_executor_service.RunTaskResponseb\x06proto3'
+    b'\n8indexify/function_executor/proto/function_executor.proto\x12\x19\x66unction_executor_service"i\n\x10SerializedObject\x12\x0f\n\x05\x62ytes\x18\x01 \x01(\x0cH\x00\x12\x10\n\x06string\x18\x02 \x01(\tH\x00\x12\x19\n\x0c\x63ontent_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x06\n\x04\x64\x61taB\x0f\n\r_content_type"\x88\x02\n\x11InitializeRequest\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x03\x88\x01\x01\x12?\n\x05graph\x18\x07 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x04\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x08\n\x06_graph"f\n\x12InitializeResponse\x12\x14\n\x07success\x18\x01 \x01(\x08H\x00\x88\x01\x01\x12\x1b\n\x0e\x63ustomer_error\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_successB\x11\n\x0f_customer_error"\x80\x01\n\x19SetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\x1c\n\x1aSetInvocationStateResponse"5\n\x19GetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_key"\x81\x01\n\x1aGetInvocationStateResponse\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\xf7\x01\n\x16InvocationStateRequest\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07task_id\x18\x02 \x01(\tH\x02\x88\x01\x01\x12\x43\n\x03set\x18\x03 \x01(\x0b\x32\x34.function_executor_service.SetInvocationStateRequestH\x00\x12\x43\n\x03get\x18\x04 \x01(\x0b\x32\x34.function_executor_service.GetInvocationStateRequestH\x00\x42\t\n\x07requestB\r\n\x0b_request_idB\n\n\x08_task_id"\xfb\x01\n\x17InvocationStateResponse\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07success\x18\x02 \x01(\x08H\x02\x88\x01\x01\x12\x44\n\x03set\x18\x03 \x01(\x0b\x32\x35.function_executor_service.SetInvocationStateResponseH\x00\x12\x44\n\x03get\x18\x04 \x01(\x0b\x32\x35.function_executor_service.GetInvocationStateResponseH\x00\x42\n\n\x08responseB\r\n\x0b_request_idB\n\n\x08_success"N\n\x0e\x46unctionOutput\x12<\n\x07outputs\x18\x01 \x03(\x0b\x32+.function_executor_service.SerializedObject"\x1d\n\x0cRouterOutput\x12\r\n\x05\x65\x64ges\x18\x01 \x03(\t"\xda\x03\n\x0eRunTaskRequest\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x14\n\x07task_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12H\n\x0e\x66unction_input\x18\x07 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x06\x88\x01\x01\x12M\n\x13\x66unction_init_value\x18\x08 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x07\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_task_idB\x11\n\x0f_function_inputB\x16\n\x14_function_init_value"\xf1\x02\n\x0fRunTaskResponse\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12G\n\x0f\x66unction_output\x18\x02 \x01(\x0b\x32).function_executor_service.FunctionOutputH\x01\x88\x01\x01\x12\x43\n\rrouter_output\x18\x03 \x01(\x0b\x32\'.function_executor_service.RouterOutputH\x02\x88\x01\x01\x12\x13\n\x06stdout\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06stderr\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x17\n\nis_reducer\x18\x06 \x01(\x08H\x05\x88\x01\x01\x12\x14\n\x07success\x18\x07 \x01(\x08H\x06\x88\x01\x01\x42\n\n\x08_task_idB\x12\n\x10_function_outputB\x10\n\x0e_router_outputB\t\n\x07_stdoutB\t\n\x07_stderrB\r\n\x0b_is_reducerB\n\n\x08_success"\x14\n\x12HealthCheckRequest"7\n\x13HealthCheckResponse\x12\x14\n\x07healthy\x18\x01 \x01(\x08H\x00\x88\x01\x01\x42\n\n\x08_healthy"\r\n\x0bInfoRequest"\xc2\x01\n\x0cInfoResponse\x12\x14\n\x07version\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x18\n\x0bsdk_version\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0csdk_language\x18\x03 \x01(\tH\x02\x88\x01\x01\x12!\n\x14sdk_language_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\n\n\x08_versionB\x0e\n\x0c_sdk_versionB\x0f\n\r_sdk_languageB\x17\n\x15_sdk_language_version2\xbe\x04\n\x10\x46unctionExecutor\x12i\n\ninitialize\x12,.function_executor_service.InitializeRequest\x1a-.function_executor_service.InitializeResponse\x12\x8f\x01\n"initialize_invocation_state_server\x12\x32.function_executor_service.InvocationStateResponse\x1a\x31.function_executor_service.InvocationStateRequest(\x01\x30\x01\x12\x61\n\x08run_task\x12).function_executor_service.RunTaskRequest\x1a*.function_executor_service.RunTaskResponse\x12m\n\x0c\x63heck_health\x12-.function_executor_service.HealthCheckRequest\x1a..function_executor_service.HealthCheckResponse\x12[\n\x08get_info\x12&.function_executor_service.InfoRequest\x1a\'.function_executor_service.InfoResponseb\x06proto3'
 )
 
 _globals = globals()
@@ -60,6 +60,14 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_RUNTASKREQUEST"]._serialized_end = 2003
     _globals["_RUNTASKRESPONSE"]._serialized_start = 2006
     _globals["_RUNTASKRESPONSE"]._serialized_end = 2375
-    _globals["_FUNCTIONEXECUTOR"]._serialized_start = 2378
-    _globals["_FUNCTIONEXECUTOR"]._serialized_end = 2748
+    _globals["_HEALTHCHECKREQUEST"]._serialized_start = 2377
+    _globals["_HEALTHCHECKREQUEST"]._serialized_end = 2397
+    _globals["_HEALTHCHECKRESPONSE"]._serialized_start = 2399
+    _globals["_HEALTHCHECKRESPONSE"]._serialized_end = 2454
+    _globals["_INFOREQUEST"]._serialized_start = 2456
+    _globals["_INFOREQUEST"]._serialized_end = 2469
+    _globals["_INFORESPONSE"]._serialized_start = 2472
+    _globals["_INFORESPONSE"]._serialized_end = 2666
+    _globals["_FUNCTIONEXECUTOR"]._serialized_start = 2669
+    _globals["_FUNCTIONEXECUTOR"]._serialized_end = 3243
 # @@protoc_insertion_point(module_scope)

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/function_executor/proto/function_executor_pb2.pyi

@@ -213,3 +213,35 @@ class RunTaskResponse(_message.Message):
         is_reducer: bool = ...,
         success: bool = ...,
     ) -> None: ...
+
+class HealthCheckRequest(_message.Message):
+    __slots__ = ()
+    def __init__(self) -> None: ...
+
+class HealthCheckResponse(_message.Message):
+    __slots__ = ("healthy",)
+    HEALTHY_FIELD_NUMBER: _ClassVar[int]
+    healthy: bool
+    def __init__(self, healthy: bool = ...) -> None: ...
+
+class InfoRequest(_message.Message):
+    __slots__ = ()
+    def __init__(self) -> None: ...
+
+class InfoResponse(_message.Message):
+    __slots__ = ("version", "sdk_version", "sdk_language", "sdk_language_version")
+    VERSION_FIELD_NUMBER: _ClassVar[int]
+    SDK_VERSION_FIELD_NUMBER: _ClassVar[int]
+    SDK_LANGUAGE_FIELD_NUMBER: _ClassVar[int]
+    SDK_LANGUAGE_VERSION_FIELD_NUMBER: _ClassVar[int]
+    version: str
+    sdk_version: str
+    sdk_language: str
+    sdk_language_version: str
+    def __init__(
+        self,
+        version: _Optional[str] = ...,
+        sdk_version: _Optional[str] = ...,
+        sdk_language: _Optional[str] = ...,
+        sdk_language_version: _Optional[str] = ...,
+    ) -> None: ...

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/function_executor/proto/function_executor_pb2_grpc.py

@@ -58,6 +58,18 @@ class FunctionExecutorStub(object):
             response_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.RunTaskResponse.FromString,
             _registered_method=True,
         )
+        self.check_health = channel.unary_unary(
+            "/function_executor_service.FunctionExecutor/check_health",
+            request_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckRequest.SerializeToString,
+            response_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckResponse.FromString,
+            _registered_method=True,
+        )
+        self.get_info = channel.unary_unary(
+            "/function_executor_service.FunctionExecutor/get_info",
+            request_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoRequest.SerializeToString,
+            response_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoResponse.FromString,
+            _registered_method=True,
+        )
 
 
 class FunctionExecutorServicer(object):
@@ -90,6 +102,18 @@ class FunctionExecutorServicer(object):
         context.set_details("Method not implemented!")
         raise NotImplementedError("Method not implemented!")
 
+    def check_health(self, request, context):
+        """Health check method to check if the Function Executor is healthy."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
+    def get_info(self, request, context):
+        """Information about this Function Executor."""
+        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+        context.set_details("Method not implemented!")
+        raise NotImplementedError("Method not implemented!")
+
 
 def add_FunctionExecutorServicer_to_server(servicer, server):
     rpc_method_handlers = {
@@ -108,6 +132,16 @@ def add_FunctionExecutorServicer_to_server(servicer, server):
             request_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.RunTaskRequest.FromString,
             response_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.RunTaskResponse.SerializeToString,
         ),
+        "check_health": grpc.unary_unary_rpc_method_handler(
+            servicer.check_health,
+            request_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckRequest.FromString,
+            response_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckResponse.SerializeToString,
+        ),
+        "get_info": grpc.unary_unary_rpc_method_handler(
+            servicer.get_info,
+            request_deserializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoRequest.FromString,
+            response_serializer=indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoResponse.SerializeToString,
+        ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
         "function_executor_service.FunctionExecutor", rpc_method_handlers
@@ -211,3 +245,63 @@ class FunctionExecutor(object):
             metadata,
             _registered_method=True,
         )
+
+    @staticmethod
+    def check_health(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/function_executor_service.FunctionExecutor/check_health",
+            indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckRequest.SerializeToString,
+            indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.HealthCheckResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True,
+        )
+
+    @staticmethod
+    def get_info(
+        request,
+        target,
+        options=(),
+        channel_credentials=None,
+        call_credentials=None,
+        insecure=False,
+        compression=None,
+        wait_for_ready=None,
+        timeout=None,
+        metadata=None,
+    ):
+        return grpc.experimental.unary_unary(
+            request,
+            target,
+            "/function_executor_service.FunctionExecutor/get_info",
+            indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoRequest.SerializeToString,
+            indexify_dot_function__executor_dot_proto_dot_function__executor__pb2.InfoResponse.FromString,
+            options,
+            channel_credentials,
+            insecure,
+            call_credentials,
+            compression,
+            wait_for_ready,
+            timeout,
+            metadata,
+            _registered_method=True,
+        )

indexify-0.3.3/src/indexify/function_executor/proto/configuration.py → indexify-0.3.5/src/indexify/function_executor/proto/server_configuration.py

@@ -17,5 +17,3 @@ GRPC_SERVER_OPTIONS = [
     ("grpc.max_send_message_length", _MAX_GRPC_MESSAGE_LENGTH),
     ("grpc.so_reuseport", _REUSE_SERVER_PORT),
 ]
-
-GRPC_CHANNEL_OPTIONS = GRPC_SERVER_OPTIONS

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/function_executor/server.py

@@ -2,8 +2,8 @@ from concurrent.futures import ThreadPoolExecutor
 
 import grpc
 
-from .proto.configuration import GRPC_SERVER_OPTIONS
 from .proto.function_executor_pb2_grpc import add_FunctionExecutorServicer_to_server
+from .proto.server_configuration import GRPC_SERVER_OPTIONS
 from .service import Service
 
 # Temporary limit until we have a better way to control this.

{indexify-0.3.3 → indexify-0.3.5}/src/indexify/function_executor/service.py

@@ -1,18 +1,23 @@
-from typing import Iterator, Optional, Union
+from typing import Iterator, Optional
 
 import grpc
 import structlog
-from tensorlake.functions_sdk.functions import TensorlakeCompute
+from tensorlake.functions_sdk.functions import TensorlakeFunctionWrapper
 from tensorlake.functions_sdk.object_serializer import get_serializer
 
 from .handlers.run_function.handler import Handler as RunTaskHandler
 from .handlers.run_function.request_validator import (
     RequestValidator as RunTaskRequestValidator,
 )
+from .info import info_response_kv_args
 from .initialize_request_validator import InitializeRequestValidator
 from .invocation_state.invocation_state_proxy_server import InvocationStateProxyServer
 from .invocation_state.proxied_invocation_state import ProxiedInvocationState
 from .proto.function_executor_pb2 import (
+    HealthCheckRequest,
+    HealthCheckResponse,
+    InfoRequest,
+    InfoResponse,
     InitializeRequest,
     InitializeResponse,
     InvocationStateResponse,
@@ -24,12 +29,14 @@ from .proto.function_executor_pb2_grpc import FunctionExecutorServicer
 
 class Service(FunctionExecutorServicer):
     def __init__(self):
-        self._logger = structlog.get_logger(module=__name__)
+        self._logger = structlog.get_logger(module=__name__).bind(
+            **info_response_kv_args()
+        )
         self._namespace: Optional[str] = None
         self._graph_name: Optional[str] = None
         self._graph_version: Optional[str] = None
         self._function_name: Optional[str] = None
-        self._function: Optional[Union[TensorlakeCompute, TensorlakeCompute]] = None
+        self._function_wrapper: Optional[TensorlakeFunctionWrapper] = None
         self._invocation_state_proxy_server: Optional[InvocationStateProxyServer] = None
 
     def initialize(
@@ -59,8 +66,10 @@ class Service(FunctionExecutorServicer):
         try:
             # Process user controlled input in a try-except block to not treat errors here as our
             # internal platform errors.
+            # TODO: capture stdout and stderr and report exceptions the same way as when we run a task.
             graph = graph_serializer.deserialize(request.graph.bytes)
-            self._function = graph_serializer.deserialize(graph[request.function_name])
+            function = graph_serializer.deserialize(graph[request.function_name])
+            self._function_wrapper = TensorlakeFunctionWrapper(function)
         except Exception as e:
             return InitializeResponse(success=False, customer_error=str(e))
 
@@ -93,10 +102,10 @@ class Service(FunctionExecutorServicer):
             graph_name=self._graph_name,
             graph_version=self._graph_version,
             function_name=self._function_name,
-            function=self._function,
             invocation_state=ProxiedInvocationState(
                 request.task_id, self._invocation_state_proxy_server
             ),
+            function_wrapper=self._function_wrapper,
             logger=self._logger,
         ).run()
 
@@ -120,3 +129,17 @@ class Service(FunctionExecutorServicer):
             raise ValueError(
                 f"This Function Executor is not initialized for this function_name {request.function_name}"
             )
+
+    def check_health(
+        self, request: HealthCheckRequest, context: grpc.ServicerContext
+    ) -> HealthCheckResponse:
+        # This health check validates that the Server:
+        # - Has its process alive (not exited).
+        # - Didn't exhaust its thread pool.
+        # - Is able to communicate over its server socket.
+        return HealthCheckResponse(healthy=True)
+
+    def get_info(
+        self, request: InfoRequest, context: grpc.ServicerContext
+    ) -> InfoResponse:
+        return InfoResponse(**info_response_kv_args())