indexify 0.3.4__tar.gz → 0.3.6__tar.gz

{indexify-0.3.4 → indexify-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.4
+Version: 0.3.6
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -14,8 +14,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: grpcio (==1.68.1)
-Requires-Dist: grpcio-tools (==1.68.1)
+Requires-Dist: grpcio (==1.70.0)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
 Requires-Dist: httpx[http2] (>=0.27,<0.28)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
@@ -23,7 +22,7 @@ Requires-Dist: pydantic (==2.10.4)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.9)
+Requires-Dist: tensorlake (>=0.1.13)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.4 → indexify-0.3.6}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.4"
+version = "0.3.6"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -11,7 +11,6 @@ repository = "https://github.com/tensorlakeai/indexify"
 
 [tool.poetry.scripts]
 indexify-cli = "indexify.cli.cli:app"
-function-executor = "indexify.function_executor.main:main"
 
 [tool.poetry.dependencies]
 # Common dependencies
@@ -19,15 +18,13 @@ python = "^3.9"
 structlog = "^24.4.0"
 pyyaml = "^6"
 httpx = { version = "^0.27", extras = ["http2"] }
-grpcio = "1.68.1"
-
-# Function Executor only
-grpcio-tools = "1.68.1"
-tensorlake = ">=0.1.9"
+grpcio = "1.70.0"
 
 # Executor only
 pydantic = "2.10.4"
 httpx-sse = "^0.4.0"
+# Adds function-executor binary and utils lib.
+tensorlake = ">=0.1.13"
 
 # CLI only
 rich = "^13.9.2"
@@ -36,7 +33,7 @@ nanoid = "^2.0.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.10.0"
-pylint = "^2.4.0"
+pylint = "^3.3.0"
 parameterized = "^0.9.0"
 
 [build-system]

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/cli/cli.py

@@ -1,4 +1,4 @@
-from indexify.utils.logging import (
+from tensorlake.utils.logging import (
     configure_development_mode_logging,
     configure_logging_early,
     configure_production_mode_logging,
@@ -219,12 +219,13 @@ def executor(
             )
 
     id = nanoid.generate()
+    executor_version = version("indexify")
     logger.info(
         "starting executor",
         server_addr=server_addr,
         config_path=config_path,
         executor_id=id,
-        executor_version=version("indexify"),
+        executor_version=executor_version,
         executor_cache=executor_cache,
         ports=ports,
         functions=function_uris,
@@ -247,7 +248,8 @@ def executor(
         exit(1)
 
     Executor(
-        id,
+        id=id,
+        version=executor_version,
         server_addr=server_addr,
         config_path=config_path,
         code_path=executor_cache,
@@ -290,7 +292,7 @@ def _parse_function_uris(uri_strs: Optional[List[str]]) -> Optional[List[Functio
 
 def _create_image(image: Image, python_sdk_path):
     console.print(
-        Text("Creating container for ", style="cyan"),
+        Text("Creating image for ", style="cyan"),
         Text(f"`{image._image_name}`", style="cyan bold"),
     )
     _build_image(image=image, python_sdk_path=python_sdk_path)

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/downloader.py

@@ -4,9 +4,8 @@ from typing import Any, Optional
 
 import httpx
 import structlog
-
-from indexify.function_executor.proto.function_executor_pb2 import SerializedObject
-from indexify.utils.http_client import get_httpx_client
+from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
+from tensorlake.utils.http_client import get_httpx_client
 
 from .api_objects import Task
 

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/executor.py

@@ -4,9 +4,8 @@ from pathlib import Path
 from typing import Any, List, Optional
 
 import structlog
-
-from indexify.function_executor.proto.function_executor_pb2 import SerializedObject
-from indexify.utils.logging import suppress as suppress_logging
+from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
+from tensorlake.utils.logging import suppress as suppress_logging
 
 from .api_objects import FunctionURI, Task
 from .downloader import Downloader
@@ -21,7 +20,8 @@ from .task_runner import TaskInput, TaskOutput, TaskRunner
 class Executor:
     def __init__(
         self,
-        executor_id: str,
+        id: str,
+        version: str,
         code_path: Path,
         function_allowlist: Optional[List[FunctionURI]],
         function_executor_server_factory: FunctionExecutorServerFactory,
@@ -48,15 +48,16 @@ class Executor:
             code_path=code_path, base_url=self._base_url, config_path=config_path
         )
         self._task_fetcher = TaskFetcher(
+            executor_id=id,
+            executor_version=version,
+            function_allowlist=function_allowlist,
             protocol=protocol,
             indexify_server_addr=self._server_addr,
-            executor_id=executor_id,
-            function_allowlist=function_allowlist,
             config_path=config_path,
         )
         self._task_reporter = TaskReporter(
             base_url=self._base_url,
-            executor_id=executor_id,
+            executor_id=id,
             config_path=self._config_path,
         )
 

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/function_executor/function_executor.py

@@ -2,15 +2,14 @@ import asyncio
 from typing import Any, Optional
 
 import grpc
-
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
     InitializeResponse,
 )
-from indexify.function_executor.proto.function_executor_pb2_grpc import (
+from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
-from indexify.utils.http_client import get_httpx_client
+from tensorlake.utils.http_client import get_httpx_client
 
 from .health_checker import HealthChecker
 from .invocation_state_client import InvocationStateClient

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/function_executor/health_checker.py

@@ -3,16 +3,16 @@ from collections.abc import Awaitable, Callable
 from typing import Any, Optional
 
 from grpc.aio import AioRpcError
-
-from indexify.function_executor.proto.configuration import HEALTH_CHECK_TIMEOUT_SEC
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     HealthCheckRequest,
     HealthCheckResponse,
 )
-from indexify.function_executor.proto.function_executor_pb2_grpc import (
+from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
 
+from .server.client_configuration import HEALTH_CHECK_TIMEOUT_SEC
+
 HEALTH_CHECK_POLL_PERIOD_SEC = 10
 
 

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/function_executor/invocation_state_client.py

@@ -3,18 +3,17 @@ from typing import Any, AsyncGenerator, Optional, Union
 
 import grpc
 import httpx
-
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     GetInvocationStateResponse,
     InvocationStateRequest,
     InvocationStateResponse,
     SerializedObject,
     SetInvocationStateResponse,
 )
-from indexify.function_executor.proto.function_executor_pb2_grpc import (
+from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
-from indexify.function_executor.proto.message_validator import MessageValidator
+from tensorlake.function_executor.proto.message_validator import MessageValidator
 
 from ..downloader import serialized_object_from_http_response
 

indexify-0.3.4/src/indexify/function_executor/proto/configuration.py → indexify-0.3.6/src/indexify/executor/function_executor/server/client_configuration.py

@@ -4,21 +4,17 @@
 # This is due to internal hard gRPC limits. When we want to increase the message sizes
 # we'll have to implement chunking for large messages.
 _MAX_GRPC_MESSAGE_LENGTH = -1
-# Disable port reuse: fail if multiple Function Executor Servers attempt to bind to the
-# same port. This happens when Indexify users misconfigure the Servers. Disabling the port
-# reuse results in a clear error message on Server startup instead of obscure errors later
-# while Indexify cluster is serving tasks.
-# If we don't disable port reuse then a random Server gets the requests so wrong tasks get
-# routed to wrong servers.
-_REUSE_SERVER_PORT = 0
 
-GRPC_SERVER_OPTIONS = [
+# Optimize the channels for low latency connection establishement as we are running on the same host.
+_RECONNECT_BACKOFF_MS = 100
+
+GRPC_CHANNEL_OPTIONS = [
     ("grpc.max_receive_message_length", _MAX_GRPC_MESSAGE_LENGTH),
     ("grpc.max_send_message_length", _MAX_GRPC_MESSAGE_LENGTH),
-    ("grpc.so_reuseport", _REUSE_SERVER_PORT),
+    ("grpc.min_reconnect_backoff_ms", _RECONNECT_BACKOFF_MS),
+    ("grpc.max_reconnect_backoff_ms", _RECONNECT_BACKOFF_MS),
+    ("grpc.initial_reconnect_backoff_ms", _RECONNECT_BACKOFF_MS),
 ]
 
-GRPC_CHANNEL_OPTIONS = GRPC_SERVER_OPTIONS
-
 # If a health check takes more than this duration then the server is considered unhealthy.
 HEALTH_CHECK_TIMEOUT_SEC = 5

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/function_executor/server/subprocess_function_executor_server.py

@@ -3,8 +3,7 @@ from typing import Any
 
 import grpc
 
-from indexify.function_executor.proto.configuration import GRPC_CHANNEL_OPTIONS
-
+from .client_configuration import GRPC_CHANNEL_OPTIONS
 from .function_executor_server import FunctionExecutorServer
 
 

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py

@@ -22,11 +22,6 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
     async def create(
         self, config: FunctionExecutorServerConfiguration, logger: Any
     ) -> SubprocessFunctionExecutorServer:
-        if config.image_uri is not None:
-            raise ValueError(
-                "SubprocessFunctionExecutorServerFactory doesn't support container images"
-            )
-
         logger = logger.bind(module=__name__)
         port: Optional[int] = None
 

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/function_executor/single_task_runner.py

@@ -2,14 +2,12 @@ from collections.abc import Awaitable, Callable
 from typing import Any, Optional
 
 import grpc
-from grpc.aio import AioRpcError
-
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
     RunTaskRequest,
     RunTaskResponse,
 )
-from indexify.function_executor.proto.function_executor_pb2_grpc import (
+from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
 

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/function_executor/task_input.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from indexify.function_executor.proto.function_executor_pb2 import SerializedObject
+from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
 
 from ..api_objects import Task
 

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/function_executor/task_output.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     FunctionOutput,
     RouterOutput,
 )

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/task_fetcher.py

@@ -4,8 +4,7 @@ from typing import AsyncGenerator, List, Optional
 
 import structlog
 from httpx_sse import aconnect_sse
-
-from indexify.utils.http_client import get_httpx_client
+from tensorlake.utils.http_client import get_httpx_client
 
 from .api_objects import ExecutorMetadata, FunctionURI, Task
 from .runtime_probes import ProbeInfo, RuntimeProbes
@@ -16,10 +15,11 @@ class TaskFetcher:
 
     def __init__(
         self,
-        protocol: str,
-        indexify_server_addr: str,
         executor_id: str,
+        executor_version: str,
         function_allowlist: Optional[List[FunctionURI]],
+        protocol: str,
+        indexify_server_addr: str,
         config_path: Optional[str] = None,
     ):
         self._protocol: str = protocol
@@ -30,7 +30,7 @@ class TaskFetcher:
         probe_info: ProbeInfo = RuntimeProbes().probe()
         self._executor_metadata: ExecutorMetadata = ExecutorMetadata(
             id=executor_id,
-            executor_version=version("indexify"),
+            executor_version=executor_version,
             addr="",
             function_allowlist=function_allowlist,
             labels=probe_info.labels,

{indexify-0.3.4 → indexify-0.3.6}/src/indexify/executor/task_reporter.py

@@ -3,11 +3,10 @@ from typing import Any, List, Optional, Tuple
 
 import nanoid
 from httpx import Timeout
+from tensorlake.function_executor.proto.function_executor_pb2 import FunctionOutput
+from tensorlake.utils.http_client import get_httpx_client
 
-from indexify.function_executor.proto.function_executor_pb2 import FunctionOutput
-from indexify.utils.http_client import get_httpx_client
-
-from .api_objects import RouterOutput, Task, TaskResult
+from .api_objects import RouterOutput, TaskResult
 from .task_runner import TaskOutput
 
 

indexify-0.3.4/src/indexify/function_executor/README.md

@@ -1,18 +0,0 @@
-## Overview
-
-Function Executor is a process with an API that allows to load and run a customer Function in Indexify.
-Each function run is a task. The tasks can be executed concurrently. The API client controls
-the desired concurrency. Killing the process allows to free all the resources that a loaded customer
-functon is using. This is helpful because the SDK doesn't provide any callbacks to customer code to free
-resources it's using. Even if there was such callback customer code still might misbehave.
-
-## Deployment
-
-A Function Executor is created and destroyed by another component called Executor. It also calls the
-Function Executor APIs. The server is not expected to be deployed or managed manually by Indexify users
-as it's a low level component.
-
-## Threat model
-
-Customer code is assumed to be not trusted. Function Executor must not obtain any credentials that grant
-access to resources not owned by the customer who owns the function.

indexify-0.3.4/src/indexify/function_executor/handlers/run_function/function_inputs_loader.py

@@ -1,53 +0,0 @@
-from typing import Optional
-
-from tensorlake.functions_sdk.data_objects import TensorlakeData
-from tensorlake.functions_sdk.object_serializer import get_serializer
-
-from ...proto.function_executor_pb2 import RunTaskRequest, SerializedObject
-
-
-class FunctionInputs:
-    def __init__(
-        self, input: TensorlakeData, init_value: Optional[TensorlakeData] = None
-    ):
-        self.input = input
-        self.init_value = init_value
-
-
-class FunctionInputsLoader:
-    def __init__(self, request: RunTaskRequest):
-        self._request = request
-
-    def load(self) -> FunctionInputs:
-        return FunctionInputs(
-            input=self._function_input(),
-            init_value=self._accumulator_input(),
-        )
-
-    def _function_input(self) -> TensorlakeData:
-        return _to_indexify_data(
-            self._request.graph_invocation_id, self._request.function_input
-        )
-
-    def _accumulator_input(self) -> Optional[TensorlakeData]:
-        return (
-            _to_indexify_data(
-                self._request.graph_invocation_id, self._request.function_init_value
-            )
-            if self._request.HasField("function_init_value")
-            else None
-        )
-
-
-def _to_indexify_data(
-    input_id: str, serialized_object: SerializedObject
-) -> TensorlakeData:
-    return TensorlakeData(
-        input_id=input_id,
-        payload=(
-            serialized_object.bytes
-            if serialized_object.HasField("bytes")
-            else serialized_object.string
-        ),
-        encoder=get_serializer(serialized_object.content_type).encoding_type,
-    )

indexify-0.3.4/src/indexify/function_executor/handlers/run_function/handler.py

@@ -1,126 +0,0 @@
-import io
-import sys
-import traceback
-from contextlib import redirect_stderr, redirect_stdout
-from typing import Any, Union
-
-from tensorlake.functions_sdk.functions import (
-    FunctionCallResult,
-    GraphInvocationContext,
-    RouterCallResult,
-    TensorlakeCompute,
-    TensorlakeFunctionWrapper,
-    TensorlakeRouter,
-)
-from tensorlake.functions_sdk.invocation_state.invocation_state import InvocationState
-
-from ...proto.function_executor_pb2 import RunTaskRequest, RunTaskResponse
-from .function_inputs_loader import FunctionInputs, FunctionInputsLoader
-from .response_helper import ResponseHelper
-
-
-class Handler:
-    def __init__(
-        self,
-        request: RunTaskRequest,
-        graph_name: str,
-        graph_version: str,
-        function_name: str,
-        function: Union[TensorlakeCompute, TensorlakeCompute],
-        invocation_state: InvocationState,
-        logger: Any,
-    ):
-        self._function_name: str = function_name
-        self._logger = logger.bind(
-            graph_invocation_id=request.graph_invocation_id,
-            task_id=request.task_id,
-        )
-        self._input_loader = FunctionInputsLoader(request)
-        self._response_helper = ResponseHelper(task_id=request.task_id)
-        # TODO: use files for stdout, stderr capturing. This puts a natural and thus reasonable
-        # rate limit on the rate of writes and allows to not consume expensive memory for function logs.
-        self._func_stdout: io.StringIO = io.StringIO()
-        self._func_stderr: io.StringIO = io.StringIO()
-
-        self._function_wrapper: TensorlakeFunctionWrapper = TensorlakeFunctionWrapper(
-            indexify_function=function,
-            context=GraphInvocationContext(
-                invocation_id=request.graph_invocation_id,
-                graph_name=graph_name,
-                graph_version=graph_version,
-                invocation_state=invocation_state,
-            ),
-        )
-
-    def run(self) -> RunTaskResponse:
-        """Runs the task.
-
-        Raises an exception if our own code failed, customer function failure doesn't result in any exception.
-        Details of customer function failure are returned in the response.
-        """
-        self._logger.info("running function")
-        inputs: FunctionInputs = self._input_loader.load()
-        self._flush_logs()
-        return self._run_func_safe_and_captured(inputs)
-
-    def _run_func_safe_and_captured(self, inputs: FunctionInputs) -> RunTaskResponse:
-        """Runs the customer function while capturing what happened in it.
-
-        Function stdout and stderr are captured so they don't get into Function Executor process stdout
-        and stderr. Never throws an Exception. Caller can determine if the function succeeded
-        using the response.
-        """
-        try:
-            with redirect_stdout(self._func_stdout), redirect_stderr(self._func_stderr):
-                return self._run_func(inputs)
-        except Exception:
-            return self._response_helper.failure_response(
-                message=traceback.format_exc(),
-                stdout=self._func_stdout.getvalue(),
-                stderr=self._func_stderr.getvalue(),
-            )
-
-    def _run_func(self, inputs: FunctionInputs) -> RunTaskResponse:
-        if _is_router(self._function_wrapper):
-            result: RouterCallResult = self._function_wrapper.invoke_router(
-                self._function_name, inputs.input
-            )
-            return self._response_helper.router_response(
-                result=result,
-                stdout=self._func_stdout.getvalue(),
-                stderr=self._func_stderr.getvalue(),
-            )
-        else:
-            result: FunctionCallResult = self._function_wrapper.invoke_fn_ser(
-                self._function_name, inputs.input, inputs.init_value
-            )
-            return self._response_helper.function_response(
-                result=result,
-                is_reducer=_func_is_reducer(self._function_wrapper),
-                stdout=self._func_stdout.getvalue(),
-                stderr=self._func_stderr.getvalue(),
-            )
-
-    def _flush_logs(self) -> None:
-        # Flush any logs buffered in memory before running the function with stdout, stderr capture.
-        # Otherwise our logs logged before this point will end up in the function's stdout.
-        # structlog.PrintLogger uses print function. This is why flushing with print works.
-        print("", flush=True)
-        sys.stdout.flush()
-        sys.stderr.flush()
-
-
-def _is_router(func_wrapper: TensorlakeFunctionWrapper) -> bool:
-    """Determines if the function is a router.
-
-    A function is a router if it is an instance of TensorlakeRouter or if it is an TensorlakeRouter class.
-    """
-    return str(
-        type(func_wrapper.indexify_function)
-    ) == "<class 'tensorlake.functions_sdk.functions.TensorlakeRouter'>" or isinstance(
-        func_wrapper.indexify_function, TensorlakeRouter
-    )
-
-
-def _func_is_reducer(func_wrapper: TensorlakeFunctionWrapper) -> bool:
-    return func_wrapper.indexify_function.accumulate is not None

indexify-0.3.4/src/indexify/function_executor/handlers/run_function/request_validator.py

@@ -1,26 +0,0 @@
-from typing import Any
-
-from ...proto.function_executor_pb2 import RunTaskRequest
-from ...proto.message_validator import MessageValidator
-
-
-class RequestValidator:
-    def __init__(self, request: RunTaskRequest):
-        self._request = request
-        self._message_validator = MessageValidator(request)
-
-    def check(self):
-        """Validates the request.
-
-        Raises: ValueError: If the request is invalid.
-        """
-        (
-            self._message_validator.required_field("namespace")
-            .required_field("graph_name")
-            .required_field("graph_version")
-            .required_field("function_name")
-            .required_field("graph_invocation_id")
-            .required_field("task_id")
-            .required_serialized_object("function_input")
-            .optional_serialized_object("function_init_value")
-        )