indexify 0.3.5__tar.gz → 0.3.6__tar.gz

{indexify-0.3.5 → indexify-0.3.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.5
+Version: 0.3.6
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -14,8 +14,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: grpcio (==1.68.1)
-Requires-Dist: grpcio-tools (==1.68.1)
+Requires-Dist: grpcio (==1.70.0)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
 Requires-Dist: httpx[http2] (>=0.27,<0.28)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
@@ -23,7 +22,7 @@ Requires-Dist: pydantic (==2.10.4)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.11)
+Requires-Dist: tensorlake (>=0.1.13)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.5 → indexify-0.3.6}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.5"
+version = "0.3.6"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -11,7 +11,6 @@ repository = "https://github.com/tensorlakeai/indexify"
 
 [tool.poetry.scripts]
 indexify-cli = "indexify.cli.cli:app"
-function-executor = "indexify.function_executor.main:main"
 
 [tool.poetry.dependencies]
 # Common dependencies
@@ -19,15 +18,13 @@ python = "^3.9"
 structlog = "^24.4.0"
 pyyaml = "^6"
 httpx = { version = "^0.27", extras = ["http2"] }
-grpcio = "1.68.1"
-
-# Function Executor only
-grpcio-tools = "1.68.1"
-tensorlake = ">=0.1.11"
+grpcio = "1.70.0"
 
 # Executor only
 pydantic = "2.10.4"
 httpx-sse = "^0.4.0"
+# Adds function-executor binary and utils lib.
+tensorlake = ">=0.1.13"
 
 # CLI only
 rich = "^13.9.2"

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/cli/cli.py

@@ -1,4 +1,4 @@
-from indexify.utils.logging import (
+from tensorlake.utils.logging import (
     configure_development_mode_logging,
     configure_logging_early,
     configure_production_mode_logging,
@@ -292,7 +292,7 @@ def _parse_function_uris(uri_strs: Optional[List[str]]) -> Optional[List[Functio
 
 def _create_image(image: Image, python_sdk_path):
     console.print(
-        Text("Creating container for ", style="cyan"),
+        Text("Creating image for ", style="cyan"),
         Text(f"`{image._image_name}`", style="cyan bold"),
     )
     _build_image(image=image, python_sdk_path=python_sdk_path)

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/downloader.py

@@ -4,9 +4,8 @@ from typing import Any, Optional
 
 import httpx
 import structlog
-
-from indexify.function_executor.proto.function_executor_pb2 import SerializedObject
-from indexify.utils.http_client import get_httpx_client
+from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
+from tensorlake.utils.http_client import get_httpx_client
 
 from .api_objects import Task
 

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/executor.py

@@ -4,9 +4,8 @@ from pathlib import Path
 from typing import Any, List, Optional
 
 import structlog
-
-from indexify.function_executor.proto.function_executor_pb2 import SerializedObject
-from indexify.utils.logging import suppress as suppress_logging
+from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
+from tensorlake.utils.logging import suppress as suppress_logging
 
 from .api_objects import FunctionURI, Task
 from .downloader import Downloader

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/function_executor/function_executor.py

@@ -2,15 +2,14 @@ import asyncio
 from typing import Any, Optional
 
 import grpc
-
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
     InitializeResponse,
 )
-from indexify.function_executor.proto.function_executor_pb2_grpc import (
+from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
-from indexify.utils.http_client import get_httpx_client
+from tensorlake.utils.http_client import get_httpx_client
 
 from .health_checker import HealthChecker
 from .invocation_state_client import InvocationStateClient

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/function_executor/health_checker.py

@@ -3,12 +3,11 @@ from collections.abc import Awaitable, Callable
 from typing import Any, Optional
 
 from grpc.aio import AioRpcError
-
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     HealthCheckRequest,
     HealthCheckResponse,
 )
-from indexify.function_executor.proto.function_executor_pb2_grpc import (
+from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
 

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/function_executor/invocation_state_client.py

@@ -3,18 +3,17 @@ from typing import Any, AsyncGenerator, Optional, Union
 
 import grpc
 import httpx
-
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     GetInvocationStateResponse,
     InvocationStateRequest,
     InvocationStateResponse,
     SerializedObject,
     SetInvocationStateResponse,
 )
-from indexify.function_executor.proto.function_executor_pb2_grpc import (
+from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
-from indexify.function_executor.proto.message_validator import MessageValidator
+from tensorlake.function_executor.proto.message_validator import MessageValidator
 
 from ..downloader import serialized_object_from_http_response
 

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/function_executor/single_task_runner.py

@@ -2,14 +2,12 @@ from collections.abc import Awaitable, Callable
 from typing import Any, Optional
 
 import grpc
-from grpc.aio import AioRpcError
-
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
     RunTaskRequest,
     RunTaskResponse,
 )
-from indexify.function_executor.proto.function_executor_pb2_grpc import (
+from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
 

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/function_executor/task_input.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from indexify.function_executor.proto.function_executor_pb2 import SerializedObject
+from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
 
 from ..api_objects import Task
 

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/function_executor/task_output.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from indexify.function_executor.proto.function_executor_pb2 import (
+from tensorlake.function_executor.proto.function_executor_pb2 import (
     FunctionOutput,
     RouterOutput,
 )

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/task_fetcher.py

@@ -4,8 +4,7 @@ from typing import AsyncGenerator, List, Optional
 
 import structlog
 from httpx_sse import aconnect_sse
-
-from indexify.utils.http_client import get_httpx_client
+from tensorlake.utils.http_client import get_httpx_client
 
 from .api_objects import ExecutorMetadata, FunctionURI, Task
 from .runtime_probes import ProbeInfo, RuntimeProbes

{indexify-0.3.5 → indexify-0.3.6}/src/indexify/executor/task_reporter.py

@@ -3,11 +3,10 @@ from typing import Any, List, Optional, Tuple
 
 import nanoid
 from httpx import Timeout
+from tensorlake.function_executor.proto.function_executor_pb2 import FunctionOutput
+from tensorlake.utils.http_client import get_httpx_client
 
-from indexify.function_executor.proto.function_executor_pb2 import FunctionOutput
-from indexify.utils.http_client import get_httpx_client
-
-from .api_objects import RouterOutput, Task, TaskResult
+from .api_objects import RouterOutput, TaskResult
 from .task_runner import TaskOutput
 
 

indexify-0.3.5/src/indexify/function_executor/README.md

@@ -1,18 +0,0 @@
-## Overview
-
-Function Executor is a process with an API that allows to load and run a customer Function in Indexify.
-Each function run is a task. The tasks can be executed concurrently. The API client controls
-the desired concurrency. Killing the process allows to free all the resources that a loaded customer
-functon is using. This is helpful because the SDK doesn't provide any callbacks to customer code to free
-resources it's using. Even if there was such callback customer code still might misbehave.
-
-## Deployment
-
-A Function Executor is created and destroyed by another component called Executor. It also calls the
-Function Executor APIs. The server is not expected to be deployed or managed manually by Indexify users
-as it's a low level component.
-
-## Threat model
-
-Customer code is assumed to be not trusted. Function Executor must not obtain any credentials that grant
-access to resources not owned by the customer who owns the function.

indexify-0.3.5/src/indexify/function_executor/handlers/run_function/function_inputs_loader.py

@@ -1,53 +0,0 @@
-from typing import Optional
-
-from tensorlake.functions_sdk.data_objects import TensorlakeData
-from tensorlake.functions_sdk.object_serializer import get_serializer
-
-from ...proto.function_executor_pb2 import RunTaskRequest, SerializedObject
-
-
-class FunctionInputs:
-    def __init__(
-        self, input: TensorlakeData, init_value: Optional[TensorlakeData] = None
-    ):
-        self.input = input
-        self.init_value = init_value
-
-
-class FunctionInputsLoader:
-    def __init__(self, request: RunTaskRequest):
-        self._request = request
-
-    def load(self) -> FunctionInputs:
-        return FunctionInputs(
-            input=self._function_input(),
-            init_value=self._accumulator_input(),
-        )
-
-    def _function_input(self) -> TensorlakeData:
-        return _to_indexify_data(
-            self._request.graph_invocation_id, self._request.function_input
-        )
-
-    def _accumulator_input(self) -> Optional[TensorlakeData]:
-        return (
-            _to_indexify_data(
-                self._request.graph_invocation_id, self._request.function_init_value
-            )
-            if self._request.HasField("function_init_value")
-            else None
-        )
-
-
-def _to_indexify_data(
-    input_id: str, serialized_object: SerializedObject
-) -> TensorlakeData:
-    return TensorlakeData(
-        input_id=input_id,
-        payload=(
-            serialized_object.bytes
-            if serialized_object.HasField("bytes")
-            else serialized_object.string
-        ),
-        encoder=get_serializer(serialized_object.content_type).encoding_type,
-    )

indexify-0.3.5/src/indexify/function_executor/handlers/run_function/handler.py

@@ -1,126 +0,0 @@
-import io
-import sys
-import traceback
-from contextlib import redirect_stderr, redirect_stdout
-from typing import Any
-
-from tensorlake.functions_sdk.functions import (
-    FunctionCallResult,
-    GraphInvocationContext,
-    RouterCallResult,
-    TensorlakeFunctionWrapper,
-    TensorlakeRouter,
-)
-from tensorlake.functions_sdk.invocation_state.invocation_state import InvocationState
-
-from ...proto.function_executor_pb2 import RunTaskRequest, RunTaskResponse
-from .function_inputs_loader import FunctionInputs, FunctionInputsLoader
-from .response_helper import ResponseHelper
-
-
-class Handler:
-    def __init__(
-        self,
-        request: RunTaskRequest,
-        graph_name: str,
-        graph_version: str,
-        function_name: str,
-        invocation_state: InvocationState,
-        function_wrapper: TensorlakeFunctionWrapper,
-        logger: Any,
-    ):
-        self._invocation_id: str = request.graph_invocation_id
-        self._graph_name: str = graph_name
-        self._graph_version: str = graph_version
-        self._function_name: str = function_name
-        self._invocation_state: InvocationState = invocation_state
-        self._logger = logger.bind(
-            graph_invocation_id=request.graph_invocation_id,
-            task_id=request.task_id,
-        )
-        self._function_wrapper = function_wrapper
-        self._input_loader = FunctionInputsLoader(request)
-        self._response_helper = ResponseHelper(task_id=request.task_id)
-        # TODO: use files for stdout, stderr capturing. This puts a natural and thus reasonable
-        # rate limit on the rate of writes and allows to not consume expensive memory for function logs.
-        self._func_stdout: io.StringIO = io.StringIO()
-        self._func_stderr: io.StringIO = io.StringIO()
-
-    def run(self) -> RunTaskResponse:
-        """Runs the task.
-
-        Raises an exception if our own code failed, customer function failure doesn't result in any exception.
-        Details of customer function failure are returned in the response.
-        """
-        self._logger.info("running function")
-        inputs: FunctionInputs = self._input_loader.load()
-        self._flush_logs()
-        return self._run_func_safe_and_captured(inputs)
-
-    def _run_func_safe_and_captured(self, inputs: FunctionInputs) -> RunTaskResponse:
-        """Runs the customer function while capturing what happened in it.
-
-        Function stdout and stderr are captured so they don't get into Function Executor process stdout
-        and stderr. Never throws an Exception. Caller can determine if the function succeeded
-        using the response.
-        """
-        try:
-            with redirect_stdout(self._func_stdout), redirect_stderr(self._func_stderr):
-                return self._run_func(inputs)
-        except Exception:
-            return self._response_helper.failure_response(
-                message=traceback.format_exc(),
-                stdout=self._func_stdout.getvalue(),
-                stderr=self._func_stderr.getvalue(),
-            )
-
-    def _run_func(self, inputs: FunctionInputs) -> RunTaskResponse:
-        ctx: GraphInvocationContext = GraphInvocationContext(
-            invocation_id=self._invocation_id,
-            graph_name=self._graph_name,
-            graph_version=self._graph_version,
-            invocation_state=self._invocation_state,
-        )
-        if _is_router(self._function_wrapper):
-            result: RouterCallResult = self._function_wrapper.invoke_router(
-                ctx, self._function_name, inputs.input
-            )
-            return self._response_helper.router_response(
-                result=result,
-                stdout=self._func_stdout.getvalue(),
-                stderr=self._func_stderr.getvalue(),
-            )
-        else:
-            result: FunctionCallResult = self._function_wrapper.invoke_fn_ser(
-                ctx, self._function_name, inputs.input, inputs.init_value
-            )
-            return self._response_helper.function_response(
-                result=result,
-                is_reducer=_function_is_reducer(self._function_wrapper),
-                stdout=self._func_stdout.getvalue(),
-                stderr=self._func_stderr.getvalue(),
-            )
-
-    def _flush_logs(self) -> None:
-        # Flush any logs buffered in memory before running the function with stdout, stderr capture.
-        # Otherwise our logs logged before this point will end up in the function's stdout.
-        # structlog.PrintLogger uses print function. This is why flushing with print works.
-        print("", flush=True)
-        sys.stdout.flush()
-        sys.stderr.flush()
-
-
-def _is_router(func_wrapper: TensorlakeFunctionWrapper) -> bool:
-    """Determines if the function is a router.
-
-    A function is a router if it is an instance of TensorlakeRouter or if it is an TensorlakeRouter class.
-    """
-    return str(
-        type(func_wrapper.indexify_function)
-    ) == "<class 'tensorlake.functions_sdk.functions.TensorlakeRouter'>" or isinstance(
-        func_wrapper.indexify_function, TensorlakeRouter
-    )
-
-
-def _function_is_reducer(func_wrapper: TensorlakeFunctionWrapper) -> bool:
-    return func_wrapper.indexify_function.accumulate is not None

indexify-0.3.5/src/indexify/function_executor/handlers/run_function/request_validator.py

@@ -1,26 +0,0 @@
-from typing import Any
-
-from ...proto.function_executor_pb2 import RunTaskRequest
-from ...proto.message_validator import MessageValidator
-
-
-class RequestValidator:
-    def __init__(self, request: RunTaskRequest):
-        self._request = request
-        self._message_validator = MessageValidator(request)
-
-    def check(self):
-        """Validates the request.
-
-        Raises: ValueError: If the request is invalid.
-        """
-        (
-            self._message_validator.required_field("namespace")
-            .required_field("graph_name")
-            .required_field("graph_version")
-            .required_field("function_name")
-            .required_field("graph_invocation_id")
-            .required_field("task_id")
-            .required_serialized_object("function_input")
-            .optional_serialized_object("function_init_value")
-        )

indexify-0.3.5/src/indexify/function_executor/handlers/run_function/response_helper.py

@@ -1,96 +0,0 @@
-from typing import List
-
-from tensorlake.functions_sdk.data_objects import TensorlakeData
-from tensorlake.functions_sdk.functions import FunctionCallResult, RouterCallResult
-from tensorlake.functions_sdk.object_serializer import get_serializer
-
-from ...proto.function_executor_pb2 import (
-    FunctionOutput,
-    RouterOutput,
-    RunTaskResponse,
-    SerializedObject,
-)
-
-
-class ResponseHelper:
-    """Helper class for generating RunFunctionResponse."""
-
-    def __init__(self, task_id: str):
-        self._task_id = task_id
-
-    def function_response(
-        self,
-        result: FunctionCallResult,
-        is_reducer: bool,
-        stdout: str = "",
-        stderr: str = "",
-    ) -> RunTaskResponse:
-        if result.traceback_msg is None:
-            return RunTaskResponse(
-                task_id=self._task_id,
-                function_output=self._to_function_output(result.ser_outputs),
-                router_output=None,
-                stdout=stdout,
-                stderr=stderr,
-                is_reducer=is_reducer,
-                success=True,
-            )
-        else:
-            return self.failure_response(
-                message=result.traceback_msg,
-                stdout=stdout,
-                stderr=stderr,
-            )
-
-    def router_response(
-        self,
-        result: RouterCallResult,
-        stdout: str = "",
-        stderr: str = "",
-    ) -> RunTaskResponse:
-        if result.traceback_msg is None:
-            return RunTaskResponse(
-                task_id=self._task_id,
-                function_output=None,
-                router_output=RouterOutput(edges=result.edges),
-                stdout=stdout,
-                stderr=stderr,
-                is_reducer=False,
-                success=True,
-            )
-        else:
-            return self.failure_response(
-                message=result.traceback_msg,
-                stdout=stdout,
-                stderr=stderr,
-            )
-
-    def failure_response(
-        self, message: str, stdout: str, stderr: str
-    ) -> RunTaskResponse:
-        stderr = "\n".join([stderr, message])
-        return RunTaskResponse(
-            task_id=self._task_id,
-            function_output=None,
-            router_output=None,
-            stdout=stdout,
-            stderr=stderr,
-            is_reducer=False,
-            success=False,
-        )
-
-    def _to_function_output(self, outputs: List[TensorlakeData]) -> FunctionOutput:
-        output = FunctionOutput(outputs=[])
-        for ix_data in outputs:
-            serialized_object: SerializedObject = SerializedObject(
-                content_type=get_serializer(ix_data.encoder).content_type,
-            )
-            if isinstance(ix_data.payload, bytes):
-                serialized_object.bytes = ix_data.payload
-            elif isinstance(ix_data.payload, str):
-                serialized_object.string = ix_data.payload
-            else:
-                raise ValueError(f"Unsupported payload type: {type(ix_data.payload)}")
-
-            output.outputs.append(serialized_object)
-        return output

indexify-0.3.5/src/indexify/function_executor/info.py

@@ -1,16 +0,0 @@
-import importlib.metadata
-import sys
-from typing import Any, Dict
-
-
-def info_response_kv_args() -> Dict[str, Any]:
-    sdk_version = importlib.metadata.version("tensorlake")
-    python_version = (
-        f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
-    )
-    return {
-        "version": "0.1.0",
-        "sdk_version": sdk_version,
-        "sdk_language": "python",
-        "sdk_language_version": python_version,
-    }

indexify-0.3.5/src/indexify/function_executor/initialize_request_validator.py

@@ -1,21 +0,0 @@
-from .proto.function_executor_pb2 import InitializeRequest
-from .proto.message_validator import MessageValidator
-
-
-class InitializeRequestValidator:
-    def __init__(self, request: InitializeRequest):
-        self._request = request
-        self._message_validator = MessageValidator(request)
-
-    def check(self):
-        """Validates the request.
-
-        Raises: ValueError: If the request is invalid.
-        """
-        (
-            self._message_validator.required_field("namespace")
-            .required_field("graph_name")
-            .required_field("graph_version")
-            .required_field("function_name")
-            .required_serialized_object("graph")
-        )