indexify 0.4.6__tar.gz → 0.4.8__tar.gz

{indexify-0.4.6 → indexify-0.4.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.6
+Version: 0.4.8
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -17,7 +17,7 @@ Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
 Requires-Dist: boto3 (>=1.37.30,<2.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: psutil (>=7.0.0,<8.0.0)
-Requires-Dist: tensorlake (==0.2.0)
+Requires-Dist: tensorlake (==0.2.6)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.4.6 → indexify-0.4.8}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.4.6"
+version = "0.4.8"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -25,7 +25,7 @@ prometheus-client = "^0.21.1"
 psutil = "^7.0.0"
 # Adds function-executor binary, utils lib, sdk used in indexify-cli commands.
 # We need to specify the tensorlake version exactly because pip install doesn't respect poetry.lock files.
-tensorlake = "0.2.0"
+tensorlake = "0.2.6"
 # Uncomment the next line to use local tensorlake package (only for development!)
 # tensorlake = { path = "../tensorlake", develop = true }
 # pydantic is provided by tensorlake

{indexify-0.4.6 → indexify-0.4.8}/src/indexify/executor/function_executor/function_executor.py

@@ -1,4 +1,6 @@
 import asyncio
+from dataclasses import dataclass
+from enum import Enum
 from typing import Any, Optional
 
 import grpc
@@ -56,12 +58,19 @@ from .server.function_executor_server_factory import (
 )
 
 
-class FunctionError(RuntimeError):
-    pass
+class FunctionExecutorInitializationError(Enum):
+    FUNCTION_TIMEOUT = 1
+    FUNCTION_ERROR = 2
 
 
-class FunctionTimeoutError(FunctionError):
-    pass
+@dataclass
+class FunctionExecutorInitializationResult:
+    """Result of FunctionExecutor initialization."""
+
+    # None error means success.
+    error: Optional[FunctionExecutorInitializationError] = None
+    stdout: Optional[str] = None
+    stderr: Optional[str] = None
 
 
 class FunctionExecutor:
@@ -83,7 +92,6 @@ class FunctionExecutor:
         self._channel: Optional[grpc.aio.Channel] = None
         self._invocation_state_client: Optional[InvocationStateClient] = None
         self._health_checker: Optional[HealthChecker] = None
-        self._initialized = False
         metric_function_executors_count.inc()
 
     async def initialize(
@@ -93,10 +101,9 @@ class FunctionExecutor:
         base_url: str,
         config_path: Optional[str],
         customer_code_timeout_sec: Optional[float] = None,
-    ):
+    ) -> FunctionExecutorInitializationResult:
         """Creates and initializes a FunctionExecutorServer and all resources associated with it.
 
-        Raises FunctionError if the server failed to initialize due to an error in customer owned code or data.
         Raises an Exception if an internal error occured."""
         try:
             with (
@@ -108,9 +115,6 @@ class FunctionExecutor:
                 await self._establish_channel()
                 stub: FunctionExecutorStub = FunctionExecutorStub(self._channel)
                 await _collect_server_info(stub)
-                await _initialize_server(
-                    stub, initialize_request, customer_code_timeout_sec
-                )
                 await self._create_invocation_state_client(
                     stub=stub,
                     base_url=base_url,
@@ -118,21 +122,21 @@ class FunctionExecutor:
                     initialize_request=initialize_request,
                 )
                 await self._create_health_checker(self._channel, stub)
-                self._initialized = True
+
+                return await _initialize_server(
+                    stub, initialize_request, customer_code_timeout_sec
+                )
         except Exception:
             await self.destroy()
             raise
 
     def channel(self) -> grpc.aio.Channel:
-        self._check_initialized()
         return self._channel
 
     def invocation_state_client(self) -> InvocationStateClient:
-        self._check_initialized()
         return self._invocation_state_client
 
     def health_checker(self) -> HealthChecker:
-        self._check_initialized()
         return self._health_checker
 
     async def destroy(self):
@@ -158,10 +162,6 @@ class FunctionExecutor:
                 exc_info=e,
             )
 
-    def _check_initialized(self) -> None:
-        if not self._initialized:
-            raise RuntimeError("FunctionExecutor is not initialized")
-
     async def _create_server(self, config: FunctionExecutorServerConfiguration) -> None:
         with (
             metric_create_server_errors.count_exceptions(),
@@ -305,7 +305,7 @@ async def _initialize_server(
     stub: FunctionExecutorStub,
     initialize_request: InitializeRequest,
     customer_code_timeout_sec: Optional[float],
-) -> None:
+) -> FunctionExecutorInitializationResult:
     with (
         metric_initialize_rpc_errors.count_exceptions(),
         metric_initialize_rpc_latency.time(),
@@ -315,15 +315,20 @@ async def _initialize_server(
                 initialize_request,
                 timeout=customer_code_timeout_sec,
             )
+            # TODO: set real stdout and stderr when their proper capturing on FE initialization is implemented.
             if initialize_response.success:
-                return
-            if initialize_response.HasField("customer_error"):
-                raise FunctionError(initialize_response.customer_error)
+                return FunctionExecutorInitializationResult()
+            elif initialize_response.HasField("customer_error"):
+                return FunctionExecutorInitializationResult(
+                    error=FunctionExecutorInitializationError.FUNCTION_ERROR,
+                    stderr=initialize_response.customer_error,
+                )
             else:
                 raise Exception("initialize RPC failed at function executor server")
         except grpc.aio.AioRpcError as e:
             if e.code() == grpc.StatusCode.DEADLINE_EXCEEDED:
-                raise FunctionTimeoutError(
-                    f"Function initialization exceeded its configured timeout of {customer_code_timeout_sec:.3f} sec."
-                ) from e
+                return FunctionExecutorInitializationResult(
+                    error=FunctionExecutorInitializationError.FUNCTION_TIMEOUT,
+                    stderr=f"Function initialization exceeded its configured timeout of {customer_code_timeout_sec:.3f} sec.",
+                )
             raise

{indexify-0.4.6 → indexify-0.4.8}/src/indexify/executor/function_executor_controller/__init__.py

@@ -1,13 +1,16 @@
 from .function_executor_controller import FunctionExecutorController
-from .loggers import function_executor_logger, task_logger
-from .message_validators import validate_function_executor_description, validate_task
+from .loggers import function_executor_logger, task_allocation_logger
+from .message_validators import (
+    validate_function_executor_description,
+    validate_task_allocation,
+)
 from .task_output import TaskOutput
 
 __all__ = [
     "function_executor_logger",
-    "task_logger",
+    "task_allocation_logger",
     "validate_function_executor_description",
-    "validate_task",
+    "validate_task_allocation",
     "FunctionExecutorController",
     "TaskOutput",
 ]

indexify-0.4.8/src/indexify/executor/function_executor_controller/create_function_executor.py

@@ -0,0 +1,256 @@
+import asyncio
+from pathlib import Path
+from typing import Any, Optional, Tuple
+
+from tensorlake.function_executor.proto.function_executor_pb2 import (
+    InitializeRequest,
+    SerializedObject,
+)
+
+from indexify.executor.blob_store.blob_store import BLOBStore
+from indexify.executor.function_executor.function_executor import (
+    FunctionExecutor,
+    FunctionExecutorInitializationError,
+    FunctionExecutorInitializationResult,
+)
+from indexify.executor.function_executor.server.function_executor_server_factory import (
+    FunctionExecutorServerConfiguration,
+    FunctionExecutorServerFactory,
+)
+from indexify.proto.executor_api_pb2 import (
+    DataPayload,
+    DataPayloadEncoding,
+    FunctionExecutorDescription,
+    FunctionExecutorTerminationReason,
+)
+
+from .downloads import download_graph
+from .events import FunctionExecutorCreated
+from .function_executor_startup_output import FunctionExecutorStartupOutput
+from .upload_task_output import compute_hash
+
+
+async def create_function_executor(
+    function_executor_description: FunctionExecutorDescription,
+    function_executor_server_factory: FunctionExecutorServerFactory,
+    blob_store: BLOBStore,
+    executor_id: str,
+    base_url: str,
+    config_path: str,
+    cache_path: Path,
+    logger: Any,
+) -> FunctionExecutorCreated:
+    """Creates a function executor.
+
+    Doesn't raise any exceptions.
+    """
+    logger = logger.bind(module=__name__)
+    try:
+        function_executor, result = await _create_function_executor(
+            function_executor_description=function_executor_description,
+            function_executor_server_factory=function_executor_server_factory,
+            blob_store=blob_store,
+            executor_id=executor_id,
+            base_url=base_url,
+            config_path=config_path,
+            cache_path=cache_path,
+            logger=logger,
+        )
+        if result.error is not None:
+            await function_executor.destroy()
+            function_executor = None
+
+        return FunctionExecutorCreated(
+            function_executor=function_executor,
+            output=await _initialization_result_to_fe_creation_output(
+                function_executor_description=function_executor_description,
+                result=result,
+                blob_store=blob_store,
+                logger=logger,
+            ),
+        )
+    except BaseException as e:
+        if isinstance(e, asyncio.CancelledError):
+            logger.info("function executor startup was cancelled")
+            return FunctionExecutorCreated(
+                function_executor=None,
+                output=FunctionExecutorStartupOutput(
+                    function_executor_description=function_executor_description,
+                    termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_REMOVED_FROM_DESIRED_STATE,
+                ),
+            )
+        else:
+            logger.error(
+                "failed to create function executor due to platform error",
+                exc_info=e,
+            )
+            return FunctionExecutorCreated(
+                function_executor=None,
+                output=FunctionExecutorStartupOutput(
+                    function_executor_description=function_executor_description,
+                    termination_reason=FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR,
+                ),
+            )
+
+
+async def _initialization_result_to_fe_creation_output(
+    function_executor_description: FunctionExecutorDescription,
+    result: FunctionExecutorInitializationResult,
+    blob_store: BLOBStore,
+    logger: Any,
+) -> FunctionExecutorStartupOutput:
+    """Converts FunctionExecutorInitializationResult to FunctionExecutorCreationOutput.
+
+    Uploads stdout and stderr to blob store if they are present. Does only one attempt to do that.
+    Doesn't raise any exceptions."""
+    termination_reason: FunctionExecutorTerminationReason = None
+    if result.error is not None:
+        if result.error == FunctionExecutorInitializationError.FUNCTION_ERROR:
+            termination_reason = (
+                FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_ERROR
+            )
+        elif result.error == FunctionExecutorInitializationError.FUNCTION_TIMEOUT:
+            termination_reason = (
+                FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_FUNCTION_TIMEOUT
+            )
+        else:
+            logger.error(
+                "unexpected function executor initialization error code",
+                error_code=FunctionExecutorInitializationError.name(result.error),
+            )
+            termination_reason = (
+                FunctionExecutorTerminationReason.FUNCTION_EXECUTOR_TERMINATION_REASON_STARTUP_FAILED_INTERNAL_ERROR
+            )
+
+    stdout: Optional[DataPayload] = None
+    if result.stdout is not None:
+        url = f"{function_executor_description.output_payload_uri_prefix}/{function_executor_description.id}/stdout"
+        stdout = await _upload_initialization_output(
+            output_name="stdout",
+            output=result.stdout,
+            output_url=url,
+            blob_store=blob_store,
+            logger=logger,
+        )
+
+    stderr: Optional[DataPayload] = None
+    if result.stderr is not None:
+        url = f"{function_executor_description.output_payload_uri_prefix}/{function_executor_description.id}/stderr"
+        stderr = await _upload_initialization_output(
+            output_name="stderr",
+            output=result.stderr,
+            output_url=url,
+            blob_store=blob_store,
+            logger=logger,
+        )
+
+    return FunctionExecutorStartupOutput(
+        function_executor_description=function_executor_description,
+        termination_reason=termination_reason,
+        stdout=stdout,
+        stderr=stderr,
+    )
+
+
+async def _upload_initialization_output(
+    output_name: str, output: str, output_url: str, blob_store: BLOBStore, logger: Any
+) -> Optional[DataPayload]:
+    """Uploads text to blob store. Returns None if the upload fails.
+
+    Doesn't raise any exceptions.
+    """
+    try:
+        output_bytes: bytes = output.encode()
+        await blob_store.put(output_url, output_bytes, logger)
+        logger.info(
+            f"function executor initialization output {output_name} uploaded to blob store",
+            size=len(output_bytes),
+        )
+        return DataPayload(
+            uri=output_url,
+            size=len(output_bytes),
+            sha256_hash=compute_hash(output_bytes),
+            encoding=DataPayloadEncoding.DATA_PAYLOAD_ENCODING_UTF8_TEXT,
+            encoding_version=0,
+        )
+    except Exception as e:
+        logger.error(
+            f"failed to upload function executor initialization output {output_name} to blob store",
+            exc_info=e,
+        )
+        return None
+
+
+async def _create_function_executor(
+    function_executor_description: FunctionExecutorDescription,
+    function_executor_server_factory: FunctionExecutorServerFactory,
+    blob_store: BLOBStore,
+    executor_id: str,
+    base_url: str,
+    config_path: str,
+    cache_path: Path,
+    logger: Any,
+) -> Tuple[FunctionExecutor, FunctionExecutorInitializationResult]:
+    """Creates a function executor.
+
+    Raises Exception on platform error.
+    """
+    graph: SerializedObject = await download_graph(
+        function_executor_description=function_executor_description,
+        cache_path=cache_path,
+        blob_store=blob_store,
+        logger=logger,
+    )
+
+    gpu_count: int = 0
+    if function_executor_description.resources.HasField("gpu"):
+        gpu_count = function_executor_description.resources.gpu.count
+
+    config: FunctionExecutorServerConfiguration = FunctionExecutorServerConfiguration(
+        executor_id=executor_id,
+        function_executor_id=function_executor_description.id,
+        namespace=function_executor_description.namespace,
+        graph_name=function_executor_description.graph_name,
+        graph_version=function_executor_description.graph_version,
+        function_name=function_executor_description.function_name,
+        image_uri=None,
+        secret_names=list(function_executor_description.secret_names),
+        cpu_ms_per_sec=function_executor_description.resources.cpu_ms_per_sec,
+        memory_bytes=function_executor_description.resources.memory_bytes,
+        disk_bytes=function_executor_description.resources.disk_bytes,
+        gpu_count=gpu_count,
+    )
+    if function_executor_description.HasField("image_uri"):
+        config.image_uri = function_executor_description.image_uri
+
+    initialize_request: InitializeRequest = InitializeRequest(
+        namespace=function_executor_description.namespace,
+        graph_name=function_executor_description.graph_name,
+        graph_version=function_executor_description.graph_version,
+        function_name=function_executor_description.function_name,
+        graph=graph,
+    )
+    customer_code_timeout_sec: Optional[float] = None
+    if function_executor_description.HasField("customer_code_timeout_ms"):
+        customer_code_timeout_sec = (
+            function_executor_description.customer_code_timeout_ms / 1000.0
+        )
+
+    function_executor: FunctionExecutor = FunctionExecutor(
+        server_factory=function_executor_server_factory, logger=logger
+    )
+
+    try:
+        result: FunctionExecutorInitializationResult = (
+            await function_executor.initialize(
+                config=config,
+                initialize_request=initialize_request,
+                base_url=base_url,
+                config_path=config_path,
+                customer_code_timeout_sec=customer_code_timeout_sec,
+            )
+        )
+        return (function_executor, result)
+    except BaseException:  # includes asyncio.CancelledError and anything else
+        await function_executor.destroy()
+        raise

{indexify-0.4.6 → indexify-0.4.8}/src/indexify/executor/function_executor_controller/destroy_function_executor.py

@@ -11,7 +11,7 @@ async def destroy_function_executor(
     termination_reason: FunctionExecutorTerminationReason,
     logger: Any,
 ) -> FunctionExecutorDestroyed:
-    """Destroys a function executor.
+    """Destroys a function executor if it's not None.
 
     Doesn't raise any exceptions.
     """

{indexify-0.4.6 → indexify-0.4.8}/src/indexify/executor/function_executor_controller/events.py

@@ -2,11 +2,11 @@ from enum import Enum
 from typing import Optional
 
 from indexify.executor.function_executor.function_executor import (
-    FunctionError,
     FunctionExecutor,
 )
 from indexify.proto.executor_api_pb2 import FunctionExecutorTerminationReason
 
+from .function_executor_startup_output import FunctionExecutorStartupOutput
 from .task_info import TaskInfo
 
 
@@ -37,21 +37,17 @@ class FunctionExecutorCreated(BaseEvent):
     """
     Event indicating that Function Executor got created or failed.
 
-    If the error is CustomerError, it indicates an error in customer code.
-    The function_executor field is None if any errors happened.
+    The function_executor field is None if the function executor was not created.
     """
 
     def __init__(
         self,
+        output: FunctionExecutorStartupOutput,
         function_executor: Optional[FunctionExecutor] = None,
-        function_error: Optional[FunctionError] = None,
-        termination_reason: FunctionExecutorTerminationReason = None,  # type: Optional[FunctionExecutorTerminationReason]
     ):
         super().__init__(EventType.FUNCTION_EXECUTOR_CREATED)
         self.function_executor: Optional[FunctionExecutor] = function_executor
-        self.function_error: Optional[FunctionError] = function_error
-        # Reason for FE termination if failed to create FE (.function_executor is None).
-        self.termination_reason = termination_reason
+        self.output: FunctionExecutorStartupOutput = output
 
 
 class FunctionExecutorDestroyed(BaseEvent):
@@ -107,8 +103,8 @@ class TaskPreparationFinished(BaseEvent):
     def __str__(self) -> str:
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.task.id}, "
-            f"allocation_id={self.task_info.allocation_id}), "
+            f"task_id={self.task_info.allocation.task.id}, "
+            f"allocation_id={self.task_info.allocation.allocation_id}), "
             f"is_success={self.is_success}"
         )
 
@@ -147,8 +143,8 @@ class TaskExecutionFinished(BaseEvent):
         )
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.task.id}, "
-            f"allocation_id={self.task_info.allocation_id}), "
+            f"task_id={self.task_info.allocation.task.id}, "
+            f"allocation_id={self.task_info.allocation.allocation_id}), "
             f"function_executor_termination_reason={function_executor_termination_reason_str}"
         )
 
@@ -166,7 +162,7 @@ class TaskOutputUploadFinished(BaseEvent):
     def __str__(self) -> str:
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.task.id}, "
-            f"allocation_id={self.task_info.allocation_id}), "
+            f"task_id={self.task_info.allocation.task.id}, "
+            f"allocation_id={self.task_info.allocation.allocation_id}), "
             f"is_success={self.is_success}"
         )