indexify 0.2.44__py3-none-any.whl → 0.2.45__py3-none-any.whl

indexify/__init__.py

@@ -3,6 +3,7 @@ from .functions_sdk.graph import Graph
 from .functions_sdk.image import Image
 from .functions_sdk.indexify_functions import (
     IndexifyFunction,
+    IndexifyRouter,
     get_ctx,
     indexify_function,
     indexify_router,
@@ -23,6 +24,7 @@ __all__ = [
     "indexify_function",
     "get_ctx",
     "IndexifyFunction",
+    "IndexifyRouter",
     "indexify_router",
     "DEFAULT_SERVICE_URL",
     "IndexifyClient",

indexify/cli.py

@@ -2,7 +2,6 @@ from .logging import configure_logging_early, configure_production_logging
 
 configure_logging_early()
 
-
 import asyncio
 import os
 import shutil
@@ -23,16 +22,14 @@ from rich.text import Text
 from rich.theme import Theme
 
 from indexify.executor.executor import Executor
+from indexify.executor.function_executor.server.subprocess_function_executor_server_factory import (
+    SubprocessFunctionExecutorServerFactory,
+)
 from indexify.function_executor.function_executor_service import (
     FunctionExecutorService,
 )
 from indexify.function_executor.server import Server as FunctionExecutorServer
-from indexify.functions_sdk.image import (
-    LOCAL_PYTHON_VERSION,
-    GetDefaultPythonImage,
-    Image,
-)
-from indexify.http_client import IndexifyClient
+from indexify.functions_sdk.image import Build, GetDefaultPythonImage, Image
 
 logger = structlog.get_logger(module=__name__)
 
@@ -48,9 +45,6 @@ custom_theme = Theme(
 console = Console(theme=custom_theme)
 
 app = typer.Typer(pretty_exceptions_enable=False, no_args_is_help=True)
-config_path_option: Optional[str] = typer.Option(
-    None, help="Path to the TLS configuration file"
-)
 
 
 @app.command(
@@ -161,6 +155,32 @@ def build_image(
                 _create_image(obj, python_sdk_path)
 
 
+@app.command(help="Build platform images for function names")
+def build_platform_image(
+    workflow_file_path: Annotated[str, typer.Argument()],
+    image_names: Optional[List[str]] = None,
+    build_service="https://api.tensorlake.ai/images/v1",
+):
+
+    globals_dict = {}
+
+    # Add the folder in the workflow file path to the current Python path
+    folder_path = os.path.dirname(workflow_file_path)
+    if folder_path not in sys.path:
+        sys.path.append(folder_path)
+
+    try:
+        exec(open(workflow_file_path).read(), globals_dict)
+    except FileNotFoundError as e:
+        raise Exception(
+            f"Could not find workflow file to execute at: " f"`{workflow_file_path}`"
+        )
+    for _, obj in globals_dict.items():
+        if type(obj) and isinstance(obj, Image):
+            if image_names is None or obj._image_name in image_names:
+                _create_platform_image(obj, build_service)
+
+
 @app.command(help="Build default image for indexify")
 def build_default_image(
     python_version: Optional[str] = typer.Option(
@@ -186,7 +206,9 @@ def executor(
     dev: Annotated[
         bool, typer.Option("--dev", "-d", help="Run the executor in development mode")
     ] = False,
-    config_path: Optional[str] = config_path_option,
+    config_path: Optional[str] = typer.Option(
+        None, help="Path to the TLS configuration file"
+    ),
     executor_cache: Optional[str] = typer.Option(
         "~/.indexify/executor_cache", help="Path to the executor cache directory"
     ),
@@ -228,9 +250,10 @@ def executor(
         code_path=executor_cache,
         name_alias=name_alias,
         image_hash=image_hash,
-        development_mode=dev,
+        function_executor_server_factory=SubprocessFunctionExecutorServerFactory(
+            development_mode=dev
+        ),
     )
-
     try:
         asyncio.get_event_loop().run_until_complete(executor.run())
     except asyncio.CancelledError:
@@ -242,11 +265,9 @@ def function_executor(
     function_executor_server_address: str = typer.Option(
         help="Function Executor server address"
     ),
-    indexify_server_address: str = typer.Option(help="Indexify server address"),
     dev: Annotated[
         bool, typer.Option("--dev", "-d", help="Run the executor in development mode")
     ] = False,
-    config_path: Optional[str] = config_path_option,
 ):
     if not dev:
         configure_production_logging()
@@ -254,15 +275,11 @@ def function_executor(
     logger.info(
         "starting function executor server",
         function_executor_server_address=function_executor_server_address,
-        indexify_server_address=indexify_server_address,
-        config_path=config_path,
     )
 
     FunctionExecutorServer(
         server_address=function_executor_server_address,
-        service=FunctionExecutorService(
-            indexify_server_address=indexify_server_address, config_path=config_path
-        ),
+        service=FunctionExecutorService(),
     ).run()
 
 
@@ -275,63 +292,7 @@ def _create_image(image: Image, python_sdk_path):
 
 
 def _build_image(image: Image, python_sdk_path: Optional[str] = None):
-
-    try:
-        import docker
-
-        client = docker.from_env()
-        client.ping()
-    except Exception as e:
-        console.print(
-            Text("Unable to connect with docker: ", style="red bold"),
-            Text(f"{e}", style="red"),
-        )
-        exit(-1)
-
-    docker_contents = [
-        f"FROM {image._base_image}",
-        "RUN mkdir -p ~/.indexify",
-        "RUN touch ~/.indexify/image_name",
-        f"RUN echo {image._image_name} > ~/.indexify/image_name",
-        f"RUN echo {image.hash()} > ~/.indexify/image_hash",
-        "WORKDIR /app",
-    ]
-
-    docker_contents.extend(["RUN " + i for i in image._run_strs])
-
-    if python_sdk_path is not None:
-        logging.info(
-            f"Building image {image._image_name} with local version of the SDK"
-        )
-        if not os.path.exists(python_sdk_path):
-            print(f"error: {python_sdk_path} does not exist")
-            os.exit(1)
-        docker_contents.append(f"COPY {python_sdk_path} /app/python-sdk")
-        docker_contents.append("RUN (cd /app/python-sdk && pip install .)")
-    else:
-        docker_contents.append(f"RUN pip install indexify=={image._sdk_version}")
-
-    docker_file = "\n".join(docker_contents)
-
-    import docker.api.build
-
-    docker.api.build.process_dockerfile = lambda dockerfile, path: (
-        "Dockerfile",
-        dockerfile,
-    )
-
-    console.print("Creating image using Dockerfile contents:", style="cyan bold")
-    print(f"{docker_file}")
-
-    client = docker.from_env()
-    image_name = f"{image._image_name}:{image._tag}"
-    (_image, generator) = client.images.build(
-        path=".",
-        dockerfile=docker_file,
-        tag=image_name,
-        rm=True,
-    )
-    for result in generator:
-        print(result)
-
-    print(f"built image: {image_name}")
+    built_image, output = image.build(python_sdk_path=python_sdk_path)
+    for line in output:
+        print(line)
+    print(f"built image: {built_image.tags[0]}")

indexify/executor/api_objects.py

@@ -12,6 +12,8 @@ class Task(BaseModel):
     input_key: str
     reducer_output_id: Optional[str] = None
     graph_version: int
+    image_uri: Optional[str] = None
+    "image_uri defines the URI of the image of this task. Optional since some executors do not require it."
 
 
 class ExecutorMetadata(BaseModel):

indexify/executor/downloader.py

@@ -13,12 +13,6 @@ from ..common_util import get_httpx_client
 from .api_objects import Task
 
 
-class DownloadedInputs:
-    def __init__(self, input: SerializedObject, init_value: Optional[SerializedObject]):
-        self.input = input
-        self.init_value = init_value
-
-
 class Downloader:
     def __init__(
         self, code_path: str, base_url: str, config_path: Optional[str] = None
@@ -78,22 +72,22 @@ class Downloader:
         # are atomic operations at filesystem level.
         os.replace(tmp_path, path)
 
-    async def download_inputs(self, task: Task) -> DownloadedInputs:
+    async def download_input(self, task: Task) -> SerializedObject:
         logger = self._task_logger(task)
 
-        input: SerializedObject
         first_function_in_graph = task.invocation_id == task.input_key.split("|")[-1]
         if first_function_in_graph:
             # The first function in Graph gets its input from graph invocation payload.
-            input = await self._fetch_graph_invocation_payload(task, logger)
+            return await self._fetch_graph_invocation_payload(task, logger)
         else:
-            input = await self._fetch_function_input(task, logger)
+            return await self._fetch_function_input(task, logger)
 
-        init_value: Optional[SerializedObject] = None
-        if task.reducer_output_id is not None:
-            init_value = await self._fetch_function_init_value(task, logger)
+    async def download_init_value(self, task: Task) -> Optional[SerializedObject]:
+        if task.reducer_output_id is None:
+            return None
 
-        return DownloadedInputs(input=input, init_value=init_value)
+        logger = self._task_logger(task)
+        return await self._fetch_function_init_value(task, logger)
 
     def _task_logger(self, task: Task) -> Any:
         return structlog.get_logger(
@@ -142,7 +136,7 @@ class Downloader:
         self, url: str, resource_description: str, logger: Any
     ) -> SerializedObject:
         logger.info(f"fetching {resource_description}", url=url)
-        response = await self._client.get(url)
+        response: httpx.Response = await self._client.get(url)
         try:
             response.raise_for_status()
         except httpx.HTTPStatusError as e:
@@ -153,13 +147,17 @@ class Downloader:
             )
             raise
 
-        # We're hardcoding the content type currently used by Python SDK. It might change in the future.
-        # There's no other way for now to determine if the response is a bytes or string.
-        if response.headers["content-type"] == "application/octet-stream":
-            return SerializedObject(
-                bytes=response.content, content_type=response.headers["content-type"]
-            )
-        else:
-            return SerializedObject(
-                string=response.text, content_type=response.headers["content-type"]
-            )
+        return serialized_object_from_http_response(response)
+
+
+def serialized_object_from_http_response(response: httpx.Response) -> SerializedObject:
+    # We're hardcoding the content type currently used by Python SDK. It might change in the future.
+    # There's no other way for now to determine if the response is a bytes or string.
+    if response.headers["content-type"] == "application/octet-stream":
+        return SerializedObject(
+            bytes=response.content, content_type=response.headers["content-type"]
+        )
+    else:
+        return SerializedObject(
+            string=response.text, content_type=response.headers["content-type"]
+        )

indexify/executor/executor.py

@@ -10,17 +10,13 @@ from indexify.function_executor.proto.function_executor_pb2 import (
 )
 
 from .api_objects import Task
-from .downloader import DownloadedInputs, Downloader
-from .function_executor.process_function_executor_factory import (
-    ProcessFunctionExecutorFactory,
-)
-from .function_worker import (
-    FunctionWorker,
-    FunctionWorkerInput,
-    FunctionWorkerOutput,
+from .downloader import Downloader
+from .function_executor.server.function_executor_server_factory import (
+    FunctionExecutorServerFactory,
 )
 from .task_fetcher import TaskFetcher
 from .task_reporter import TaskReporter
+from .task_runner import TaskInput, TaskOutput, TaskRunner
 
 
 class Executor:
@@ -28,8 +24,8 @@ class Executor:
         self,
         executor_id: str,
         code_path: Path,
+        function_executor_server_factory: FunctionExecutorServerFactory,
         server_addr: str = "localhost:8900",
-        development_mode: bool = False,
         config_path: Optional[str] = None,
         name_alias: Optional[str] = None,
         image_hash: Optional[str] = None,
@@ -42,16 +38,14 @@ class Executor:
             self._logger.info("running the extractor with TLS enabled")
             protocol = "https"
 
-        self._function_worker = FunctionWorker(
-            function_executor_factory=ProcessFunctionExecutorFactory(
-                indexify_server_address=server_addr,
-                development_mode=development_mode,
-                config_path=config_path,
-            )
-        )
         self._server_addr = server_addr
         self._base_url = f"{protocol}://{self._server_addr}"
         self._code_path = code_path
+        self._task_runnner = TaskRunner(
+            function_executor_server_factory=function_executor_server_factory,
+            base_url=self._base_url,
+            config_path=config_path,
+        )
         self._downloader = Downloader(
             code_path=code_path, base_url=self._base_url, config_path=config_path
         )
@@ -92,39 +86,39 @@ class Executor:
 
         Doesn't raise any Exceptions. All errors are reported to the server."""
         logger = self._task_logger(task)
-        output: Optional[FunctionWorkerOutput] = None
+        output: Optional[TaskOutput] = None
 
         try:
             graph: SerializedObject = await self._downloader.download_graph(task)
-            input: DownloadedInputs = await self._downloader.download_inputs(task)
-            output = await self._function_worker.run(
-                input=FunctionWorkerInput(
+            input: SerializedObject = await self._downloader.download_input(task)
+            init_value: Optional[SerializedObject] = (
+                await self._downloader.download_init_value(task)
+            )
+            logger.info("task_execution_started")
+            output: TaskOutput = await self._task_runnner.run(
+                TaskInput(
                     task=task,
                     graph=graph,
-                    function_input=input,
-                )
+                    input=input,
+                    init_value=init_value,
+                ),
+                logger=logger,
             )
             logger.info("task_execution_finished", success=output.success)
         except Exception as e:
-            logger.error("failed running the task", exc_info=e)
+            output = TaskOutput.internal_error(task)
+            logger.error("task_execution_failed", exc_info=e)
 
-        await self._report_task_outcome(task=task, output=output, logger=logger)
+        await self._report_task_outcome(output=output, logger=logger)
 
-    async def _report_task_outcome(
-        self, task: Task, output: Optional[FunctionWorkerOutput], logger: Any
-    ) -> None:
-        """Reports the task with the given output to the server.
-
-        None output means that the task execution didn't finish due to an internal error.
-        Doesn't raise any exceptions."""
+    async def _report_task_outcome(self, output: TaskOutput, logger: Any) -> None:
+        """Reports the task with the given output to the server."""
         reporting_retries: int = 0
 
         while True:
             logger = logger.bind(retries=reporting_retries)
             try:
-                await self._task_reporter.report(
-                    task=task, output=output, logger=logger
-                )
+                await self._task_reporter.report(output=output, logger=logger)
                 break
             except Exception as e:
                 logger.error(
@@ -137,7 +131,7 @@ class Executor:
     async def _shutdown(self, loop):
         self._logger.info("shutting_down")
         self._should_run = False
-        await self._function_worker.shutdown()
+        await self._task_runnner.shutdown()
         for task in asyncio.all_tasks(loop):
             task.cancel()
 

indexify/executor/function_executor/function_executor.py

@@ -1,32 +1,133 @@
+import asyncio
 from typing import Any, Optional
 
 import grpc
 
-# Timeout for Function Executor startup in seconds.
-# The timeout is counted from the moment when the Function Executor environment
-# is fully prepared and the Function Executor gets started.
-FUNCTION_EXECUTOR_READY_TIMEOUT_SEC = 5
+from indexify.common_util import get_httpx_client
+from indexify.function_executor.proto.function_executor_pb2 import (
+    InitializeRequest,
+    InitializeResponse,
+)
+from indexify.function_executor.proto.function_executor_pb2_grpc import (
+    FunctionExecutorStub,
+)
+
+from .invocation_state_client import InvocationStateClient
+from .server.function_executor_server import (
+    FUNCTION_EXECUTOR_SERVER_READY_TIMEOUT_SEC,
+    FunctionExecutorServer,
+)
+from .server.function_executor_server_factory import (
+    FunctionExecutorServerConfiguration,
+    FunctionExecutorServerFactory,
+)
 
 
 class FunctionExecutor:
-    """Abstract interface for a FunctionExecutor.
+    """Executor side class supporting a running FunctionExecutorServer.
+
+    FunctionExecutor primary responsibility is creation and initialization
+    of all resources associated with a particular Function Executor Server
+    including the Server itself. FunctionExecutor owns all these resources
+    and provides other Executor components with access to them.
 
-    FunctionExecutor is a class that executes tasks for a particular function.
-    FunctionExecutor implements the gRPC server that listens for incoming tasks.
+    Addition of any business logic besides resource management is discouraged.
+    Please add such logic to other classes managed by this class.
     """
 
-    async def channel(self) -> grpc.aio.Channel:
-        """Returns a async gRPC channel to the Function Executor.
+    def __init__(self, server_factory: FunctionExecutorServerFactory, logger: Any):
+        self._server_factory: FunctionExecutorServerFactory = server_factory
+        self._logger = logger.bind(module=__name__)
+        self._server: Optional[FunctionExecutorServer] = None
+        self._channel: Optional[grpc.aio.Channel] = None
+        self._invocation_state_client: Optional[InvocationStateClient] = None
+        self._initialized = False
+
+    async def initialize(
+        self,
+        config: FunctionExecutorServerConfiguration,
+        initialize_request: InitializeRequest,
+        base_url: str,
+        config_path: Optional[str],
+    ):
+        """Creates and initializes a FunctionExecutorServer and all resources associated with it."""
+        try:
+            self._server = await self._server_factory.create(
+                config=config, logger=self._logger
+            )
+            self._channel = await self._server.create_channel(self._logger)
+            await _channel_ready(self._channel)
+
+            stub: FunctionExecutorStub = FunctionExecutorStub(self._channel)
+            await _initialize_server(stub, initialize_request)
+
+            self._invocation_state_client = InvocationStateClient(
+                stub=stub,
+                base_url=base_url,
+                http_client=get_httpx_client(config_path=config_path, make_async=True),
+                graph=initialize_request.graph_name,
+                namespace=initialize_request.namespace,
+                logger=self._logger,
+            )
+            await self._invocation_state_client.start()
+
+            self._initialized = True
+        except Exception:
+            await self.destroy()
+            raise
+
+    def channel(self) -> grpc.aio.Channel:
+        self._check_initialized()
+        return self._channel
+
+    def invocation_state_client(self) -> InvocationStateClient:
+        self._check_initialized()
+        return self._invocation_state_client
+
+    async def destroy(self):
+        """Destroys all resources owned by this FunctionExecutor.
+
+        Never raises any exceptions but logs them."""
+        try:
+            if self._invocation_state_client is not None:
+                await self._invocation_state_client.destroy()
+                self._invocation_state_client = None
+        except Exception as e:
+            self._logger.error(
+                "failed to destroy FunctionExecutor invocation state client", exc_info=e
+            )
+
+        try:
+            if self._channel is not None:
+                await self._channel.close()
+                self._channel = None
+        except Exception as e:
+            self._logger.error(
+                "failed to close FunctionExecutorServer channel", exc_info=e
+            )
+
+        try:
+            if self._server is not None:
+                await self._server_factory.destroy(self._server, self._logger)
+                self._server = None
+        except Exception as e:
+            self._logger.error("failed to destroy FunctionExecutorServer", exc_info=e)
+
+    def _check_initialized(self):
+        if not self._initialized:
+            raise RuntimeError("FunctionExecutor is not initialized")
+
 
-        The channel is in ready state and can be used for all gRPC communication with the Function Executor
-        and can be shared among coroutines running in the same event loop in the same thread. Users should
-        not close the channel as it's reused for all requests.
-        Raises Exception if an error occurred."""
-        raise NotImplementedError
+async def _channel_ready(channel: grpc.aio.Channel):
+    await asyncio.wait_for(
+        channel.channel_ready(),
+        timeout=FUNCTION_EXECUTOR_SERVER_READY_TIMEOUT_SEC,
+    )
 
-    def state(self) -> Optional[Any]:
-        """Returns optional state object.
 
-        The state object can be used to associate any data with the Function Executor.
-        """
-        raise NotImplementedError
+async def _initialize_server(
+    stub: FunctionExecutorStub, initialize_request: InitializeRequest
+):
+    initialize_response: InitializeResponse = await stub.initialize(initialize_request)
+    if not initialize_response.success:
+        raise Exception("initialize RPC failed at function executor server")

indexify/executor/function_executor/function_executor_state.py

@@ -0,0 +1,75 @@
+import asyncio
+from typing import Optional
+
+from .function_executor import FunctionExecutor
+
+
+class FunctionExecutorState:
+    """State of a Function Executor with a particular ID.
+
+    The Function Executor might not exist, i.e. not yet created or destroyed.
+    This object represents all such states. Any state modification must be done
+    under the lock.
+    """
+
+    def __init__(self, function_id_with_version: str, function_id_without_version: str):
+        self.function_id_with_version: str = function_id_with_version
+        self.function_id_without_version: str = function_id_without_version
+        self.function_executor: Optional[FunctionExecutor] = None
+        self.running_tasks: int = 0
+        self.lock: asyncio.Lock = asyncio.Lock()
+        self.running_tasks_change_notifier: asyncio.Condition = asyncio.Condition(
+            lock=self.lock
+        )
+
+    def increment_running_tasks(self) -> None:
+        """Increments the number of running tasks.
+
+        The caller must hold the lock.
+        """
+        self.check_locked()
+        self.running_tasks += 1
+        self.running_tasks_change_notifier.notify_all()
+
+    def decrement_running_tasks(self) -> None:
+        """Decrements the number of running tasks.
+
+        The caller must hold the lock.
+        """
+        self.check_locked()
+        self.running_tasks -= 1
+        self.running_tasks_change_notifier.notify_all()
+
+    async def wait_running_tasks_less(self, value: int) -> None:
+        """Waits until the number of running tasks is less than the supplied value.
+
+        The caller must hold the lock.
+        """
+        self.check_locked()
+        while self.running_tasks >= value:
+            await self.running_tasks_change_notifier.wait()
+
+    async def destroy_function_executor(self) -> None:
+        """Destroys the Function Executor if it exists.
+
+        The caller must hold the lock."""
+        self.check_locked()
+        if self.function_executor is not None:
+            await self.function_executor.destroy()
+            self.function_executor = None
+
+    async def destroy_function_executor_not_locked(self) -> None:
+        """Destroys the Function Executor if it exists.
+
+        The caller doesn't need to hold the lock but this call
+        might make the state inconsistent."""
+        if self.function_executor is not None:
+            # Atomically hide the destroyed Function Executor from other asyncio tasks.
+            ref = self.function_executor
+            self.function_executor = None
+            await ref.destroy()
+
+    def check_locked(self) -> None:
+        """Raises an exception if the lock is not held."""
+        if not self.lock.locked():
+            raise RuntimeError("The FunctionExecutorState lock must be held.")