indexify 0.2.48__py3-none-any.whl → 0.3.0__py3-none-any.whl

indexify/{cli.py → cli/cli.py}

@@ -1,8 +1,11 @@
-from .logging import configure_logging_early, configure_production_logging
+from indexify.utils.logging import (
+    configure_development_mode_logging,
+    configure_logging_early,
+    configure_production_mode_logging,
+)
 
 configure_logging_early()
 
-import asyncio
 import os
 import shutil
 import signal
@@ -11,25 +14,22 @@ import sys
 import threading
 import time
 from importlib.metadata import version
-from typing import Annotated, List, Optional
+from pathlib import Path
+from typing import Annotated, List, Optional, Tuple
 
 import nanoid
 import structlog
 import typer
 from rich.console import Console
-from rich.panel import Panel
 from rich.text import Text
 from rich.theme import Theme
+from tensorlake.functions_sdk.image import GetDefaultPythonImage, Image
 
+from indexify.executor.api_objects import FunctionURI
 from indexify.executor.executor import Executor
 from indexify.executor.function_executor.server.subprocess_function_executor_server_factory import (
     SubprocessFunctionExecutorServerFactory,
 )
-from indexify.function_executor.function_executor_service import (
-    FunctionExecutorService,
-)
-from indexify.function_executor.server import Server as FunctionExecutorServer
-from indexify.functions_sdk.image import Build, GetDefaultPythonImage, Image
 
 logger = structlog.get_logger(module=__name__)
 
@@ -71,8 +71,10 @@ def server_dev_mode():
     print("starting indexify server and executor in dev mode...")
     print("press Ctrl+C to stop the server and executor.")
     print(f"server binary path: {indexify_server_path}")
-    commands = [indexify_server_path, "indexify-cli executor"]
-
+    commands: List[List[str]] = [
+        [indexify_server_path, "--dev"],
+        ["indexify-cli", "executor", "--dev"],
+    ]
     processes = []
     stop_event = threading.Event()
 
@@ -103,7 +105,7 @@ def server_dev_mode():
 
     for cmd in commands:
         process = subprocess.Popen(
-            cmd.split(),
+            cmd,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             bufsize=1,
@@ -155,32 +157,6 @@ def build_image(
                 _create_image(obj, python_sdk_path)
 
 
-@app.command(help="Build platform images for function names")
-def build_platform_image(
-    workflow_file_path: Annotated[str, typer.Argument()],
-    image_names: Optional[List[str]] = None,
-    build_service="https://api.tensorlake.ai/images/v1",
-):
-
-    globals_dict = {}
-
-    # Add the folder in the workflow file path to the current Python path
-    folder_path = os.path.dirname(workflow_file_path)
-    if folder_path not in sys.path:
-        sys.path.append(folder_path)
-
-    try:
-        exec(open(workflow_file_path).read(), globals_dict)
-    except FileNotFoundError as e:
-        raise Exception(
-            f"Could not find workflow file to execute at: " f"`{workflow_file_path}`"
-        )
-    for _, obj in globals_dict.items():
-        if type(obj) and isinstance(obj, Image):
-            if image_names is None or obj._image_name in image_names:
-                _create_platform_image(obj, build_service)
-
-
 @app.command(help="Build default image for indexify")
 def build_default_image(
     python_version: Optional[str] = typer.Option(
@@ -200,87 +176,104 @@ def build_default_image(
     )
 
 
-@app.command(help="Joins the extractors to the coordinator server")
+@app.command(
+    help="Runs Executor that connects to the Indexify server and starts running its tasks"
+)
 def executor(
     server_addr: str = "localhost:8900",
     dev: Annotated[
         bool, typer.Option("--dev", "-d", help="Run the executor in development mode")
     ] = False,
+    function_uris: Annotated[
+        Optional[List[str]],
+        typer.Option(
+            "--function",
+            "-f",
+            help="Function that the executor will run "
+            "specified as <namespace>:<workflow>:<function>:<version>",
+        ),
+    ] = None,
     config_path: Optional[str] = typer.Option(
         None, help="Path to the TLS configuration file"
     ),
     executor_cache: Optional[str] = typer.Option(
         "~/.indexify/executor_cache", help="Path to the executor cache directory"
     ),
-    name_alias: Optional[str] = typer.Option(
-        None, help="Image name override for the executor"
-    ),
-    image_hash: Optional[str] = typer.Option(
-        None, help="Image hash override for the executor"
+    # Registred ports range ends at 49151.
+    ports: Tuple[int, int] = typer.Option(
+        (50000, 51000), help="Range of localhost TCP ports to be used by the executor"
     ),
 ):
-    if not dev:
-        configure_production_logging()
+    if dev:
+        configure_development_mode_logging()
+    else:
+        configure_production_mode_logging()
+        if function_uris is None:
+            raise typer.BadParameter(
+                "At least one function must be specified when not running in development mode"
+            )
 
     id = nanoid.generate()
-    executor_version = version("indexify")
     logger.info(
-        "executor started",
+        "starting executor",
         server_addr=server_addr,
         config_path=config_path,
         executor_id=id,
-        executor_version=executor_version,
+        executor_version=version("indexify"),
         executor_cache=executor_cache,
-        name_alias=name_alias,
-        image_hash=image_hash,
+        ports=ports,
+        functions=function_uris,
         dev_mode=dev,
     )
 
-    from pathlib import Path
-
     executor_cache = Path(executor_cache).expanduser().absolute()
     if os.path.exists(executor_cache):
         shutil.rmtree(executor_cache)
     Path(executor_cache).mkdir(parents=True, exist_ok=True)
 
-    executor = Executor(
+    start_port: int = ports[0]
+    end_port: int = ports[1]
+    if start_port >= end_port:
+        console.print(
+            Text(
+                f"start port {start_port} should be less than {end_port}", style="red"
+            ),
+        )
+        exit(1)
+
+    Executor(
         id,
         server_addr=server_addr,
         config_path=config_path,
         code_path=executor_cache,
-        name_alias=name_alias,
-        image_hash=image_hash,
+        function_allowlist=_parse_function_uris(function_uris),
         function_executor_server_factory=SubprocessFunctionExecutorServerFactory(
-            development_mode=dev
+            development_mode=dev,
+            server_ports=range(ports[0], ports[1]),
         ),
-    )
-    try:
-        asyncio.get_event_loop().run_until_complete(executor.run())
-    except asyncio.CancelledError:
-        logger.info("graceful shutdown")
+    ).run()
 
 
-@app.command(help="Runs a Function Executor server")
-def function_executor(
-    function_executor_server_address: str = typer.Option(
-        help="Function Executor server address"
-    ),
-    dev: Annotated[
-        bool, typer.Option("--dev", "-d", help="Run the executor in development mode")
-    ] = False,
-):
-    if not dev:
-        configure_production_logging()
+def _parse_function_uris(uri_strs: Optional[List[str]]) -> Optional[List[FunctionURI]]:
+    if uri_strs is None:
+        return None
 
-    logger.info(
-        "starting function executor server",
-        function_executor_server_address=function_executor_server_address,
-    )
-
-    FunctionExecutorServer(
-        server_address=function_executor_server_address,
-        service=FunctionExecutorService(),
-    ).run()
+    uris: List[FunctionURI] = []
+    for uri_str in uri_strs:
+        tokens = uri_str.split(":")
+        if len(tokens) != 4:
+            raise typer.BadParameter(
+                "Function should be specified as <namespace>:<workflow>:<function>:<version>"
+            )
+        uris.append(
+            FunctionURI(
+                namespace=tokens[0],
+                compute_graph=tokens[1],
+                compute_fn=tokens[2],
+                version=tokens[3],
+            )
+        )
+    return uris
 
 
 def _create_image(image: Image, python_sdk_path):

indexify/executor/README.md

@@ -0,0 +1,35 @@
+## Overview
+
+Executor registers at Indexify Server and continuously pulls tasks assigned to it from the Indexify Server
+and executes them. While registering it shares its capabilities like available hardware with the Indexify
+Server and periodically updates the Server about its current state. Executor spins up Function Executors
+to run customer functions. Executor should never link with Tensorlake Python-SDK. It should not know anything
+about programming languages and runtime environments used by Tensorlake Functions. Function Executor is
+responsible for this.
+
+This subpackage doesn't provide an executable entry point that runs an Executor. This is intentional
+as Executor has many configurable sub-components. indexify cli subpackage provides `executor`
+command that runs Executor with functionality available in Open Source offering.
+
+## Deployment
+
+### Production setup
+
+A single Executor runs in a Virtual Machine, container or a in bare metal host. An Indexify cluster
+is scaled by adding more Executor hosts. Open Source users manage and scale the hosts themselves e.g.
+using Kubernetes, any other orchestrator or even manually. E.g. the users provision secrets,
+persistent volumes to each host using the orchestrator or manually. Each Executor runs a single function.
+The function name and other qualifiers are defined in Executor arguments.
+
+### Development setup
+
+To make Indexify development and testing easier an Executor in development mode can run any function.
+Running multiple Executors on the same host is supported too. In this case each Executor requires a
+unique port range passed to it in its arguments.
+
+## Threat model
+
+A VM/container/bare metal host where an Executor is running is fully trusted. This works well for single
+tenant deployments where customer functions' code is fully trusted. If this is not the case then Function
+Executors that run customer functions need to get isolated from Executor using e.g. Virtual Machines.
+This functionality is not included into the Open Source offering.

indexify/executor/api_objects.py

@@ -11,17 +11,23 @@ class Task(BaseModel):
     invocation_id: str
     input_key: str
     reducer_output_id: Optional[str] = None
-    graph_version: int
+    graph_version: str
     image_uri: Optional[str] = None
     "image_uri defines the URI of the image of this task. Optional since some executors do not require it."
 
 
+class FunctionURI(BaseModel):
+    namespace: str
+    compute_graph: str
+    compute_fn: str
+    version: str
+
+
 class ExecutorMetadata(BaseModel):
     id: str
     executor_version: str
     addr: str
-    image_name: str
-    image_hash: str
+    function_allowlist: Optional[List[FunctionURI]] = None
     labels: Dict[str, Any]
 
 

indexify/executor/downloader.py

@@ -5,11 +5,9 @@ from typing import Any, Optional
 import httpx
 import structlog
 
-from indexify.function_executor.proto.function_executor_pb2 import (
-    SerializedObject,
-)
+from indexify.function_executor.proto.function_executor_pb2 import SerializedObject
+from indexify.utils.http_client import get_httpx_client
 
-from ..common_util import get_httpx_client
 from .api_objects import Task
 
 
@@ -27,7 +25,8 @@ class Downloader:
             self.code_path,
             "graph_cache",
             task.namespace,
-            f"{task.compute_graph}.{task.graph_version}",
+            task.compute_graph,
+            task.graph_version,
         )
         # Filesystem operations are synchronous.
         # Run in a separate thread to not block the main event loop.
@@ -70,6 +69,7 @@ class Downloader:
         # Atomically rename the fully written file at tmp path.
         # This allows us to not use any locking because file link/unlink
         # are atomic operations at filesystem level.
+        # This also allows to share the same cache between multiple Executors.
         os.replace(tmp_path, path)
 
     async def download_input(self, task: Task) -> SerializedObject:

indexify/executor/executor.py

@@ -1,15 +1,14 @@
 import asyncio
 import signal
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any, List, Optional
 
 import structlog
 
-from indexify.function_executor.proto.function_executor_pb2 import (
-    SerializedObject,
-)
+from indexify.function_executor.proto.function_executor_pb2 import SerializedObject
+from indexify.utils.logging import suppress as suppress_logging
 
-from .api_objects import Task
+from .api_objects import FunctionURI, Task
 from .downloader import Downloader
 from .function_executor.server.function_executor_server_factory import (
     FunctionExecutorServerFactory,
@@ -24,14 +23,13 @@ class Executor:
         self,
         executor_id: str,
         code_path: Path,
+        function_allowlist: Optional[List[FunctionURI]],
         function_executor_server_factory: FunctionExecutorServerFactory,
         server_addr: str = "localhost:8900",
         config_path: Optional[str] = None,
-        name_alias: Optional[str] = None,
-        image_hash: Optional[str] = None,
     ):
         self._logger = structlog.get_logger(module=__name__)
-        self._should_run = True
+        self._is_shutdown: bool = False
         self._config_path = config_path
         protocol: str = "http"
         if config_path:
@@ -41,7 +39,7 @@ class Executor:
         self._server_addr = server_addr
         self._base_url = f"{protocol}://{self._server_addr}"
         self._code_path = code_path
-        self._task_runnner = TaskRunner(
+        self._task_runner = TaskRunner(
             function_executor_server_factory=function_executor_server_factory,
             base_url=self._base_url,
             config_path=config_path,
@@ -53,8 +51,7 @@ class Executor:
             protocol=protocol,
             indexify_server_addr=self._server_addr,
             executor_id=executor_id,
-            name_alias=name_alias,
-            image_hash=image_hash,
+            function_allowlist=function_allowlist,
             config_path=config_path,
         )
         self._task_reporter = TaskReporter(
@@ -63,15 +60,25 @@ class Executor:
             config_path=self._config_path,
         )
 
-    async def run(self):
-        asyncio.get_event_loop().add_signal_handler(
-            signal.SIGINT, self.shutdown, asyncio.get_event_loop()
-        )
-        asyncio.get_event_loop().add_signal_handler(
-            signal.SIGTERM, self.shutdown, asyncio.get_event_loop()
-        )
+    def run(self):
+        for signum in [
+            signal.SIGABRT,
+            signal.SIGINT,
+            signal.SIGTERM,
+            signal.SIGQUIT,
+            signal.SIGHUP,
+        ]:
+            asyncio.get_event_loop().add_signal_handler(
+                signum, self.shutdown, asyncio.get_event_loop()
+            )
+
+        try:
+            asyncio.get_event_loop().run_until_complete(self._run_async())
+        except asyncio.CancelledError:
+            pass  # Suppress this expected exception and return without error (normally).
 
-        while self._should_run:
+    async def _run_async(self):
+        while not self._is_shutdown:
             try:
                 async for task in self._task_fetcher.run():
                     asyncio.create_task(self._run_task(task))
@@ -95,7 +102,7 @@ class Executor:
                 await self._downloader.download_init_value(task)
             )
             logger.info("task_execution_started")
-            output: TaskOutput = await self._task_runnner.run(
+            output: TaskOutput = await self._task_runner.run(
                 TaskInput(
                     task=task,
                     graph=graph,
@@ -130,8 +137,14 @@ class Executor:
 
     async def _shutdown(self, loop):
         self._logger.info("shutting_down")
-        self._should_run = False
-        await self._task_runnner.shutdown()
+        # There will be lots of task cancellation exceptions and "X is shutting down"
+        # exceptions logged during Executor shutdown. Suppress their logs as they are
+        # expected and are confusing for users.
+        suppress_logging()
+
+        self._is_shutdown = True
+        await self._task_runner.shutdown()
+        # We mainly need to cancel the task that runs _run_async() loop.
         for task in asyncio.all_tasks(loop):
             task.cancel()
 

indexify/executor/function_executor/function_executor.py

@@ -3,7 +3,6 @@ from typing import Any, Optional
 
 import grpc
 
-from indexify.common_util import get_httpx_client
 from indexify.function_executor.proto.function_executor_pb2 import (
     InitializeRequest,
     InitializeResponse,
@@ -11,6 +10,7 @@ from indexify.function_executor.proto.function_executor_pb2 import (
 from indexify.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
+from indexify.utils.http_client import get_httpx_client
 
 from .invocation_state_client import InvocationStateClient
 from .server.function_executor_server import (
@@ -23,6 +23,10 @@ from .server.function_executor_server_factory import (
 )
 
 
+class CustomerError(RuntimeError):
+    pass
+
+
 class FunctionExecutor:
     """Executor side class supporting a running FunctionExecutorServer.
 
@@ -50,7 +54,10 @@ class FunctionExecutor:
         base_url: str,
         config_path: Optional[str],
     ):
-        """Creates and initializes a FunctionExecutorServer and all resources associated with it."""
+        """Creates and initializes a FunctionExecutorServer and all resources associated with it.
+
+        Raises CustomerError if the server failed to initialize due to an error in customer owned code or data.
+        Raises an Exception if an internal error occured."""
         try:
             self._server = await self._server_factory.create(
                 config=config, logger=self._logger
@@ -129,5 +136,9 @@ async def _initialize_server(
     stub: FunctionExecutorStub, initialize_request: InitializeRequest
 ):
     initialize_response: InitializeResponse = await stub.initialize(initialize_request)
-    if not initialize_response.success:
+    if initialize_response.success:
+        return
+    if initialize_response.HasField("customer_error"):
+        raise CustomerError(initialize_response.customer_error)
+    else:
         raise Exception("initialize RPC failed at function executor server")

indexify/executor/function_executor/function_executor_state.py

@@ -15,9 +15,11 @@ class FunctionExecutorState:
     def __init__(self, function_id_with_version: str, function_id_without_version: str):
         self.function_id_with_version: str = function_id_with_version
         self.function_id_without_version: str = function_id_without_version
+        # All the fields below are protected by the lock.
+        self.lock: asyncio.Lock = asyncio.Lock()
+        self.is_shutdown: bool = False
         self.function_executor: Optional[FunctionExecutor] = None
         self.running_tasks: int = 0
-        self.lock: asyncio.Lock = asyncio.Lock()
         self.running_tasks_change_notifier: asyncio.Condition = asyncio.Condition(
             lock=self.lock
         )
@@ -58,16 +60,17 @@ class FunctionExecutorState:
             await self.function_executor.destroy()
             self.function_executor = None
 
-    async def destroy_function_executor_not_locked(self) -> None:
-        """Destroys the Function Executor if it exists.
+    async def shutdown(self) -> None:
+        """Shuts down the state.
 
-        The caller doesn't need to hold the lock but this call
-        might make the state inconsistent."""
-        if self.function_executor is not None:
-            # Atomically hide the destroyed Function Executor from other asyncio tasks.
-            ref = self.function_executor
-            self.function_executor = None
-            await ref.destroy()
+        Called only during Executor shutdown so it's okay to fail all running and pending
+        Function Executor tasks. The state is not valid anymore after this call.
+        The caller must hold the lock.
+        """
+        self.check_locked()
+        # Pending tasks will not create a new Function Executor and won't run.
+        self.is_shutdown = True
+        await self.destroy_function_executor()
 
     def check_locked(self) -> None:
         """Raises an exception if the lock is not held."""

indexify/executor/function_executor/invocation_state_client.py

@@ -4,7 +4,6 @@ from typing import Any, AsyncGenerator, Optional, Union
 import grpc
 import httpx
 
-from indexify.executor.downloader import serialized_object_from_http_response
 from indexify.function_executor.proto.function_executor_pb2 import (
     GetInvocationStateResponse,
     InvocationStateRequest,
@@ -17,6 +16,8 @@ from indexify.function_executor.proto.function_executor_pb2_grpc import (
 )
 from indexify.function_executor.proto.message_validator import MessageValidator
 
+from ..downloader import serialized_object_from_http_response
+
 
 class InvocationStateClient:
     """InvocationStateClient is a client for the invocation state server of a Function Executor.

indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py

@@ -1,23 +1,23 @@
 import asyncio
-from typing import Any, Optional
+import os
+import signal
+from typing import Any, List, Optional
 
 from .function_executor_server_factory import (
     FunctionExecutorServerConfiguration,
     FunctionExecutorServerFactory,
 )
-from .subprocess_function_executor_server import (
-    SubprocessFunctionExecutorServer,
-)
+from .subprocess_function_executor_server import SubprocessFunctionExecutorServer
 
 
 class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
     def __init__(
         self,
         development_mode: bool,
+        server_ports: range,
     ):
         self._development_mode: bool = development_mode
-        # Registred ports range end at 49151. We start from 50000 to hopefully avoid conflicts.
-        self._free_ports = set(range(50000, 51000))
+        self._free_ports: List[int] = list(reversed(server_ports))
 
     async def create(
         self, config: FunctionExecutorServerConfiguration, logger: Any
@@ -33,8 +33,7 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
         try:
             port = self._allocate_port()
             args = [
-                "function-executor",
-                "--function-executor-server-address",
+                "--address",
                 _server_address(port),
             ]
             if self._development_mode:
@@ -44,8 +43,10 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
             # so we won't see it in our process outputs. This is the right behavior as customer function stdout and stderr
             # contains private customer data.
             proc: asyncio.subprocess.Process = await asyncio.create_subprocess_exec(
-                "indexify-cli",
+                "function-executor",
                 *args,
+                # TODO: pass `process_group=0` instead of the depricated `preexec_fn` once we only support Python 3.11+.
+                preexec_fn=_new_process_group,
             )
             return SubprocessFunctionExecutorServer(
                 process=proc,
@@ -77,6 +78,10 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
                 # The process already exited and was waited() sucessfully.
                 return
 
+            if os.name == "posix":
+                # On POSIX systems, we can kill the whole process group so processes forked by customer code are also killed.
+                # This should be done before proc.kill because PG processes get their own PG when their PG leader dies.
+                os.killpg(proc.pid, signal.SIGKILL)
             proc.kill()
             await proc.wait()
         except Exception as e:
@@ -95,8 +100,15 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
     def _release_port(self, port: int) -> None:
         # No asyncio.Lock is required here because this operation never awaits
         # and it is always called from the same thread where the event loop is running.
-        self._free_ports.add(port)
+        #
+        # Prefer port reuse to repro as many possible issues deterministically as possible.
+        self._free_ports.append(port)
 
 
 def _server_address(port: int) -> str:
     return f"localhost:{port}"
+
+
+def _new_process_group() -> None:
+    """Creates a new process group with ID equal to the current process PID. POSIX only."""
+    os.setpgid(0, 0)