indexify 0.3.17__py3-none-any.whl → 0.3.18__py3-none-any.whl

indexify/cli/cli.py

@@ -13,7 +13,7 @@ import sys
 from importlib.metadata import version
 from pathlib import Path
 from socket import gethostname
-from typing import Annotated, List, Optional, Tuple
+from typing import Annotated, Dict, List, Optional, Tuple
 
 import nanoid
 import prometheus_client
@@ -26,6 +26,7 @@ from tensorlake.functions_sdk.image import Image
 
 from indexify.executor.api_objects import FunctionURI
 from indexify.executor.executor import Executor
+from indexify.executor.executor_flavor import ExecutorFlavor
 from indexify.executor.function_executor.server.subprocess_function_executor_server_factory import (
     SubprocessFunctionExecutorServerFactory,
 )
@@ -119,7 +120,6 @@ def executor(
             help="Port where to run Executor Monitoring server",
         ),
     ] = 7000,
-    # TODO: Figure out mTLS for gRPC.
     grpc_server_addr: Annotated[
         Optional[str],
         typer.Option(
@@ -140,6 +140,15 @@ def executor(
             ),
         ),
     ] = False,
+    labels: Annotated[
+        List[str],
+        typer.Option(
+            "--label",
+            "-l",
+            help="Executor key-value label to be sent to the Server. "
+            "Specified as <key>=<value>",
+        ),
+    ] = [],
 ):
     if dev:
         configure_development_mode_logging()
@@ -162,6 +171,11 @@ def executor(
             "--grpc-server-addr must be set when --enable-grpc-state-reconciler is set"
         )
 
+    kv_labels: Dict[str, str] = {}
+    for label in labels:
+        key, value = label.split("=")
+        kv_labels[key] = value
+
     executor_version = version("indexify")
     logger = structlog.get_logger(module=__name__, executor_id=executor_id)
 
@@ -171,6 +185,7 @@ def executor(
         server_addr=server_addr,
         config_path=config_path,
         executor_version=executor_version,
+        labels=kv_labels,
         executor_cache=executor_cache,
         ports=ports,
         functions=function_uris,
@@ -205,7 +220,9 @@ def executor(
     Executor(
         id=executor_id,
         development_mode=dev,
+        flavor=ExecutorFlavor.OSS,
         version=executor_version,
+        labels=kv_labels,
         health_checker=GenericHealthChecker(),
         code_path=executor_cache,
         function_allowlist=_parse_function_uris(function_uris),

indexify/executor/executor.py

@@ -9,17 +9,18 @@ import structlog
 from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
 from tensorlake.utils.logging import suppress as suppress_logging
 
-from indexify.proto.task_scheduler_pb2 import ExecutorStatus
+from indexify.proto.executor_api_pb2 import ExecutorStatus
 
 from .api_objects import FunctionURI, Task
 from .downloader import Downloader
+from .executor_flavor import ExecutorFlavor
 from .function_executor.function_executor_states_container import (
     FunctionExecutorStatesContainer,
 )
 from .function_executor.server.function_executor_server_factory import (
     FunctionExecutorServerFactory,
 )
-from .grpc.channel_creator import ChannelCreator
+from .grpc.channel_manager import ChannelManager
 from .grpc.state_reconciler import ExecutorStateReconciler
 from .grpc.state_reporter import ExecutorStateReporter
 from .metrics.executor import (
@@ -55,7 +56,9 @@ class Executor:
         self,
         id: str,
         development_mode: bool,
+        flavor: ExecutorFlavor,
         version: str,
+        labels: Dict[str, str],
         code_path: Path,
         health_checker: HealthChecker,
         function_allowlist: Optional[List[FunctionURI]],
@@ -106,18 +109,25 @@ class Executor:
         self._task_runner: Optional[TaskRunner] = None
         self._task_fetcher: Optional[TaskFetcher] = None
         # gRPC mode services
-        self._channel_creator: Optional[ChannelCreator] = None
+        self._channel_manager: Optional[ChannelManager] = None
         self._state_reporter: Optional[ExecutorStateReporter] = None
         self._state_reconciler: Optional[ExecutorStateReconciler] = None
 
         if grpc_server_addr is not None:
-            self._channel_creator = ChannelCreator(grpc_server_addr, self._logger)
+            self._channel_manager = ChannelManager(
+                server_address=grpc_server_addr,
+                config_path=config_path,
+                logger=self._logger,
+            )
             self._state_reporter = ExecutorStateReporter(
                 executor_id=id,
+                flavor=flavor,
+                version=version,
+                labels=labels,
                 development_mode=development_mode,
                 function_allowlist=self._function_allowlist,
                 function_executor_states=self._function_executor_states,
-                channel_creator=self._channel_creator,
+                channel_manager=self._channel_manager,
                 logger=self._logger,
             )
             self._state_reporter.update_executor_status(
@@ -133,7 +143,8 @@ class Executor:
                 config_path=config_path,
                 downloader=self._downloader,
                 task_reporter=self._task_reporter,
-                channel_creator=self._channel_creator,
+                channel_manager=self._channel_manager,
+                state_reporter=self._state_reporter,
                 logger=self._logger,
             )
         else:
@@ -147,6 +158,7 @@ class Executor:
             self._task_fetcher = TaskFetcher(
                 executor_id=id,
                 executor_version=version,
+                labels=labels,
                 function_allowlist=function_allowlist,
                 protocol=protocol,
                 indexify_server_addr=self._server_addr,
@@ -326,7 +338,9 @@ class Executor:
             ).inc()
 
     async def _shutdown(self, loop):
-        self._logger.info("shutting down")
+        self._logger.info(
+            "shutting down, all Executor logs are suppressed, no task outcomes will be reported to Server from this point"
+        )
         if self._state_reporter is not None:
             self._state_reporter.update_executor_status(
                 ExecutorStatus.EXECUTOR_STATUS_STOPPING
@@ -339,12 +353,13 @@ class Executor:
 
         self._is_shutdown = True
         await self._monitoring_server.shutdown()
+        await self._task_reporter.shutdown()
 
         if self._task_runner is not None:
             await self._task_runner.shutdown()
 
-        if self._channel_creator is not None:
-            await self._channel_creator.shutdown()
+        if self._channel_manager is not None:
+            await self._channel_manager.shutdown()
         if self._state_reporter is not None:
             await self._state_reporter.shutdown()
         if self._state_reconciler is not None:

indexify/executor/executor_flavor.py

@@ -0,0 +1,7 @@
+from enum import Enum
+
+
+class ExecutorFlavor(Enum):
+    UNKNOWN = "unknown"
+    OSS = "oss"
+    PLATFORM = "platform"

indexify/executor/function_executor/health_checker.py

@@ -70,8 +70,10 @@ class HealthChecker:
                 # code is not involved when TCP connections are established to FE. Problems reestablishing
                 # the TCP connection are usually due to the FE process crashing and its gRPC server socket
                 # not being available anymore or due to prolonged local networking failures on Executor.
-                channel_connectivity = self._channel.get_state()
-                if channel_connectivity == grpc.ChannelConnectivity.TRANSIENT_FAILURE:
+                if (
+                    _channel_state(self._channel, self._logger)
+                    == grpc.ChannelConnectivity.TRANSIENT_FAILURE
+                ):
                     return HealthCheckResult(
                         is_healthy=False,
                         reason="Channel is in TRANSIENT_FAILURE state, assuming Function Executor crashed.",
@@ -126,3 +128,19 @@ class HealthChecker:
 
         asyncio.create_task(self._health_check_failed_callback(result))
         self._health_check_loop_task = None
+
+
+def _channel_state(channel: grpc.aio.Channel, logger: Any) -> grpc.ChannelConnectivity:
+    """Get channel connectivity state and suppresses all exceptions.
+
+    Suppressing the exceptions is important because the channel connectivity state is an experimental
+    feature. On error fallse back to READY state which assumes that the channel is okay.
+    """
+    try:
+        return channel.get_state()
+    except Exception as e:
+        logger.error(
+            "Failed getting channel state, falling back to default READY state",
+            exc_info=e,
+        )
+        return grpc.ChannelConnectivity.READY

indexify/executor/grpc/channel_manager.py

@@ -0,0 +1,160 @@
+import asyncio
+from typing import Any, Dict, Optional
+
+import grpc.aio
+import yaml
+
+from .metrics.channel_manager import (
+    metric_grpc_server_channel_creation_latency,
+    metric_grpc_server_channel_creation_retries,
+    metric_grpc_server_channel_creations,
+)
+
+_RETRY_INTERVAL_SEC = 5
+_CONNECT_TIMEOUT_SEC = 5
+
+
+class ChannelManager:
+    def __init__(self, server_address: str, config_path: Optional[str], logger: Any):
+        self._logger: Any = logger.bind(module=__name__, server_address=server_address)
+        self._server_address: str = server_address
+        self._channel_credentials: Optional[grpc.ChannelCredentials] = None
+        # This lock protects the fields below.
+        self._lock = asyncio.Lock()
+        self._channel: Optional[grpc.aio.Channel] = None
+
+        self._init_tls(config_path)
+
+    def _init_tls(self, config_path: Optional[str]):
+        if config_path is None:
+            return
+
+        # The same config file format as in Tensorlake SDK HTTP client, see:
+        # https://github.com/tensorlakeai/tensorlake/blob/main/src/tensorlake/utils/http_client.py
+        with open(config_path, "r") as config_file:
+            config = yaml.safe_load(config_file)
+
+        if not config.get("use_tls", False):
+            return
+
+        tls_config: Dict[str, str] = config["tls_config"]
+        cert_path: Optional[str] = tls_config.get("cert_path", None)
+        key_path: Optional[str] = tls_config.get("key_path", None)
+        ca_bundle_path: Optional[str] = tls_config.get("ca_bundle_path", None)
+
+        self._logger = self._logger.bind(
+            cert_path=cert_path,
+            key_path=key_path,
+            ca_bundle_path=ca_bundle_path,
+        )
+        self._logger.info("TLS is enabled for grpc channels to server")
+
+        private_key: Optional[bytes] = None
+        certificate_chain: Optional[bytes] = None
+        root_certificates: Optional[bytes] = None
+
+        if cert_path is not None:
+            with open(cert_path, "rb") as cert_file:
+                certificate_chain = cert_file.read()
+        if key_path is not None:
+            with open(key_path, "rb") as key_file:
+                private_key = key_file.read()
+        if ca_bundle_path is not None:
+            with open(ca_bundle_path, "rb") as ca_bundle_file:
+                root_certificates = ca_bundle_file.read()
+
+        self._channel_credentials = grpc.ssl_channel_credentials(
+            root_certificates=root_certificates,
+            private_key=private_key,
+            certificate_chain=certificate_chain,
+        )
+
+    async def get_channel(self) -> grpc.aio.Channel:
+        """Returns a channel to the gRPC server.
+
+        Returns a ready to use channel. Blocks until the channel is ready,
+        never raises any exceptions.
+        If previously returned channel is healthy then returns it again.
+        Otherwise, returns a new channel but closes the previously returned one.
+        """
+        # Use the lock to ensure that we only create one channel without race conditions.
+        async with self._lock:
+            if self._channel is None:
+                self._channel = await self._create_channel()
+            elif not await self._locked_channel_is_healthy():
+                self._logger.info("grpc channel to server is unhealthy")
+                await self._destroy_locked_channel()
+                self._channel = await self._create_channel()
+
+            return self._channel
+
+    async def _create_channel(self) -> grpc.aio.Channel:
+        """Creates a new channel to the gRPC server."
+
+        Returns a ready to use channel. Blocks until the channel
+        is ready, never raises any exceptions.
+        """
+        self._logger.info("creating new grpc server channel")
+
+        with metric_grpc_server_channel_creation_latency.time():
+            metric_grpc_server_channel_creations.inc()
+            while True:
+                try:
+                    if self._channel_credentials is None:
+                        channel = grpc.aio.insecure_channel(target=self._server_address)
+                    else:
+                        channel = grpc.aio.secure_channel(
+                            target=self._server_address,
+                            credentials=self._channel_credentials,
+                        )
+
+                    await asyncio.wait_for(
+                        channel.channel_ready(),
+                        timeout=_CONNECT_TIMEOUT_SEC,
+                    )
+                    return channel
+                except Exception:
+                    self._logger.error(
+                        f"failed establishing grpc server channel in {_CONNECT_TIMEOUT_SEC} sec, retrying in {_RETRY_INTERVAL_SEC} sec"
+                    )
+                    try:
+                        await channel.close()
+                    except Exception as e:
+                        self._logger.error(
+                            "failed closing not established channel", exc_info=e
+                        )
+
+                    metric_grpc_server_channel_creation_retries.inc()
+                    await asyncio.sleep(_RETRY_INTERVAL_SEC)
+
+    async def _locked_channel_is_healthy(self) -> bool:
+        """Checks if the channel is healthy.
+
+        Returns True if the channel is healthy, False otherwise.
+        self._lock must be acquired before calling this method.
+        Never raises any exceptions.
+        """
+        try:
+            return self._channel.get_state() == grpc.ChannelConnectivity.READY
+        except Exception as e:
+            # Assume that the channel is healthy because get_state() method is marked as experimental
+            # so we can't fully trust it.
+            self._logger.error(
+                "failed getting channel state, assuming channel is healthy", exc_info=e
+            )
+            return True
+
+    async def _destroy_locked_channel(self):
+        """Closes the existing channel.
+
+        self._lock must be acquired before calling this method.
+        Never raises any exceptions.
+        """
+        try:
+            await self._channel.close()
+        except Exception as e:
+            self._logger.error("failed closing channel", exc_info=e)
+        self._channel = None
+
+    async def shutdown(self):
+        pass

indexify/executor/grpc/state_reconciler.py

@@ -7,14 +7,14 @@ from tensorlake.function_executor.proto.function_executor_pb2 import (
     SerializedObject,
 )
 
-from indexify.proto.task_scheduler_pb2 import (
+from indexify.proto.executor_api_pb2 import (
     DesiredExecutorState,
     FunctionExecutorDescription,
     FunctionExecutorStatus,
     GetDesiredExecutorStatesRequest,
 )
-from indexify.proto.task_scheduler_pb2_grpc import (
-    TaskSchedulerServiceStub,
+from indexify.proto.executor_api_pb2_grpc import (
+    ExecutorAPIStub,
 )
 
 from ..downloader import Downloader
@@ -43,7 +43,8 @@ from ..metrics.executor import (
     metric_tasks_reporting_outcome,
 )
 from ..task_reporter import TaskReporter
-from .channel_creator import ChannelCreator
+from .channel_manager import ChannelManager
+from .state_reporter import ExecutorStateReporter
 
 _RECONCILE_STREAM_BACKOFF_INTERVAL_SEC = 5
 
@@ -58,7 +59,8 @@ class ExecutorStateReconciler:
         config_path: Optional[str],
         downloader: Downloader,
         task_reporter: TaskReporter,
-        channel_creator: ChannelCreator,
+        channel_manager: ChannelManager,
+        state_reporter: ExecutorStateReporter,
         logger: Any,
     ):
         self._executor_id: str = executor_id
@@ -72,7 +74,8 @@ class ExecutorStateReconciler:
         self._function_executor_states: FunctionExecutorStatesContainer = (
             function_executor_states
         )
-        self._channel_creator = channel_creator
+        self._channel_manager: ChannelManager = channel_manager
+        self._state_reporter: ExecutorStateReporter = state_reporter
         self._logger: Any = logger.bind(module=__name__)
         self._is_shutdown: bool = False
         self._server_last_clock: Optional[int] = None
@@ -83,12 +86,14 @@ class ExecutorStateReconciler:
         Never raises any exceptions.
         """
         while not self._is_shutdown:
-            async with await self._channel_creator.create() as server_channel:
+            async with await self._channel_manager.get_channel() as server_channel:
                 server_channel: grpc.aio.Channel
-                stub = TaskSchedulerServiceStub(server_channel)
+                stub = ExecutorAPIStub(server_channel)
                 while not self._is_shutdown:
                     try:
-                        # TODO: Report state once before starting the stream.
+                        # Report state once before starting the stream so Server
+                        # doesn't use old state it knew about this Executor in the past.
+                        await self._state_reporter.report_state(stub)
                         desired_states_stream: AsyncGenerator[
                             DesiredExecutorState, None
                         ] = stub.get_desired_executor_states(

indexify/executor/grpc/state_reporter.py

@@ -1,37 +1,44 @@
 import asyncio
+import hashlib
+from socket import gethostname
 from typing import Any, Dict, List, Optional
 
 import grpc
 
-from indexify.proto.task_scheduler_pb2 import (
+from indexify.proto.executor_api_pb2 import (
     AllowedFunction,
+)
+from indexify.proto.executor_api_pb2 import ExecutorFlavor as ExecutorFlavorProto
+from indexify.proto.executor_api_pb2 import (
     ExecutorState,
     ExecutorStatus,
     FunctionExecutorDescription,
 )
-from indexify.proto.task_scheduler_pb2 import (
+from indexify.proto.executor_api_pb2 import (
     FunctionExecutorState as FunctionExecutorStateProto,
 )
-from indexify.proto.task_scheduler_pb2 import (
+from indexify.proto.executor_api_pb2 import (
     FunctionExecutorStatus as FunctionExecutorStatusProto,
 )
-from indexify.proto.task_scheduler_pb2 import (
+from indexify.proto.executor_api_pb2 import (
     GPUModel,
     GPUResources,
     HostResources,
     ReportExecutorStateRequest,
 )
-from indexify.proto.task_scheduler_pb2_grpc import (
-    TaskSchedulerServiceStub,
+from indexify.proto.executor_api_pb2_grpc import (
+    ExecutorAPIStub,
 )
 
 from ..api_objects import FunctionURI
+from ..executor_flavor import ExecutorFlavor
 from ..function_executor.function_executor_state import FunctionExecutorState
 from ..function_executor.function_executor_states_container import (
     FunctionExecutorStatesContainer,
 )
 from ..function_executor.function_executor_status import FunctionExecutorStatus
-from .channel_creator import ChannelCreator
+from ..runtime_probes import RuntimeProbes
+from .channel_manager import ChannelManager
 from .metrics.state_reporter import (
     metric_state_report_errors,
     metric_state_report_latency,
@@ -47,24 +54,32 @@ class ExecutorStateReporter:
     def __init__(
         self,
         executor_id: str,
+        flavor: ExecutorFlavor,
+        version: str,
+        labels: Dict[str, str],
         development_mode: bool,
         function_allowlist: Optional[List[FunctionURI]],
         function_executor_states: FunctionExecutorStatesContainer,
-        channel_creator: ChannelCreator,
+        channel_manager: ChannelManager,
         logger: Any,
     ):
         self._executor_id: str = executor_id
+        self._flavor: ExecutorFlavor = flavor
+        self._version: str = version
+        self._labels: Dict[str, str] = labels.copy()
         self._development_mode: bool = development_mode
+        self._hostname: str = gethostname()
         self._function_executor_states: FunctionExecutorStatesContainer = (
             function_executor_states
         )
-        self._channel_creator = channel_creator
+        self._channel_manager = channel_manager
         self._logger: Any = logger.bind(module=__name__)
         self._is_shutdown: bool = False
         self._executor_status: ExecutorStatus = ExecutorStatus.EXECUTOR_STATUS_UNKNOWN
         self._allowed_functions: List[AllowedFunction] = _to_grpc_allowed_functions(
             function_allowlist
         )
+        self._labels.update(_label_values_to_strings(RuntimeProbes().probe().labels))
 
     def update_executor_status(self, value: ExecutorStatus):
         self._executor_status = value
@@ -75,12 +90,16 @@ class ExecutorStateReporter:
         Never raises any exceptions.
         """
         while not self._is_shutdown:
-            async with await self._channel_creator.create() as server_channel:
+            async with await self._channel_manager.get_channel() as server_channel:
                 server_channel: grpc.aio.Channel
-                stub = TaskSchedulerServiceStub(server_channel)
+                stub = ExecutorAPIStub(server_channel)
                 while not self._is_shutdown:
                     try:
-                        await self._report_state(stub)
+                        # The periodic state reports serve as channel health monitoring requests
+                        # (same as TCP keep-alive). Channel Manager returns the same healthy channel
+                        # for all RPCs that we do from Executor to Server. So all the RPCs benefit
+                        # from this channel health monitoring.
+                        await self.report_state(stub)
                         await asyncio.sleep(_REPORTING_INTERVAL_SEC)
                     except Exception as e:
                         self._logger.error(
@@ -92,7 +111,11 @@ class ExecutorStateReporter:
 
         self._logger.info("State reporter shutdown")
 
-    async def _report_state(self, stub: TaskSchedulerServiceStub):
+    async def report_state(self, stub: ExecutorAPIStub):
+        """Reports the current state to the server represented by the supplied stub.
+
+        Raises exceptions on failure.
+        """
         with (
             metric_state_report_errors.count_exceptions(),
             metric_state_report_latency.time(),
@@ -101,11 +124,16 @@ class ExecutorStateReporter:
             state = ExecutorState(
                 executor_id=self._executor_id,
                 development_mode=self._development_mode,
-                executor_status=self._executor_status,
+                hostname=self._hostname,
+                flavor=_to_grpc_executor_flavor(self._flavor, self._logger),
+                version=self._version,
+                status=self._executor_status,
                 free_resources=await self._fetch_free_host_resources(),
                 allowed_functions=self._allowed_functions,
                 function_executor_states=await self._fetch_function_executor_states(),
+                labels=self._labels,
             )
+            state.state_hash = _state_hash(state)
 
             await stub.report_executor_state(
                 ReportExecutorStateRequest(executor_state=state),
@@ -197,3 +225,33 @@ def _to_grpc_function_executor_status(
         logger.error("Unexpected Function Executor status", status=status)
 
     return result
+
+
+_FLAVOR_MAPPING = {
+    ExecutorFlavor.OSS: ExecutorFlavorProto.EXECUTOR_FLAVOR_OSS,
+    ExecutorFlavor.PLATFORM: ExecutorFlavorProto.EXECUTOR_FLAVOR_PLATFORM,
+}
+
+
+def _to_grpc_executor_flavor(
+    flavor: ExecutorFlavor, logger: Any
+) -> ExecutorFlavorProto:
+    result: ExecutorFlavorProto = _FLAVOR_MAPPING.get(
+        flavor, ExecutorFlavorProto.EXECUTOR_FLAVOR_UNKNOWN
+    )
+
+    if result == ExecutorFlavorProto.EXECUTOR_FLAVOR_UNKNOWN:
+        logger.error("Unexpected Executor flavor", flavor=flavor)
+
+    return result
+
+
+def _label_values_to_strings(labels: Dict[str, Any]) -> Dict[str, str]:
+    return {k: str(v) for k, v in labels.items()}
+
+
+def _state_hash(state: ExecutorState) -> str:
+    serialized_state: bytes = state.SerializeToString(deterministic=True)
+    hasher = hashlib.sha256(usedforsecurity=False)
+    hasher.update(serialized_state)
+    return hasher.hexdigest()

indexify/executor/task_fetcher.py

@@ -1,6 +1,7 @@
 import json
 import time
-from typing import AsyncGenerator, List, Optional
+from socket import gethostname
+from typing import AsyncGenerator, Dict, List, Optional
 
 import structlog
 from httpx_sse import aconnect_sse
@@ -22,6 +23,7 @@ class TaskFetcher:
         self,
         executor_id: str,
         executor_version: str,
+        labels: Dict[str, str],
         function_allowlist: Optional[List[FunctionURI]],
         protocol: str,
         indexify_server_addr: str,
@@ -33,12 +35,15 @@ class TaskFetcher:
         self._logger = structlog.get_logger(module=__name__)
 
         probe_info: ProbeInfo = RuntimeProbes().probe()
+        all_labels = probe_info.labels.copy()
+        all_labels.update(labels)
+
         self._executor_metadata: ExecutorMetadata = ExecutorMetadata(
             id=executor_id,
             executor_version=executor_version,
-            addr="",
+            addr=gethostname(),
             function_allowlist=function_allowlist,
-            labels=probe_info.labels,
+            labels=all_labels,
         )
 
     async def run(self) -> AsyncGenerator[Task, None]:

indexify/executor/task_reporter.py

@@ -49,6 +49,7 @@ class TaskReporter:
     ):
         self._base_url = base_url
         self._executor_id = executor_id
+        self._is_shutdown = False
         # Use thread-safe sync client due to issues with async client.
         # Async client attempts to use connections it already closed.
         # See e.g. https://github.com/encode/httpx/issues/2337.
@@ -56,9 +57,25 @@ class TaskReporter:
         # results in not reusing established TCP connections to server.
         self._client = get_httpx_client(config_path, make_async=False)
 
+    async def shutdown(self):
+        """Shuts down the task reporter.
+
+        Task reporter stops reporting all task outcomes to the Server.
+        There are many task failures due to Executor shutdown. We give wrong
+        signals to Server if we report such failures.
+        """
+        self._is_shutdown = True
+
     async def report(self, output: TaskOutput, logger: Any):
         """Reports result of the supplied task."""
         logger = logger.bind(module=__name__)
+
+        if self._is_shutdown:
+            logger.warning(
+                "task reporter got shutdown, skipping task outcome reporting"
+            )
+            return
+
         task_result, output_files, output_summary = self._process_task_output(output)
         task_result_data = task_result.model_dump_json(exclude_none=True)
 

indexify/proto/{task_scheduler.proto → executor_api.proto}

@@ -1,6 +1,8 @@
 syntax = "proto3";
 
-package task_scheduler_service;
+// Rename with caution. The package name is part of gRPC service name.
+// Existing clients won't find the service if the package name changes.
+package executor_api_pb;
 
 // ===== ReportExecutorState RPC =====
 
@@ -81,15 +83,26 @@ enum ExecutorStatus {
     EXECUTOR_STATUS_STOPPED = 5;
 }
 
+enum ExecutorFlavor {
+    EXECUTOR_FLAVOR_UNKNOWN = 0;
+    EXECUTOR_FLAVOR_OSS = 1;
+    EXECUTOR_FLAVOR_PLATFORM = 2;
+}
+
 message ExecutorState {
     optional string executor_id = 1;
     optional bool development_mode = 2;
-    optional ExecutorStatus executor_status = 3;
+    optional string hostname = 3;
+    optional ExecutorFlavor flavor = 4;
+    optional string version = 5;
+    optional ExecutorStatus status = 6;
     // Free resources available at the Executor.
-    optional HostResources free_resources = 4;
+    optional HostResources free_resources = 7;
     // Empty allowed_functions list means that any function can run on the Executor.
-    repeated AllowedFunction allowed_functions = 5;
-    repeated FunctionExecutorState function_executor_states = 6;
+    repeated AllowedFunction allowed_functions = 8;
+    repeated FunctionExecutorState function_executor_states = 9;
+    map<string, string> labels = 10;
+    optional string state_hash = 11;
 }
 
 // A message sent by Executor to report its up to date state to Server.
@@ -136,7 +149,11 @@ message DesiredExecutorState {
 
 // Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
 // Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
-service TaskSchedulerService {
+//
+// Rename with caution. Existing clients won't find the service if the service name changes. A HTTP2 ingress proxy
+// might use the service name in it HTTP2 path based routing rules. See how gRPC uses service names in its HTTP2 paths
+// at https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md.
+service ExecutorAPI {
     // Called by Executor every 5 seconds to report that it's still alive and provide its current state.
     //
     // Missing 3 reports will result in the Executor being deregistered by Server.

indexify/proto/executor_api_pb2.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# NO CHECKED-IN PROTOBUF GENCODE
+# source: indexify/proto/executor_api.proto
+# Protobuf Python Version: 5.29.0
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import runtime_version as _runtime_version
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+
+_runtime_version.ValidateProtobufRuntimeVersion(
+    _runtime_version.Domain.PUBLIC, 5, 29, 0, "", "indexify/proto/executor_api.proto"
+)
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
+    b'\n!indexify/proto/executor_api.proto\x12\x0f\x65xecutor_api_pb"e\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12-\n\x05model\x18\x02 \x01(\x0e\x32\x19.executor_api_pb.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc2\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12/\n\x03gpu\x18\x04 \x01(\x0b\x32\x1d.executor_api_pb.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xed\x02\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12<\n\x0fresource_limits\x18\x08 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x12\n\x10_resource_limits"\xb8\x01\n\x15\x46unctionExecutorState\x12\x46\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32,.executor_api_pb.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12<\n\x06status\x18\x02 \x01(\x0e\x32\'.executor_api_pb.FunctionExecutorStatusH\x01\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_status"\x9f\x05\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1d\n\x10\x64\x65velopment_mode\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x15\n\x08hostname\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x34\n\x06\x66lavor\x18\x04 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorFlavorH\x03\x88\x01\x01\x12\x14\n\x07version\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x34\n\x06status\x18\x06 \x01(\x0e\x32\x1f.executor_api_pb.ExecutorStatusH\x05\x88\x01\x01\x12;\n\x0e\x66ree_resources\x18\x07 \x01(\x0b\x32\x1e.executor_api_pb.HostResourcesH\x06\x88\x01\x01\x12;\n\x11\x61llowed_functions\x18\x08 \x03(\x0b\x32 .executor_api_pb.AllowedFunction\x12H\n\x18\x66unction_executor_states\x18\t \x03(\x0b\x32&.executor_api_pb.FunctionExecutorState\x12:\n\x06labels\x18\n \x03(\x0b\x32*.executor_api_pb.ExecutorState.LabelsEntry\x12\x17\n\nstate_hash\x18\x0b \x01(\tH\x07\x88\x01\x01\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0c_executor_idB\x13\n\x11_development_modeB\x0b\n\t_hostnameB\t\n\x07_flavorB\n\n\x08_versionB\t\n\x07_statusB\x11\n\x0f_free_resourcesB\r\n\x0b_state_hash"l\n\x1aReportExecutorStateRequest\x12;\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32\x1e.executor_api_pb.ExecutorStateH\x00\x88\x01\x01\x42\x11\n\x0f_executor_state"\x1d\n\x1bReportExecutorStateResponse"\x88\x03\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x16\n\tinput_key\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x1f\n\x12reducer_output_key\x18\t \x01(\tH\x07\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\tH\x08\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\x0c\n\n_input_keyB\x15\n\x13_reducer_output_keyB\r\n\x0b_timeout_ms"\x7f\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12(\n\x04task\x18\x02 \x01(\x0b\x32\x15.executor_api_pb.TaskH\x01\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_task"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xb9\x01\n\x14\x44\x65siredExecutorState\x12H\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32,.executor_api_pb.FunctionExecutorDescription\x12\x39\n\x10task_allocations\x18\x02 \x03(\x0b\x32\x1f.executor_api_pb.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock*\x86\x03\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12"\n\x1eGPU_MODEL_NVIDIA_TESLA_T4_16GB\x10\n\x12$\n GPU_MODEL_NVIDIA_TESLA_V100_16GB\x10\x14\x12\x1d\n\x19GPU_MODEL_NVIDIA_A10_24GB\x10\x1e\x12\x1f\n\x1bGPU_MODEL_NVIDIA_A6000_48GB\x10(\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_40GB\x10\x32\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_80GB\x10\x33\x12"\n\x1eGPU_MODEL_NVIDIA_A100_PCI_40GB\x10\x34\x12#\n\x1fGPU_MODEL_NVIDIA_H100_SXM5_80GB\x10<\x12"\n\x1eGPU_MODEL_NVIDIA_H100_PCI_80GB\x10=\x12"\n\x1eGPU_MODEL_NVIDIA_RTX_6000_24GB\x10>*\xa3\x03\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12(\n$FUNCTION_EXECUTOR_STATUS_STARTING_UP\x10\x01\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR\x10\x02\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_PLATFORM_ERROR\x10\x03\x12!\n\x1d\x46UNCTION_EXECUTOR_STATUS_IDLE\x10\x04\x12)\n%FUNCTION_EXECUTOR_STATUS_RUNNING_TASK\x10\x05\x12&\n"FUNCTION_EXECUTOR_STATUS_UNHEALTHY\x10\x06\x12%\n!FUNCTION_EXECUTOR_STATUS_STOPPING\x10\x07\x12$\n FUNCTION_EXECUTOR_STATUS_STOPPED\x10\x08*\xc3\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1c\n\x18\x45XECUTOR_STATUS_STOPPING\x10\x04\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x05*d\n\x0e\x45xecutorFlavor\x12\x1b\n\x17\x45XECUTOR_FLAVOR_UNKNOWN\x10\x00\x12\x17\n\x13\x45XECUTOR_FLAVOR_OSS\x10\x01\x12\x1c\n\x18\x45XECUTOR_FLAVOR_PLATFORM\x10\x02\x32\xff\x01\n\x0b\x45xecutorAPI\x12t\n\x15report_executor_state\x12+.executor_api_pb.ReportExecutorStateRequest\x1a,.executor_api_pb.ReportExecutorStateResponse"\x00\x12z\n\x1bget_desired_executor_states\x12\x30.executor_api_pb.GetDesiredExecutorStatesRequest\x1a%.executor_api_pb.DesiredExecutorState"\x00\x30\x01\x62\x06proto3'
+)
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(
+    DESCRIPTOR, "indexify.proto.executor_api_pb2", _globals
+)
+if not _descriptor._USE_C_DESCRIPTORS:
+    DESCRIPTOR._loaded_options = None
+    _globals["_EXECUTORSTATE_LABELSENTRY"]._loaded_options = None
+    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_options = b"8\001"
+    _globals["_GPUMODEL"]._serialized_start = 2704
+    _globals["_GPUMODEL"]._serialized_end = 3094
+    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 3097
+    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 3516
+    _globals["_EXECUTORSTATUS"]._serialized_start = 3519
+    _globals["_EXECUTORSTATUS"]._serialized_end = 3714
+    _globals["_EXECUTORFLAVOR"]._serialized_start = 3716
+    _globals["_EXECUTORFLAVOR"]._serialized_end = 3816
+    _globals["_GPURESOURCES"]._serialized_start = 54
+    _globals["_GPURESOURCES"]._serialized_end = 155
+    _globals["_HOSTRESOURCES"]._serialized_start = 158
+    _globals["_HOSTRESOURCES"]._serialized_end = 352
+    _globals["_ALLOWEDFUNCTION"]._serialized_start = 355
+    _globals["_ALLOWEDFUNCTION"]._serialized_end = 542
+    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_start = 545
+    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_end = 910
+    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_start = 913
+    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_end = 1097
+    _globals["_EXECUTORSTATE"]._serialized_start = 1100
+    _globals["_EXECUTORSTATE"]._serialized_end = 1771
+    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_start = 1608
+    _globals["_EXECUTORSTATE_LABELSENTRY"]._serialized_end = 1653
+    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_start = 1773
+    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_end = 1881
+    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_start = 1883
+    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_end = 1912
+    _globals["_TASK"]._serialized_start = 1915
+    _globals["_TASK"]._serialized_end = 2307
+    _globals["_TASKALLOCATION"]._serialized_start = 2309
+    _globals["_TASKALLOCATION"]._serialized_end = 2436
+    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_start = 2438
+    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 2513
+    _globals["_DESIREDEXECUTORSTATE"]._serialized_start = 2516
+    _globals["_DESIREDEXECUTORSTATE"]._serialized_end = 2701
+    _globals["_EXECUTORAPI"]._serialized_start = 3819
+    _globals["_EXECUTORAPI"]._serialized_end = 4074
+# @@protoc_insertion_point(module_scope)

indexify/proto/{task_scheduler_pb2.pyi → executor_api_pb2.pyi}

@@ -50,6 +50,12 @@ class ExecutorStatus(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     EXECUTOR_STATUS_STOPPING: _ClassVar[ExecutorStatus]
     EXECUTOR_STATUS_STOPPED: _ClassVar[ExecutorStatus]
 
+class ExecutorFlavor(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    EXECUTOR_FLAVOR_UNKNOWN: _ClassVar[ExecutorFlavor]
+    EXECUTOR_FLAVOR_OSS: _ClassVar[ExecutorFlavor]
+    EXECUTOR_FLAVOR_PLATFORM: _ClassVar[ExecutorFlavor]
+
 GPU_MODEL_UNKNOWN: GPUModel
 GPU_MODEL_NVIDIA_TESLA_T4_16GB: GPUModel
 GPU_MODEL_NVIDIA_TESLA_V100_16GB: GPUModel
@@ -76,6 +82,9 @@ EXECUTOR_STATUS_RUNNING: ExecutorStatus
 EXECUTOR_STATUS_DRAINED: ExecutorStatus
 EXECUTOR_STATUS_STOPPING: ExecutorStatus
 EXECUTOR_STATUS_STOPPED: ExecutorStatus
+EXECUTOR_FLAVOR_UNKNOWN: ExecutorFlavor
+EXECUTOR_FLAVOR_OSS: ExecutorFlavor
+EXECUTOR_FLAVOR_PLATFORM: ExecutorFlavor
 
 class GPUResources(_message.Message):
     __slots__ = ("count", "model")
@@ -178,30 +187,59 @@ class ExecutorState(_message.Message):
     __slots__ = (
         "executor_id",
         "development_mode",
-        "executor_status",
+        "hostname",
+        "flavor",
+        "version",
+        "status",
         "free_resources",
         "allowed_functions",
         "function_executor_states",
+        "labels",
+        "state_hash",
     )
+
+    class LabelsEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(
+            self, key: _Optional[str] = ..., value: _Optional[str] = ...
+        ) -> None: ...
+
     EXECUTOR_ID_FIELD_NUMBER: _ClassVar[int]
     DEVELOPMENT_MODE_FIELD_NUMBER: _ClassVar[int]
-    EXECUTOR_STATUS_FIELD_NUMBER: _ClassVar[int]
+    HOSTNAME_FIELD_NUMBER: _ClassVar[int]
+    FLAVOR_FIELD_NUMBER: _ClassVar[int]
+    VERSION_FIELD_NUMBER: _ClassVar[int]
+    STATUS_FIELD_NUMBER: _ClassVar[int]
     FREE_RESOURCES_FIELD_NUMBER: _ClassVar[int]
     ALLOWED_FUNCTIONS_FIELD_NUMBER: _ClassVar[int]
     FUNCTION_EXECUTOR_STATES_FIELD_NUMBER: _ClassVar[int]
+    LABELS_FIELD_NUMBER: _ClassVar[int]
+    STATE_HASH_FIELD_NUMBER: _ClassVar[int]
     executor_id: str
     development_mode: bool
-    executor_status: ExecutorStatus
+    hostname: str
+    flavor: ExecutorFlavor
+    version: str
+    status: ExecutorStatus
     free_resources: HostResources
     allowed_functions: _containers.RepeatedCompositeFieldContainer[AllowedFunction]
     function_executor_states: _containers.RepeatedCompositeFieldContainer[
         FunctionExecutorState
     ]
+    labels: _containers.ScalarMap[str, str]
+    state_hash: str
     def __init__(
         self,
         executor_id: _Optional[str] = ...,
         development_mode: bool = ...,
-        executor_status: _Optional[_Union[ExecutorStatus, str]] = ...,
+        hostname: _Optional[str] = ...,
+        flavor: _Optional[_Union[ExecutorFlavor, str]] = ...,
+        version: _Optional[str] = ...,
+        status: _Optional[_Union[ExecutorStatus, str]] = ...,
         free_resources: _Optional[_Union[HostResources, _Mapping]] = ...,
         allowed_functions: _Optional[
             _Iterable[_Union[AllowedFunction, _Mapping]]
@@ -209,6 +247,8 @@ class ExecutorState(_message.Message):
         function_executor_states: _Optional[
             _Iterable[_Union[FunctionExecutorState, _Mapping]]
         ] = ...,
+        labels: _Optional[_Mapping[str, str]] = ...,
+        state_hash: _Optional[str] = ...,
     ) -> None: ...
 
 class ReportExecutorStateRequest(_message.Message):

indexify/proto/{task_scheduler_pb2_grpc.py → executor_api_pb2_grpc.py}

@@ -4,9 +4,7 @@ import warnings
 
 import grpc
 
-from indexify.proto import (
-    task_scheduler_pb2 as indexify_dot_proto_dot_task__scheduler__pb2,
-)
+from indexify.proto import executor_api_pb2 as indexify_dot_proto_dot_executor__api__pb2
 
 GRPC_GENERATED_VERSION = "1.70.0"
 GRPC_VERSION = grpc.__version__
@@ -24,16 +22,20 @@ except ImportError:
 if _version_not_supported:
     raise RuntimeError(
         f"The grpc package installed is at version {GRPC_VERSION},"
-        + f" but the generated code in indexify/proto/task_scheduler_pb2_grpc.py depends on"
+        + f" but the generated code in indexify/proto/executor_api_pb2_grpc.py depends on"
         + f" grpcio>={GRPC_GENERATED_VERSION}."
         + f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}"
         + f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}."
     )
 
 
-class TaskSchedulerServiceStub(object):
+class ExecutorAPIStub(object):
     """Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
     Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
+
+    Rename with caution. Existing clients won't find the service if the service name changes. A HTTP2 ingress proxy
+    might use the service name in it HTTP2 path based routing rules. See how gRPC uses service names in its HTTP2 paths
+    at https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md.
     """
 
     def __init__(self, channel):
@@ -43,22 +45,26 @@ class TaskSchedulerServiceStub(object):
             channel: A grpc.Channel.
         """
         self.report_executor_state = channel.unary_unary(
-            "/task_scheduler_service.TaskSchedulerService/report_executor_state",
-            request_serializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateRequest.SerializeToString,
-            response_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateResponse.FromString,
+            "/executor_api_pb.ExecutorAPI/report_executor_state",
+            request_serializer=indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateRequest.SerializeToString,
+            response_deserializer=indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateResponse.FromString,
             _registered_method=True,
         )
         self.get_desired_executor_states = channel.unary_stream(
-            "/task_scheduler_service.TaskSchedulerService/get_desired_executor_states",
-            request_serializer=indexify_dot_proto_dot_task__scheduler__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
-            response_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.DesiredExecutorState.FromString,
+            "/executor_api_pb.ExecutorAPI/get_desired_executor_states",
+            request_serializer=indexify_dot_proto_dot_executor__api__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
+            response_deserializer=indexify_dot_proto_dot_executor__api__pb2.DesiredExecutorState.FromString,
             _registered_method=True,
         )
 
 
-class TaskSchedulerServiceServicer(object):
+class ExecutorAPIServicer(object):
     """Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
     Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
+
+    Rename with caution. Existing clients won't find the service if the service name changes. A HTTP2 ingress proxy
+    might use the service name in it HTTP2 path based routing rules. See how gRPC uses service names in its HTTP2 paths
+    at https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md.
     """
 
     def report_executor_state(self, request, context):
@@ -81,32 +87,36 @@ class TaskSchedulerServiceServicer(object):
         raise NotImplementedError("Method not implemented!")
 
 
-def add_TaskSchedulerServiceServicer_to_server(servicer, server):
+def add_ExecutorAPIServicer_to_server(servicer, server):
     rpc_method_handlers = {
         "report_executor_state": grpc.unary_unary_rpc_method_handler(
             servicer.report_executor_state,
-            request_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateRequest.FromString,
-            response_serializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateResponse.SerializeToString,
+            request_deserializer=indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateRequest.FromString,
+            response_serializer=indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateResponse.SerializeToString,
         ),
         "get_desired_executor_states": grpc.unary_stream_rpc_method_handler(
             servicer.get_desired_executor_states,
-            request_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.GetDesiredExecutorStatesRequest.FromString,
-            response_serializer=indexify_dot_proto_dot_task__scheduler__pb2.DesiredExecutorState.SerializeToString,
+            request_deserializer=indexify_dot_proto_dot_executor__api__pb2.GetDesiredExecutorStatesRequest.FromString,
+            response_serializer=indexify_dot_proto_dot_executor__api__pb2.DesiredExecutorState.SerializeToString,
         ),
     }
     generic_handler = grpc.method_handlers_generic_handler(
-        "task_scheduler_service.TaskSchedulerService", rpc_method_handlers
+        "executor_api_pb.ExecutorAPI", rpc_method_handlers
     )
     server.add_generic_rpc_handlers((generic_handler,))
     server.add_registered_method_handlers(
-        "task_scheduler_service.TaskSchedulerService", rpc_method_handlers
+        "executor_api_pb.ExecutorAPI", rpc_method_handlers
     )
 
 
 # This class is part of an EXPERIMENTAL API.
-class TaskSchedulerService(object):
+class ExecutorAPI(object):
     """Internal API for scheduling and running tasks on Executors. Executors are acting as clients of this API.
     Server is responsible for scheduling tasks on Executors and Executors are responsible for running the tasks.
+
+    Rename with caution. Existing clients won't find the service if the service name changes. A HTTP2 ingress proxy
+    might use the service name in it HTTP2 path based routing rules. See how gRPC uses service names in its HTTP2 paths
+    at https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md.
     """
 
     @staticmethod
@@ -125,9 +135,9 @@ class TaskSchedulerService(object):
         return grpc.experimental.unary_unary(
             request,
             target,
-            "/task_scheduler_service.TaskSchedulerService/report_executor_state",
-            indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateRequest.SerializeToString,
-            indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateResponse.FromString,
+            "/executor_api_pb.ExecutorAPI/report_executor_state",
+            indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateRequest.SerializeToString,
+            indexify_dot_proto_dot_executor__api__pb2.ReportExecutorStateResponse.FromString,
             options,
             channel_credentials,
             insecure,
@@ -155,9 +165,9 @@ class TaskSchedulerService(object):
         return grpc.experimental.unary_stream(
             request,
             target,
-            "/task_scheduler_service.TaskSchedulerService/get_desired_executor_states",
-            indexify_dot_proto_dot_task__scheduler__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
-            indexify_dot_proto_dot_task__scheduler__pb2.DesiredExecutorState.FromString,
+            "/executor_api_pb.ExecutorAPI/get_desired_executor_states",
+            indexify_dot_proto_dot_executor__api__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
+            indexify_dot_proto_dot_executor__api__pb2.DesiredExecutorState.FromString,
             options,
             channel_credentials,
             insecure,

{indexify-0.3.17.dist-info → indexify-0.3.18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.17
+Version: 0.3.18
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.3.17.dist-info → indexify-0.3.18.dist-info}/RECORD

@@ -1,13 +1,14 @@
-indexify/cli/cli.py,sha256=-ngLINNhZ2Req18_dQ1rHEnusY2feQ-9gYxOP8jcorA,8768
+indexify/cli/cli.py,sha256=YGIpXDtWnA3uj9VYhH8PeFGiRBbGBWLb8SyyzIVRKFg,9255
 indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
 indexify/executor/api_objects.py,sha256=oUlH-GQPuPmwgcBzMpI2HehXeElBTCULECk-oHiBHwU,1263
 indexify/executor/downloader.py,sha256=LkvAXfKxddnDzgfmwHcpDB_n795-eVKzn-hLjq4nUEM,9412
-indexify/executor/executor.py,sha256=FTp05YxuKlMUbI99OV7NkL3KuFD12evKcqxzb-fXWBk,14641
+indexify/executor/executor.py,sha256=0wsWDchr4ocLBk2JVVFEA9be-8Qz09kbxPLDUnrJuV0,15198
+indexify/executor/executor_flavor.py,sha256=uilzDQVVYlQGR1MVnrUC4NevUActDWHdnJkr38M6kTk,118
 indexify/executor/function_executor/function_executor.py,sha256=s1mc7g6b8ilc98Fp7RFElEBSLJl0UGNQY0iZzCpuR2A,11334
 indexify/executor/function_executor/function_executor_state.py,sha256=b2taGClg0BUnlD_rYGkpom6syXBMUp7UWWrjLrUCwyo,3966
 indexify/executor/function_executor/function_executor_states_container.py,sha256=RclJDJqIr8ywKipPBC6_idnPAqYi0dPa1d4QUAaXqbw,3460
 indexify/executor/function_executor/function_executor_status.py,sha256=U4p1fcdVWlHr7uPY7e7ZSb2_WelUmPeH-WgboQQ9mw4,3336
-indexify/executor/function_executor/health_checker.py,sha256=CPUWvvtJtJCwbEsfr_BOhSKkRki4qOoSk1oeyBosWz0,5464
+indexify/executor/function_executor/health_checker.py,sha256=Fvd1gmrcjyJqP-8vcsUxfnTHQIMNlHeMWCS70PAVr9E,6095
 indexify/executor/function_executor/invocation_state_client.py,sha256=p-xgM4__cHR1ApvMV9hShrGWee_Je0VDhICZUGjpQY4,9644
 indexify/executor/function_executor/metrics/function_executor.py,sha256=TDksxLRJr-P9ZKhF2Orsaxzzb4lVIBxFEjd_9Zv53Ng,6313
 indexify/executor/function_executor/metrics/function_executor_state.py,sha256=qheMhnoiYLiZB7ky5EyegfDy4Mr0Zh83bOE0gJ38YmU,1607
@@ -23,11 +24,11 @@ indexify/executor/function_executor/server/subprocess_function_executor_server_f
 indexify/executor/function_executor/single_task_runner.py,sha256=iWnJsB2BGqdgAkrlJHbOvSIhVXc88X0AYbB2_o-bB-E,13547
 indexify/executor/function_executor/task_input.py,sha256=wSrHR4m0juiGClQyeVdhRC37QzDt6Rrjq-ZXJkfBi9k,584
 indexify/executor/function_executor/task_output.py,sha256=SQJSlrknB7Ylf5IOeINfBEgiplS5hAPJh1hYulhyvfU,1962
-indexify/executor/grpc/channel_creator.py,sha256=Z_DU212-wkaU_m-I14OBbWKVeHo3aG5vPmF_ebJaZGc,1849
-indexify/executor/grpc/metrics/channel_creator.py,sha256=k-WArgklmP5WhjcmFmrgRblB7yc3XlaOXO8owRyV-mw,649
+indexify/executor/grpc/channel_manager.py,sha256=THamn5VghCxRkXDlu2WEXtC6-SNKGc0xoa718bw9A4k,6257
+indexify/executor/grpc/metrics/channel_manager.py,sha256=k-WArgklmP5WhjcmFmrgRblB7yc3XlaOXO8owRyV-mw,649
 indexify/executor/grpc/metrics/state_reporter.py,sha256=GggBEjMzQUYIG95LtTS4fUg1u9jYowkaXoUXppAXucs,543
-indexify/executor/grpc/state_reconciler.py,sha256=g7Qi6t79vuldh4y0Ue2mfnU2Jj8J-rsDtQzNZZlbLfE,12973
-indexify/executor/grpc/state_reporter.py,sha256=kk1kqvuUfgNZQriWj2FWjkPyloLcWPq7cCuNCK7mb5I,7669
+indexify/executor/grpc/state_reconciler.py,sha256=RvlY2k6QwxryjOYxhf1AMb1T8BRadEYzsU03mS0nQFY,13300
+indexify/executor/grpc/state_reporter.py,sha256=tpbg4A3nMyvwEsrYd-whET821a2ZuS8OLyu89Y3DvBw,9876
 indexify/executor/metrics/downloader.py,sha256=lctPh8xjkXeLEFJnl1hNrD1yEhLhIl5sggsR4Yoe_Zc,2746
 indexify/executor/metrics/executor.py,sha256=ua-Vv_k1CB4juJdF7tEBQbBMksqWAA3iXKKMKXZUCLk,2369
 indexify/executor/metrics/task_fetcher.py,sha256=iJEwCLzYr2cuz7hRvNiqaa2nvQP4OrA0hm0iJY0YKG0,736
@@ -43,14 +44,14 @@ indexify/executor/monitoring/prometheus_metrics_handler.py,sha256=KiGqSf7rkXTfbD
 indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFseOmHyQpTo,1254
 indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpxWDfed1VHtKKnwvLQ8-frpG98,425
 indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
-indexify/executor/task_fetcher.py,sha256=NpFfHgaY99bSL-K2D5kcDAMNUG2FArq0-qF_mgF-LBQ,3375
-indexify/executor/task_reporter.py,sha256=mYgwozUO95PEwYMmeeIS0-HfMrO4z3Nhy6IduMsMahM,7367
+indexify/executor/task_fetcher.py,sha256=p3iEsWyGi0ZMPAv0183smzOUD1KycQ_dXsyd9mpB9IU,3529
+indexify/executor/task_reporter.py,sha256=0D6ToLhDvd9U0ZPRaDMsZJYBsdzZUqcdkpIxHDUrvdk,7892
 indexify/executor/task_runner.py,sha256=1zYH03yS_FaFk9xXBl-ioM74-L2xdW3vHJt522mseds,7073
-indexify/proto/task_scheduler.proto,sha256=kxMIJCj1pXG-fHeJGHXlthZTsB1dy_yvshQLt0UJRTM,5672
-indexify/proto/task_scheduler_pb2.py,sha256=X97JBJZ2n6ToDtUlDjPFV66_vZ05-vO8wPATrpzAonA,9085
-indexify/proto/task_scheduler_pb2.pyi,sha256=aXrB7-eNwgchy2OVlvEfPXtr9EyYoU-sgbdSRVNEI8s,11357
-indexify/proto/task_scheduler_pb2_grpc.py,sha256=STtk9XrBzLbmWdLwpL55Obyf9ehUesfxxysxER32SEE,6854
-indexify-0.3.17.dist-info/METADATA,sha256=hREMWJfSrd4Vcclp2w8fcUnjtvkiXHw6jMCTWECKAtw,1158
-indexify-0.3.17.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.17.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
-indexify-0.3.17.dist-info/RECORD,,
+indexify/proto/executor_api.proto,sha256=-dSnBE35OUoknDDV0HvSOVk11-pPqIjbQ5X22uJ_rSs,6399
+indexify/proto/executor_api_pb2.py,sha256=WffUqYV39xoDmdaHYpckR3XX-pdbOzMoaYO5ghcA1Lg,9949
+indexify/proto/executor_api_pb2.pyi,sha256=lXP79CootL4pHghuVIv1wgR0Y0YPl0wIVUiHKY7PM2s,12677
+indexify/proto/executor_api_pb2_grpc.py,sha256=i8LEPG6esub6C-xxJ7S3vEJSgWCOxSqElNjMW3Imqg8,7607
+indexify-0.3.18.dist-info/METADATA,sha256=T7_EDOfiMyAn0dpZ-m96vgiDVT2oGJDn0N7UYzDYNSA,1158
+indexify-0.3.18.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.18.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
+indexify-0.3.18.dist-info/RECORD,,

indexify/executor/grpc/channel_creator.py

@@ -1,53 +0,0 @@
-import asyncio
-from typing import Any
-
-import grpc.aio
-
-from .metrics.channel_creator import (
-    metric_grpc_server_channel_creation_latency,
-    metric_grpc_server_channel_creation_retries,
-    metric_grpc_server_channel_creations,
-)
-
-_RETRY_INTERVAL_SEC = 5
-_CONNECT_TIMEOUT_SEC = 5
-
-
-class ChannelCreator:
-    def __init__(self, server_address: str, logger: Any):
-        self._logger = logger.bind(module=__name__)
-        self._server_address = server_address
-        self._is_shutdown = False
-
-    async def create(self) -> grpc.aio.Channel:
-        """Creates a channel to the gRPC server.
-
-        Blocks until the channel is ready.
-        Never raises any exceptions.
-        """
-        with metric_grpc_server_channel_creation_latency.time():
-            metric_grpc_server_channel_creations.inc()
-            while not self._is_shutdown:
-                try:
-                    channel = grpc.aio.insecure_channel(self._server_address)
-                    await asyncio.wait_for(
-                        channel.channel_ready(),
-                        timeout=_CONNECT_TIMEOUT_SEC,
-                    )
-                    return channel
-                except Exception:
-                    self._logger.error(
-                        f"failed establishing grpc server channel in {_CONNECT_TIMEOUT_SEC} sec, retrying in {_RETRY_INTERVAL_SEC} sec"
-                    )
-                    try:
-                        await channel.close()
-                    except Exception as e:
-                        self._logger.error(
-                            "failed closing not established channel", exc_info=e
-                        )
-
-                    metric_grpc_server_channel_creation_retries.inc()
-                    await asyncio.sleep(_RETRY_INTERVAL_SEC)
-
-    async def shutdown(self):
-        self._is_shutdown = True

indexify/proto/task_scheduler_pb2.py

@@ -1,64 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# NO CHECKED-IN PROTOBUF GENCODE
-# source: indexify/proto/task_scheduler.proto
-# Protobuf Python Version: 5.29.0
-"""Generated protocol buffer code."""
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import runtime_version as _runtime_version
-from google.protobuf import symbol_database as _symbol_database
-from google.protobuf.internal import builder as _builder
-
-_runtime_version.ValidateProtobufRuntimeVersion(
-    _runtime_version.Domain.PUBLIC, 5, 29, 0, "", "indexify/proto/task_scheduler.proto"
-)
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n#indexify/proto/task_scheduler.proto\x12\x16task_scheduler_service"l\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x34\n\x05model\x18\x02 \x01(\x0e\x32 .task_scheduler_service.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc9\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12\x36\n\x03gpu\x18\x04 \x01(\x0b\x32$.task_scheduler_service.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\xf4\x02\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x14\n\x0csecret_names\x18\x07 \x03(\t\x12\x43\n\x0fresource_limits\x18\x08 \x01(\x0b\x32%.task_scheduler_service.HostResourcesH\x06\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uriB\x12\n\x10_resource_limits"\xc6\x01\n\x15\x46unctionExecutorState\x12M\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32\x33.task_scheduler_service.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12\x43\n\x06status\x18\x02 \x01(\x0e\x32..task_scheduler_service.FunctionExecutorStatusH\x01\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_status"\xb3\x03\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1d\n\x10\x64\x65velopment_mode\x18\x02 \x01(\x08H\x01\x88\x01\x01\x12\x44\n\x0f\x65xecutor_status\x18\x03 \x01(\x0e\x32&.task_scheduler_service.ExecutorStatusH\x02\x88\x01\x01\x12\x42\n\x0e\x66ree_resources\x18\x04 \x01(\x0b\x32%.task_scheduler_service.HostResourcesH\x03\x88\x01\x01\x12\x42\n\x11\x61llowed_functions\x18\x05 \x03(\x0b\x32\'.task_scheduler_service.AllowedFunction\x12O\n\x18\x66unction_executor_states\x18\x06 \x03(\x0b\x32-.task_scheduler_service.FunctionExecutorStateB\x0e\n\x0c_executor_idB\x13\n\x11_development_modeB\x12\n\x10_executor_statusB\x11\n\x0f_free_resources"s\n\x1aReportExecutorStateRequest\x12\x42\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32%.task_scheduler_service.ExecutorStateH\x00\x88\x01\x01\x42\x11\n\x0f_executor_state"\x1d\n\x1bReportExecutorStateResponse"\x88\x03\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x16\n\tinput_key\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x1f\n\x12reducer_output_key\x18\t \x01(\tH\x07\x88\x01\x01\x12\x17\n\ntimeout_ms\x18\n \x01(\tH\x08\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\x0c\n\n_input_keyB\x15\n\x13_reducer_output_keyB\r\n\x0b_timeout_ms"\x86\x01\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12/\n\x04task\x18\x02 \x01(\x0b\x32\x1c.task_scheduler_service.TaskH\x01\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_task"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xc7\x01\n\x14\x44\x65siredExecutorState\x12O\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32\x33.task_scheduler_service.FunctionExecutorDescription\x12@\n\x10task_allocations\x18\x02 \x03(\x0b\x32&.task_scheduler_service.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock*\x86\x03\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12"\n\x1eGPU_MODEL_NVIDIA_TESLA_T4_16GB\x10\n\x12$\n GPU_MODEL_NVIDIA_TESLA_V100_16GB\x10\x14\x12\x1d\n\x19GPU_MODEL_NVIDIA_A10_24GB\x10\x1e\x12\x1f\n\x1bGPU_MODEL_NVIDIA_A6000_48GB\x10(\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_40GB\x10\x32\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_80GB\x10\x33\x12"\n\x1eGPU_MODEL_NVIDIA_A100_PCI_40GB\x10\x34\x12#\n\x1fGPU_MODEL_NVIDIA_H100_SXM5_80GB\x10<\x12"\n\x1eGPU_MODEL_NVIDIA_H100_PCI_80GB\x10=\x12"\n\x1eGPU_MODEL_NVIDIA_RTX_6000_24GB\x10>*\xa3\x03\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12(\n$FUNCTION_EXECUTOR_STATUS_STARTING_UP\x10\x01\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR\x10\x02\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_PLATFORM_ERROR\x10\x03\x12!\n\x1d\x46UNCTION_EXECUTOR_STATUS_IDLE\x10\x04\x12)\n%FUNCTION_EXECUTOR_STATUS_RUNNING_TASK\x10\x05\x12&\n"FUNCTION_EXECUTOR_STATUS_UNHEALTHY\x10\x06\x12%\n!FUNCTION_EXECUTOR_STATUS_STOPPING\x10\x07\x12$\n FUNCTION_EXECUTOR_STATUS_STOPPED\x10\x08*\xc3\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1f\n\x1b\x45XECUTOR_STATUS_STARTING_UP\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12\x1c\n\x18\x45XECUTOR_STATUS_STOPPING\x10\x04\x12\x1b\n\x17\x45XECUTOR_STATUS_STOPPED\x10\x05\x32\xa6\x02\n\x14TaskSchedulerService\x12\x82\x01\n\x15report_executor_state\x12\x32.task_scheduler_service.ReportExecutorStateRequest\x1a\x33.task_scheduler_service.ReportExecutorStateResponse"\x00\x12\x88\x01\n\x1bget_desired_executor_states\x12\x37.task_scheduler_service.GetDesiredExecutorStatesRequest\x1a,.task_scheduler_service.DesiredExecutorState"\x00\x30\x01\x62\x06proto3'
-)
-
-_globals = globals()
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
-_builder.BuildTopDescriptorsAndMessages(
-    DESCRIPTOR, "indexify.proto.task_scheduler_pb2", _globals
-)
-if not _descriptor._USE_C_DESCRIPTORS:
-    DESCRIPTOR._loaded_options = None
-    _globals["_GPUMODEL"]._serialized_start = 2541
-    _globals["_GPUMODEL"]._serialized_end = 2931
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 2934
-    _globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 3353
-    _globals["_EXECUTORSTATUS"]._serialized_start = 3356
-    _globals["_EXECUTORSTATUS"]._serialized_end = 3551
-    _globals["_GPURESOURCES"]._serialized_start = 63
-    _globals["_GPURESOURCES"]._serialized_end = 171
-    _globals["_HOSTRESOURCES"]._serialized_start = 174
-    _globals["_HOSTRESOURCES"]._serialized_end = 375
-    _globals["_ALLOWEDFUNCTION"]._serialized_start = 378
-    _globals["_ALLOWEDFUNCTION"]._serialized_end = 565
-    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_start = 568
-    _globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_end = 940
-    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_start = 943
-    _globals["_FUNCTIONEXECUTORSTATE"]._serialized_end = 1141
-    _globals["_EXECUTORSTATE"]._serialized_start = 1144
-    _globals["_EXECUTORSTATE"]._serialized_end = 1579
-    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_start = 1581
-    _globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_end = 1696
-    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_start = 1698
-    _globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_end = 1727
-    _globals["_TASK"]._serialized_start = 1730
-    _globals["_TASK"]._serialized_end = 2122
-    _globals["_TASKALLOCATION"]._serialized_start = 2125
-    _globals["_TASKALLOCATION"]._serialized_end = 2259
-    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_start = 2261
-    _globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 2336
-    _globals["_DESIREDEXECUTORSTATE"]._serialized_start = 2339
-    _globals["_DESIREDEXECUTORSTATE"]._serialized_end = 2538
-    _globals["_TASKSCHEDULERSERVICE"]._serialized_start = 3554
-    _globals["_TASKSCHEDULERSERVICE"]._serialized_end = 3848
-# @@protoc_insertion_point(module_scope)