indexify 0.3.9__tar.gz → 0.3.11__tar.gz

{indexify-0.3.9 → indexify-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.9
+Version: 0.3.11
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -14,15 +14,17 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
 Requires-Dist: grpcio (==1.70.0)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
 Requires-Dist: httpx[http2] (>=0.27,<0.28)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
+Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
 Requires-Dist: pydantic (==2.10.4)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.16)
+Requires-Dist: tensorlake (>=0.1.20)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.9 → indexify-0.3.11}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.9"
+version = "0.3.11"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -23,8 +23,10 @@ grpcio = "1.70.0"
 # Executor only
 pydantic = "2.10.4"
 httpx-sse = "^0.4.0"
+aiohttp = "^3.11.0"
+prometheus-client = "^0.21.1"
 # Adds function-executor binary and utils lib.
-tensorlake = ">=0.1.16"
+tensorlake = ">=0.1.20"
 
 # CLI only
 rich = "^13.9.2"

{indexify-0.3.9 → indexify-0.3.11}/src/indexify/cli/cli.py

@@ -6,8 +6,6 @@ from tensorlake.utils.logging import (
 
 configure_logging_early()
 
-import importlib.metadata
-import json
 import os
 import shutil
 import signal
@@ -17,22 +15,26 @@ import threading
 import time
 from importlib.metadata import version
 from pathlib import Path
+from socket import gethostname
 from typing import Annotated, List, Optional, Tuple
 
-import docker
 import nanoid
+import prometheus_client
 import structlog
 import typer
 from rich.console import Console
 from rich.text import Text
 from rich.theme import Theme
-from tensorlake.functions_sdk.image import GetDefaultPythonImage, Image
+from tensorlake.functions_sdk.image import Image
 
 from indexify.executor.api_objects import FunctionURI
 from indexify.executor.executor import Executor
 from indexify.executor.function_executor.server.subprocess_function_executor_server_factory import (
     SubprocessFunctionExecutorServerFactory,
 )
+from indexify.executor.monitoring.health_checker.generic_health_checker import (
+    GenericHealthChecker,
+)
 
 custom_theme = Theme(
     {
@@ -185,8 +187,23 @@ def executor(
     ),
     # Registred ports range ends at 49151.
     ports: Tuple[int, int] = typer.Option(
-        (50000, 51000), help="Range of localhost TCP ports to be used by the executor"
+        (50000, 51000),
+        help="Range of localhost TCP ports to be used by Function Executors",
     ),
+    monitoring_server_host: Annotated[
+        str,
+        typer.Option(
+            "--monitoring-server-host",
+            help="IP address or hostname where to run Executor Monitoring server",
+        ),
+    ] = "localhost",
+    monitoring_server_port: Annotated[
+        int,
+        typer.Option(
+            "--monitoring-server-port",
+            help="Port where to run Executor Monitoring server",
+        ),
+    ] = 7000,
     disable_automatic_function_executor_management: Annotated[
         bool,
         typer.Option(
@@ -210,6 +227,7 @@ def executor(
 
     logger.info(
         "starting executor",
+        hostname=gethostname(),
         server_addr=server_addr,
         config_path=config_path,
         executor_version=executor_version,
@@ -217,6 +235,8 @@ def executor(
         ports=ports,
         functions=function_uris,
         dev_mode=dev,
+        monitoring_server_host=monitoring_server_host,
+        monitoring_server_port=monitoring_server_port,
         disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     )
 
@@ -235,17 +255,26 @@ def executor(
         )
         exit(1)
 
+    prometheus_client.Info("cli", "CLI information").info(
+        {
+            "package": "indexify",
+        }
+    )
+
     Executor(
         id=id,
         version=executor_version,
-        server_addr=server_addr,
-        config_path=config_path,
+        health_checker=GenericHealthChecker(),
         code_path=executor_cache,
         function_allowlist=_parse_function_uris(function_uris),
         function_executor_server_factory=SubprocessFunctionExecutorServerFactory(
             development_mode=dev,
             server_ports=range(ports[0], ports[1]),
         ),
+        server_addr=server_addr,
+        config_path=config_path,
+        monitoring_server_host=monitoring_server_host,
+        monitoring_server_port=monitoring_server_port,
         disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     ).run()
 

{indexify-0.3.9 → indexify-0.3.11}/src/indexify/executor/api_objects.py

@@ -45,3 +45,7 @@ class TaskResult(BaseModel):
     executor_id: str
     task_id: str
     reducer: bool = False
+
+
+TASK_OUTCOME_SUCCESS = "success"
+TASK_OUTCOME_FAILURE = "failure"

{indexify-0.3.9 → indexify-0.3.11}/src/indexify/executor/downloader.py

@@ -8,6 +8,21 @@ from tensorlake.function_executor.proto.function_executor_pb2 import SerializedO
 from tensorlake.utils.http_client import get_httpx_client
 
 from .api_objects import Task
+from .metrics.downloader import (
+    metric_graph_download_errors,
+    metric_graph_download_latency,
+    metric_graph_downloads,
+    metric_graphs_from_cache,
+    metric_reducer_init_value_download_errors,
+    metric_reducer_init_value_download_latency,
+    metric_reducer_init_value_downloads,
+    metric_task_input_download_errors,
+    metric_task_input_download_latency,
+    metric_task_input_downloads,
+    metric_tasks_downloading_graphs,
+    metric_tasks_downloading_inputs,
+    metric_tasks_downloading_reducer_init_value,
+)
 
 
 class Downloader:
@@ -19,6 +34,33 @@ class Downloader:
         self._client = get_httpx_client(config_path, make_async=True)
 
     async def download_graph(self, task: Task) -> SerializedObject:
+        with (
+            metric_graph_download_errors.count_exceptions(),
+            metric_tasks_downloading_graphs.track_inprogress(),
+            metric_graph_download_latency.time(),
+        ):
+            metric_graph_downloads.inc()
+            return await self._download_graph(task)
+
+    async def download_input(self, task: Task) -> SerializedObject:
+        with (
+            metric_task_input_download_errors.count_exceptions(),
+            metric_tasks_downloading_inputs.track_inprogress(),
+            metric_task_input_download_latency.time(),
+        ):
+            metric_task_input_downloads.inc()
+            return await self._download_input(task)
+
+    async def download_init_value(self, task: Task) -> SerializedObject:
+        with (
+            metric_reducer_init_value_download_errors.count_exceptions(),
+            metric_tasks_downloading_reducer_init_value.track_inprogress(),
+            metric_reducer_init_value_download_latency.time(),
+        ):
+            metric_reducer_init_value_downloads.inc()
+            return await self._download_init_value(task)
+
+    async def _download_graph(self, task: Task) -> SerializedObject:
         # Cache graph to reduce load on the server.
         graph_path = os.path.join(
             self.code_path,
@@ -33,6 +75,7 @@ class Downloader:
             self._read_cached_graph, graph_path
         )
         if graph is not None:
+            metric_graphs_from_cache.inc()
             return graph
 
         logger = self._task_logger(task)
@@ -71,7 +114,7 @@ class Downloader:
         # This also allows to share the same cache between multiple Executors.
         os.replace(tmp_path, path)
 
-    async def download_input(self, task: Task) -> SerializedObject:
+    async def _download_input(self, task: Task) -> SerializedObject:
         logger = self._task_logger(task)
 
         first_function_in_graph = task.invocation_id == task.input_key.split("|")[-1]
@@ -81,10 +124,7 @@ class Downloader:
         else:
             return await self._fetch_function_input(task, logger)
 
-    async def download_init_value(self, task: Task) -> Optional[SerializedObject]:
-        if task.reducer_output_id is None:
-            return None
-
+    async def _download_init_value(self, task: Task) -> SerializedObject:
         logger = self._task_logger(task)
         return await self._fetch_function_init_value(task, logger)
 

{indexify-0.3.9 → indexify-0.3.11}/src/indexify/executor/executor.py

@@ -1,7 +1,8 @@
 import asyncio
 import signal
 from pathlib import Path
-from typing import Any, List, Optional
+from socket import gethostname
+from typing import Any, Dict, List, Optional
 
 import structlog
 from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
@@ -9,13 +10,38 @@ from tensorlake.utils.logging import suppress as suppress_logging
 
 from .api_objects import FunctionURI, Task
 from .downloader import Downloader
+from .function_executor.function_executor_states_container import (
+    FunctionExecutorStatesContainer,
+)
 from .function_executor.server.function_executor_server_factory import (
     FunctionExecutorServerFactory,
 )
+from .metrics.executor import (
+    METRIC_TASKS_COMPLETED_OUTCOME_ALL,
+    METRIC_TASKS_COMPLETED_OUTCOME_ERROR_CUSTOMER_CODE,
+    METRIC_TASKS_COMPLETED_OUTCOME_ERROR_PLATFORM,
+    METRIC_TASKS_COMPLETED_OUTCOME_SUCCESS,
+    metric_executor_info,
+    metric_executor_state,
+    metric_task_outcome_report_latency,
+    metric_task_outcome_report_retries,
+    metric_task_outcome_reports,
+    metric_tasks_completed,
+    metric_tasks_fetched,
+    metric_tasks_reporting_outcome,
+)
+from .monitoring.function_allowlist import function_allowlist_to_info_dict
+from .monitoring.health_check_handler import HealthCheckHandler
+from .monitoring.health_checker.health_checker import HealthChecker
+from .monitoring.prometheus_metrics_handler import PrometheusMetricsHandler
+from .monitoring.server import MonitoringServer
+from .monitoring.startup_probe_handler import StartupProbeHandler
 from .task_fetcher import TaskFetcher
 from .task_reporter import TaskReporter
 from .task_runner import TaskInput, TaskOutput, TaskRunner
 
+metric_executor_state.state("starting")
+
 
 class Executor:
     def __init__(
@@ -23,11 +49,14 @@ class Executor:
         id: str,
         version: str,
         code_path: Path,
+        health_checker: HealthChecker,
         function_allowlist: Optional[List[FunctionURI]],
         function_executor_server_factory: FunctionExecutorServerFactory,
-        server_addr: str = "localhost:8900",
-        config_path: Optional[str] = None,
-        disable_automatic_function_executor_management: bool = False,
+        server_addr: str,
+        config_path: Optional[str],
+        monitoring_server_host: str,
+        monitoring_server_port: int,
+        disable_automatic_function_executor_management: bool,
     ):
         self._logger = structlog.get_logger(module=__name__)
         self._is_shutdown: bool = False
@@ -40,12 +69,25 @@ class Executor:
         self._server_addr = server_addr
         self._base_url = f"{protocol}://{self._server_addr}"
         self._code_path = code_path
+        self._startup_probe_handler = StartupProbeHandler()
+        self._monitoring_server = MonitoringServer(
+            host=monitoring_server_host,
+            port=monitoring_server_port,
+            startup_probe_handler=self._startup_probe_handler,
+            health_probe_handler=HealthCheckHandler(health_checker),
+            metrics_handler=PrometheusMetricsHandler(),
+        )
+        self._function_executor_states = FunctionExecutorStatesContainer()
+        health_checker.set_function_executor_states_container(
+            self._function_executor_states
+        )
         self._task_runner = TaskRunner(
             executor_id=id,
             function_executor_server_factory=function_executor_server_factory,
             base_url=self._base_url,
-            config_path=config_path,
             disable_automatic_function_executor_management=disable_automatic_function_executor_management,
+            function_executor_states=self._function_executor_states,
+            config_path=config_path,
         )
         self._downloader = Downloader(
             code_path=code_path, base_url=self._base_url, config_path=config_path
@@ -63,8 +105,22 @@ class Executor:
             executor_id=id,
             config_path=self._config_path,
         )
+        executor_info: Dict[str, str] = {
+            "id": id,
+            "version": version,
+            "code_path": str(code_path),
+            "server_addr": server_addr,
+            "config_path": str(config_path),
+            "disable_automatic_function_executor_management": str(
+                disable_automatic_function_executor_management
+            ),
+            "hostname": gethostname(),
+        }
+        executor_info.update(function_allowlist_to_info_dict(function_allowlist))
+        metric_executor_info.info(executor_info)
 
     def run(self):
+        asyncio.new_event_loop()
         for signum in [
             signal.SIGABRT,
             signal.SIGINT,
@@ -76,15 +132,20 @@ class Executor:
                 signum, self.shutdown, asyncio.get_event_loop()
             )
 
+        asyncio.get_event_loop().create_task(self._monitoring_server.run())
+
         try:
-            asyncio.get_event_loop().run_until_complete(self._run_async())
+            asyncio.get_event_loop().run_until_complete(self._run_tasks_loop())
         except asyncio.CancelledError:
             pass  # Suppress this expected exception and return without error (normally).
 
-    async def _run_async(self):
+    async def _run_tasks_loop(self):
+        metric_executor_state.state("running")
+        self._startup_probe_handler.set_ready()
         while not self._is_shutdown:
             try:
                 async for task in self._task_fetcher.run():
+                    metric_tasks_fetched.inc()
                     asyncio.create_task(self._run_task(task))
             except Exception as e:
                 self._logger.error(
@@ -103,9 +164,10 @@ class Executor:
             graph: SerializedObject = await self._downloader.download_graph(task)
             input: SerializedObject = await self._downloader.download_input(task)
             init_value: Optional[SerializedObject] = (
-                await self._downloader.download_init_value(task)
+                None
+                if task.reducer_output_id is None
+                else (await self._downloader.download_init_value(task))
             )
-            logger.info("task_execution_started")
             output: TaskOutput = await self._task_runner.run(
                 TaskInput(
                     task=task,
@@ -115,15 +177,22 @@ class Executor:
                 ),
                 logger=logger,
             )
-            logger.info("task_execution_finished", success=output.success)
+            logger.info("task execution finished", success=output.success)
         except Exception as e:
             output = TaskOutput.internal_error(task)
-            logger.error("task_execution_failed", exc_info=e)
+            logger.error("task execution failed", exc_info=e)
 
-        await self._report_task_outcome(output=output, logger=logger)
+        with (
+            metric_tasks_reporting_outcome.track_inprogress(),
+            metric_task_outcome_report_latency.time(),
+        ):
+            metric_task_outcome_reports.inc()
+            await self._report_task_outcome(output=output, logger=logger)
 
     async def _report_task_outcome(self, output: TaskOutput, logger: Any) -> None:
-        """Reports the task with the given output to the server."""
+        """Reports the task with the given output to the server.
+
+        Doesn't raise any Exceptions. Runs till the reporting is successful."""
         reporting_retries: int = 0
 
         while True:
@@ -133,22 +202,40 @@ class Executor:
                 break
             except Exception as e:
                 logger.error(
-                    "failed_to_report_task",
+                    "failed to report task",
                     exc_info=e,
                 )
                 reporting_retries += 1
+                metric_task_outcome_report_retries.inc()
                 await asyncio.sleep(5)
 
+        metric_tasks_completed.labels(outcome=METRIC_TASKS_COMPLETED_OUTCOME_ALL).inc()
+        if output.is_internal_error:
+            metric_tasks_completed.labels(
+                outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_PLATFORM
+            ).inc()
+        elif output.success:
+            metric_tasks_completed.labels(
+                outcome=METRIC_TASKS_COMPLETED_OUTCOME_SUCCESS
+            ).inc()
+        else:
+            metric_tasks_completed.labels(
+                outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_CUSTOMER_CODE
+            ).inc()
+
     async def _shutdown(self, loop):
-        self._logger.info("shutting_down")
+        self._logger.info("shutting down")
+        metric_executor_state.state("shutting_down")
         # There will be lots of task cancellation exceptions and "X is shutting down"
         # exceptions logged during Executor shutdown. Suppress their logs as they are
         # expected and are confusing for users.
         suppress_logging()
 
         self._is_shutdown = True
+        await self._monitoring_server.shutdown()
         await self._task_runner.shutdown()
-        # We mainly need to cancel the task that runs _run_async() loop.
+        await self._function_executor_states.shutdown()
+        # We mainly need to cancel the task that runs _run_tasks_loop().
         for task in asyncio.all_tasks(loop):
             task.cancel()