indexify 0.3.8__py3-none-any.whl → 0.3.10__py3-none-any.whl

indexify/cli/cli.py

@@ -6,8 +6,6 @@ from tensorlake.utils.logging import (
 
 configure_logging_early()
 
-import importlib.metadata
-import json
 import os
 import shutil
 import signal
@@ -17,10 +15,11 @@ import threading
 import time
 from importlib.metadata import version
 from pathlib import Path
+from socket import gethostname
 from typing import Annotated, List, Optional, Tuple
 
-import docker
 import nanoid
+import prometheus_client
 import structlog
 import typer
 from rich.console import Console
@@ -33,6 +32,9 @@ from indexify.executor.executor import Executor
 from indexify.executor.function_executor.server.subprocess_function_executor_server_factory import (
     SubprocessFunctionExecutorServerFactory,
 )
+from indexify.executor.monitoring.health_checker.generic_health_checker import (
+    GenericHealthChecker,
+)
 
 custom_theme = Theme(
     {
@@ -158,25 +160,6 @@ def build_image(
                 _create_image(obj, python_sdk_path)
 
 
-@app.command(help="Build default image for indexify")
-def build_default_image(
-    python_version: Optional[str] = typer.Option(
-        f"{sys.version_info.major}.{sys.version_info.minor}",
-        help="Python version to use in the base image",
-    )
-):
-    image = GetDefaultPythonImage(python_version)
-
-    _build_image(image=image)
-
-    console.print(
-        Text(f"Built default indexify image with hash {image.hash()}\n", style="cyan"),
-        Text(
-            f"Don't forget to update your executors to run this image!", style="yellow"
-        ),
-    )
-
-
 @app.command(
     help="Runs Executor that connects to the Indexify server and starts running its tasks"
 )
@@ -204,8 +187,23 @@ def executor(
     ),
     # Registred ports range ends at 49151.
     ports: Tuple[int, int] = typer.Option(
-        (50000, 51000), help="Range of localhost TCP ports to be used by the executor"
+        (50000, 51000),
+        help="Range of localhost TCP ports to be used by Function Executors",
     ),
+    monitoring_server_host: Annotated[
+        str,
+        typer.Option(
+            "--monitoring-server-host",
+            help="IP address or hostname where to run Executor Monitoring server",
+        ),
+    ] = "localhost",
+    monitoring_server_port: Annotated[
+        int,
+        typer.Option(
+            "--monitoring-server-port",
+            help="Port where to run Executor Monitoring server",
+        ),
+    ] = 7000,
     disable_automatic_function_executor_management: Annotated[
         bool,
         typer.Option(
@@ -229,6 +227,7 @@ def executor(
 
     logger.info(
         "starting executor",
+        hostname=gethostname(),
         server_addr=server_addr,
         config_path=config_path,
         executor_version=executor_version,
@@ -236,6 +235,8 @@ def executor(
         ports=ports,
         functions=function_uris,
         dev_mode=dev,
+        monitoring_server_host=monitoring_server_host,
+        monitoring_server_port=monitoring_server_port,
         disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     )
 
@@ -254,17 +255,26 @@ def executor(
         )
         exit(1)
 
+    prometheus_client.Info("cli", "CLI information").info(
+        {
+            "package": "indexify",
+        }
+    )
+
     Executor(
         id=id,
         version=executor_version,
-        server_addr=server_addr,
-        config_path=config_path,
+        health_checker=GenericHealthChecker(),
         code_path=executor_cache,
         function_allowlist=_parse_function_uris(function_uris),
         function_executor_server_factory=SubprocessFunctionExecutorServerFactory(
             development_mode=dev,
             server_ports=range(ports[0], ports[1]),
         ),
+        server_addr=server_addr,
+        config_path=config_path,
+        monitoring_server_host=monitoring_server_host,
+        monitoring_server_port=monitoring_server_port,
         disable_automatic_function_executor_management=disable_automatic_function_executor_management,
     ).run()
 
@@ -307,57 +317,7 @@ def _create_image(image: Image, python_sdk_path):
 
 
 def _build_image(image: Image, python_sdk_path: Optional[str] = None):
-    docker_file = _generate_dockerfile(image, python_sdk_path=python_sdk_path)
-    image_name = f"{image._image_name}:{image._tag}"
-
-    # low_level_client = docker.APIClient(base_url=docker_client.api.base_url)
-    docker_host = os.getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
-    low_level_client = docker.APIClient(base_url=docker_host)
-    docker.api.build.process_dockerfile = lambda dockerfile, path: (
-        "Dockerfile",
-        dockerfile,
-    )
-    generator = low_level_client.build(
-        dockerfile=docker_file,
-        rm=True,
-        path=".",
-        tag=image_name,
-    )
-
+    built_image, generator = image.build(python_sdk_path=python_sdk_path)
     for output in generator:
-        for line in output.decode().splitlines():
-            json_line = json.loads(line)
-            if "stream" in json_line:
-                print(json_line["stream"], end="")
-
-            elif "errorDetail" in json_line:
-                print(json_line["errorDetail"]["message"])
-
-
-def _generate_dockerfile(image, python_sdk_path: Optional[str] = None):
-    docker_contents = [
-        f"FROM {image._base_image}",
-        "RUN mkdir -p ~/.indexify",
-        f"RUN echo {image._image_name} > ~/.indexify/image_name",  # TODO: Do we still use this in executors?
-        f"RUN echo {image.hash()} > ~/.indexify/image_hash",  # TODO: Do we still use this in executors?
-        "WORKDIR /app",
-    ]
-
-    for build_op in image._build_ops:
-        docker_contents.append(build_op.render())
-
-    if python_sdk_path is not None:
-        print(f"Building image {image._image_name} with local version of the SDK")
-
-        if not os.path.exists(python_sdk_path):
-            print(f"error: {python_sdk_path} does not exist")
-            os.exit(1)
-        docker_contents.append(f"COPY {python_sdk_path} /app/python-sdk")
-        docker_contents.append("RUN (cd /app/python-sdk && pip install .)")
-    else:
-        docker_contents.append(
-            f"RUN pip install indexify=={importlib.metadata.version('indexify')}"
-        )
-
-    docker_file = "\n".join(docker_contents)
-    return docker_file
+        print(output)
+    print(f"built image: {built_image.tags[0]}")

indexify/executor/api_objects.py

@@ -45,3 +45,7 @@ class TaskResult(BaseModel):
     executor_id: str
     task_id: str
     reducer: bool = False
+
+
+TASK_OUTCOME_SUCCESS = "success"
+TASK_OUTCOME_FAILURE = "failure"

indexify/executor/downloader.py

@@ -8,6 +8,21 @@ from tensorlake.function_executor.proto.function_executor_pb2 import SerializedO
 from tensorlake.utils.http_client import get_httpx_client
 
 from .api_objects import Task
+from .metrics.downloader import (
+    metric_graph_download_errors,
+    metric_graph_download_latency,
+    metric_graph_downloads,
+    metric_graphs_from_cache,
+    metric_reducer_init_value_download_errors,
+    metric_reducer_init_value_download_latency,
+    metric_reducer_init_value_downloads,
+    metric_task_input_download_errors,
+    metric_task_input_download_latency,
+    metric_task_input_downloads,
+    metric_tasks_downloading_graphs,
+    metric_tasks_downloading_inputs,
+    metric_tasks_downloading_reducer_init_value,
+)
 
 
 class Downloader:
@@ -19,6 +34,33 @@ class Downloader:
         self._client = get_httpx_client(config_path, make_async=True)
 
     async def download_graph(self, task: Task) -> SerializedObject:
+        with (
+            metric_graph_download_errors.count_exceptions(),
+            metric_tasks_downloading_graphs.track_inprogress(),
+            metric_graph_download_latency.time(),
+        ):
+            metric_graph_downloads.inc()
+            return await self._download_graph(task)
+
+    async def download_input(self, task: Task) -> SerializedObject:
+        with (
+            metric_task_input_download_errors.count_exceptions(),
+            metric_tasks_downloading_inputs.track_inprogress(),
+            metric_task_input_download_latency.time(),
+        ):
+            metric_task_input_downloads.inc()
+            return await self._download_input(task)
+
+    async def download_init_value(self, task: Task) -> SerializedObject:
+        with (
+            metric_reducer_init_value_download_errors.count_exceptions(),
+            metric_tasks_downloading_reducer_init_value.track_inprogress(),
+            metric_reducer_init_value_download_latency.time(),
+        ):
+            metric_reducer_init_value_downloads.inc()
+            return await self._download_init_value(task)
+
+    async def _download_graph(self, task: Task) -> SerializedObject:
         # Cache graph to reduce load on the server.
         graph_path = os.path.join(
             self.code_path,
@@ -33,6 +75,7 @@ class Downloader:
             self._read_cached_graph, graph_path
         )
         if graph is not None:
+            metric_graphs_from_cache.inc()
             return graph
 
         logger = self._task_logger(task)
@@ -71,7 +114,7 @@ class Downloader:
         # This also allows to share the same cache between multiple Executors.
         os.replace(tmp_path, path)
 
-    async def download_input(self, task: Task) -> SerializedObject:
+    async def _download_input(self, task: Task) -> SerializedObject:
         logger = self._task_logger(task)
 
         first_function_in_graph = task.invocation_id == task.input_key.split("|")[-1]
@@ -81,10 +124,7 @@ class Downloader:
         else:
             return await self._fetch_function_input(task, logger)
 
-    async def download_init_value(self, task: Task) -> Optional[SerializedObject]:
-        if task.reducer_output_id is None:
-            return None
-
+    async def _download_init_value(self, task: Task) -> SerializedObject:
         logger = self._task_logger(task)
         return await self._fetch_function_init_value(task, logger)
 

indexify/executor/executor.py

@@ -1,7 +1,8 @@
 import asyncio
 import signal
 from pathlib import Path
-from typing import Any, List, Optional
+from socket import gethostname
+from typing import Any, Dict, List, Optional
 
 import structlog
 from tensorlake.function_executor.proto.function_executor_pb2 import SerializedObject
@@ -9,13 +10,38 @@ from tensorlake.utils.logging import suppress as suppress_logging
 
 from .api_objects import FunctionURI, Task
 from .downloader import Downloader
+from .function_executor.function_executor_states_container import (
+    FunctionExecutorStatesContainer,
+)
 from .function_executor.server.function_executor_server_factory import (
     FunctionExecutorServerFactory,
 )
+from .metrics.executor import (
+    METRIC_TASKS_COMPLETED_OUTCOME_ALL,
+    METRIC_TASKS_COMPLETED_OUTCOME_ERROR_CUSTOMER_CODE,
+    METRIC_TASKS_COMPLETED_OUTCOME_ERROR_PLATFORM,
+    METRIC_TASKS_COMPLETED_OUTCOME_SUCCESS,
+    metric_executor_info,
+    metric_executor_state,
+    metric_task_outcome_report_latency,
+    metric_task_outcome_report_retries,
+    metric_task_outcome_reports,
+    metric_tasks_completed,
+    metric_tasks_fetched,
+    metric_tasks_reporting_outcome,
+)
+from .monitoring.function_allowlist import function_allowlist_to_info_dict
+from .monitoring.health_check_handler import HealthCheckHandler
+from .monitoring.health_checker.health_checker import HealthChecker
+from .monitoring.prometheus_metrics_handler import PrometheusMetricsHandler
+from .monitoring.server import MonitoringServer
+from .monitoring.startup_probe_handler import StartupProbeHandler
 from .task_fetcher import TaskFetcher
 from .task_reporter import TaskReporter
 from .task_runner import TaskInput, TaskOutput, TaskRunner
 
+metric_executor_state.state("starting")
+
 
 class Executor:
     def __init__(
@@ -23,11 +49,14 @@ class Executor:
         id: str,
         version: str,
         code_path: Path,
+        health_checker: HealthChecker,
         function_allowlist: Optional[List[FunctionURI]],
         function_executor_server_factory: FunctionExecutorServerFactory,
-        server_addr: str = "localhost:8900",
-        config_path: Optional[str] = None,
-        disable_automatic_function_executor_management: bool = False,
+        server_addr: str,
+        config_path: Optional[str],
+        monitoring_server_host: str,
+        monitoring_server_port: int,
+        disable_automatic_function_executor_management: bool,
     ):
         self._logger = structlog.get_logger(module=__name__)
         self._is_shutdown: bool = False
@@ -40,12 +69,25 @@ class Executor:
         self._server_addr = server_addr
         self._base_url = f"{protocol}://{self._server_addr}"
         self._code_path = code_path
+        self._startup_probe_handler = StartupProbeHandler()
+        self._monitoring_server = MonitoringServer(
+            host=monitoring_server_host,
+            port=monitoring_server_port,
+            startup_probe_handler=self._startup_probe_handler,
+            health_probe_handler=HealthCheckHandler(health_checker),
+            metrics_handler=PrometheusMetricsHandler(),
+        )
+        self._function_executor_states = FunctionExecutorStatesContainer()
+        health_checker.set_function_executor_states_container(
+            self._function_executor_states
+        )
         self._task_runner = TaskRunner(
             executor_id=id,
             function_executor_server_factory=function_executor_server_factory,
             base_url=self._base_url,
-            config_path=config_path,
             disable_automatic_function_executor_management=disable_automatic_function_executor_management,
+            function_executor_states=self._function_executor_states,
+            config_path=config_path,
         )
         self._downloader = Downloader(
             code_path=code_path, base_url=self._base_url, config_path=config_path
@@ -63,8 +105,22 @@ class Executor:
             executor_id=id,
             config_path=self._config_path,
         )
+        executor_info: Dict[str, str] = {
+            "id": id,
+            "version": version,
+            "code_path": str(code_path),
+            "server_addr": server_addr,
+            "config_path": str(config_path),
+            "disable_automatic_function_executor_management": str(
+                disable_automatic_function_executor_management
+            ),
+            "hostname": gethostname(),
+        }
+        executor_info.update(function_allowlist_to_info_dict(function_allowlist))
+        metric_executor_info.info(executor_info)
 
     def run(self):
+        asyncio.new_event_loop()
         for signum in [
             signal.SIGABRT,
             signal.SIGINT,
@@ -76,15 +132,20 @@ class Executor:
                 signum, self.shutdown, asyncio.get_event_loop()
             )
 
+        asyncio.get_event_loop().create_task(self._monitoring_server.run())
+
         try:
-            asyncio.get_event_loop().run_until_complete(self._run_async())
+            asyncio.get_event_loop().run_until_complete(self._run_tasks_loop())
         except asyncio.CancelledError:
             pass  # Suppress this expected exception and return without error (normally).
 
-    async def _run_async(self):
+    async def _run_tasks_loop(self):
+        metric_executor_state.state("running")
+        self._startup_probe_handler.set_ready()
         while not self._is_shutdown:
             try:
                 async for task in self._task_fetcher.run():
+                    metric_tasks_fetched.inc()
                     asyncio.create_task(self._run_task(task))
             except Exception as e:
                 self._logger.error(
@@ -103,9 +164,10 @@ class Executor:
             graph: SerializedObject = await self._downloader.download_graph(task)
             input: SerializedObject = await self._downloader.download_input(task)
             init_value: Optional[SerializedObject] = (
-                await self._downloader.download_init_value(task)
+                None
+                if task.reducer_output_id is None
+                else (await self._downloader.download_init_value(task))
             )
-            logger.info("task_execution_started")
             output: TaskOutput = await self._task_runner.run(
                 TaskInput(
                     task=task,
@@ -115,15 +177,22 @@ class Executor:
                 ),
                 logger=logger,
             )
-            logger.info("task_execution_finished", success=output.success)
+            logger.info("task execution finished", success=output.success)
         except Exception as e:
             output = TaskOutput.internal_error(task)
-            logger.error("task_execution_failed", exc_info=e)
+            logger.error("task execution failed", exc_info=e)
 
-        await self._report_task_outcome(output=output, logger=logger)
+        with (
+            metric_tasks_reporting_outcome.track_inprogress(),
+            metric_task_outcome_report_latency.time(),
+        ):
+            metric_task_outcome_reports.inc()
+            await self._report_task_outcome(output=output, logger=logger)
 
     async def _report_task_outcome(self, output: TaskOutput, logger: Any) -> None:
-        """Reports the task with the given output to the server."""
+        """Reports the task with the given output to the server.
+
+        Doesn't raise any Exceptions. Runs till the reporting is successful."""
         reporting_retries: int = 0
 
         while True:
@@ -133,22 +202,40 @@ class Executor:
                 break
             except Exception as e:
                 logger.error(
-                    "failed_to_report_task",
+                    "failed to report task",
                     exc_info=e,
                 )
                 reporting_retries += 1
+                metric_task_outcome_report_retries.inc()
                 await asyncio.sleep(5)
 
+        metric_tasks_completed.labels(outcome=METRIC_TASKS_COMPLETED_OUTCOME_ALL).inc()
+        if output.is_internal_error:
+            metric_tasks_completed.labels(
+                outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_PLATFORM
+            ).inc()
+        elif output.success:
+            metric_tasks_completed.labels(
+                outcome=METRIC_TASKS_COMPLETED_OUTCOME_SUCCESS
+            ).inc()
+        else:
+            metric_tasks_completed.labels(
+                outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_CUSTOMER_CODE
+            ).inc()
+
     async def _shutdown(self, loop):
-        self._logger.info("shutting_down")
+        self._logger.info("shutting down")
+        metric_executor_state.state("shutting_down")
         # There will be lots of task cancellation exceptions and "X is shutting down"
         # exceptions logged during Executor shutdown. Suppress their logs as they are
         # expected and are confusing for users.
         suppress_logging()
 
         self._is_shutdown = True
+        await self._monitoring_server.shutdown()
         await self._task_runner.shutdown()
-        # We mainly need to cancel the task that runs _run_async() loop.
+        await self._function_executor_states.shutdown()
+        # We mainly need to cancel the task that runs _run_tasks_loop().
         for task in asyncio.all_tasks(loop):
             task.cancel()