indexify 0.3.11__py3-none-any.whl → 0.3.13__py3-none-any.whl

indexify/cli/cli.py

@@ -7,6 +7,7 @@ from tensorlake.utils.logging import (
 configure_logging_early()
 
 import os
+import re
 import shutil
 import signal
 import subprocess
@@ -185,6 +186,9 @@ def executor(
     executor_cache: Optional[str] = typer.Option(
         "~/.indexify/executor_cache", help="Path to the executor cache directory"
     ),
+    executor_id: Optional[str] = typer.Option(
+        None, help="ID of the executor, if not provided, a random ID will be generated"
+    ),
     # Registred ports range ends at 49151.
     ports: Tuple[int, int] = typer.Option(
         (50000, 51000),
@@ -221,9 +225,15 @@ def executor(
                 "At least one function must be specified when not running in development mode"
             )
 
+    if executor_id is None:
+        executor_id = nanoid.generate()
+    elif not re.compile(r"^[a-zA-Z0-9_-]{10,}$").match(executor_id):
+        raise typer.BadParameter(
+            "--executor-id should be at least 10 characters long and only include characters _-[0-9][a-z][A-Z]"
+        )
+
     executor_version = version("indexify")
-    id = nanoid.generate()
-    logger = structlog.get_logger(module=__name__, executor_id=id)
+    logger = structlog.get_logger(module=__name__, executor_id=executor_id)
 
     logger.info(
         "starting executor",
@@ -262,7 +272,7 @@ def executor(
     )
 
     Executor(
-        id=id,
+        id=executor_id,
         version=executor_version,
         health_checker=GenericHealthChecker(),
         code_path=executor_cache,

indexify/executor/executor.py

@@ -1,5 +1,6 @@
 import asyncio
 import signal
+import time
 from pathlib import Path
 from socket import gethostname
 from typing import Any, Dict, List, Optional
@@ -23,6 +24,7 @@ from .metrics.executor import (
     METRIC_TASKS_COMPLETED_OUTCOME_SUCCESS,
     metric_executor_info,
     metric_executor_state,
+    metric_task_completion_latency,
     metric_task_outcome_report_latency,
     metric_task_outcome_report_retries,
     metric_task_outcome_reports,
@@ -157,26 +159,12 @@ class Executor:
         """Runs the supplied task.
 
         Doesn't raise any Exceptions. All errors are reported to the server."""
+        start_time: float = time.monotonic()
         logger = self._task_logger(task)
         output: Optional[TaskOutput] = None
 
         try:
-            graph: SerializedObject = await self._downloader.download_graph(task)
-            input: SerializedObject = await self._downloader.download_input(task)
-            init_value: Optional[SerializedObject] = (
-                None
-                if task.reducer_output_id is None
-                else (await self._downloader.download_init_value(task))
-            )
-            output: TaskOutput = await self._task_runner.run(
-                TaskInput(
-                    task=task,
-                    graph=graph,
-                    input=input,
-                    init_value=init_value,
-                ),
-                logger=logger,
-            )
+            output = await self._run_task_and_get_output(task, logger)
             logger.info("task execution finished", success=output.success)
         except Exception as e:
             output = TaskOutput.internal_error(task)
@@ -189,6 +177,26 @@ class Executor:
             metric_task_outcome_reports.inc()
             await self._report_task_outcome(output=output, logger=logger)
 
+        metric_task_completion_latency.observe(time.monotonic() - start_time)
+
+    async def _run_task_and_get_output(self, task: Task, logger: Any) -> TaskOutput:
+        graph: SerializedObject = await self._downloader.download_graph(task)
+        input: SerializedObject = await self._downloader.download_input(task)
+        init_value: Optional[SerializedObject] = (
+            None
+            if task.reducer_output_id is None
+            else (await self._downloader.download_init_value(task))
+        )
+        return await self._task_runner.run(
+            TaskInput(
+                task=task,
+                graph=graph,
+                input=input,
+                init_value=init_value,
+            ),
+            logger=logger,
+        )
+
     async def _report_task_outcome(self, output: TaskOutput, logger: Any) -> None:
         """Reports the task with the given output to the server.
 

indexify/executor/function_executor/function_executor.py

@@ -3,12 +3,15 @@ from typing import Any, Optional
 
 import grpc
 from tensorlake.function_executor.proto.function_executor_pb2 import (
+    InfoRequest,
+    InfoResponse,
     InitializeRequest,
     InitializeResponse,
 )
 from tensorlake.function_executor.proto.function_executor_pb2_grpc import (
     FunctionExecutorStub,
 )
+from tensorlake.function_executor.proto.message_validator import MessageValidator
 from tensorlake.utils.http_client import get_httpx_client
 
 from .health_checker import HealthChecker
@@ -36,7 +39,10 @@ from .metrics.function_executor import (
     metric_destroys,
     metric_establish_channel_errors,
     metric_establish_channel_latency,
+    metric_function_executor_infos,
     metric_function_executors_count,
+    metric_get_info_rpc_errors,
+    metric_get_info_rpc_latency,
     metric_initialize_rpc_errors,
     metric_initialize_rpc_latency,
 )
@@ -96,6 +102,7 @@ class FunctionExecutor:
                 await self._create_server(config)
                 await self._establish_channel()
                 stub: FunctionExecutorStub = FunctionExecutorStub(self._channel)
+                await _collect_server_info(stub)
                 await _initialize_server(stub, initialize_request)
                 await self._create_invocation_state_client(
                     stub=stub,
@@ -262,9 +269,29 @@ class FunctionExecutor:
             self._health_checker = None
 
 
+async def _collect_server_info(stub: FunctionExecutorStub) -> None:
+    with (
+        metric_get_info_rpc_errors.count_exceptions(),
+        metric_get_info_rpc_latency.time(),
+    ):
+        info: InfoResponse = await stub.get_info(InfoRequest())
+        validator = MessageValidator(info)
+        validator.required_field("version")
+        validator.required_field("sdk_version")
+        validator.required_field("sdk_language")
+        validator.required_field("sdk_language_version")
+
+        metric_function_executor_infos.labels(
+            version=info.version,
+            sdk_version=info.sdk_version,
+            sdk_language=info.sdk_language,
+            sdk_language_version=info.sdk_language_version,
+        ).inc()
+
+
 async def _initialize_server(
     stub: FunctionExecutorStub, initialize_request: InitializeRequest
-):
+) -> None:
     with (
         metric_initialize_rpc_errors.count_exceptions(),
         metric_initialize_rpc_latency.time(),

indexify/executor/function_executor/metrics/function_executor.py

@@ -78,6 +78,22 @@ metric_destroy_channel_errors: prometheus_client.Counter = prometheus_client.Cou
     "Number of Function Executor channel destruction errors",
 )
 
+# FE get_info RPC metrics.
+metric_get_info_rpc_latency: prometheus_client.Histogram = (
+    latency_metric_for_fast_operation(
+        "function_executor_get_info_rpc", "Function Executor get_info RPC"
+    )
+)
+metric_get_info_rpc_errors: prometheus_client.Counter = prometheus_client.Counter(
+    "function_executor_get_info_rpc_errors",
+    "Number of Function Executor get_info RPC errors",
+)
+metric_function_executor_infos: prometheus_client.Counter = prometheus_client.Counter(
+    "function_executor_infos",
+    "Number of Function Executors with particular info",
+    ["version", "sdk_version", "sdk_language", "sdk_language_version"],
+)
+
 # FE initialization RPC metrics.
 metric_initialize_rpc_latency: prometheus_client.Histogram = (
     latency_metric_for_customer_controlled_operation(

indexify/executor/metrics/executor.py

@@ -1,6 +1,9 @@
 import prometheus_client
 
-from ..monitoring.metrics import latency_metric_for_fast_operation
+from ..monitoring.metrics import (
+    latency_metric_for_customer_controlled_operation,
+    latency_metric_for_fast_operation,
+)
 
 # This file contains all metrics used by Executor.
 
@@ -31,6 +34,12 @@ metric_tasks_completed.labels(
     outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_CUSTOMER_CODE
 )
 metric_tasks_completed.labels(outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_PLATFORM)
+metric_task_completion_latency: prometheus_client.Histogram = (
+    latency_metric_for_customer_controlled_operation(
+        "task_completion",
+        "task completion from the moment it got fetched until its outcome got reported",
+    )
+)
 
 # Task outcome reporting metrics.
 metric_task_outcome_reports: prometheus_client.Counter = prometheus_client.Counter(

{indexify-0.3.11.dist-info → indexify-0.3.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.11
+Version: 0.3.13
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -15,16 +15,9 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
-Requires-Dist: grpcio (==1.70.0)
-Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
-Requires-Dist: httpx[http2] (>=0.27,<0.28)
-Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
-Requires-Dist: pydantic (==2.10.4)
-Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
-Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.20)
+Requires-Dist: tensorlake (>=0.1)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.11.dist-info → indexify-0.3.13.dist-info}/RECORD

@@ -1,14 +1,14 @@
-indexify/cli/cli.py,sha256=c2Y2ozNXKhEtLaAL6wrovGSkN9dGn-MA5F56MPcxwT4,10509
+indexify/cli/cli.py,sha256=SQOVWIr-ZNPilFAVQw1qRC42V8T0VEbVt_E1eClsnpM,10946
 indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
 indexify/executor/api_objects.py,sha256=TaYwDoo7EjuLBusxH512-KdvAJRtBwbEP2IObWraabU,1100
 indexify/executor/downloader.py,sha256=XjaGCzsGM3ex2HxbKGkIsB50OhlXgdnywWGCjUBdW2k,8127
-indexify/executor/executor.py,sha256=hij_EkB1_lueakCf3pfi_0O3wxSLYFBkvbpwlsMm-ao,9784
-indexify/executor/function_executor/function_executor.py,sha256=BxNhsW0uXxKjdKT5ixMAyIo5F7otdIT2c0FMQ09Lsrs,10234
+indexify/executor/executor.py,sha256=r5do6k7Izp9SIzebgOn2psYjzKZmNqHx3iuTW3vWrhg,10035
+indexify/executor/function_executor/function_executor.py,sha256=sQ5FOdrjybDDsjagghlfjV06IXTpWWDBSTHqQXI-w9M,11245
 indexify/executor/function_executor/function_executor_state.py,sha256=IWPLWa7LaN0Eq8PDu-0kFzkuKJB0COShu7wCO1oyiNA,3141
 indexify/executor/function_executor/function_executor_states_container.py,sha256=x4hlF7ZBWswk9dkA06Rvgeqar6H9TWjZ7Etyy2CzBDE,2682
 indexify/executor/function_executor/health_checker.py,sha256=YT24ajPLdYOpdt2UNyJGviGbivCDXIJpeQOxQofcl50,3258
 indexify/executor/function_executor/invocation_state_client.py,sha256=p-xgM4__cHR1ApvMV9hShrGWee_Je0VDhICZUGjpQY4,9644
-indexify/executor/function_executor/metrics/function_executor.py,sha256=y36cgZEp949HPCBVFF6uFQIxk2PvnSI74QSbxcAalI4,5659
+indexify/executor/function_executor/metrics/function_executor.py,sha256=KHzf4cMh_HqnOAE7TZ6_oIqUH3nExOkt9LSekAEN69w,6304
 indexify/executor/function_executor/metrics/function_executor_state.py,sha256=M7cMA7JY8_8FW9xjuSqtp6o2xxUgB31LJowo7kzcexg,352
 indexify/executor/function_executor/metrics/function_executor_state_container.py,sha256=6rrAfml-TivjkHatCM4BLY7jmVs523Wzb6QIysncc-0,302
 indexify/executor/function_executor/metrics/health_checker.py,sha256=EaeIYJPrQ-qqNMGZVGkvjPoeQSCl4FzPKXEv3Cly1NE,456
@@ -23,7 +23,7 @@ indexify/executor/function_executor/single_task_runner.py,sha256=Td28vF6pNemKZEs
 indexify/executor/function_executor/task_input.py,sha256=wSrHR4m0juiGClQyeVdhRC37QzDt6Rrjq-ZXJkfBi9k,584
 indexify/executor/function_executor/task_output.py,sha256=Qg7vojYi0WmeHRf2qlC-5h46jcwis13EgiN63OmWpcg,1229
 indexify/executor/metrics/downloader.py,sha256=lctPh8xjkXeLEFJnl1hNrD1yEhLhIl5sggsR4Yoe_Zc,2746
-indexify/executor/metrics/executor.py,sha256=UkJTa8NerbdC1p7jrwl4obLUS2xGSzEYwONPZpoKhT0,2064
+indexify/executor/metrics/executor.py,sha256=ua-Vv_k1CB4juJdF7tEBQbBMksqWAA3iXKKMKXZUCLk,2369
 indexify/executor/metrics/task_fetcher.py,sha256=iJEwCLzYr2cuz7hRvNiqaa2nvQP4OrA0hm0iJY0YKG0,736
 indexify/executor/metrics/task_reporter.py,sha256=zUA9RpkSgx5lG_ZqDDuela5VuhtsnC0IKoQcEvHND0Y,730
 indexify/executor/metrics/task_runner.py,sha256=o5ERNePKPmVKknFoSZUr-r597dEOOWvWn3ocbiL2jxI,1699
@@ -40,7 +40,7 @@ indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3V
 indexify/executor/task_fetcher.py,sha256=NpFfHgaY99bSL-K2D5kcDAMNUG2FArq0-qF_mgF-LBQ,3375
 indexify/executor/task_reporter.py,sha256=t7FzSKV7fG1fEtxyTcFh-sTWZ8WXApbG-qXaAMq_gRQ,7363
 indexify/executor/task_runner.py,sha256=tcS7hHdSVQexC3XEIsDTwX51bJK4iSQqC8CbWeNoud0,5590
-indexify-0.3.11.dist-info/METADATA,sha256=Uv7Dj89CyUGzdH4_c7YKSQNMxJSGzkXCXt3huTC4pR8,1428
-indexify-0.3.11.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.11.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
-indexify-0.3.11.dist-info/RECORD,,
+indexify-0.3.13.dist-info/METADATA,sha256=zaJ7N4LFWTxgUPXsA01xxhPY8Dvo8D7tcPwNH6x2F5U,1158
+indexify-0.3.13.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.13.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
+indexify-0.3.13.dist-info/RECORD,,