indexify 0.3.10__py3-none-any.whl → 0.3.12__py3-none-any.whl

indexify/cli/cli.py

@@ -7,6 +7,7 @@ from tensorlake.utils.logging import (
 configure_logging_early()
 
 import os
+import re
 import shutil
 import signal
 import subprocess
@@ -25,7 +26,7 @@ import typer
 from rich.console import Console
 from rich.text import Text
 from rich.theme import Theme
-from tensorlake.functions_sdk.image import GetDefaultPythonImage, Image
+from tensorlake.functions_sdk.image import Image
 
 from indexify.executor.api_objects import FunctionURI
 from indexify.executor.executor import Executor
@@ -185,6 +186,9 @@ def executor(
     executor_cache: Optional[str] = typer.Option(
         "~/.indexify/executor_cache", help="Path to the executor cache directory"
     ),
+    executor_id: Optional[str] = typer.Option(
+        None, help="ID of the executor, if not provided, a random ID will be generated"
+    ),
     # Registred ports range ends at 49151.
     ports: Tuple[int, int] = typer.Option(
         (50000, 51000),
@@ -221,9 +225,15 @@ def executor(
                 "At least one function must be specified when not running in development mode"
             )
 
+    if executor_id is None:
+        executor_id = nanoid.generate()
+    elif not re.compile(r"^[a-zA-Z0-9_-]{10,}$").match(executor_id):
+        raise typer.BadParameter(
+            "--executor-id should be at least 10 characters long and only include characters _-[0-9][a-z][A-Z]"
+        )
+
     executor_version = version("indexify")
-    id = nanoid.generate()
-    logger = structlog.get_logger(module=__name__, executor_id=id)
+    logger = structlog.get_logger(module=__name__, executor_id=executor_id)
 
     logger.info(
         "starting executor",
@@ -262,7 +272,7 @@ def executor(
     )
 
     Executor(
-        id=id,
+        id=executor_id,
         version=executor_version,
         health_checker=GenericHealthChecker(),
         code_path=executor_cache,

indexify/executor/executor.py

@@ -1,5 +1,6 @@
 import asyncio
 import signal
+import time
 from pathlib import Path
 from socket import gethostname
 from typing import Any, Dict, List, Optional
@@ -23,6 +24,7 @@ from .metrics.executor import (
     METRIC_TASKS_COMPLETED_OUTCOME_SUCCESS,
     metric_executor_info,
     metric_executor_state,
+    metric_task_completion_latency,
     metric_task_outcome_report_latency,
     metric_task_outcome_report_retries,
     metric_task_outcome_reports,
@@ -157,26 +159,12 @@ class Executor:
         """Runs the supplied task.
 
         Doesn't raise any Exceptions. All errors are reported to the server."""
+        start_time: float = time.monotonic()
         logger = self._task_logger(task)
         output: Optional[TaskOutput] = None
 
         try:
-            graph: SerializedObject = await self._downloader.download_graph(task)
-            input: SerializedObject = await self._downloader.download_input(task)
-            init_value: Optional[SerializedObject] = (
-                None
-                if task.reducer_output_id is None
-                else (await self._downloader.download_init_value(task))
-            )
-            output: TaskOutput = await self._task_runner.run(
-                TaskInput(
-                    task=task,
-                    graph=graph,
-                    input=input,
-                    init_value=init_value,
-                ),
-                logger=logger,
-            )
+            output = await self._run_task_and_get_output(task, logger)
             logger.info("task execution finished", success=output.success)
         except Exception as e:
             output = TaskOutput.internal_error(task)
@@ -189,6 +177,26 @@ class Executor:
             metric_task_outcome_reports.inc()
             await self._report_task_outcome(output=output, logger=logger)
 
+        metric_task_completion_latency.observe(time.monotonic() - start_time)
+
+    async def _run_task_and_get_output(self, task: Task, logger: Any) -> TaskOutput:
+        graph: SerializedObject = await self._downloader.download_graph(task)
+        input: SerializedObject = await self._downloader.download_input(task)
+        init_value: Optional[SerializedObject] = (
+            None
+            if task.reducer_output_id is None
+            else (await self._downloader.download_init_value(task))
+        )
+        return await self._task_runner.run(
+            TaskInput(
+                task=task,
+                graph=graph,
+                input=input,
+                init_value=init_value,
+            ),
+            logger=logger,
+        )
+
     async def _report_task_outcome(self, output: TaskOutput, logger: Any) -> None:
         """Reports the task with the given output to the server.
 

indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py

@@ -28,8 +28,7 @@ class SubprocessFunctionExecutorServerFactory(FunctionExecutorServerFactory):
         try:
             port = self._allocate_port()
             args = [
-                "--executor-id",
-                config.executor_id,
+                f"--executor-id={config.executor_id}",  # use = as executor_id can start with -
                 "--address",
                 _server_address(port),
             ]

indexify/executor/metrics/executor.py

@@ -1,6 +1,9 @@
 import prometheus_client
 
-from ..monitoring.metrics import latency_metric_for_fast_operation
+from ..monitoring.metrics import (
+    latency_metric_for_customer_controlled_operation,
+    latency_metric_for_fast_operation,
+)
 
 # This file contains all metrics used by Executor.
 
@@ -31,6 +34,12 @@ metric_tasks_completed.labels(
     outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_CUSTOMER_CODE
 )
 metric_tasks_completed.labels(outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_PLATFORM)
+metric_task_completion_latency: prometheus_client.Histogram = (
+    latency_metric_for_customer_controlled_operation(
+        "task_completion",
+        "task completion from the moment it got fetched until its outcome got reported",
+    )
+)
 
 # Task outcome reporting metrics.
 metric_task_outcome_reports: prometheus_client.Counter = prometheus_client.Counter(

{indexify-0.3.10.dist-info → indexify-0.3.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.10
+Version: 0.3.12
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -15,16 +15,9 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
-Requires-Dist: grpcio (==1.70.0)
-Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
-Requires-Dist: httpx[http2] (>=0.27,<0.28)
-Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
-Requires-Dist: pydantic (==2.10.4)
-Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
-Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.20)
+Requires-Dist: tensorlake (>=0.1)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.10.dist-info → indexify-0.3.12.dist-info}/RECORD

@@ -1,8 +1,8 @@
-indexify/cli/cli.py,sha256=ssXc0zalHS7vbCvetZ0cX4BkxIuPIw590jam3c2YVME,10532
+indexify/cli/cli.py,sha256=SQOVWIr-ZNPilFAVQw1qRC42V8T0VEbVt_E1eClsnpM,10946
 indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
 indexify/executor/api_objects.py,sha256=TaYwDoo7EjuLBusxH512-KdvAJRtBwbEP2IObWraabU,1100
 indexify/executor/downloader.py,sha256=XjaGCzsGM3ex2HxbKGkIsB50OhlXgdnywWGCjUBdW2k,8127
-indexify/executor/executor.py,sha256=hij_EkB1_lueakCf3pfi_0O3wxSLYFBkvbpwlsMm-ao,9784
+indexify/executor/executor.py,sha256=r5do6k7Izp9SIzebgOn2psYjzKZmNqHx3iuTW3vWrhg,10035
 indexify/executor/function_executor/function_executor.py,sha256=BxNhsW0uXxKjdKT5ixMAyIo5F7otdIT2c0FMQ09Lsrs,10234
 indexify/executor/function_executor/function_executor_state.py,sha256=IWPLWa7LaN0Eq8PDu-0kFzkuKJB0COShu7wCO1oyiNA,3141
 indexify/executor/function_executor/function_executor_states_container.py,sha256=x4hlF7ZBWswk9dkA06Rvgeqar6H9TWjZ7Etyy2CzBDE,2682
@@ -18,12 +18,12 @@ indexify/executor/function_executor/server/client_configuration.py,sha256=gOywMu
 indexify/executor/function_executor/server/function_executor_server.py,sha256=_DLivLDikupZusRk8gVWDk7fWPT9XjZ4un1yWSlOObs,883
 indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=oBEuOwuGsZrvOZsv9v8M4JEtfproirE-cH28XCr0HoY,1739
 indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=JekDOqF7oFD4J6zcN3xB0Dxd1cgpEXMOsb_rKZOeBlI,668
-indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=xm_EL2Gouwi0qAuKsyJVAQJRBZ0VB-NYsWUM4mOi3nQ,4187
+indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=xbH0-73tBM-i0dh1x_k0fDrNAi8WgDKjqiQ9xveU4Zg,4214
 indexify/executor/function_executor/single_task_runner.py,sha256=Td28vF6pNemKZEs4mNWeLKzy22XRmmMVgp2ipYO_OXU,9262
 indexify/executor/function_executor/task_input.py,sha256=wSrHR4m0juiGClQyeVdhRC37QzDt6Rrjq-ZXJkfBi9k,584
 indexify/executor/function_executor/task_output.py,sha256=Qg7vojYi0WmeHRf2qlC-5h46jcwis13EgiN63OmWpcg,1229
 indexify/executor/metrics/downloader.py,sha256=lctPh8xjkXeLEFJnl1hNrD1yEhLhIl5sggsR4Yoe_Zc,2746
-indexify/executor/metrics/executor.py,sha256=UkJTa8NerbdC1p7jrwl4obLUS2xGSzEYwONPZpoKhT0,2064
+indexify/executor/metrics/executor.py,sha256=ua-Vv_k1CB4juJdF7tEBQbBMksqWAA3iXKKMKXZUCLk,2369
 indexify/executor/metrics/task_fetcher.py,sha256=iJEwCLzYr2cuz7hRvNiqaa2nvQP4OrA0hm0iJY0YKG0,736
 indexify/executor/metrics/task_reporter.py,sha256=zUA9RpkSgx5lG_ZqDDuela5VuhtsnC0IKoQcEvHND0Y,730
 indexify/executor/metrics/task_runner.py,sha256=o5ERNePKPmVKknFoSZUr-r597dEOOWvWn3ocbiL2jxI,1699
@@ -40,7 +40,7 @@ indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3V
 indexify/executor/task_fetcher.py,sha256=NpFfHgaY99bSL-K2D5kcDAMNUG2FArq0-qF_mgF-LBQ,3375
 indexify/executor/task_reporter.py,sha256=t7FzSKV7fG1fEtxyTcFh-sTWZ8WXApbG-qXaAMq_gRQ,7363
 indexify/executor/task_runner.py,sha256=tcS7hHdSVQexC3XEIsDTwX51bJK4iSQqC8CbWeNoud0,5590
-indexify-0.3.10.dist-info/METADATA,sha256=WZfdgioEIs_yVFdyELZb9NLcHeKxgsxmxouLpKsZpZM,1428
-indexify-0.3.10.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
-indexify-0.3.10.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
-indexify-0.3.10.dist-info/RECORD,,
+indexify-0.3.12.dist-info/METADATA,sha256=OkLYjGB3yomMTO21dsw1YR5q3swzqRsaWvsOzrPQTKg,1158
+indexify-0.3.12.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.12.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
+indexify-0.3.12.dist-info/RECORD,,