indexify 0.3.11__tar.gz → 0.3.12__tar.gz

{indexify-0.3.11 → indexify-0.3.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.3.11
+Version: 0.3.12
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -15,16 +15,9 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.11.0,<4.0.0)
-Requires-Dist: grpcio (==1.70.0)
-Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
-Requires-Dist: httpx[http2] (>=0.27,<0.28)
-Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: prometheus-client (>=0.21.1,<0.22.0)
-Requires-Dist: pydantic (==2.10.4)
-Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
-Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: tensorlake (>=0.1.20)
+Requires-Dist: tensorlake (>=0.1)
 Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.3.11 → indexify-0.3.12}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.3.11"
+version = "0.3.12"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -15,23 +15,23 @@ indexify-cli = "indexify.cli.cli:app"
 [tool.poetry.dependencies]
 # Common dependencies
 python = "^3.9"
-structlog = "^24.4.0"
-pyyaml = "^6"
-httpx = { version = "^0.27", extras = ["http2"] }
-grpcio = "1.70.0"
+# structlog is provided by tensorlake
+# pyyaml is provided by tensorlake
+# httpx is provided by tensorlake
+# grpcio is provided by tensorlake
 
 # Executor only
-pydantic = "2.10.4"
-httpx-sse = "^0.4.0"
 aiohttp = "^3.11.0"
 prometheus-client = "^0.21.1"
 # Adds function-executor binary and utils lib.
-tensorlake = ">=0.1.20"
+tensorlake = ">=0.1"
+# pydantic is provided by tensorlake
+# httpx-sse is provided by tensorlake
 
 # CLI only
 rich = "^13.9.2"
 typer = "^0.12"
-nanoid = "^2.0.0"
+# nanoid is provided by tensorlake
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.10.0"

{indexify-0.3.11 → indexify-0.3.12}/src/indexify/cli/cli.py

@@ -7,6 +7,7 @@ from tensorlake.utils.logging import (
 configure_logging_early()
 
 import os
+import re
 import shutil
 import signal
 import subprocess
@@ -185,6 +186,9 @@ def executor(
     executor_cache: Optional[str] = typer.Option(
         "~/.indexify/executor_cache", help="Path to the executor cache directory"
     ),
+    executor_id: Optional[str] = typer.Option(
+        None, help="ID of the executor, if not provided, a random ID will be generated"
+    ),
     # Registred ports range ends at 49151.
     ports: Tuple[int, int] = typer.Option(
         (50000, 51000),
@@ -221,9 +225,15 @@ def executor(
                 "At least one function must be specified when not running in development mode"
             )
 
+    if executor_id is None:
+        executor_id = nanoid.generate()
+    elif not re.compile(r"^[a-zA-Z0-9_-]{10,}$").match(executor_id):
+        raise typer.BadParameter(
+            "--executor-id should be at least 10 characters long and only include characters _-[0-9][a-z][A-Z]"
+        )
+
     executor_version = version("indexify")
-    id = nanoid.generate()
-    logger = structlog.get_logger(module=__name__, executor_id=id)
+    logger = structlog.get_logger(module=__name__, executor_id=executor_id)
 
     logger.info(
         "starting executor",
@@ -262,7 +272,7 @@ def executor(
     )
 
     Executor(
-        id=id,
+        id=executor_id,
         version=executor_version,
         health_checker=GenericHealthChecker(),
         code_path=executor_cache,

{indexify-0.3.11 → indexify-0.3.12}/src/indexify/executor/executor.py

@@ -1,5 +1,6 @@
 import asyncio
 import signal
+import time
 from pathlib import Path
 from socket import gethostname
 from typing import Any, Dict, List, Optional
@@ -23,6 +24,7 @@ from .metrics.executor import (
     METRIC_TASKS_COMPLETED_OUTCOME_SUCCESS,
     metric_executor_info,
     metric_executor_state,
+    metric_task_completion_latency,
     metric_task_outcome_report_latency,
     metric_task_outcome_report_retries,
     metric_task_outcome_reports,
@@ -157,26 +159,12 @@ class Executor:
         """Runs the supplied task.
 
         Doesn't raise any Exceptions. All errors are reported to the server."""
+        start_time: float = time.monotonic()
         logger = self._task_logger(task)
         output: Optional[TaskOutput] = None
 
         try:
-            graph: SerializedObject = await self._downloader.download_graph(task)
-            input: SerializedObject = await self._downloader.download_input(task)
-            init_value: Optional[SerializedObject] = (
-                None
-                if task.reducer_output_id is None
-                else (await self._downloader.download_init_value(task))
-            )
-            output: TaskOutput = await self._task_runner.run(
-                TaskInput(
-                    task=task,
-                    graph=graph,
-                    input=input,
-                    init_value=init_value,
-                ),
-                logger=logger,
-            )
+            output = await self._run_task_and_get_output(task, logger)
             logger.info("task execution finished", success=output.success)
         except Exception as e:
             output = TaskOutput.internal_error(task)
@@ -189,6 +177,26 @@ class Executor:
             metric_task_outcome_reports.inc()
             await self._report_task_outcome(output=output, logger=logger)
 
+        metric_task_completion_latency.observe(time.monotonic() - start_time)
+
+    async def _run_task_and_get_output(self, task: Task, logger: Any) -> TaskOutput:
+        graph: SerializedObject = await self._downloader.download_graph(task)
+        input: SerializedObject = await self._downloader.download_input(task)
+        init_value: Optional[SerializedObject] = (
+            None
+            if task.reducer_output_id is None
+            else (await self._downloader.download_init_value(task))
+        )
+        return await self._task_runner.run(
+            TaskInput(
+                task=task,
+                graph=graph,
+                input=input,
+                init_value=init_value,
+            ),
+            logger=logger,
+        )
+
     async def _report_task_outcome(self, output: TaskOutput, logger: Any) -> None:
         """Reports the task with the given output to the server.
 

{indexify-0.3.11 → indexify-0.3.12}/src/indexify/executor/metrics/executor.py

@@ -1,6 +1,9 @@
 import prometheus_client
 
-from ..monitoring.metrics import latency_metric_for_fast_operation
+from ..monitoring.metrics import (
+    latency_metric_for_customer_controlled_operation,
+    latency_metric_for_fast_operation,
+)
 
 # This file contains all metrics used by Executor.
 
@@ -31,6 +34,12 @@ metric_tasks_completed.labels(
     outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_CUSTOMER_CODE
 )
 metric_tasks_completed.labels(outcome=METRIC_TASKS_COMPLETED_OUTCOME_ERROR_PLATFORM)
+metric_task_completion_latency: prometheus_client.Histogram = (
+    latency_metric_for_customer_controlled_operation(
+        "task_completion",
+        "task completion from the moment it got fetched until its outcome got reported",
+    )
+)
 
 # Task outcome reporting metrics.
 metric_task_outcome_reports: prometheus_client.Counter = prometheus_client.Counter(