indexify 0.4.28__tar.gz → 0.4.30__tar.gz

{indexify-0.4.28 → indexify-0.4.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: indexify
-Version: 0.4.28
+Version: 0.4.30
 Summary: Open Source Indexify components and helper tools
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -14,7 +14,7 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: aiohttp (>=3.12.15,<4.0.0)
-Requires-Dist: boto3 (>=1.40.12,<2.0.0)
+Requires-Dist: boto3 (>=1.40.15,<2.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: httpx[http2] (==0.27.2)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)

{indexify-0.4.28 → indexify-0.4.30}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "indexify"
 # Incremented if any of the components provided in this packages are updated.
-version = "0.4.28"
+version = "0.4.30"
 description = "Open Source Indexify components and helper tools"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -25,7 +25,7 @@ httpx = { version = "0.27.2", extras = ["http2"] }
 pydantic = "^2.11"
 prometheus-client = "^0.22.1"
 psutil = "^7.0.0"
-boto3 = "^1.40.12"
+boto3 = "^1.40.15"
 structlog = "25.4.0"
 # Adds function-executor binary, utils lib, sdk used in indexify-cli commands.
 # We need to specify the tensorlake version exactly because pip install doesn't respect poetry.lock files.

{indexify-0.4.28 → indexify-0.4.30}/src/indexify/executor/executor.py

@@ -23,9 +23,11 @@ from .metrics.executor import (
     metric_executor_info,
     metric_executor_state,
 )
+from .monitoring.desired_state_handler import DesiredStateHandler
 from .monitoring.health_check_handler import HealthCheckHandler
 from .monitoring.health_checker.health_checker import HealthChecker
 from .monitoring.prometheus_metrics_handler import PrometheusMetricsHandler
+from .monitoring.reported_state_handler import ReportedStateHandler
 from .monitoring.server import MonitoringServer
 from .monitoring.startup_probe_handler import StartupProbeHandler
 from .state_reconciler import ExecutorStateReconciler
@@ -59,13 +61,6 @@ class Executor:
             protocol = "https"
 
         self._startup_probe_handler = StartupProbeHandler()
-        self._monitoring_server = MonitoringServer(
-            host=monitoring_server_host,
-            port=monitoring_server_port,
-            startup_probe_handler=self._startup_probe_handler,
-            health_probe_handler=HealthCheckHandler(health_checker),
-            metrics_handler=PrometheusMetricsHandler(),
-        )
         self._channel_manager = ChannelManager(
             server_address=grpc_server_addr,
             config_path=config_path,
@@ -96,6 +91,15 @@ class Executor:
             state_reporter=self._state_reporter,
             logger=self._logger,
         )
+        self._monitoring_server = MonitoringServer(
+            host=monitoring_server_host,
+            port=monitoring_server_port,
+            startup_probe_handler=self._startup_probe_handler,
+            health_probe_handler=HealthCheckHandler(health_checker),
+            metrics_handler=PrometheusMetricsHandler(),
+            reported_state_handler=ReportedStateHandler(self._state_reporter),
+            desired_state_handler=DesiredStateHandler(self._state_reconciler),
+        )
         self._run_aio_task: Optional[asyncio.Task] = None
         self._shutdown_aio_task: Optional[asyncio.Task] = None
 

{indexify-0.4.28 → indexify-0.4.30}/src/indexify/executor/function_executor_controller/__init__.py

@@ -4,7 +4,7 @@ from .message_validators import (
     validate_function_executor_description,
     validate_task_allocation,
 )
-from .task_output import TaskOutput
+from .task_allocation_output import TaskAllocationOutput
 
 __all__ = [
     "function_executor_logger",
@@ -12,5 +12,5 @@ __all__ = [
     "validate_function_executor_description",
     "validate_task_allocation",
     "FunctionExecutorController",
-    "TaskOutput",
+    "TaskAllocationOutput",
 ]

indexify-0.4.30/src/indexify/executor/function_executor_controller/completed_task_allocation_metrics.py

@@ -0,0 +1,87 @@
+import time
+from typing import Any
+
+from indexify.proto.executor_api_pb2 import (
+    TaskFailureReason,
+    TaskOutcomeCode,
+)
+
+from .metrics.completed_task_allocation_metrics import (
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_ALL,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_NONE,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_TASK_CANCELLED,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_UNKNOWN,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_ALL,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_SUCCESS,
+    metric_task_allocation_completion_latency,
+    metric_task_allocations_completed,
+)
+from .task_allocation_info import TaskAllocationInfo
+
+
+def emit_completed_task_allocation_metrics(
+    alloc_info: TaskAllocationInfo, logger: Any
+) -> None:
+    """Emits Prometheus metrics for a completed task allocation.
+
+    Doesn't raise any exceptions.
+    """
+    logger = logger.bind(module=__name__)
+    metric_task_allocation_completion_latency.observe(
+        time.monotonic() - alloc_info.start_time
+    )
+
+    task_outcome_code: TaskOutcomeCode = alloc_info.output.outcome_code
+    task_failure_reason: TaskFailureReason = alloc_info.output.failure_reason
+    metric_task_allocations_completed.labels(
+        outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_ALL,
+        failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_ALL,
+    ).inc()
+    if task_outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_SUCCESS:
+        metric_task_allocations_completed.labels(
+            outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_SUCCESS,
+            failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_NONE,
+        ).inc()
+    elif task_outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE:
+        if task_failure_reason == TaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR:
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR,
+            ).inc()
+        elif (
+            task_failure_reason
+            == TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED
+        ):
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
+            ).inc()
+        elif (
+            task_failure_reason == TaskFailureReason.TASK_FAILURE_REASON_TASK_CANCELLED
+        ):
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_TASK_CANCELLED,
+            ).inc()
+        elif task_failure_reason in [
+            TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR,
+            TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_TIMEOUT,
+            TaskFailureReason.TASK_FAILURE_REASON_INVOCATION_ERROR,
+        ]:
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR,
+            ).inc()
+        else:
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_UNKNOWN,
+            ).inc()
+            logger.warning(
+                "unexpected task allocation failure reason",
+                failure_reason=TaskFailureReason.Name(task_failure_reason),
+            )

{indexify-0.4.28 → indexify-0.4.30}/src/indexify/executor/function_executor_controller/events.py

@@ -1,26 +1,22 @@
 from enum import Enum
 from typing import List, Optional
 
-from tensorlake.function_executor.proto.function_executor_pb2 import (
-    FunctionInputs,
-)
-
 from indexify.executor.function_executor.function_executor import (
     FunctionExecutor,
 )
 from indexify.proto.executor_api_pb2 import FunctionExecutorTerminationReason
 
-from .task_info import TaskInfo
+from .task_allocation_info import TaskAllocationInfo
 
 
 class EventType(Enum):
     FUNCTION_EXECUTOR_CREATED = 1
     FUNCTION_EXECUTOR_TERMINATED = 2
     SHUTDOWN_INITIATED = 3
-    TASK_PREPARATION_FINISHED = 4
-    SCHEDULE_TASK_EXECUTION = 5
-    TASK_EXECUTION_FINISHED = 6
-    TASK_OUTPUT_UPLOAD_FINISHED = 7
+    TASK_ALLOCATION_PREPARATION_FINISHED = 4
+    SCHEDULE_TASK_ALLOCATION_EXECUTION = 5
+    TASK_ALLOCATION_EXECUTION_FINISHED = 6
+    TASK_ALLOCATION_FINALIZATION_FINISHED = 7
 
 
 class BaseEvent:
@@ -94,50 +90,50 @@ class ShutdownInitiated(BaseEvent):
         super().__init__(EventType.SHUTDOWN_INITIATED)
 
 
-class TaskPreparationFinished(BaseEvent):
+class TaskAllocationPreparationFinished(BaseEvent):
     """
-    Event indicating that a task has been prepared for execution or failed to do that.
+    Event indicating that a task allocation has been prepared for execution or failed to do that.
     """
 
     def __init__(
         self,
-        task_info: TaskInfo,
+        alloc_info: TaskAllocationInfo,
         is_success: bool,
     ):
-        super().__init__(EventType.TASK_PREPARATION_FINISHED)
-        self.task_info: TaskInfo = task_info
+        super().__init__(EventType.TASK_ALLOCATION_PREPARATION_FINISHED)
+        self.alloc_info: TaskAllocationInfo = alloc_info
         self.is_success: bool = is_success
 
     def __str__(self) -> str:
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.allocation.task.id}, "
-            f"allocation_id={self.task_info.allocation.allocation_id}), "
+            f"task_id={self.alloc_info.allocation.task.id}, "
+            f"allocation_id={self.alloc_info.allocation.allocation_id}), "
             f"is_success={self.is_success}"
         )
 
 
-class ScheduleTaskExecution(BaseEvent):
+class ScheduleTaskAllocationExecution(BaseEvent):
     """
-    Event indicating that a task execution has been scheduled.
+    Event indicating that a task allocation has been scheduled.
     """
 
     def __init__(self):
-        super().__init__(EventType.SCHEDULE_TASK_EXECUTION)
+        super().__init__(EventType.SCHEDULE_TASK_ALLOCATION_EXECUTION)
 
 
-class TaskExecutionFinished(BaseEvent):
+class TaskAllocationExecutionFinished(BaseEvent):
     """
-    Event indicating that a task execution has been finished on Function Executor.
+    Event indicating that a task allocation execution has been finished on Function Executor.
     """
 
     def __init__(
         self,
-        task_info: TaskInfo,
+        alloc_info: TaskAllocationInfo,
         function_executor_termination_reason: FunctionExecutorTerminationReason,  # type: Optional[FunctionExecutorTerminationReason]
     ):
-        super().__init__(EventType.TASK_EXECUTION_FINISHED)
-        self.task_info: TaskInfo = task_info
+        super().__init__(EventType.TASK_ALLOCATION_EXECUTION_FINISHED)
+        self.alloc_info: TaskAllocationInfo = alloc_info
         # Not None if the FE needs to get destroyed after running the task.
         self.function_executor_termination_reason = function_executor_termination_reason
 
@@ -151,26 +147,26 @@ class TaskExecutionFinished(BaseEvent):
         )
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.allocation.task.id}, "
-            f"allocation_id={self.task_info.allocation.allocation_id}), "
+            f"task_id={self.alloc_info.allocation.task.id}, "
+            f"allocation_id={self.alloc_info.allocation.allocation_id}), "
             f"function_executor_termination_reason={function_executor_termination_reason_str}"
         )
 
 
-class TaskFinalizationFinished(BaseEvent):
+class TaskAllocationFinalizationFinished(BaseEvent):
     """
-    Event indicating that a task finalization is finished.
+    Event indicating that a task allocation finalization is finished.
     """
 
-    def __init__(self, task_info: TaskInfo, is_success: bool):
-        super().__init__(EventType.TASK_OUTPUT_UPLOAD_FINISHED)
-        self.task_info: TaskInfo = task_info
+    def __init__(self, alloc_info: TaskAllocationInfo, is_success: bool):
+        super().__init__(EventType.TASK_ALLOCATION_FINALIZATION_FINISHED)
+        self.alloc_info: TaskAllocationInfo = alloc_info
         self.is_success: bool = is_success
 
     def __str__(self) -> str:
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.allocation.task.id}, "
-            f"allocation_id={self.task_info.allocation.allocation_id}), "
+            f"task_id={self.alloc_info.allocation.task.id}, "
+            f"allocation_id={self.alloc_info.allocation.allocation_id}), "
             f"is_success={self.is_success}"
         )

indexify-0.4.28/src/indexify/executor/function_executor_controller/finalize_task.py → indexify-0.4.30/src/indexify/executor/function_executor_controller/finalize_task_allocation.py

@@ -8,21 +8,21 @@ from indexify.proto.executor_api_pb2 import (
     TaskOutcomeCode,
 )
 
-from .events import TaskFinalizationFinished
-from .metrics.finalize_task import (
-    metric_task_finalization_errors,
-    metric_task_finalization_latency,
-    metric_task_finalizations,
-    metric_tasks_finalizing,
+from .events import TaskAllocationFinalizationFinished
+from .metrics.finalize_task_allocation import (
+    metric_task_allocation_finalization_errors,
+    metric_task_allocation_finalization_latency,
+    metric_task_allocation_finalizations,
+    metric_task_allocations_finalizing,
 )
-from .task_info import TaskInfo
-from .task_input import TaskInput
-from .task_output import TaskOutput
+from .task_allocation_info import TaskAllocationInfo
+from .task_allocation_input import TaskAllocationInput
+from .task_allocation_output import TaskAllocationOutput
 
 
-async def finalize_task(
-    task_info: TaskInfo, blob_store: BLOBStore, logger: Any
-) -> TaskFinalizationFinished:
+async def finalize_task_allocation(
+    task_alloc: TaskAllocationInfo, blob_store: BLOBStore, logger: Any
+) -> TaskAllocationFinalizationFinished:
     """Prepares the task output for getting it reported to Server.
 
     The task output is either coming from a failed task or from its finished execution on the Function Executor.
@@ -32,34 +32,40 @@ async def finalize_task(
     start_time = time.monotonic()
 
     with (
-        metric_tasks_finalizing.track_inprogress(),
-        metric_task_finalization_latency.time(),
-        metric_task_finalization_errors.count_exceptions(),
+        metric_task_allocations_finalizing.track_inprogress(),
+        metric_task_allocation_finalization_latency.time(),
+        metric_task_allocation_finalization_errors.count_exceptions(),
     ):
-        metric_task_finalizations.inc()
+        metric_task_allocation_finalizations.inc()
         try:
-            await _finalize_task_output(
-                task_info=task_info,
+            await _finalize_task_alloc_output(
+                alloc_info=task_alloc,
                 blob_store=blob_store,
                 logger=logger,
             )
             logger.info(
-                "task finalized",
+                "task allocation finalized",
                 duration=time.monotonic() - start_time,
             )
-            return TaskFinalizationFinished(task_info=task_info, is_success=True)
+            return TaskAllocationFinalizationFinished(
+                alloc_info=task_alloc, is_success=True
+            )
         except asyncio.CancelledError:
-            return TaskFinalizationFinished(task_info=task_info, is_success=False)
+            return TaskAllocationFinalizationFinished(
+                alloc_info=task_alloc, is_success=False
+            )
         except BaseException as e:
             logger.error(
-                "failed to finalize task",
+                "failed to finalize task allocation",
                 exc_info=e,
                 duration=time.monotonic() - start_time,
             )
-            return TaskFinalizationFinished(task_info=task_info, is_success=False)
+            return TaskAllocationFinalizationFinished(
+                alloc_info=task_alloc, is_success=False
+            )
 
 
-class _TaskOutputSummary:
+class _TaskAllocationOutputSummary:
     def __init__(self):
         self.output_count: int = 0
         self.output_bytes: int = 0
@@ -68,27 +74,27 @@ class _TaskOutputSummary:
         self.next_functions_count: int = 0
 
 
-async def _finalize_task_output(
-    task_info: TaskInfo, blob_store: BLOBStore, logger: Any
+async def _finalize_task_alloc_output(
+    alloc_info: TaskAllocationInfo, blob_store: BLOBStore, logger: Any
 ) -> None:
     """Finalizes the task output.
 
     Raises exception on error."""
-    if task_info.input is None:
+    if alloc_info.input is None:
         raise Exception(
-            "task input is None, this should never happen",
+            "task allocation input is None, this should never happen",
         )
-    if task_info.output is None:
+    if alloc_info.output is None:
         raise Exception(
-            "task output is None, this should never happen",
+            "task allocation output is None, this should never happen",
         )
 
-    input: TaskInput = task_info.input
-    output: TaskOutput = task_info.output
+    input: TaskAllocationInput = alloc_info.input
+    output: TaskAllocationOutput = alloc_info.output
 
-    output_summary: _TaskOutputSummary = _task_output_summary(output)
+    output_summary: _TaskAllocationOutputSummary = _task_output_summary(output)
     logger.info(
-        "task output summary",
+        "task allocation output summary",
         output_count=output_summary.output_count,
         output_bytes=output_summary.output_bytes,
         invocation_error_output_count=output_summary.invocation_error_output_count,
@@ -152,8 +158,10 @@ async def _finalize_task_output(
         )
 
 
-def _task_output_summary(task_output: TaskOutput) -> _TaskOutputSummary:
-    summary: _TaskOutputSummary = _TaskOutputSummary()
+def _task_output_summary(
+    task_output: TaskAllocationOutput,
+) -> _TaskAllocationOutputSummary:
+    summary: _TaskAllocationOutputSummary = _TaskAllocationOutputSummary()
 
     for output in task_output.function_outputs:
         summary.output_count += 1
@@ -172,7 +180,7 @@ def _task_output_summary(task_output: TaskOutput) -> _TaskOutputSummary:
 
 # Temporary workaround is logging customer metrics until we store them somewhere
 # for future retrieval and processing.
-def _log_function_metrics(output: TaskOutput, logger: Any):
+def _log_function_metrics(output: TaskAllocationOutput, logger: Any):
     if output.metrics is None:
         return