indexify 0.4.29__py3-none-any.whl → 0.4.30__py3-none-any.whl

indexify/executor/function_executor_controller/__init__.py

@@ -4,7 +4,7 @@ from .message_validators import (
     validate_function_executor_description,
     validate_task_allocation,
 )
-from .task_output import TaskOutput
+from .task_allocation_output import TaskAllocationOutput
 
 __all__ = [
     "function_executor_logger",
@@ -12,5 +12,5 @@ __all__ = [
     "validate_function_executor_description",
     "validate_task_allocation",
     "FunctionExecutorController",
-    "TaskOutput",
+    "TaskAllocationOutput",
 ]

indexify/executor/function_executor_controller/completed_task_allocation_metrics.py

@@ -0,0 +1,87 @@
+import time
+from typing import Any
+
+from indexify.proto.executor_api_pb2 import (
+    TaskFailureReason,
+    TaskOutcomeCode,
+)
+
+from .metrics.completed_task_allocation_metrics import (
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_ALL,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_NONE,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_TASK_CANCELLED,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_UNKNOWN,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_ALL,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_SUCCESS,
+    metric_task_allocation_completion_latency,
+    metric_task_allocations_completed,
+)
+from .task_allocation_info import TaskAllocationInfo
+
+
+def emit_completed_task_allocation_metrics(
+    alloc_info: TaskAllocationInfo, logger: Any
+) -> None:
+    """Emits Prometheus metrics for a completed task allocation.
+
+    Doesn't raise any exceptions.
+    """
+    logger = logger.bind(module=__name__)
+    metric_task_allocation_completion_latency.observe(
+        time.monotonic() - alloc_info.start_time
+    )
+
+    task_outcome_code: TaskOutcomeCode = alloc_info.output.outcome_code
+    task_failure_reason: TaskFailureReason = alloc_info.output.failure_reason
+    metric_task_allocations_completed.labels(
+        outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_ALL,
+        failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_ALL,
+    ).inc()
+    if task_outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_SUCCESS:
+        metric_task_allocations_completed.labels(
+            outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_SUCCESS,
+            failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_NONE,
+        ).inc()
+    elif task_outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE:
+        if task_failure_reason == TaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR:
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR,
+            ).inc()
+        elif (
+            task_failure_reason
+            == TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED
+        ):
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
+            ).inc()
+        elif (
+            task_failure_reason == TaskFailureReason.TASK_FAILURE_REASON_TASK_CANCELLED
+        ):
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_TASK_CANCELLED,
+            ).inc()
+        elif task_failure_reason in [
+            TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR,
+            TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_TIMEOUT,
+            TaskFailureReason.TASK_FAILURE_REASON_INVOCATION_ERROR,
+        ]:
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR,
+            ).inc()
+        else:
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_UNKNOWN,
+            ).inc()
+            logger.warning(
+                "unexpected task allocation failure reason",
+                failure_reason=TaskFailureReason.Name(task_failure_reason),
+            )

indexify/executor/function_executor_controller/events.py

@@ -1,26 +1,22 @@
 from enum import Enum
 from typing import List, Optional
 
-from tensorlake.function_executor.proto.function_executor_pb2 import (
-    FunctionInputs,
-)
-
 from indexify.executor.function_executor.function_executor import (
     FunctionExecutor,
 )
 from indexify.proto.executor_api_pb2 import FunctionExecutorTerminationReason
 
-from .task_info import TaskInfo
+from .task_allocation_info import TaskAllocationInfo
 
 
 class EventType(Enum):
     FUNCTION_EXECUTOR_CREATED = 1
     FUNCTION_EXECUTOR_TERMINATED = 2
     SHUTDOWN_INITIATED = 3
-    TASK_PREPARATION_FINISHED = 4
-    SCHEDULE_TASK_EXECUTION = 5
-    TASK_EXECUTION_FINISHED = 6
-    TASK_OUTPUT_UPLOAD_FINISHED = 7
+    TASK_ALLOCATION_PREPARATION_FINISHED = 4
+    SCHEDULE_TASK_ALLOCATION_EXECUTION = 5
+    TASK_ALLOCATION_EXECUTION_FINISHED = 6
+    TASK_ALLOCATION_FINALIZATION_FINISHED = 7
 
 
 class BaseEvent:
@@ -94,50 +90,50 @@ class ShutdownInitiated(BaseEvent):
         super().__init__(EventType.SHUTDOWN_INITIATED)
 
 
-class TaskPreparationFinished(BaseEvent):
+class TaskAllocationPreparationFinished(BaseEvent):
     """
-    Event indicating that a task has been prepared for execution or failed to do that.
+    Event indicating that a task allocation has been prepared for execution or failed to do that.
     """
 
     def __init__(
         self,
-        task_info: TaskInfo,
+        alloc_info: TaskAllocationInfo,
         is_success: bool,
     ):
-        super().__init__(EventType.TASK_PREPARATION_FINISHED)
-        self.task_info: TaskInfo = task_info
+        super().__init__(EventType.TASK_ALLOCATION_PREPARATION_FINISHED)
+        self.alloc_info: TaskAllocationInfo = alloc_info
         self.is_success: bool = is_success
 
     def __str__(self) -> str:
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.allocation.task.id}, "
-            f"allocation_id={self.task_info.allocation.allocation_id}), "
+            f"task_id={self.alloc_info.allocation.task.id}, "
+            f"allocation_id={self.alloc_info.allocation.allocation_id}), "
             f"is_success={self.is_success}"
         )
 
 
-class ScheduleTaskExecution(BaseEvent):
+class ScheduleTaskAllocationExecution(BaseEvent):
     """
-    Event indicating that a task execution has been scheduled.
+    Event indicating that a task allocation has been scheduled.
     """
 
     def __init__(self):
-        super().__init__(EventType.SCHEDULE_TASK_EXECUTION)
+        super().__init__(EventType.SCHEDULE_TASK_ALLOCATION_EXECUTION)
 
 
-class TaskExecutionFinished(BaseEvent):
+class TaskAllocationExecutionFinished(BaseEvent):
     """
-    Event indicating that a task execution has been finished on Function Executor.
+    Event indicating that a task allocation execution has been finished on Function Executor.
     """
 
     def __init__(
         self,
-        task_info: TaskInfo,
+        alloc_info: TaskAllocationInfo,
         function_executor_termination_reason: FunctionExecutorTerminationReason,  # type: Optional[FunctionExecutorTerminationReason]
     ):
-        super().__init__(EventType.TASK_EXECUTION_FINISHED)
-        self.task_info: TaskInfo = task_info
+        super().__init__(EventType.TASK_ALLOCATION_EXECUTION_FINISHED)
+        self.alloc_info: TaskAllocationInfo = alloc_info
         # Not None if the FE needs to get destroyed after running the task.
         self.function_executor_termination_reason = function_executor_termination_reason
 
@@ -151,26 +147,26 @@ class TaskExecutionFinished(BaseEvent):
         )
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.allocation.task.id}, "
-            f"allocation_id={self.task_info.allocation.allocation_id}), "
+            f"task_id={self.alloc_info.allocation.task.id}, "
+            f"allocation_id={self.alloc_info.allocation.allocation_id}), "
             f"function_executor_termination_reason={function_executor_termination_reason_str}"
         )
 
 
-class TaskFinalizationFinished(BaseEvent):
+class TaskAllocationFinalizationFinished(BaseEvent):
     """
-    Event indicating that a task finalization is finished.
+    Event indicating that a task allocation finalization is finished.
     """
 
-    def __init__(self, task_info: TaskInfo, is_success: bool):
-        super().__init__(EventType.TASK_OUTPUT_UPLOAD_FINISHED)
-        self.task_info: TaskInfo = task_info
+    def __init__(self, alloc_info: TaskAllocationInfo, is_success: bool):
+        super().__init__(EventType.TASK_ALLOCATION_FINALIZATION_FINISHED)
+        self.alloc_info: TaskAllocationInfo = alloc_info
         self.is_success: bool = is_success
 
     def __str__(self) -> str:
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.allocation.task.id}, "
-            f"allocation_id={self.task_info.allocation.allocation_id}), "
+            f"task_id={self.alloc_info.allocation.task.id}, "
+            f"allocation_id={self.alloc_info.allocation.allocation_id}), "
             f"is_success={self.is_success}"
         )

indexify/executor/function_executor_controller/{finalize_task.py → finalize_task_allocation.py}

@@ -8,21 +8,21 @@ from indexify.proto.executor_api_pb2 import (
     TaskOutcomeCode,
 )
 
-from .events import TaskFinalizationFinished
-from .metrics.finalize_task import (
-    metric_task_finalization_errors,
-    metric_task_finalization_latency,
-    metric_task_finalizations,
-    metric_tasks_finalizing,
+from .events import TaskAllocationFinalizationFinished
+from .metrics.finalize_task_allocation import (
+    metric_task_allocation_finalization_errors,
+    metric_task_allocation_finalization_latency,
+    metric_task_allocation_finalizations,
+    metric_task_allocations_finalizing,
 )
-from .task_info import TaskInfo
-from .task_input import TaskInput
-from .task_output import TaskOutput
+from .task_allocation_info import TaskAllocationInfo
+from .task_allocation_input import TaskAllocationInput
+from .task_allocation_output import TaskAllocationOutput
 
 
-async def finalize_task(
-    task_info: TaskInfo, blob_store: BLOBStore, logger: Any
-) -> TaskFinalizationFinished:
+async def finalize_task_allocation(
+    task_alloc: TaskAllocationInfo, blob_store: BLOBStore, logger: Any
+) -> TaskAllocationFinalizationFinished:
     """Prepares the task output for getting it reported to Server.
 
     The task output is either coming from a failed task or from its finished execution on the Function Executor.
@@ -32,34 +32,40 @@ async def finalize_task(
     start_time = time.monotonic()
 
     with (
-        metric_tasks_finalizing.track_inprogress(),
-        metric_task_finalization_latency.time(),
-        metric_task_finalization_errors.count_exceptions(),
+        metric_task_allocations_finalizing.track_inprogress(),
+        metric_task_allocation_finalization_latency.time(),
+        metric_task_allocation_finalization_errors.count_exceptions(),
     ):
-        metric_task_finalizations.inc()
+        metric_task_allocation_finalizations.inc()
         try:
-            await _finalize_task_output(
-                task_info=task_info,
+            await _finalize_task_alloc_output(
+                alloc_info=task_alloc,
                 blob_store=blob_store,
                 logger=logger,
             )
             logger.info(
-                "task finalized",
+                "task allocation finalized",
                 duration=time.monotonic() - start_time,
             )
-            return TaskFinalizationFinished(task_info=task_info, is_success=True)
+            return TaskAllocationFinalizationFinished(
+                alloc_info=task_alloc, is_success=True
+            )
         except asyncio.CancelledError:
-            return TaskFinalizationFinished(task_info=task_info, is_success=False)
+            return TaskAllocationFinalizationFinished(
+                alloc_info=task_alloc, is_success=False
+            )
         except BaseException as e:
             logger.error(
-                "failed to finalize task",
+                "failed to finalize task allocation",
                 exc_info=e,
                 duration=time.monotonic() - start_time,
             )
-            return TaskFinalizationFinished(task_info=task_info, is_success=False)
+            return TaskAllocationFinalizationFinished(
+                alloc_info=task_alloc, is_success=False
+            )
 
 
-class _TaskOutputSummary:
+class _TaskAllocationOutputSummary:
     def __init__(self):
         self.output_count: int = 0
         self.output_bytes: int = 0
@@ -68,27 +74,27 @@ class _TaskOutputSummary:
         self.next_functions_count: int = 0
 
 
-async def _finalize_task_output(
-    task_info: TaskInfo, blob_store: BLOBStore, logger: Any
+async def _finalize_task_alloc_output(
+    alloc_info: TaskAllocationInfo, blob_store: BLOBStore, logger: Any
 ) -> None:
     """Finalizes the task output.
 
     Raises exception on error."""
-    if task_info.input is None:
+    if alloc_info.input is None:
         raise Exception(
-            "task input is None, this should never happen",
+            "task allocation input is None, this should never happen",
         )
-    if task_info.output is None:
+    if alloc_info.output is None:
         raise Exception(
-            "task output is None, this should never happen",
+            "task allocation output is None, this should never happen",
         )
 
-    input: TaskInput = task_info.input
-    output: TaskOutput = task_info.output
+    input: TaskAllocationInput = alloc_info.input
+    output: TaskAllocationOutput = alloc_info.output
 
-    output_summary: _TaskOutputSummary = _task_output_summary(output)
+    output_summary: _TaskAllocationOutputSummary = _task_output_summary(output)
     logger.info(
-        "task output summary",
+        "task allocation output summary",
         output_count=output_summary.output_count,
         output_bytes=output_summary.output_bytes,
         invocation_error_output_count=output_summary.invocation_error_output_count,
@@ -152,8 +158,10 @@ async def _finalize_task_output(
         )
 
 
-def _task_output_summary(task_output: TaskOutput) -> _TaskOutputSummary:
-    summary: _TaskOutputSummary = _TaskOutputSummary()
+def _task_output_summary(
+    task_output: TaskAllocationOutput,
+) -> _TaskAllocationOutputSummary:
+    summary: _TaskAllocationOutputSummary = _TaskAllocationOutputSummary()
 
     for output in task_output.function_outputs:
         summary.output_count += 1
@@ -172,7 +180,7 @@ def _task_output_summary(task_output: TaskOutput) -> _TaskOutputSummary:
 
 # Temporary workaround is logging customer metrics until we store them somewhere
 # for future retrieval and processing.
-def _log_function_metrics(output: TaskOutput, logger: Any):
+def _log_function_metrics(output: TaskAllocationOutput, logger: Any):
     if output.metrics is None:
         return