indexify 0.4.29__py3-none-any.whl → 0.4.31__py3-none-any.whl

indexify/cli/build_image.py

@@ -4,8 +4,7 @@ from typing import Any, Generator, Tuple
 import click
 import docker
 import docker.api.build
-import docker.models
-import docker.models.images
+from docker.errors import BuildError
 from tensorlake.functions_sdk.image import Image
 from tensorlake.functions_sdk.workflow_module import (
     WorkflowModuleInfo,
@@ -16,7 +15,6 @@ from tensorlake.functions_sdk.workflow_module import (
 @click.command(
     short_help="Build images for graphs/workflows defined in the workflow file"
 )
-# Path to the file where the graphs/workflows are defined as global variables
 @click.argument(
     "workflow-file-path",
     type=click.Path(exists=True, file_okay=True, dir_okay=False),
@@ -31,6 +29,11 @@ def build_image(
     workflow_file_path: str,
     image_names: tuple[str, ...] = None,
 ):
+    """
+    Build the images associated to an Indexify workflow
+
+    A workflow is defined in a Python file, and the images are built using the local Docker daemon.
+    """
     try:
         workflow_module_info: WorkflowModuleInfo = load_workflow_module_info(
             workflow_file_path
@@ -48,7 +51,7 @@ def build_image(
     indexify_version: str = importlib.metadata.version("indexify")
     for image in workflow_module_info.images.keys():
         image: Image
-        if image_names is not None and image.image_name not in image_names:
+        if len(image_names) > 0 and image.image_name not in image_names:
             click.echo(
                 f"Skipping image `{image.image_name}` as it is not in the provided image names."
             )
@@ -57,28 +60,65 @@ def build_image(
         click.echo(f"Building image `{image.image_name}`")
 
         image.run(f"pip install 'indexify=={indexify_version}'")
-        built_image, logs_generator = image.build()
-        built_image: docker.models.images.Image
-        for output in logs_generator:
-            click.secho(output)
+        built_image, logs_generator = _build(image=image, docker_client=docker_client)
+        try:
+            built_image, logs_generator = _build(
+                image=image, docker_client=docker_client
+            )
+            _print_build_log(logs_generator)
+            click.secho(f"built image: {built_image.tags[0]}", fg="green")
+        except BuildError as e:
+            raise click.Abort() from e
 
         click.secho(f"built image: {built_image.tags[0]}", fg="green")
 
 
-def build(
+def _build(
     image: Image, docker_client: docker.DockerClient
 ) -> Tuple[docker.models.images.Image, Generator[str, Any, None]]:
     docker_file = image.dockerfile()
-    image_name = f"{image.image_name}:{image.image_tag}"
+    image_name = (
+        image.image_name
+        if ":" in image.image_name
+        else f"{image.image_name}:{image.image_tag}"
+    )
 
     docker.api.build.process_dockerfile = lambda dockerfile, path: (
         "Dockerfile",
         dockerfile,
     )
 
-    return docker_client.images.build(
-        path=".",
-        dockerfile=docker_file,
-        tag=image_name,
-        rm=True,
-    )
+    try:
+        built_image, logs_generator = docker_client.images.build(
+            path=".",
+            dockerfile=docker_file,
+            tag=image_name,
+            rm=True,
+            # pull=True,  # optional: ensures fresh base images
+            # forcerm=True,  # optional: always remove intermediate containers
+        )
+        return built_image, logs_generator
+    except BuildError as e:
+        click.secho("Docker build failed:", fg="red")
+        _print_build_log(e.build_log or [])
+        click.secho(str(e), fg="red")
+        raise
+
+
+def _print_build_log(build_logs):
+    for log_entry in build_logs:
+        if isinstance(log_entry, dict):
+            if "stream" in log_entry:
+                click.echo(log_entry["stream"].rstrip("\n"))
+            elif "status" in log_entry:
+                if "id" in log_entry:
+                    click.echo(f"{log_entry['status']}: {log_entry['id']}")
+                else:
+                    click.echo(log_entry["status"])
+            if "errorDetail" in log_entry:
+                # This is the most useful bit when a RUN command fails
+                msg = log_entry["errorDetail"].get("message") or log_entry.get("error")
+                if msg:
+                    click.secho(msg.rstrip("\n"), fg="red")
+        elif isinstance(log_entry, str):
+            click.echo(log_entry.rstrip("\n"))

indexify/cli/deploy.py

@@ -18,7 +18,7 @@ from tensorlake.functions_sdk.workflow_module import (
 )
 @click.option(
     "-u",
-    "--upgrade-queued-requests",
+    "--upgrade-queued-invocations",
     is_flag=True,
     default=False,
     help="Upgrade invocations that are already queued or running to use the deployed version of the graphs/workflows",

indexify/executor/function_executor_controller/__init__.py

@@ -4,7 +4,7 @@ from .message_validators import (
     validate_function_executor_description,
     validate_task_allocation,
 )
-from .task_output import TaskOutput
+from .task_allocation_output import TaskAllocationOutput
 
 __all__ = [
     "function_executor_logger",
@@ -12,5 +12,5 @@ __all__ = [
     "validate_function_executor_description",
     "validate_task_allocation",
     "FunctionExecutorController",
-    "TaskOutput",
+    "TaskAllocationOutput",
 ]

indexify/executor/function_executor_controller/completed_task_allocation_metrics.py

@@ -0,0 +1,87 @@
+import time
+from typing import Any
+
+from indexify.proto.executor_api_pb2 import (
+    TaskFailureReason,
+    TaskOutcomeCode,
+)
+
+from .metrics.completed_task_allocation_metrics import (
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_ALL,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_NONE,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_TASK_CANCELLED,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_UNKNOWN,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_ALL,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+    METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_SUCCESS,
+    metric_task_allocation_completion_latency,
+    metric_task_allocations_completed,
+)
+from .task_allocation_info import TaskAllocationInfo
+
+
+def emit_completed_task_allocation_metrics(
+    alloc_info: TaskAllocationInfo, logger: Any
+) -> None:
+    """Emits Prometheus metrics for a completed task allocation.
+
+    Doesn't raise any exceptions.
+    """
+    logger = logger.bind(module=__name__)
+    metric_task_allocation_completion_latency.observe(
+        time.monotonic() - alloc_info.start_time
+    )
+
+    task_outcome_code: TaskOutcomeCode = alloc_info.output.outcome_code
+    task_failure_reason: TaskFailureReason = alloc_info.output.failure_reason
+    metric_task_allocations_completed.labels(
+        outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_ALL,
+        failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_ALL,
+    ).inc()
+    if task_outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_SUCCESS:
+        metric_task_allocations_completed.labels(
+            outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_SUCCESS,
+            failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_NONE,
+        ).inc()
+    elif task_outcome_code == TaskOutcomeCode.TASK_OUTCOME_CODE_FAILURE:
+        if task_failure_reason == TaskFailureReason.TASK_FAILURE_REASON_INTERNAL_ERROR:
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_INTERNAL_ERROR,
+            ).inc()
+        elif (
+            task_failure_reason
+            == TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED
+        ):
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_EXECUTOR_TERMINATED,
+            ).inc()
+        elif (
+            task_failure_reason == TaskFailureReason.TASK_FAILURE_REASON_TASK_CANCELLED
+        ):
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_TASK_CANCELLED,
+            ).inc()
+        elif task_failure_reason in [
+            TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_ERROR,
+            TaskFailureReason.TASK_FAILURE_REASON_FUNCTION_TIMEOUT,
+            TaskFailureReason.TASK_FAILURE_REASON_INVOCATION_ERROR,
+        ]:
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_FUNCTION_ERROR,
+            ).inc()
+        else:
+            metric_task_allocations_completed.labels(
+                outcome_code=METRIC_TASK_ALLOCATIONS_COMPLETED_OUTCOME_CODE_FAILURE,
+                failure_reason=METRIC_TASK_ALLOCATIONS_COMPLETED_FAILURE_REASON_UNKNOWN,
+            ).inc()
+            logger.warning(
+                "unexpected task allocation failure reason",
+                failure_reason=TaskFailureReason.Name(task_failure_reason),
+            )

indexify/executor/function_executor_controller/events.py

@@ -1,26 +1,22 @@
 from enum import Enum
 from typing import List, Optional
 
-from tensorlake.function_executor.proto.function_executor_pb2 import (
-    FunctionInputs,
-)
-
 from indexify.executor.function_executor.function_executor import (
     FunctionExecutor,
 )
 from indexify.proto.executor_api_pb2 import FunctionExecutorTerminationReason
 
-from .task_info import TaskInfo
+from .task_allocation_info import TaskAllocationInfo
 
 
 class EventType(Enum):
     FUNCTION_EXECUTOR_CREATED = 1
     FUNCTION_EXECUTOR_TERMINATED = 2
     SHUTDOWN_INITIATED = 3
-    TASK_PREPARATION_FINISHED = 4
-    SCHEDULE_TASK_EXECUTION = 5
-    TASK_EXECUTION_FINISHED = 6
-    TASK_OUTPUT_UPLOAD_FINISHED = 7
+    TASK_ALLOCATION_PREPARATION_FINISHED = 4
+    SCHEDULE_TASK_ALLOCATION_EXECUTION = 5
+    TASK_ALLOCATION_EXECUTION_FINISHED = 6
+    TASK_ALLOCATION_FINALIZATION_FINISHED = 7
 
 
 class BaseEvent:
@@ -94,50 +90,50 @@ class ShutdownInitiated(BaseEvent):
         super().__init__(EventType.SHUTDOWN_INITIATED)
 
 
-class TaskPreparationFinished(BaseEvent):
+class TaskAllocationPreparationFinished(BaseEvent):
     """
-    Event indicating that a task has been prepared for execution or failed to do that.
+    Event indicating that a task allocation has been prepared for execution or failed to do that.
     """
 
     def __init__(
         self,
-        task_info: TaskInfo,
+        alloc_info: TaskAllocationInfo,
         is_success: bool,
     ):
-        super().__init__(EventType.TASK_PREPARATION_FINISHED)
-        self.task_info: TaskInfo = task_info
+        super().__init__(EventType.TASK_ALLOCATION_PREPARATION_FINISHED)
+        self.alloc_info: TaskAllocationInfo = alloc_info
         self.is_success: bool = is_success
 
     def __str__(self) -> str:
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.allocation.task.id}, "
-            f"allocation_id={self.task_info.allocation.allocation_id}), "
+            f"task_id={self.alloc_info.allocation.task.id}, "
+            f"allocation_id={self.alloc_info.allocation.allocation_id}), "
             f"is_success={self.is_success}"
         )
 
 
-class ScheduleTaskExecution(BaseEvent):
+class ScheduleTaskAllocationExecution(BaseEvent):
     """
-    Event indicating that a task execution has been scheduled.
+    Event indicating that a task allocation has been scheduled.
     """
 
     def __init__(self):
-        super().__init__(EventType.SCHEDULE_TASK_EXECUTION)
+        super().__init__(EventType.SCHEDULE_TASK_ALLOCATION_EXECUTION)
 
 
-class TaskExecutionFinished(BaseEvent):
+class TaskAllocationExecutionFinished(BaseEvent):
     """
-    Event indicating that a task execution has been finished on Function Executor.
+    Event indicating that a task allocation execution has been finished on Function Executor.
     """
 
     def __init__(
         self,
-        task_info: TaskInfo,
+        alloc_info: TaskAllocationInfo,
         function_executor_termination_reason: FunctionExecutorTerminationReason,  # type: Optional[FunctionExecutorTerminationReason]
     ):
-        super().__init__(EventType.TASK_EXECUTION_FINISHED)
-        self.task_info: TaskInfo = task_info
+        super().__init__(EventType.TASK_ALLOCATION_EXECUTION_FINISHED)
+        self.alloc_info: TaskAllocationInfo = alloc_info
         # Not None if the FE needs to get destroyed after running the task.
         self.function_executor_termination_reason = function_executor_termination_reason
 
@@ -151,26 +147,26 @@ class TaskExecutionFinished(BaseEvent):
         )
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.allocation.task.id}, "
-            f"allocation_id={self.task_info.allocation.allocation_id}), "
+            f"task_id={self.alloc_info.allocation.task.id}, "
+            f"allocation_id={self.alloc_info.allocation.allocation_id}), "
             f"function_executor_termination_reason={function_executor_termination_reason_str}"
         )
 
 
-class TaskFinalizationFinished(BaseEvent):
+class TaskAllocationFinalizationFinished(BaseEvent):
     """
-    Event indicating that a task finalization is finished.
+    Event indicating that a task allocation finalization is finished.
     """
 
-    def __init__(self, task_info: TaskInfo, is_success: bool):
-        super().__init__(EventType.TASK_OUTPUT_UPLOAD_FINISHED)
-        self.task_info: TaskInfo = task_info
+    def __init__(self, alloc_info: TaskAllocationInfo, is_success: bool):
+        super().__init__(EventType.TASK_ALLOCATION_FINALIZATION_FINISHED)
+        self.alloc_info: TaskAllocationInfo = alloc_info
         self.is_success: bool = is_success
 
     def __str__(self) -> str:
         return (
             f"Event(type={self.event_type.name}, "
-            f"task_id={self.task_info.allocation.task.id}, "
-            f"allocation_id={self.task_info.allocation.allocation_id}), "
+            f"task_id={self.alloc_info.allocation.task.id}, "
+            f"allocation_id={self.alloc_info.allocation.allocation_id}), "
             f"is_success={self.is_success}"
         )

indexify/executor/function_executor_controller/{finalize_task.py → finalize_task_allocation.py}

@@ -8,21 +8,21 @@ from indexify.proto.executor_api_pb2 import (
     TaskOutcomeCode,
 )
 
-from .events import TaskFinalizationFinished
-from .metrics.finalize_task import (
-    metric_task_finalization_errors,
-    metric_task_finalization_latency,
-    metric_task_finalizations,
-    metric_tasks_finalizing,
+from .events import TaskAllocationFinalizationFinished
+from .metrics.finalize_task_allocation import (
+    metric_task_allocation_finalization_errors,
+    metric_task_allocation_finalization_latency,
+    metric_task_allocation_finalizations,
+    metric_task_allocations_finalizing,
 )
-from .task_info import TaskInfo
-from .task_input import TaskInput
-from .task_output import TaskOutput
+from .task_allocation_info import TaskAllocationInfo
+from .task_allocation_input import TaskAllocationInput
+from .task_allocation_output import TaskAllocationOutput
 
 
-async def finalize_task(
-    task_info: TaskInfo, blob_store: BLOBStore, logger: Any
-) -> TaskFinalizationFinished:
+async def finalize_task_allocation(
+    task_alloc: TaskAllocationInfo, blob_store: BLOBStore, logger: Any
+) -> TaskAllocationFinalizationFinished:
     """Prepares the task output for getting it reported to Server.
 
     The task output is either coming from a failed task or from its finished execution on the Function Executor.
@@ -32,34 +32,40 @@ async def finalize_task(
     start_time = time.monotonic()
 
     with (
-        metric_tasks_finalizing.track_inprogress(),
-        metric_task_finalization_latency.time(),
-        metric_task_finalization_errors.count_exceptions(),
+        metric_task_allocations_finalizing.track_inprogress(),
+        metric_task_allocation_finalization_latency.time(),
+        metric_task_allocation_finalization_errors.count_exceptions(),
     ):
-        metric_task_finalizations.inc()
+        metric_task_allocation_finalizations.inc()
         try:
-            await _finalize_task_output(
-                task_info=task_info,
+            await _finalize_task_alloc_output(
+                alloc_info=task_alloc,
                 blob_store=blob_store,
                 logger=logger,
             )
             logger.info(
-                "task finalized",
+                "task allocation finalized",
                 duration=time.monotonic() - start_time,
             )
-            return TaskFinalizationFinished(task_info=task_info, is_success=True)
+            return TaskAllocationFinalizationFinished(
+                alloc_info=task_alloc, is_success=True
+            )
         except asyncio.CancelledError:
-            return TaskFinalizationFinished(task_info=task_info, is_success=False)
+            return TaskAllocationFinalizationFinished(
+                alloc_info=task_alloc, is_success=False
+            )
         except BaseException as e:
             logger.error(
-                "failed to finalize task",
+                "failed to finalize task allocation",
                 exc_info=e,
                 duration=time.monotonic() - start_time,
             )
-            return TaskFinalizationFinished(task_info=task_info, is_success=False)
+            return TaskAllocationFinalizationFinished(
+                alloc_info=task_alloc, is_success=False
+            )
 
 
-class _TaskOutputSummary:
+class _TaskAllocationOutputSummary:
     def __init__(self):
         self.output_count: int = 0
         self.output_bytes: int = 0
@@ -68,27 +74,27 @@ class _TaskOutputSummary:
         self.next_functions_count: int = 0
 
 
-async def _finalize_task_output(
-    task_info: TaskInfo, blob_store: BLOBStore, logger: Any
+async def _finalize_task_alloc_output(
+    alloc_info: TaskAllocationInfo, blob_store: BLOBStore, logger: Any
 ) -> None:
     """Finalizes the task output.
 
     Raises exception on error."""
-    if task_info.input is None:
+    if alloc_info.input is None:
         raise Exception(
-            "task input is None, this should never happen",
+            "task allocation input is None, this should never happen",
         )
-    if task_info.output is None:
+    if alloc_info.output is None:
         raise Exception(
-            "task output is None, this should never happen",
+            "task allocation output is None, this should never happen",
         )
 
-    input: TaskInput = task_info.input
-    output: TaskOutput = task_info.output
+    input: TaskAllocationInput = alloc_info.input
+    output: TaskAllocationOutput = alloc_info.output
 
-    output_summary: _TaskOutputSummary = _task_output_summary(output)
+    output_summary: _TaskAllocationOutputSummary = _task_output_summary(output)
     logger.info(
-        "task output summary",
+        "task allocation output summary",
         output_count=output_summary.output_count,
         output_bytes=output_summary.output_bytes,
         invocation_error_output_count=output_summary.invocation_error_output_count,
@@ -152,8 +158,10 @@ async def _finalize_task_output(
         )
 
 
-def _task_output_summary(task_output: TaskOutput) -> _TaskOutputSummary:
-    summary: _TaskOutputSummary = _TaskOutputSummary()
+def _task_output_summary(
+    task_output: TaskAllocationOutput,
+) -> _TaskAllocationOutputSummary:
+    summary: _TaskAllocationOutputSummary = _TaskAllocationOutputSummary()
 
     for output in task_output.function_outputs:
         summary.output_count += 1
@@ -172,7 +180,7 @@ def _task_output_summary(task_output: TaskOutput) -> _TaskOutputSummary:
 
 # Temporary workaround is logging customer metrics until we store them somewhere
 # for future retrieval and processing.
-def _log_function_metrics(output: TaskOutput, logger: Any):
+def _log_function_metrics(output: TaskAllocationOutput, logger: Any):
     if output.metrics is None:
         return