indexify 0.2.20__tar.gz → 0.2.22__tar.gz

{indexify-0.2.20 → indexify-0.2.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.20
+Version: 0.2.22
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.2.20 → indexify-0.2.22}/indexify/cli.py

@@ -1,5 +1,4 @@
 import asyncio
-import io
 import os
 import shutil
 import signal
@@ -168,6 +167,9 @@ def executor(
     executor_cache: Optional[str] = typer.Option(
         "~/.indexify/executor_cache", help="Path to the executor cache directory"
     ),
+    name_alias: Optional[str] = typer.Option(
+        None, help="Name alias for the executor if it's spun up with the base image"
+    ),
 ):
     id = nanoid.generate()
     console.print(
@@ -176,7 +178,8 @@ def executor(
             f"Config path: {config_path}\n"
             f"Server address: {server_addr}\n"
             f"Executor ID: {id}\n"
-            f"Executor cache: {executor_cache}",
+            f"Executor cache: {executor_cache}\n"
+            f"Name Alias: {name_alias}",
             title="Agent Configuration",
             border_style="info",
         )
@@ -197,6 +200,7 @@ def executor(
         server_addr=server_addr,
         config_path=config_path,
         code_path=executor_cache,
+        name_alias=name_alias,
     )
 
     try:

indexify-0.2.22/indexify/error.py

@@ -0,0 +1,8 @@
+class ApiException(Exception):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class GraphStillProcessing(Exception):
+    def __init__(self) -> None:
+        super().__init__("graph is still processing")

{indexify-0.2.20 → indexify-0.2.22}/indexify/executor/agent.py

@@ -3,9 +3,12 @@ import json
 import ssl
 from concurrent.futures.process import BrokenProcessPool
 from importlib.metadata import version
+import traceback
 from typing import Dict, List, Optional
 
 import httpx
+from indexify.functions_sdk.graph_definition import ComputeGraphMetadata
+from indexify.http_client import IndexifyClient
 import yaml
 from httpx_sse import aconnect_sse
 from pydantic import BaseModel
@@ -19,6 +22,7 @@ from indexify.functions_sdk.data_objects import (
     IndexifyData,
     RouterOutput,
 )
+from . import image_dependency_installer
 
 from .api_objects import ExecutorMetadata, Task
 from .downloader import DownloadedInputs, Downloader
@@ -27,6 +31,7 @@ from .function_worker import FunctionWorker
 from .runtime_probes import ProbeInfo, RuntimeProbes
 from .task_reporter import TaskReporter
 from .task_store import CompletedTask, TaskStore
+from ..functions_sdk.image import ImageInformation
 
 custom_theme = Theme(
     {
@@ -58,7 +63,24 @@ class ExtractorAgent:
         function_worker: FunctionWorker,
         server_addr: str = "localhost:8900",
         config_path: Optional[str] = None,
+        name_alias: Optional[str] = None,
     ):
+        self.name_alias = name_alias
+
+        self._probe = RuntimeProbes()
+
+        runtime_probe: ProbeInfo = self._probe.probe()
+        self._require_image_bootstrap = (
+            True
+            if (runtime_probe.is_default_executor and self.name_alias is not None)
+            else False
+        )
+        self._executor_bootstrap_failed = False
+
+        console.print(
+            f"Require Bootstrap? {self._require_image_bootstrap}", style="cyan bold"
+        )
+
         self.num_workers = num_workers
         self._use_tls = False
         if config_path:
@@ -99,7 +121,6 @@ class ExtractorAgent:
         self._task_reporter = TaskReporter(
             base_url=self._base_url, executor_id=self._executor_id
         )
-        self._probe = RuntimeProbes()
 
     async def task_completion_reporter(self):
         console.print(Text("Starting task completion reporter", style="bold cyan"))
@@ -145,15 +166,55 @@ class ExtractorAgent:
     async def task_launcher(self):
         async_tasks: List[asyncio.Task] = []
         fn_queue: List[FunctionInput] = []
+
         async_tasks.append(
             asyncio.create_task(
                 self._task_store.get_runnable_tasks(), name="get_runnable_tasks"
             )
         )
+
         while True:
             fn: FunctionInput
             for fn in fn_queue:
                 task: Task = self._task_store.get_task(fn.task_id)
+
+                if self._executor_bootstrap_failed:
+                    completed_task = CompletedTask(
+                        task=task,
+                        outputs=[],
+                        task_outcome="failure",
+                    )
+                    self._task_store.complete(outcome=completed_task)
+
+                    continue
+
+                # Bootstrap this executor. Fail the task if we can't.
+                if self._require_image_bootstrap:
+                    try:
+                        image_info = await _get_image_info_for_compute_graph(
+                            task, self._protocol, self._server_addr
+                        )
+                        image_dependency_installer.executor_image_builder(
+                            image_info, self.name_alias
+                        )
+                        self._require_image_bootstrap = False
+                    except Exception as e:
+                        console.print(
+                            Text("Failed to bootstrap the executor ", style="red bold")
+                            + Text(f"Exception: {traceback.format_exc()}", style="red")
+                        )
+
+                        self._executor_bootstrap_failed = True
+
+                        completed_task = CompletedTask(
+                            task=task,
+                            outputs=[],
+                            task_outcome="failure",
+                        )
+                        self._task_store.complete(outcome=completed_task)
+
+                        continue
+
                 async_tasks.append(
                     ExtractTask(
                         function_worker=self._function_worker,
@@ -309,18 +370,19 @@ class ExtractorAgent:
 
             runtime_probe: ProbeInfo = self._probe.probe()
 
-            # Inspect the image
-            if runtime_probe.is_default_executor:
-                # install dependencies
-                # rewrite the image name
-                pass
-
             executor_version = version("indexify")
+
+            image_name = (
+                self.name_alias
+                if self.name_alias is not None
+                else runtime_probe.image_name
+            )
+
             data = ExecutorMetadata(
                 id=self._executor_id,
                 executor_version=executor_version,
                 addr="",
-                image_name=runtime_probe.image_name,
+                image_name=image_name,
                 labels=runtime_probe.labels,
             ).model_dump()
 
@@ -372,3 +434,25 @@ class ExtractorAgent:
     def shutdown(self, loop):
         self._function_worker.shutdown()
         loop.create_task(self._shutdown(loop))
+
+
+async def _get_image_info_for_compute_graph(
+    task: Task, protocol, server_addr
+) -> ImageInformation:
+    namespace = task.namespace
+    graph_name: str = task.compute_graph
+    compute_fn_name: str = task.compute_fn
+
+    http_client = IndexifyClient(
+        service_url=f"{protocol}://{server_addr}", namespace=namespace
+    )
+    compute_graph: ComputeGraphMetadata = http_client.graph(graph_name)
+
+    console.print(
+        Text(
+            f"Compute_fn name {compute_fn_name}, ComputeGraph {compute_graph} \n",
+            style="red yellow",
+        )
+    )
+
+    return compute_graph.nodes[compute_fn_name].compute_fn.image_information

indexify-0.2.22/indexify/executor/image_dependency_installer.py

@@ -0,0 +1,52 @@
+import os
+import subprocess
+
+from rich.console import Console
+from rich.text import Text
+from rich.theme import Theme
+
+from indexify.functions_sdk.image import ImageInformation
+
+
+custom_theme = Theme(
+    {
+        "info": "cyan",
+        "warning": "yellow",
+        "error": "red",
+        "success": "green",
+    }
+)
+
+console = Console(theme=custom_theme)
+
+
+def _record_image_name(name: str):
+    dir_path = os.path.expanduser("~/.indexify/")
+    file_path = os.path.expanduser("~/.indexify/image_name")
+    os.makedirs(dir_path, exist_ok=True)
+
+    with open(file_path, "w") as file:
+        file.write(name)
+
+
+def _install_dependencies(run_str: str):
+    # Throw error to the caller if these subprocesses fail.
+    proc = subprocess.run(run_str.split())
+    if proc.returncode != 0:
+        raise Exception(f"Unable to install dep `{run_str}`")
+
+
+def executor_image_builder(image_info: ImageInformation, name_alias: str):
+    console.print(Text("Attempting Executor Bootstrap.", style="red bold"))
+
+    run_strs = image_info.run_strs
+    console.print(Text("Attempting to install dependencies.", style="red bold"))
+
+    for run_str in run_strs:
+        console.print(Text(f"Attempting {run_str}", style="red bold"))
+        _install_dependencies(run_str)
+
+    console.print(Text("Install dependencies done.", style="red bold"))
+
+    console.print(Text(f"Recording image name {name_alias}", style="red bold"))
+    _record_image_name(name_alias)

{indexify-0.2.20 → indexify-0.2.22}/indexify/executor/task_reporter.py

@@ -30,7 +30,7 @@ class TaskReporter:
         fn_outputs = []
         for output in completed_task.outputs or []:
             print(
-                f"[bold]task-reporter[/bold] uploading output of size: {len(output_bytes)} bytes"
+                f"[bold]task-reporter[/bold] uploading output of size: {len(output.payload)} bytes"
             )
             output_bytes = MsgPackSerializer.serialize(output) 
             fn_outputs.append(

{indexify-0.2.20 → indexify-0.2.22}/indexify/http_client.py

@@ -10,7 +10,7 @@ from httpx_sse import connect_sse
 from pydantic import BaseModel, Json
 from rich import print
 
-from indexify.error import ApiException
+from indexify.error import ApiException, GraphStillProcessing
 from indexify.functions_sdk.data_objects import IndexifyData
 from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
 from indexify.functions_sdk.indexify_functions import IndexifyFunction
@@ -36,7 +36,9 @@ class GraphOutputMetadata(BaseModel):
 
 
 class GraphOutputs(BaseModel):
+    status: str
     outputs: List[GraphOutputMetadata]
+    cursor: Optional[str] = None
 
 
 class IndexifyClient:
@@ -329,6 +331,8 @@ class IndexifyClient:
         )
         response.raise_for_status()
         graph_outputs = GraphOutputs(**response.json())
+        if graph_outputs.status == "pending":
+            raise GraphStillProcessing()
         outputs = []
         for output in graph_outputs.outputs:
             if output.compute_fn == fn_name:

{indexify-0.2.20 → indexify-0.2.22}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "indexify"
-version = "0.2.20"
+version = "0.2.22"
 description = "Python Client for Indexify"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"

indexify-0.2.20/indexify/error.py

@@ -1,3 +0,0 @@
-class ApiException(Exception):
-    def __init__(self, message: str) -> None:
-        super().__init__(message)