indexify 0.2.20__py3-none-any.whl → 0.2.22__py3-none-any.whl

indexify/cli.py

@@ -1,5 +1,4 @@
 import asyncio
-import io
 import os
 import shutil
 import signal
@@ -168,6 +167,9 @@ def executor(
     executor_cache: Optional[str] = typer.Option(
         "~/.indexify/executor_cache", help="Path to the executor cache directory"
     ),
+    name_alias: Optional[str] = typer.Option(
+        None, help="Name alias for the executor if it's spun up with the base image"
+    ),
 ):
     id = nanoid.generate()
     console.print(
@@ -176,7 +178,8 @@ def executor(
             f"Config path: {config_path}\n"
             f"Server address: {server_addr}\n"
             f"Executor ID: {id}\n"
-            f"Executor cache: {executor_cache}",
+            f"Executor cache: {executor_cache}\n"
+            f"Name Alias: {name_alias}",
             title="Agent Configuration",
             border_style="info",
         )
@@ -197,6 +200,7 @@ def executor(
         server_addr=server_addr,
         config_path=config_path,
         code_path=executor_cache,
+        name_alias=name_alias,
     )
 
     try:

indexify/error.py

@@ -1,3 +1,8 @@
 class ApiException(Exception):
     def __init__(self, message: str) -> None:
         super().__init__(message)
+
+
+class GraphStillProcessing(Exception):
+    def __init__(self) -> None:
+        super().__init__("graph is still processing")

indexify/executor/agent.py

@@ -3,9 +3,12 @@ import json
 import ssl
 from concurrent.futures.process import BrokenProcessPool
 from importlib.metadata import version
+import traceback
 from typing import Dict, List, Optional
 
 import httpx
+from indexify.functions_sdk.graph_definition import ComputeGraphMetadata
+from indexify.http_client import IndexifyClient
 import yaml
 from httpx_sse import aconnect_sse
 from pydantic import BaseModel
@@ -19,6 +22,7 @@ from indexify.functions_sdk.data_objects import (
     IndexifyData,
     RouterOutput,
 )
+from . import image_dependency_installer
 
 from .api_objects import ExecutorMetadata, Task
 from .downloader import DownloadedInputs, Downloader
@@ -27,6 +31,7 @@ from .function_worker import FunctionWorker
 from .runtime_probes import ProbeInfo, RuntimeProbes
 from .task_reporter import TaskReporter
 from .task_store import CompletedTask, TaskStore
+from ..functions_sdk.image import ImageInformation
 
 custom_theme = Theme(
     {
@@ -58,7 +63,24 @@ class ExtractorAgent:
         function_worker: FunctionWorker,
         server_addr: str = "localhost:8900",
         config_path: Optional[str] = None,
+        name_alias: Optional[str] = None,
     ):
+        self.name_alias = name_alias
+
+        self._probe = RuntimeProbes()
+
+        runtime_probe: ProbeInfo = self._probe.probe()
+        self._require_image_bootstrap = (
+            True
+            if (runtime_probe.is_default_executor and self.name_alias is not None)
+            else False
+        )
+        self._executor_bootstrap_failed = False
+
+        console.print(
+            f"Require Bootstrap? {self._require_image_bootstrap}", style="cyan bold"
+        )
+
         self.num_workers = num_workers
         self._use_tls = False
         if config_path:
@@ -99,7 +121,6 @@ class ExtractorAgent:
         self._task_reporter = TaskReporter(
             base_url=self._base_url, executor_id=self._executor_id
         )
-        self._probe = RuntimeProbes()
 
     async def task_completion_reporter(self):
         console.print(Text("Starting task completion reporter", style="bold cyan"))
@@ -145,15 +166,55 @@ class ExtractorAgent:
     async def task_launcher(self):
         async_tasks: List[asyncio.Task] = []
         fn_queue: List[FunctionInput] = []
+
         async_tasks.append(
             asyncio.create_task(
                 self._task_store.get_runnable_tasks(), name="get_runnable_tasks"
             )
         )
+
         while True:
             fn: FunctionInput
             for fn in fn_queue:
                 task: Task = self._task_store.get_task(fn.task_id)
+
+                if self._executor_bootstrap_failed:
+                    completed_task = CompletedTask(
+                        task=task,
+                        outputs=[],
+                        task_outcome="failure",
+                    )
+                    self._task_store.complete(outcome=completed_task)
+
+                    continue
+
+                # Bootstrap this executor. Fail the task if we can't.
+                if self._require_image_bootstrap:
+                    try:
+                        image_info = await _get_image_info_for_compute_graph(
+                            task, self._protocol, self._server_addr
+                        )
+                        image_dependency_installer.executor_image_builder(
+                            image_info, self.name_alias
+                        )
+                        self._require_image_bootstrap = False
+                    except Exception as e:
+                        console.print(
+                            Text("Failed to bootstrap the executor ", style="red bold")
+                            + Text(f"Exception: {traceback.format_exc()}", style="red")
+                        )
+
+                        self._executor_bootstrap_failed = True
+
+                        completed_task = CompletedTask(
+                            task=task,
+                            outputs=[],
+                            task_outcome="failure",
+                        )
+                        self._task_store.complete(outcome=completed_task)
+
+                        continue
+
                 async_tasks.append(
                     ExtractTask(
                         function_worker=self._function_worker,
@@ -309,18 +370,19 @@ class ExtractorAgent:
 
             runtime_probe: ProbeInfo = self._probe.probe()
 
-            # Inspect the image
-            if runtime_probe.is_default_executor:
-                # install dependencies
-                # rewrite the image name
-                pass
-
             executor_version = version("indexify")
+
+            image_name = (
+                self.name_alias
+                if self.name_alias is not None
+                else runtime_probe.image_name
+            )
+
             data = ExecutorMetadata(
                 id=self._executor_id,
                 executor_version=executor_version,
                 addr="",
-                image_name=runtime_probe.image_name,
+                image_name=image_name,
                 labels=runtime_probe.labels,
             ).model_dump()
 
@@ -372,3 +434,25 @@ class ExtractorAgent:
     def shutdown(self, loop):
         self._function_worker.shutdown()
         loop.create_task(self._shutdown(loop))
+
+
+async def _get_image_info_for_compute_graph(
+    task: Task, protocol, server_addr
+) -> ImageInformation:
+    namespace = task.namespace
+    graph_name: str = task.compute_graph
+    compute_fn_name: str = task.compute_fn
+
+    http_client = IndexifyClient(
+        service_url=f"{protocol}://{server_addr}", namespace=namespace
+    )
+    compute_graph: ComputeGraphMetadata = http_client.graph(graph_name)
+
+    console.print(
+        Text(
+            f"Compute_fn name {compute_fn_name}, ComputeGraph {compute_graph} \n",
+            style="red yellow",
+        )
+    )
+
+    return compute_graph.nodes[compute_fn_name].compute_fn.image_information

indexify/executor/image_dependency_installer.py

@@ -0,0 +1,52 @@
+import os
+import subprocess
+
+from rich.console import Console
+from rich.text import Text
+from rich.theme import Theme
+
+from indexify.functions_sdk.image import ImageInformation
+
+
+custom_theme = Theme(
+    {
+        "info": "cyan",
+        "warning": "yellow",
+        "error": "red",
+        "success": "green",
+    }
+)
+
+console = Console(theme=custom_theme)
+
+
+def _record_image_name(name: str):
+    dir_path = os.path.expanduser("~/.indexify/")
+    file_path = os.path.expanduser("~/.indexify/image_name")
+    os.makedirs(dir_path, exist_ok=True)
+
+    with open(file_path, "w") as file:
+        file.write(name)
+
+
+def _install_dependencies(run_str: str):
+    # Throw error to the caller if these subprocesses fail.
+    proc = subprocess.run(run_str.split())
+    if proc.returncode != 0:
+        raise Exception(f"Unable to install dep `{run_str}`")
+
+
+def executor_image_builder(image_info: ImageInformation, name_alias: str):
+    console.print(Text("Attempting Executor Bootstrap.", style="red bold"))
+
+    run_strs = image_info.run_strs
+    console.print(Text("Attempting to install dependencies.", style="red bold"))
+
+    for run_str in run_strs:
+        console.print(Text(f"Attempting {run_str}", style="red bold"))
+        _install_dependencies(run_str)
+
+    console.print(Text("Install dependencies done.", style="red bold"))
+
+    console.print(Text(f"Recording image name {name_alias}", style="red bold"))
+    _record_image_name(name_alias)

indexify/executor/task_reporter.py

@@ -30,7 +30,7 @@ class TaskReporter:
         fn_outputs = []
         for output in completed_task.outputs or []:
             print(
-                f"[bold]task-reporter[/bold] uploading output of size: {len(output_bytes)} bytes"
+                f"[bold]task-reporter[/bold] uploading output of size: {len(output.payload)} bytes"
             )
             output_bytes = MsgPackSerializer.serialize(output) 
             fn_outputs.append(

indexify/http_client.py

@@ -10,7 +10,7 @@ from httpx_sse import connect_sse
 from pydantic import BaseModel, Json
 from rich import print
 
-from indexify.error import ApiException
+from indexify.error import ApiException, GraphStillProcessing
 from indexify.functions_sdk.data_objects import IndexifyData
 from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
 from indexify.functions_sdk.indexify_functions import IndexifyFunction
@@ -36,7 +36,9 @@ class GraphOutputMetadata(BaseModel):
 
 
 class GraphOutputs(BaseModel):
+    status: str
     outputs: List[GraphOutputMetadata]
+    cursor: Optional[str] = None
 
 
 class IndexifyClient:
@@ -329,6 +331,8 @@ class IndexifyClient:
         )
         response.raise_for_status()
         graph_outputs = GraphOutputs(**response.json())
+        if graph_outputs.status == "pending":
+            raise GraphStillProcessing()
         outputs = []
         for output in graph_outputs.outputs:
             if output.compute_fn == fn_name:

{indexify-0.2.20.dist-info → indexify-0.2.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.20
+Version: 0.2.22
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.2.20.dist-info → indexify-0.2.22.dist-info}/RECORD

@@ -1,17 +1,18 @@
 indexify/__init__.py,sha256=xYjdqZQ9CEtV-s4PJhj_6Vx_dfg_qknWHkLqOSVom8g,623
-indexify/cli.py,sha256=E7NIWMAJcGHeOFG0rE2mCgT_V0e1l2M6tOPDSjKzrzg,8219
+indexify/cli.py,sha256=sR-9wc3J_46TsOV424VD6TwKZkEg9MHiflvAJB1VyeY,8420
 indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
 indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
 indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
-indexify/error.py,sha256=vjd5SPPNFIEW35GorSIodsqvm9RKHQm9kdp8t9gv-WM,111
-indexify/executor/agent.py,sha256=IeKaJw1qgoRXjhLZINp-J3Boq0AeR61cPst9sx6KiTY,15099
+indexify/error.py,sha256=mBEz_fj2sdtTQol-VfwvD8Syu7x9uvYSy2nne4Gr2GI,237
+indexify/executor/agent.py,sha256=nSJ9vIWqY74pL5cEy7Rvb_xDatqtgEmxT4sJ65z3Rb0,17984
 indexify/executor/api_objects.py,sha256=QMlHbcL-ZOAMVD71XOJqD_ZIy-NjrUf1DAvi0hFYVg0,836
 indexify/executor/downloader.py,sha256=3mEDdluTzspsLGAZtFHZOVuyKOzT3CSema2kIK6Z1yU,4005
 indexify/executor/executor_tasks.py,sha256=gAZ2pvza1YwGlaR1o_tJW4SXtdCgK7sLJgp4W7rOjR0,1834
 indexify/executor/function_worker.py,sha256=pCGn13rg4dEykzmwYNyCTkewrpyQXQR1cH6n2Hx5Lfc,5813
+indexify/executor/image_dependency_installer.py,sha256=ditkQQ8rMrtQgZZ7Nw5wDn4I0d-qk8Q2irL-cU-4gOY,1449
 indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
 indexify/executor/runtime_probes.py,sha256=JY0FoxtlQ9sgsE8gBKWM5h3R1TWkYENhNF0HR2KkV4Q,1704
-indexify/executor/task_reporter.py,sha256=OsF-D88BIY3wzhBjZeKl4O5-BH8Hl6C8s1R8i4MHi8I,3719
+indexify/executor/task_reporter.py,sha256=0ujOJrN7aW9PiVH0gr1rGqadfZE9yfV318Ll3eahBng,3721
 indexify/executor/task_store.py,sha256=q8s2gImsFffWeXQR0mk1Xlo1Aj_2GfclNPjQ2EA_YBo,3984
 indexify/functions_sdk/data_objects.py,sha256=CQZMzYiV7l6dyzFkYquWQHqdte6JnC7XA3i2ZyvPvgQ,844
 indexify/functions_sdk/graph.py,sha256=L8jtuSGaUk05JgT_cQX-n_1H9t84lcPa1v4XXlpuJa0,11143
@@ -22,12 +23,12 @@ indexify/functions_sdk/indexify_functions.py,sha256=KLeRQY5IcrVp5T3K4yQ8Uu_CLL-C
 indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
 indexify/functions_sdk/object_serializer.py,sha256=Zz4GobW3ZamBBtFDF76QxU3TP6oJNdWnhsfKd0OUFoc,1660
 indexify/functions_sdk/pipeline.py,sha256=7hDatRK-SCHYvttf2Vj5YFyiJEVU0OOXEZBOIQenSmk,847
-indexify/http_client.py,sha256=MflqHbkkzYWw64nnmZCeJE76Q9p-o5hFnlI3h7kbJFI,13729
+indexify/http_client.py,sha256=pFJ4i40_GgzGJP9ih_iCHNUJQlSsWUiCLBMT3aIegOg,13887
 indexify/remote_graph.py,sha256=NOotayhw_x4-mAw1c3ooNe7gFE_q3QJ9qaO0t0Sb8sE,4397
 indexify/remote_pipeline.py,sha256=FW7IAv3r24OOpiqlprw3kuFrpdkqi6Ic4_tT26FThjA,761
 indexify/settings.py,sha256=LSaWZ0ADIVmUv6o6dHWRC3-Ry5uLbCw2sBSg1e_U7UM,99
-indexify-0.2.20.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.20.dist-info/METADATA,sha256=T8Qbsc59Y_uLGCVlRgUw_N-OblvDCM0g7SscDDVZ6jg,6199
-indexify-0.2.20.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-indexify-0.2.20.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.20.dist-info/RECORD,,
+indexify-0.2.22.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.22.dist-info/METADATA,sha256=PTlq-SzEMf1hcd09SwyB7aBF4BvsQgVPyPK8PQoGrCU,6199
+indexify-0.2.22.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+indexify-0.2.22.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.22.dist-info/RECORD,,