indexify 0.2__py3-none-any.whl → 0.2.1__py3-none-any.whl

indexify/__init__.py

@@ -1,6 +1,7 @@
 from . import data_loaders
 from .client import create_client
 from .functions_sdk.graph import Graph
+from .functions_sdk.image import Image
 from .functions_sdk.indexify_functions import (
     indexify_function,
     indexify_router,
@@ -12,6 +13,7 @@ from .settings import DEFAULT_SERVICE_URL
 __all__ = [
     "data_loaders",
     "Graph",
+    "Image",
     "indexify_function",
     "indexify_router",
     "DEFAULT_SERVICE_URL",

indexify/cli.py

@@ -19,7 +19,7 @@ from rich.theme import Theme
 
 from indexify.executor.agent import ExtractorAgent
 from indexify.executor.function_worker import FunctionWorker
-from indexify.functions_sdk.image import Image
+from indexify.functions_sdk.image import Image, DEFAULT_IMAGE
 
 custom_theme = Theme(
     {
@@ -130,7 +130,6 @@ def build_image(workflow_file_path: str, func_names: List[str]):
         )
 
     found_funcs = []
-    graph = None
     for name, obj in globals_dict.items():
         for func_name in func_names:
             if name == func_name:
@@ -143,6 +142,15 @@ def build_image(workflow_file_path: str, func_names: List[str]):
     )
 
 
+@app.command(help="Build default image for indexify")
+def build_default_image():
+    _build_image(image=DEFAULT_IMAGE)
+
+    console.print(
+        Text(f"Built default indexify image", style="cyan"),
+    )
+
+
 @app.command(help="Joins the extractors to the coordinator server")
 def executor(
     server_addr: str = "localhost:8900",
@@ -211,25 +219,29 @@ def _build_image(image: Image, func_name: str = None):
         )
         exit(-1)
 
-    docker_file_str_template = """
-FROM {base_image}
+    docker_file = f"""
+FROM {image._base_image}
+
+RUN mkdir -p ~/.indexify
+
+RUN touch ~/.indexify/image_name
+
+RUN  echo {image._image_name} > ~/.indexify/image_name
 
 WORKDIR /app
 
 """
 
-    docker_file_str = docker_file_str_template.format(base_image=image._base_image)
-
     run_strs = ["RUN " + i for i in image._run_strs]
 
-    docker_file_str += "\n".join(run_strs)
+    docker_file += "\n".join(run_strs)
 
     console.print("Creating image using Dockerfile contents:", style="cyan bold")
-    console.print(f"{docker_file_str}", style="magenta")
+    console.print(f"{docker_file}", style="magenta")
 
     client = docker.from_env()
     client.images.build(
-        fileobj=io.BytesIO(docker_file_str.encode()),
+        fileobj=io.BytesIO(docker_file.encode()),
         tag=f"{image._image_name}:{image._tag}",
         rm=True,
     )

indexify/client.py

@@ -9,10 +9,10 @@ from .settings import DEFAULT_SERVICE_URL
 def create_client(
     service_url: str = DEFAULT_SERVICE_URL,
     config_path: Optional[str] = None,
-    local: bool = False,
+    in_process: bool = False,
     *args,
     **kwargs,
 ) -> IndexifyClient:
-    if local:
+    if in_process:
         return LocalClient()
     return RemoteClient(config_path=config_path, service_url=service_url, **kwargs)

indexify/executor/agent.py

@@ -1,7 +1,6 @@
 import asyncio
 import json
 import ssl
-import traceback
 from concurrent.futures.process import BrokenProcessPool
 from typing import Dict, List, Optional
 
@@ -24,6 +23,7 @@ from .api_objects import ExecutorMetadata, Task
 from .downloader import DownloadedInputs, Downloader
 from .executor_tasks import DownloadGraphTask, DownloadInputTask, ExtractTask
 from .function_worker import FunctionWorker
+from .runtime_probes import ProbeInfo, RuntimeProbes
 from .task_reporter import TaskReporter
 from .task_store import CompletedTask, TaskStore
 
@@ -98,6 +98,7 @@ class ExtractorAgent:
         self._task_reporter = TaskReporter(
             base_url=self._base_url, executor_id=self._executor_id
         )
+        self._probe = RuntimeProbes()
 
     async def task_completion_reporter(self):
         console.print(Text("Starting task completion reporter", style="bold cyan"))
@@ -305,11 +306,12 @@ class ExtractorAgent:
                 words = snake_str.split("_")
                 return words[0].capitalize() + "" + " ".join(words[1:])
 
+            runtime_probe: ProbeInfo = self._probe.probe()
             data = ExecutorMetadata(
                 id=self._executor_id,
-                address="",
-                runner_name="extractor",
-                labels={},
+                addr="",
+                image_name=runtime_probe.image_name,
+                labels=runtime_probe.labels,
             ).model_dump()
 
             panel_content = "\n".join(
@@ -318,7 +320,7 @@ class ExtractorAgent:
             console.print(
                 Panel(
                     panel_content,
-                    title="Attempting to Register Executor",
+                    title="attempting to Register Executor",
                     border_style="cyan",
                 )
             )
@@ -333,7 +335,7 @@ class ExtractorAgent:
                         headers={"Content-Type": "application/json"},
                     ) as event_source:
                         console.print(
-                            Text("Executor registered successfully", style="bold green")
+                            Text("executor registered successfully", style="bold green")
                         )
                         async for sse in event_source.aiter_sse():
                             data = json.loads(sse.data)
@@ -345,14 +347,14 @@ class ExtractorAgent:
                             self._task_store.add_tasks(tasks)
             except Exception as e:
                 console.print(
-                    Text("Registration Error: ", style="red bold")
-                    + Text(f"Failed to register: {e}", style="red")
+                    Text("registration Error: ", style="red bold")
+                    + Text(f"failed to register: {e}", style="red")
                 )
                 await asyncio.sleep(5)
                 continue
 
     async def _shutdown(self, loop):
-        console.print(Text("Shutting down agent...", style="bold yellow"))
+        console.print(Text("shutting down agent...", style="bold yellow"))
         self._should_run = False
         for task in asyncio.all_tasks(loop):
             task.cancel()

indexify/executor/api_objects.py

@@ -18,8 +18,8 @@ class Task(BaseModel):
 
 class ExecutorMetadata(BaseModel):
     id: str
-    address: str
-    runner_name: str
+    addr: str
+    image_name: str
     labels: Dict[str, Any]
 
 

indexify/executor/runtime_probes.py

@@ -0,0 +1,48 @@
+import os
+import platform
+import sys
+from typing import Any, Dict, Tuple
+
+from pydantic import BaseModel
+
+
+class ProbeInfo(BaseModel):
+    image_name: str
+    python_major_version: int
+    labels: Dict[str, Any] = {}
+
+
+class RuntimeProbes:
+    def __init__(self) -> None:
+        self._image_name = self._read_image_name()
+        self._os_name = platform.system()
+        self._architecture = platform.machine()
+        (
+            self._python_version_major,
+            self._python_version_minor,
+        ) = self._get_python_version()
+
+    def _read_image_name(self) -> str:
+        file_path = os.path.expanduser("~/.indexify/image_name")
+        if os.path.exists(file_path):
+            with open(file_path, "r") as file:
+                return file.read().strip()
+        return "indexify-executor-default"
+
+    def _get_python_version(self) -> Tuple[int, int]:
+        version_info = sys.version_info
+        return version_info.major, version_info.minor
+
+    def probe(self) -> ProbeInfo:
+        labels = {
+            "os": self._os_name,
+            "image_name": self._image_name,
+            "architecture": self._architecture,
+            "python_major_version": self._python_version_major,
+            "python_minor_version": self._python_version_minor,
+        }
+        return ProbeInfo(
+            image_name=self._image_name,
+            python_major_version=self._python_version_major,
+            labels=labels,
+        )

indexify/functions_sdk/graph.py

@@ -66,6 +66,7 @@ class FunctionMetadata(BaseModel):
     fn_name: str
     description: str
     reducer: bool = False
+    image_name: str
     payload_encoder: str = "cloudpickle"
 
 
@@ -74,6 +75,7 @@ class RouterMetadata(BaseModel):
     description: str
     source_fn: str
     target_fns: List[str]
+    image_name: str
     payload_encoder: str = "cloudpickle"
 
 
@@ -244,6 +246,7 @@ class Graph:
             fn_name=start_node.fn_name,
             description=start_node.description,
             reducer=start_node.accumulate is not None,
+            image_name=start_node.image._image_name,
         )
         metadata_edges = self.edges.copy()
         metadata_nodes = {}
@@ -256,6 +259,7 @@ class Graph:
                         source_fn=node_name,
                         target_fns=self.routers[node_name],
                         payload_encoder=node.payload_encoder,
+                        image_name=node.image._image_name,
                     )
                 )
             else:
@@ -265,6 +269,7 @@ class Graph:
                         fn_name=node.fn_name,
                         description=node.description,
                         reducer=node.accumulate is not None,
+                        image_name=node.image._image_name,
                     )
                 )
         return ComputeGraphMetadata(

indexify/functions_sdk/graph_validation.py

@@ -42,28 +42,25 @@ def validate_route(
 
     if signature.return_annotation == inspect.Signature.empty:
         raise Exception(f"Function {from_node.name} has empty return type annotation")
+    
+    return_annotation = signature.return_annotation
 
-    # We lose the exact type string when the object is created
-    source = inspect.getsource(from_node.run)
+    if hasattr(return_annotation, '__origin__') and return_annotation.__origin__ is Union:
+        for arg in return_annotation.__args__:
+            if hasattr(arg, 'name'):
+                if arg not in to_nodes:
+                    raise Exception(
+                        f"Unable to find {arg.name} in to_nodes {[node.name for node in to_nodes]}"
+                    )
 
-    union_pattern = r"Union\[((?:\w+(?:,\s*)?)+)\]"
-    union_match = re.search(union_pattern, source)
-
-    src_route_nodes = None
-    if union_match:
-        # nodes = re.findall(r'\w+', match.group(1))
-        src_route_nodes = [node.strip() for node in union_match.group(1).split(",")]
-        if len(src_route_nodes) <= 1:
-            raise Exception(f"Invalid router for {from_node.name}, lte 1 route.")
+    if hasattr(return_annotation, '__origin__') and return_annotation.__origin__ is list:
+        union_args = return_annotation.__args__[0].__args__
+        for arg in union_args:
+            if hasattr(arg, 'name'):
+                if arg not in to_nodes:
+                    raise Exception(
+                        f"Unable to find {arg.name} in to_nodes {[node.name for node in to_nodes]}"
+                    )
     else:
-        raise Exception(
-            f"Invalid router for {from_node.name}, cannot find output nodes"
-        )
+        raise Exception(f"Return type of {from_node.name} is not a Union")
 
-    to_node_names = [i.name for i in to_nodes]
-
-    for src_node in src_route_nodes:
-        if src_node not in to_node_names:
-            raise Exception(
-                f"Unable to find {src_node} in to_nodes " f"{to_node_names}"
-            )

indexify/functions_sdk/image.py

@@ -7,9 +7,8 @@ class Image:
         self._base_image = None
 
         self._run_strs = []
-        pass
 
-    def image_name(self, image_name):
+    def name(self, image_name):
         self._image_name = image_name
         return self
 
@@ -24,3 +23,12 @@ class Image:
     def run(self, run_str):
         self._run_strs.append(run_str)
         return self
+
+
+DEFAULT_IMAGE = (
+    Image()
+    .name("indexify-executor-default")
+    .base_image("python:3.10.15-slim-bookworm")
+    .tag("latest")
+    .run("pip install indexify")
+)

indexify/functions_sdk/indexify_functions.py

@@ -16,7 +16,7 @@ from pydantic import BaseModel
 from typing_extensions import get_type_hints
 
 from .data_objects import IndexifyData, RouterOutput
-from .image import Image
+from .image import DEFAULT_IMAGE, Image
 
 
 class EmbeddingIndexes(BaseModel):
@@ -34,8 +34,8 @@ class PlacementConstraints(BaseModel):
 
 class IndexifyFunction(ABC):
     name: str = ""
-    base_image: Optional[str] = None
     description: str = ""
+    image: Optional[Image] = DEFAULT_IMAGE
     placement_constraints: List[PlacementConstraints] = []
     accumulate: Optional[Type[Any]] = None
     payload_encoder: Optional[str] = "cloudpickle"
@@ -53,7 +53,7 @@ class IndexifyFunction(ABC):
 class IndexifyRouter(ABC):
     name: str = ""
     description: str = ""
-    image: Image = None
+    image: Optional[Image] = DEFAULT_IMAGE
     placement_constraints: List[PlacementConstraints] = []
     payload_encoder: Optional[str] = "cloudpickle"
 
@@ -65,9 +65,9 @@ class IndexifyRouter(ABC):
 def indexify_router(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Image = None,
+    image: Optional[Image] = DEFAULT_IMAGE,
     placement_constraints: List[PlacementConstraints] = [],
-    output_encoder: Optional[str] = "cloudpickle",
+    payload_encoder: Optional[str] = "cloudpickle",
 ):
     def construct(fn):
         args = locals().copy()
@@ -90,7 +90,7 @@ def indexify_router(
                 setattr(IndexifyRo, key, value)
 
         IndexifyRo.image = image
-        IndexifyRo.payload_encoder = output_encoder
+        IndexifyRo.payload_encoder = payload_encoder
         return IndexifyRo
 
     return construct
@@ -99,11 +99,9 @@ def indexify_router(
 def indexify_function(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Image = None,
+    image: Optional[Image] = DEFAULT_IMAGE,
     accumulate: Optional[Type[BaseModel]] = None,
-    min_batch_size: Optional[int] = None,
-    max_batch_size: Optional[int] = None,
-    output_encoder: Optional[str] = "cloudpickle",
+    payload_encoder: Optional[str] = "cloudpickle",
     placement_constraints: List[PlacementConstraints] = [],
 ):
     def construct(fn):
@@ -128,9 +126,7 @@ def indexify_function(
 
         IndexifyFn.image = image
         IndexifyFn.accumulate = accumulate
-        IndexifyFn.min_batch_size = min_batch_size
-        IndexifyFn.max_batch_size = max_batch_size
-        IndexifyFn.payload_encoder = output_encoder
+        IndexifyFn.payload_encoder = payload_encoder
         return IndexifyFn
 
     return construct

indexify/functions_sdk/object_serializer.py

@@ -48,7 +48,6 @@ class MsgPackSerializer:
     @staticmethod
     def deserialize(data: bytes) -> IndexifyData:
         cached_output = msgpack.unpackb(data)
-        print(cached_output)
         return IndexifyData(**cached_output)
 
     @staticmethod

{indexify-0.2.dist-info → indexify-0.2.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2
+Version: 0.2.1
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.2.dist-info → indexify-0.2.1.dist-info}/RECORD

@@ -1,32 +1,33 @@
-indexify/__init__.py,sha256=fD9E-i9gawBNwbMd8ZIfiNsxUtL1S5VBCiyvKU5Yslc,497
+indexify/__init__.py,sha256=yJ3K_TyocH2EpFr6kEbKaOqfk8dA33fpoCB_QXV-rko,549
 indexify/base_client.py,sha256=Si1XnZ6X_mFvkYCnS6qx6axFsBpkrGiorqmKohFwvLQ,3324
-indexify/cli.py,sha256=XZYU9iMVHqNzecMU6LdAyfa_JkHb0Xx35akaW319tTI,7091
-indexify/client.py,sha256=sopeSA9hc7QHIHdpGq5rehSkr0tiPvdsLSJsaVzw3kY,491
+indexify/cli.py,sha256=kLKruCRNlo1xdezxYQoN6c9EpGWAeNiSs7kAjYfxCao,7311
+indexify/client.py,sha256=6cwCxBky6IJYu4caq0E6SMWIxf3nn5SX795moHfS4Cw,501
 indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
 indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
 indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
 indexify/error.py,sha256=vjd5SPPNFIEW35GorSIodsqvm9RKHQm9kdp8t9gv-WM,111
-indexify/executor/agent.py,sha256=RxpmnZ4ne-2BCv6Lv0kkkBJX3Dz71bruvaFC8V3O1D0,14563
-indexify/executor/api_objects.py,sha256=nvRmKYngjrv8sc_GoYM1MSLdFMYB6hoq7XK4LMx2U-s,814
+indexify/executor/agent.py,sha256=I08CiOWeJ_mz8OHr9_iJfp07Ma1VMQirZ2MsDp8lDZw,14723
+indexify/executor/api_objects.py,sha256=SysjlGYu4JtYdqfexZHHN1IW4TtaDdFUF3hYZ5mpUJU,810
 indexify/executor/downloader.py,sha256=0MPiKw0AWs3Z7ReC9l2z-3515yqq85ghPzdh485dnuw,3998
 indexify/executor/executor_tasks.py,sha256=gAZ2pvza1YwGlaR1o_tJW4SXtdCgK7sLJgp4W7rOjR0,1834
 indexify/executor/function_worker.py,sha256=83ih8TjAJHtrH6LqqDoSfzAj4zB7ZZPR2Voq0RMZ1T8,5410
 indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
+indexify/executor/runtime_probes.py,sha256=tvi8KCaQTVJqcyBJ4-jzEUAnQ01ZbMmjCxV2KJ96_PI,1449
 indexify/executor/task_reporter.py,sha256=gnnse0v6rjjni8lNzeb-ZYq6iF2DgafKoT7dcGUZhQ4,3716
 indexify/executor/task_store.py,sha256=q8s2gImsFffWeXQR0mk1Xlo1Aj_2GfclNPjQ2EA_YBo,3984
 indexify/foo,sha256=e385Ws-u8zx-LOq3tdfTa-siK9pMaccdAE8_0rrp_k4,5165
 indexify/functions_sdk/data_objects.py,sha256=2LqAWJ_S2Xkp4OQTmhd3InVIrBs7juV41udnSQFMMfM,840
-indexify/functions_sdk/graph.py,sha256=ilImZsALzOQLwioPYqva0_FxHtrn3FscCmtVniPJecU,9764
-indexify/functions_sdk/graph_validation.py,sha256=UgP_iMlUsH3jO0aBkQOwfxGQV1_x2qx9_iZvSxvuJBo,2419
-indexify/functions_sdk/image.py,sha256=0A8xTjLv0jITRu0u06iCFplS_cu47Zu3fUCXq9Rc7zI,525
-indexify/functions_sdk/indexify_functions.py,sha256=sJclmxvOAiAPUONbcwmUwywsUEzMvMA6gIE6zaURr1s,5955
+indexify/functions_sdk/graph.py,sha256=qy9zVbf26oTLW0d0jt991Hd8MP6N0F1CEBLL9VbwLBA,9975
+indexify/functions_sdk/graph_validation.py,sha256=y-f0ZNiGYl_fjPA7v9OJWtoUMPELgtVR_ifpgqZ0IoY,2465
+indexify/functions_sdk/image.py,sha256=euuz2QTZQoS-JmwnPmWJ8lfIgKzrSEsfkUc2qU26xjM,679
+indexify/functions_sdk/indexify_functions.py,sha256=xxgvnw0MQ_csIksunIdero8be0PR4mfwgoHp3UlkMZU,5851
 indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
-indexify/functions_sdk/object_serializer.py,sha256=HPH0Ym6-Los8lg_RHxmerrAh5fSLBHDeTZikom0eXxk,1689
+indexify/functions_sdk/object_serializer.py,sha256=Zz4GobW3ZamBBtFDF76QxU3TP6oJNdWnhsfKd0OUFoc,1660
 indexify/local_client.py,sha256=9wPYHjG516ZX6q5sPj5Pnn9C-Fi0sghYiLaNMe7LPPk,7220
 indexify/remote_client.py,sha256=oKgTqLbIxQVDqkMjQmNCOOEIM156UeYMC1jDWWSqBAQ,12297
 indexify/settings.py,sha256=LSaWZ0ADIVmUv6o6dHWRC3-Ry5uLbCw2sBSg1e_U7UM,99
-indexify-0.2.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.dist-info/METADATA,sha256=GuigbP7vmakmIXhhijeYAiCbhvaFZX-Np2gZAcHYwrc,6127
-indexify-0.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-indexify-0.2.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.dist-info/RECORD,,
+indexify-0.2.1.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.1.dist-info/METADATA,sha256=ZwqwbjyxS8vszZH5L_X_1zNJM9OyAr-B0ezxpUDeLJo,6129
+indexify-0.2.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+indexify-0.2.1.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.1.dist-info/RECORD,,