indexify 0.2.11__py3-none-any.whl → 0.2.13__py3-none-any.whl

indexify/__init__.py

@@ -5,14 +5,18 @@ from .functions_sdk.indexify_functions import (
     indexify_function,
     indexify_router,
 )
+from .functions_sdk.pipeline import Pipeline
 from .http_client import IndexifyClient
 from .remote_graph import RemoteGraph
+from .remote_pipeline import RemotePipeline
 from .settings import DEFAULT_SERVICE_URL
 
 __all__ = [
     "data_loaders",
     "Graph",
     "RemoteGraph",
+    "Pipeline",
+    "RemotePipeline",
     "Image",
     "indexify_function",
     "indexify_router",

indexify/cli.py

@@ -18,7 +18,11 @@ from rich.theme import Theme
 
 from indexify.executor.agent import ExtractorAgent
 from indexify.executor.function_worker import FunctionWorker
-from indexify.functions_sdk.image import DEFAULT_IMAGE, Image
+from indexify.functions_sdk.image import (
+    DEFAULT_IMAGE_3_10,
+    DEFAULT_IMAGE_3_11,
+    Image,
+)
 
 custom_theme = Theme(
     {
@@ -120,7 +124,7 @@ def server_dev_mode():
 @app.command(help="Build image for function names")
 def build_image(
     workflow_file_path: str,
-    func_names: List[str],
+    image_names: Optional[List[str]] = None,
     python_sdk_path: Optional[str] = None,
 ):
     globals_dict = {}
@@ -136,25 +140,16 @@ def build_image(
         raise Exception(
             f"Could not find workflow file to execute at: " f"`{workflow_file_path}`"
         )
-
-    found_funcs = []
-    for name, obj in globals_dict.items():
-        for func_name in func_names:
-            if name == func_name:
-                found_funcs.append(name)
-                _create_image_for_func(
-                    func_name=func_name, func_obj=obj, python_sdk_path=python_sdk_path
-                )
-
-    console.print(
-        Text(f"Processed functions: ", style="cyan"),
-        Text(f"{found_funcs}", style="green"),
-    )
+    for _, obj in globals_dict.items():
+        if type(obj) and isinstance(obj, Image):
+            if image_names is None or obj._image_name in image_names:
+                _create_image(obj, python_sdk_path)
 
 
 @app.command(help="Build default image for indexify")
 def build_default_image():
-    _build_image(image=DEFAULT_IMAGE)
+    _build_image(image=DEFAULT_IMAGE_3_10)
+    _build_image(image=DEFAULT_IMAGE_3_11)
 
     console.print(
         Text(f"Built default indexify image", style="cyan"),
@@ -210,12 +205,12 @@ def executor(
         console.print(Text(f"Exiting gracefully: {ex}", style="bold yellow"))
 
 
-def _create_image_for_func(func_name, func_obj, python_sdk_path):
+def _create_image(image: Image, python_sdk_path):
     console.print(
         Text("Creating container for ", style="cyan"),
-        Text(f"`{func_name}`", style="cyan bold"),
+        Text(f"`{image._image_name}`", style="cyan bold"),
     )
-    _build_image(image=func_obj.image, python_sdk_path=python_sdk_path)
+    _build_image(image=image, python_sdk_path=python_sdk_path)
 
 
 def _build_image(image: Image, python_sdk_path: Optional[str] = None):

indexify/executor/downloader.py

@@ -75,7 +75,7 @@ class Downloader:
 
         reducer_url = None
         if task.reducer_output_id:
-            reducer_url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/fn/{task.compute_fn}/{task.reducer_output_id}"
+            reducer_url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/fn/{task.compute_fn}/output/{task.reducer_output_id}"
 
         console.print(
             Panel(

indexify/functions_sdk/image.py

@@ -4,7 +4,7 @@ class Image:
 
         self._tag = "latest"
 
-        self._base_image = "python:3.10.15-slim-bookworm"
+        self._base_image = "python:3.11.10-slim-bookworm"
 
         self._run_strs = []
 
@@ -25,10 +25,18 @@ class Image:
         return self
 
 
-DEFAULT_IMAGE = (
+DEFAULT_IMAGE_3_10 = (
     Image()
     .name("tensorlake/indexify-executor-default")
     .base_image("python:3.10.15-slim-bookworm")
-    .tag("latest")
+    .tag("3.10")
+    .run("pip install indexify")
+)
+
+DEFAULT_IMAGE_3_11 = (
+    Image()
+    .name("tensorlake/indexify-executor-default")
+    .base_image("python:3.11.10-slim-bookworm")
+    .tag("3.11")
     .run("pip install indexify")
 )

indexify/functions_sdk/indexify_functions.py

@@ -18,7 +18,7 @@ from pydantic import BaseModel
 from typing_extensions import get_type_hints
 
 from .data_objects import IndexifyData, RouterOutput
-from .image import DEFAULT_IMAGE, Image
+from .image import DEFAULT_IMAGE_3_10, Image
 from .object_serializer import CloudPickleSerializer, get_serializer
 
 
@@ -65,7 +65,7 @@ class PlacementConstraints(BaseModel):
 class IndexifyFunction(ABC):
     name: str = ""
     description: str = ""
-    image: Optional[Image] = DEFAULT_IMAGE
+    image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
     accumulate: Optional[Type[Any]] = None
     payload_encoder: Optional[str] = "cloudpickle"
@@ -83,7 +83,7 @@ class IndexifyFunction(ABC):
 class IndexifyRouter(ABC):
     name: str = ""
     description: str = ""
-    image: Optional[Image] = DEFAULT_IMAGE
+    image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
     payload_encoder: Optional[str] = "cloudpickle"
 
@@ -95,7 +95,7 @@ class IndexifyRouter(ABC):
 def indexify_router(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Optional[Image] = DEFAULT_IMAGE,
+    image: Optional[Image] = DEFAULT_IMAGE_3_10,
     placement_constraints: List[PlacementConstraints] = [],
     payload_encoder: Optional[str] = "cloudpickle",
 ):
@@ -129,7 +129,7 @@ def indexify_router(
 def indexify_function(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Optional[Image] = DEFAULT_IMAGE,
+    image: Optional[Image] = DEFAULT_IMAGE_3_10,
     accumulate: Optional[Type[BaseModel]] = None,
     payload_encoder: Optional[str] = "cloudpickle",
     placement_constraints: List[PlacementConstraints] = [],

indexify/functions_sdk/pipeline.py

@@ -0,0 +1,30 @@
+from typing import Union
+
+from indexify.functions_sdk.indexify_functions import (
+    IndexifyFunction,
+    IndexifyRouter,
+)
+
+from .graph import Graph
+
+
+class Pipeline:
+    def __init__(self, name: str, description: str):
+        self.name = name
+        self.description = description
+        self._graph: Graph = None
+        self._last_step = None
+
+    def add_step(self, function: Union[IndexifyFunction, IndexifyRouter]):
+        if self._graph is None:
+            self._graph = Graph(
+                name=self.name, description=self.description, start_node=function
+            )
+            self._last_step = function
+            return
+        self._graph.add_edge(self._last_step, function)
+        self._last_step = function
+
+    def run(self, **kwargs):
+        invocation_id = self._graph.run(**kwargs)
+        return invocation_id

indexify/http_client.py

@@ -300,7 +300,7 @@ class IndexifyClient:
         output_id: str,
     ) -> IndexifyData:
         response = self._get(
-            f"namespaces/{namespace}/compute_graphs/{graph}/invocations/{invocation_id}/fn/{fn_name}/{output_id}",
+            f"namespaces/{namespace}/compute_graphs/{graph}/invocations/{invocation_id}/fn/{fn_name}/output/{output_id}",
         )
         response.raise_for_status()
         data_dict = msgpack.unpackb(response.content)

indexify/remote_pipeline.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from indexify.functions_sdk.pipeline import Pipeline
+
+from .http_client import IndexifyClient
+from .remote_graph import RemoteGraph
+
+
+class RemotePipeline(RemoteGraph):
+    @classmethod
+    def deploy(
+        cls,
+        p: Pipeline,
+        additional_modules=[],
+        server_url: Optional[str] = "http://localhost:8900",
+    ):
+        """
+        Create a new RemoteGraph from a local Graph object.
+        :param g: The local Graph object.
+        :param server_url: The URL of the server where the graph will be registered.
+        """
+        client = IndexifyClient(service_url=server_url)
+        client.register_compute_graph(p._graph, additional_modules)
+        return cls(name=p._graph.name, server_url=server_url)

{indexify-0.2.11.dist-info → indexify-0.2.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.11
+Version: 0.2.13
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -42,12 +42,12 @@ pip install indexify
 ```
 
 ## Examples 
-**[PDF Document Extraction](./examples/pdf_document_extraction/workflow.py)**
+**[PDF Document Extraction](../examples/pdf_document_extraction/workflow.py)**
 1. Extracts text, tables and images from an ingested PDF file
 2. Indexes the text using MiniLM-L6-v2, the images with CLIP
 3. Writes the results into a vector database.
 
-**[Youtube Transcription Summarizer](./examples/video_summarization/workflow.py)**
+**[Youtube Transcription Summarizer](../examples/video_summarization/workflow.py)**
 1. Downloads Youtube Video
 2. Extracts audio from the video and transcribes using `Faster Whisper` 
 3. Uses Llama 3.1 backed by `Llama.cpp` to understand and classify the nature of the video.

{indexify-0.2.11.dist-info → indexify-0.2.13.dist-info}/RECORD

@@ -1,12 +1,12 @@
-indexify/__init__.py,sha256=qgMBKVrM_tI-tFeWpE8ktlC5rcExk05nbWyFqxxqeEU,496
-indexify/cli.py,sha256=zSJ5dk5mqmRWkXX150v9ziGqPvFd8gmQDf77Gcc3ZYY,8391
+indexify/__init__.py,sha256=xYjdqZQ9CEtV-s4PJhj_6Vx_dfg_qknWHkLqOSVom8g,623
+indexify/cli.py,sha256=tfqXvOYlrseZMRUxM70zalRz-O1IGE1cFuO_HiTZ1oU,8244
 indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
 indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
 indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
 indexify/error.py,sha256=vjd5SPPNFIEW35GorSIodsqvm9RKHQm9kdp8t9gv-WM,111
 indexify/executor/agent.py,sha256=I08CiOWeJ_mz8OHr9_iJfp07Ma1VMQirZ2MsDp8lDZw,14723
 indexify/executor/api_objects.py,sha256=SysjlGYu4JtYdqfexZHHN1IW4TtaDdFUF3hYZ5mpUJU,810
-indexify/executor/downloader.py,sha256=0MPiKw0AWs3Z7ReC9l2z-3515yqq85ghPzdh485dnuw,3998
+indexify/executor/downloader.py,sha256=3mEDdluTzspsLGAZtFHZOVuyKOzT3CSema2kIK6Z1yU,4005
 indexify/executor/executor_tasks.py,sha256=gAZ2pvza1YwGlaR1o_tJW4SXtdCgK7sLJgp4W7rOjR0,1834
 indexify/executor/function_worker.py,sha256=RQh-lJD2UtUuYXyCDkGohe8rVpHEriJg-AjBKSgD5KI,5436
 indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
@@ -17,15 +17,17 @@ indexify/functions_sdk/data_objects.py,sha256=CQZMzYiV7l6dyzFkYquWQHqdte6JnC7XA3
 indexify/functions_sdk/graph.py,sha256=t35z26jRCBgoz7fncdlAL7MVTOYjeJuoPpz-xI7WjBw,11827
 indexify/functions_sdk/graph_definition.py,sha256=EJfC0MdKEbFF1CBaU0htrveSlcAQJCH96DLSNfZ02V4,1178
 indexify/functions_sdk/graph_validation.py,sha256=XLHiC9PAtZungJLysU3hIUOPNDkO5TXUDZ_jiZ0H4hg,2508
-indexify/functions_sdk/image.py,sha256=RcFhVjmQv83S2wuf7Yn9RMhdpT-WfRqr1-krCF5UjNM,716
-indexify/functions_sdk/indexify_functions.py,sha256=oUssAADTt3byhz3Azl_GiYxV1BmhwuGE1CjmN-oiBpg,9575
+indexify/functions_sdk/image.py,sha256=Rsol3LfE15Kca78ruPfDrDPqCkeZPxZx0ULYrFFS5ig,905
+indexify/functions_sdk/indexify_functions.py,sha256=QnumxE0Ln_ryyyjLMopjCGZQUGSX7N25PRpisIdEEXI,9600
 indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
 indexify/functions_sdk/object_serializer.py,sha256=Zz4GobW3ZamBBtFDF76QxU3TP6oJNdWnhsfKd0OUFoc,1660
-indexify/http_client.py,sha256=0nB-2yQPoVfvTRGvRru_pW6SWjqeT8jtXJPm-MC71pg,13636
+indexify/functions_sdk/pipeline.py,sha256=7hDatRK-SCHYvttf2Vj5YFyiJEVU0OOXEZBOIQenSmk,847
+indexify/http_client.py,sha256=x9ABHWc_oQH8_fa5qwFi53D10m9ibhrjvfw_OkTJfhc,13643
 indexify/remote_graph.py,sha256=ILg6IY6EFgyvnfz1DSzicBZhqvPkg2-UUwgI6lxp6sA,3094
+indexify/remote_pipeline.py,sha256=FW7IAv3r24OOpiqlprw3kuFrpdkqi6Ic4_tT26FThjA,761
 indexify/settings.py,sha256=LSaWZ0ADIVmUv6o6dHWRC3-Ry5uLbCw2sBSg1e_U7UM,99
-indexify-0.2.11.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.11.dist-info/METADATA,sha256=FRk9YjzPFT37qHRdRiYOMS-SMvrDH51Fo8aIa_OW10E,6130
-indexify-0.2.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-indexify-0.2.11.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.11.dist-info/RECORD,,
+indexify-0.2.13.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.13.dist-info/METADATA,sha256=urBxY_dtcmMIfTd_UQvHq0efxq4-LHJahRGEOM0WYhE,6132
+indexify-0.2.13.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+indexify-0.2.13.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.13.dist-info/RECORD,,