indexify 0.2.12__py3-none-any.whl → 0.2.14__py3-none-any.whl

indexify/__init__.py

@@ -5,14 +5,18 @@ from .functions_sdk.indexify_functions import (
     indexify_function,
     indexify_router,
 )
+from .functions_sdk.pipeline import Pipeline
 from .http_client import IndexifyClient
 from .remote_graph import RemoteGraph
+from .remote_pipeline import RemotePipeline
 from .settings import DEFAULT_SERVICE_URL
 
 __all__ = [
     "data_loaders",
     "Graph",
     "RemoteGraph",
+    "Pipeline",
+    "RemotePipeline",
     "Image",
     "indexify_function",
     "indexify_router",

indexify/cli.py

@@ -18,7 +18,11 @@ from rich.theme import Theme
 
 from indexify.executor.agent import ExtractorAgent
 from indexify.executor.function_worker import FunctionWorker
-from indexify.functions_sdk.image import DEFAULT_IMAGE, Image
+from indexify.functions_sdk.image import (
+    DEFAULT_IMAGE_3_10,
+    DEFAULT_IMAGE_3_11,
+    Image,
+)
 
 custom_theme = Theme(
     {
@@ -120,7 +124,7 @@ def server_dev_mode():
 @app.command(help="Build image for function names")
 def build_image(
     workflow_file_path: str,
-    image_names: List[str],
+    image_names: Optional[List[str]] = None,
     python_sdk_path: Optional[str] = None,
 ):
     globals_dict = {}
@@ -136,14 +140,16 @@ def build_image(
         raise Exception(
             f"Could not find workflow file to execute at: " f"`{workflow_file_path}`"
         )
-    for name, obj in globals_dict.items():
-        if type(obj) and isinstance(obj, Image) and obj._image_name in image_names:
-            _create_image(obj, python_sdk_path)
+    for _, obj in globals_dict.items():
+        if type(obj) and isinstance(obj, Image):
+            if image_names is None or obj._image_name in image_names:
+                _create_image(obj, python_sdk_path)
 
 
 @app.command(help="Build default image for indexify")
 def build_default_image():
-    _build_image(image=DEFAULT_IMAGE)
+    _build_image(image=DEFAULT_IMAGE_3_10)
+    _build_image(image=DEFAULT_IMAGE_3_11)
 
     console.print(
         Text(f"Built default indexify image", style="cyan"),

indexify/executor/downloader.py

@@ -75,7 +75,7 @@ class Downloader:
 
         reducer_url = None
         if task.reducer_output_id:
-            reducer_url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/fn/{task.compute_fn}/{task.reducer_output_id}"
+            reducer_url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/fn/{task.compute_fn}/output/{task.reducer_output_id}"
 
         console.print(
             Panel(

indexify/functions_sdk/graph.py

@@ -153,7 +153,7 @@ class Graph:
         start_node = self.nodes[self._start_node]
         start_node = FunctionMetadata(
             name=start_node.name,
-            fn_name=start_node.fn_name,
+            fn_name=start_node.name,
             description=start_node.description,
             reducer=start_node.accumulate is not None,
             image_name=start_node.image._image_name,
@@ -209,13 +209,15 @@ class Graph:
                 k: IndexifyData(payload=serializer.serialize(v))
             }
         self._results[input.id] = outputs
-        self._run(input, outputs)
+        enable_cache = kwargs.get('enable_cache', True)
+        self._run(input, outputs,enable_cache)
         return input.id
 
     def _run(
         self,
         initial_input: IndexifyData,
         outputs: Dict[str, List[bytes]],
+        enable_cache: bool
     ):
         accumulator_values = self._accumulator_values[initial_input.id]
         queue = deque([(self._start_node, initial_input)])
@@ -227,7 +229,7 @@ class Graph:
             cached_output_bytes: Optional[bytes] = self._cache.get(
                 self.name, node_name, input_bytes
             )
-            if cached_output_bytes is not None:
+            if cached_output_bytes is not None and enable_cache:
                 print(
                     f"ran {node_name}: num outputs: {len(cached_output_bytes)} (cache hit)"
                 )

indexify/functions_sdk/image.py

@@ -25,11 +25,11 @@ class Image:
         return self
 
 
-DEFAULT_IMAGE = (
+DEFAULT_IMAGE_3_10 = (
     Image()
     .name("tensorlake/indexify-executor-default")
     .base_image("python:3.10.15-slim-bookworm")
-    .tag("3:10")
+    .tag("3.10")
     .run("pip install indexify")
 )
 
@@ -37,6 +37,6 @@ DEFAULT_IMAGE_3_11 = (
     Image()
     .name("tensorlake/indexify-executor-default")
     .base_image("python:3.11.10-slim-bookworm")
-    .tag("3:11")
+    .tag("3.11")
     .run("pip install indexify")
 )

indexify/functions_sdk/indexify_functions.py

@@ -18,7 +18,7 @@ from pydantic import BaseModel
 from typing_extensions import get_type_hints
 
 from .data_objects import IndexifyData, RouterOutput
-from .image import DEFAULT_IMAGE, Image
+from .image import DEFAULT_IMAGE_3_10, Image
 from .object_serializer import CloudPickleSerializer, get_serializer
 
 
@@ -65,7 +65,7 @@ class PlacementConstraints(BaseModel):
 class IndexifyFunction(ABC):
     name: str = ""
     description: str = ""
-    image: Optional[Image] = DEFAULT_IMAGE
+    image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
     accumulate: Optional[Type[Any]] = None
     payload_encoder: Optional[str] = "cloudpickle"
@@ -83,7 +83,7 @@ class IndexifyFunction(ABC):
 class IndexifyRouter(ABC):
     name: str = ""
     description: str = ""
-    image: Optional[Image] = DEFAULT_IMAGE
+    image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
     payload_encoder: Optional[str] = "cloudpickle"
 
@@ -95,7 +95,7 @@ class IndexifyRouter(ABC):
 def indexify_router(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Optional[Image] = DEFAULT_IMAGE,
+    image: Optional[Image] = DEFAULT_IMAGE_3_10,
     placement_constraints: List[PlacementConstraints] = [],
     payload_encoder: Optional[str] = "cloudpickle",
 ):
@@ -129,7 +129,7 @@ def indexify_router(
 def indexify_function(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Optional[Image] = DEFAULT_IMAGE,
+    image: Optional[Image] = DEFAULT_IMAGE_3_10,
     accumulate: Optional[Type[BaseModel]] = None,
     payload_encoder: Optional[str] = "cloudpickle",
     placement_constraints: List[PlacementConstraints] = [],

indexify/functions_sdk/pipeline.py

@@ -0,0 +1,30 @@
+from typing import Union
+
+from indexify.functions_sdk.indexify_functions import (
+    IndexifyFunction,
+    IndexifyRouter,
+)
+
+from .graph import Graph
+
+
+class Pipeline:
+    def __init__(self, name: str, description: str):
+        self.name = name
+        self.description = description
+        self._graph: Graph = None
+        self._last_step = None
+
+    def add_step(self, function: Union[IndexifyFunction, IndexifyRouter]):
+        if self._graph is None:
+            self._graph = Graph(
+                name=self.name, description=self.description, start_node=function
+            )
+            self._last_step = function
+            return
+        self._graph.add_edge(self._last_step, function)
+        self._last_step = function
+
+    def run(self, **kwargs):
+        invocation_id = self._graph.run(**kwargs)
+        return invocation_id

indexify/http_client.py

@@ -79,7 +79,7 @@ class IndexifyClient:
             status_code = str(response.status_code)
             if status_code.startswith("4"):
                 raise ApiException(
-                    "status code: " + status_code + " request args: " + str(kwargs)
+                    "status code: " + status_code + " message: " + response.text
                 )
             if status_code.startswith("5"):
                 raise ApiException(response.text)
@@ -300,7 +300,7 @@ class IndexifyClient:
         output_id: str,
     ) -> IndexifyData:
         response = self._get(
-            f"namespaces/{namespace}/compute_graphs/{graph}/invocations/{invocation_id}/fn/{fn_name}/{output_id}",
+            f"namespaces/{namespace}/compute_graphs/{graph}/invocations/{invocation_id}/fn/{fn_name}/output/{output_id}",
         )
         response.raise_for_status()
         data_dict = msgpack.unpackb(response.content)

indexify/remote_pipeline.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from indexify.functions_sdk.pipeline import Pipeline
+
+from .http_client import IndexifyClient
+from .remote_graph import RemoteGraph
+
+
+class RemotePipeline(RemoteGraph):
+    @classmethod
+    def deploy(
+        cls,
+        p: Pipeline,
+        additional_modules=[],
+        server_url: Optional[str] = "http://localhost:8900",
+    ):
+        """
+        Create a new RemoteGraph from a local Graph object.
+        :param g: The local Graph object.
+        :param server_url: The URL of the server where the graph will be registered.
+        """
+        client = IndexifyClient(service_url=server_url)
+        client.register_compute_graph(p._graph, additional_modules)
+        return cls(name=p._graph.name, server_url=server_url)

{indexify-0.2.12.dist-info → indexify-0.2.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.12
+Version: 0.2.14
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -13,7 +13,8 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: cloudpickle (>=3,<4)
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: cloudpickle (>=3.1.0,<4.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
 Requires-Dist: httpx[http2] (>=0,<1)
@@ -21,7 +22,7 @@ Requires-Dist: msgpack (>=1.1.0,<2.0.0)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: pydantic (>=2.9.2,<3.0.0)
 Requires-Dist: pyyaml (>=6,<7)
-Requires-Dist: rich (>=13,<14)
+Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: typer (>=0.12.5,<0.13.0)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
@@ -42,12 +43,12 @@ pip install indexify
 ```
 
 ## Examples 
-**[PDF Document Extraction](./examples/pdf_document_extraction/workflow.py)**
+**[PDF Document Extraction](../examples/pdf_document_extraction/workflow.py)**
 1. Extracts text, tables and images from an ingested PDF file
 2. Indexes the text using MiniLM-L6-v2, the images with CLIP
 3. Writes the results into a vector database.
 
-**[Youtube Transcription Summarizer](./examples/video_summarization/workflow.py)**
+**[Youtube Transcription Summarizer](../examples/video_summarization/workflow.py)**
 1. Downloads Youtube Video
 2. Extracts audio from the video and transcribes using `Faster Whisper` 
 3. Uses Llama 3.1 backed by `Llama.cpp` to understand and classify the nature of the video.

{indexify-0.2.12.dist-info → indexify-0.2.14.dist-info}/RECORD

@@ -1,12 +1,12 @@
-indexify/__init__.py,sha256=qgMBKVrM_tI-tFeWpE8ktlC5rcExk05nbWyFqxxqeEU,496
-indexify/cli.py,sha256=SR4AWCxObIwHZzKb2iKMwtOQ3BcJUJIZ3SRiDvhv1sQ,8101
+indexify/__init__.py,sha256=xYjdqZQ9CEtV-s4PJhj_6Vx_dfg_qknWHkLqOSVom8g,623
+indexify/cli.py,sha256=tfqXvOYlrseZMRUxM70zalRz-O1IGE1cFuO_HiTZ1oU,8244
 indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
 indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
 indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
 indexify/error.py,sha256=vjd5SPPNFIEW35GorSIodsqvm9RKHQm9kdp8t9gv-WM,111
 indexify/executor/agent.py,sha256=I08CiOWeJ_mz8OHr9_iJfp07Ma1VMQirZ2MsDp8lDZw,14723
 indexify/executor/api_objects.py,sha256=SysjlGYu4JtYdqfexZHHN1IW4TtaDdFUF3hYZ5mpUJU,810
-indexify/executor/downloader.py,sha256=0MPiKw0AWs3Z7ReC9l2z-3515yqq85ghPzdh485dnuw,3998
+indexify/executor/downloader.py,sha256=3mEDdluTzspsLGAZtFHZOVuyKOzT3CSema2kIK6Z1yU,4005
 indexify/executor/executor_tasks.py,sha256=gAZ2pvza1YwGlaR1o_tJW4SXtdCgK7sLJgp4W7rOjR0,1834
 indexify/executor/function_worker.py,sha256=RQh-lJD2UtUuYXyCDkGohe8rVpHEriJg-AjBKSgD5KI,5436
 indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
@@ -14,18 +14,20 @@ indexify/executor/runtime_probes.py,sha256=L-nB4D2zBZ287xxPwErv5R_gxFOmTEvg61FMe
 indexify/executor/task_reporter.py,sha256=gnnse0v6rjjni8lNzeb-ZYq6iF2DgafKoT7dcGUZhQ4,3716
 indexify/executor/task_store.py,sha256=q8s2gImsFffWeXQR0mk1Xlo1Aj_2GfclNPjQ2EA_YBo,3984
 indexify/functions_sdk/data_objects.py,sha256=CQZMzYiV7l6dyzFkYquWQHqdte6JnC7XA3i2ZyvPvgQ,844
-indexify/functions_sdk/graph.py,sha256=t35z26jRCBgoz7fncdlAL7MVTOYjeJuoPpz-xI7WjBw,11827
+indexify/functions_sdk/graph.py,sha256=3hUZjdCesiAHJ9y1_heQsEFO-WHROR4kRYxBtVnb7Y4,11937
 indexify/functions_sdk/graph_definition.py,sha256=EJfC0MdKEbFF1CBaU0htrveSlcAQJCH96DLSNfZ02V4,1178
 indexify/functions_sdk/graph_validation.py,sha256=XLHiC9PAtZungJLysU3hIUOPNDkO5TXUDZ_jiZ0H4hg,2508
-indexify/functions_sdk/image.py,sha256=0-QrvMZSFT0_RGb8aeUfSYpSlh6QQkssFHLibGP9ywI,900
-indexify/functions_sdk/indexify_functions.py,sha256=oUssAADTt3byhz3Azl_GiYxV1BmhwuGE1CjmN-oiBpg,9575
+indexify/functions_sdk/image.py,sha256=Rsol3LfE15Kca78ruPfDrDPqCkeZPxZx0ULYrFFS5ig,905
+indexify/functions_sdk/indexify_functions.py,sha256=QnumxE0Ln_ryyyjLMopjCGZQUGSX7N25PRpisIdEEXI,9600
 indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
 indexify/functions_sdk/object_serializer.py,sha256=Zz4GobW3ZamBBtFDF76QxU3TP6oJNdWnhsfKd0OUFoc,1660
-indexify/http_client.py,sha256=0nB-2yQPoVfvTRGvRru_pW6SWjqeT8jtXJPm-MC71pg,13636
+indexify/functions_sdk/pipeline.py,sha256=7hDatRK-SCHYvttf2Vj5YFyiJEVU0OOXEZBOIQenSmk,847
+indexify/http_client.py,sha256=EBzqwT3r90W4w0ADG8oRy8tCh47BHuMPB-vVh-bjUO8,13640
 indexify/remote_graph.py,sha256=ILg6IY6EFgyvnfz1DSzicBZhqvPkg2-UUwgI6lxp6sA,3094
+indexify/remote_pipeline.py,sha256=FW7IAv3r24OOpiqlprw3kuFrpdkqi6Ic4_tT26FThjA,761
 indexify/settings.py,sha256=LSaWZ0ADIVmUv6o6dHWRC3-Ry5uLbCw2sBSg1e_U7UM,99
-indexify-0.2.12.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.12.dist-info/METADATA,sha256=ryNoVKX3M3qm7MjZ_briF2E_BBF9kCx0eaKgDgmPpiA,6130
-indexify-0.2.12.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-indexify-0.2.12.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.12.dist-info/RECORD,,
+indexify-0.2.14.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.14.dist-info/METADATA,sha256=zMLk24iy5hXk8BGlvSxkAMHenl2JNxSeXQuENZLNsuY,6199
+indexify-0.2.14.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+indexify-0.2.14.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.14.dist-info/RECORD,,

{indexify-0.2.12.dist-info → indexify-0.2.14.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.0
+Generator: poetry-core 1.9.1
 Root-Is-Purelib: true
 Tag: py3-none-any