indexify 0.2.12__tar.gz → 0.2.14__tar.gz

{indexify-0.2.12 → indexify-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.12
+Version: 0.2.14
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -13,7 +13,8 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
-Requires-Dist: cloudpickle (>=3,<4)
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: cloudpickle (>=3.1.0,<4.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
 Requires-Dist: httpx[http2] (>=0,<1)
@@ -21,7 +22,7 @@ Requires-Dist: msgpack (>=1.1.0,<2.0.0)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: pydantic (>=2.9.2,<3.0.0)
 Requires-Dist: pyyaml (>=6,<7)
-Requires-Dist: rich (>=13,<14)
+Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: typer (>=0.12.5,<0.13.0)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
@@ -42,12 +43,12 @@ pip install indexify
 ```
 
 ## Examples 
-**[PDF Document Extraction](./examples/pdf_document_extraction/workflow.py)**
+**[PDF Document Extraction](../examples/pdf_document_extraction/workflow.py)**
 1. Extracts text, tables and images from an ingested PDF file
 2. Indexes the text using MiniLM-L6-v2, the images with CLIP
 3. Writes the results into a vector database.
 
-**[Youtube Transcription Summarizer](./examples/video_summarization/workflow.py)**
+**[Youtube Transcription Summarizer](../examples/video_summarization/workflow.py)**
 1. Downloads Youtube Video
 2. Extracts audio from the video and transcribes using `Faster Whisper` 
 3. Uses Llama 3.1 backed by `Llama.cpp` to understand and classify the nature of the video.

{indexify-0.2.12 → indexify-0.2.14}/README.md

@@ -14,12 +14,12 @@ pip install indexify
 ```
 
 ## Examples 
-**[PDF Document Extraction](./examples/pdf_document_extraction/workflow.py)**
+**[PDF Document Extraction](../examples/pdf_document_extraction/workflow.py)**
 1. Extracts text, tables and images from an ingested PDF file
 2. Indexes the text using MiniLM-L6-v2, the images with CLIP
 3. Writes the results into a vector database.
 
-**[Youtube Transcription Summarizer](./examples/video_summarization/workflow.py)**
+**[Youtube Transcription Summarizer](../examples/video_summarization/workflow.py)**
 1. Downloads Youtube Video
 2. Extracts audio from the video and transcribes using `Faster Whisper` 
 3. Uses Llama 3.1 backed by `Llama.cpp` to understand and classify the nature of the video.

{indexify-0.2.12 → indexify-0.2.14}/indexify/__init__.py

@@ -5,14 +5,18 @@ from .functions_sdk.indexify_functions import (
     indexify_function,
     indexify_router,
 )
+from .functions_sdk.pipeline import Pipeline
 from .http_client import IndexifyClient
 from .remote_graph import RemoteGraph
+from .remote_pipeline import RemotePipeline
 from .settings import DEFAULT_SERVICE_URL
 
 __all__ = [
     "data_loaders",
     "Graph",
     "RemoteGraph",
+    "Pipeline",
+    "RemotePipeline",
     "Image",
     "indexify_function",
     "indexify_router",

{indexify-0.2.12 → indexify-0.2.14}/indexify/cli.py

@@ -18,7 +18,11 @@ from rich.theme import Theme
 
 from indexify.executor.agent import ExtractorAgent
 from indexify.executor.function_worker import FunctionWorker
-from indexify.functions_sdk.image import DEFAULT_IMAGE, Image
+from indexify.functions_sdk.image import (
+    DEFAULT_IMAGE_3_10,
+    DEFAULT_IMAGE_3_11,
+    Image,
+)
 
 custom_theme = Theme(
     {
@@ -120,7 +124,7 @@ def server_dev_mode():
 @app.command(help="Build image for function names")
 def build_image(
     workflow_file_path: str,
-    image_names: List[str],
+    image_names: Optional[List[str]] = None,
     python_sdk_path: Optional[str] = None,
 ):
     globals_dict = {}
@@ -136,14 +140,16 @@ def build_image(
         raise Exception(
             f"Could not find workflow file to execute at: " f"`{workflow_file_path}`"
         )
-    for name, obj in globals_dict.items():
-        if type(obj) and isinstance(obj, Image) and obj._image_name in image_names:
-            _create_image(obj, python_sdk_path)
+    for _, obj in globals_dict.items():
+        if type(obj) and isinstance(obj, Image):
+            if image_names is None or obj._image_name in image_names:
+                _create_image(obj, python_sdk_path)
 
 
 @app.command(help="Build default image for indexify")
 def build_default_image():
-    _build_image(image=DEFAULT_IMAGE)
+    _build_image(image=DEFAULT_IMAGE_3_10)
+    _build_image(image=DEFAULT_IMAGE_3_11)
 
     console.print(
         Text(f"Built default indexify image", style="cyan"),

{indexify-0.2.12 → indexify-0.2.14}/indexify/executor/downloader.py

@@ -75,7 +75,7 @@ class Downloader:
 
         reducer_url = None
         if task.reducer_output_id:
-            reducer_url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/fn/{task.compute_fn}/{task.reducer_output_id}"
+            reducer_url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/fn/{task.compute_fn}/output/{task.reducer_output_id}"
 
         console.print(
             Panel(

{indexify-0.2.12 → indexify-0.2.14}/indexify/functions_sdk/graph.py

@@ -153,7 +153,7 @@ class Graph:
         start_node = self.nodes[self._start_node]
         start_node = FunctionMetadata(
             name=start_node.name,
-            fn_name=start_node.fn_name,
+            fn_name=start_node.name,
             description=start_node.description,
             reducer=start_node.accumulate is not None,
             image_name=start_node.image._image_name,
@@ -209,13 +209,15 @@ class Graph:
                 k: IndexifyData(payload=serializer.serialize(v))
             }
         self._results[input.id] = outputs
-        self._run(input, outputs)
+        enable_cache = kwargs.get('enable_cache', True)
+        self._run(input, outputs,enable_cache)
         return input.id
 
     def _run(
         self,
         initial_input: IndexifyData,
         outputs: Dict[str, List[bytes]],
+        enable_cache: bool
     ):
         accumulator_values = self._accumulator_values[initial_input.id]
         queue = deque([(self._start_node, initial_input)])
@@ -227,7 +229,7 @@ class Graph:
             cached_output_bytes: Optional[bytes] = self._cache.get(
                 self.name, node_name, input_bytes
             )
-            if cached_output_bytes is not None:
+            if cached_output_bytes is not None and enable_cache:
                 print(
                     f"ran {node_name}: num outputs: {len(cached_output_bytes)} (cache hit)"
                 )

{indexify-0.2.12 → indexify-0.2.14}/indexify/functions_sdk/image.py

@@ -25,11 +25,11 @@ class Image:
         return self
 
 
-DEFAULT_IMAGE = (
+DEFAULT_IMAGE_3_10 = (
     Image()
     .name("tensorlake/indexify-executor-default")
     .base_image("python:3.10.15-slim-bookworm")
-    .tag("3:10")
+    .tag("3.10")
     .run("pip install indexify")
 )
 
@@ -37,6 +37,6 @@ DEFAULT_IMAGE_3_11 = (
     Image()
     .name("tensorlake/indexify-executor-default")
     .base_image("python:3.11.10-slim-bookworm")
-    .tag("3:11")
+    .tag("3.11")
     .run("pip install indexify")
 )

{indexify-0.2.12 → indexify-0.2.14}/indexify/functions_sdk/indexify_functions.py

@@ -18,7 +18,7 @@ from pydantic import BaseModel
 from typing_extensions import get_type_hints
 
 from .data_objects import IndexifyData, RouterOutput
-from .image import DEFAULT_IMAGE, Image
+from .image import DEFAULT_IMAGE_3_10, Image
 from .object_serializer import CloudPickleSerializer, get_serializer
 
 
@@ -65,7 +65,7 @@ class PlacementConstraints(BaseModel):
 class IndexifyFunction(ABC):
     name: str = ""
     description: str = ""
-    image: Optional[Image] = DEFAULT_IMAGE
+    image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
     accumulate: Optional[Type[Any]] = None
     payload_encoder: Optional[str] = "cloudpickle"
@@ -83,7 +83,7 @@ class IndexifyFunction(ABC):
 class IndexifyRouter(ABC):
     name: str = ""
     description: str = ""
-    image: Optional[Image] = DEFAULT_IMAGE
+    image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
     payload_encoder: Optional[str] = "cloudpickle"
 
@@ -95,7 +95,7 @@ class IndexifyRouter(ABC):
 def indexify_router(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Optional[Image] = DEFAULT_IMAGE,
+    image: Optional[Image] = DEFAULT_IMAGE_3_10,
     placement_constraints: List[PlacementConstraints] = [],
     payload_encoder: Optional[str] = "cloudpickle",
 ):
@@ -129,7 +129,7 @@ def indexify_router(
 def indexify_function(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Optional[Image] = DEFAULT_IMAGE,
+    image: Optional[Image] = DEFAULT_IMAGE_3_10,
     accumulate: Optional[Type[BaseModel]] = None,
     payload_encoder: Optional[str] = "cloudpickle",
     placement_constraints: List[PlacementConstraints] = [],

indexify-0.2.14/indexify/functions_sdk/pipeline.py

@@ -0,0 +1,30 @@
+from typing import Union
+
+from indexify.functions_sdk.indexify_functions import (
+    IndexifyFunction,
+    IndexifyRouter,
+)
+
+from .graph import Graph
+
+
+class Pipeline:
+    def __init__(self, name: str, description: str):
+        self.name = name
+        self.description = description
+        self._graph: Graph = None
+        self._last_step = None
+
+    def add_step(self, function: Union[IndexifyFunction, IndexifyRouter]):
+        if self._graph is None:
+            self._graph = Graph(
+                name=self.name, description=self.description, start_node=function
+            )
+            self._last_step = function
+            return
+        self._graph.add_edge(self._last_step, function)
+        self._last_step = function
+
+    def run(self, **kwargs):
+        invocation_id = self._graph.run(**kwargs)
+        return invocation_id

{indexify-0.2.12 → indexify-0.2.14}/indexify/http_client.py

@@ -79,7 +79,7 @@ class IndexifyClient:
             status_code = str(response.status_code)
             if status_code.startswith("4"):
                 raise ApiException(
-                    "status code: " + status_code + " request args: " + str(kwargs)
+                    "status code: " + status_code + " message: " + response.text
                 )
             if status_code.startswith("5"):
                 raise ApiException(response.text)
@@ -300,7 +300,7 @@ class IndexifyClient:
         output_id: str,
     ) -> IndexifyData:
         response = self._get(
-            f"namespaces/{namespace}/compute_graphs/{graph}/invocations/{invocation_id}/fn/{fn_name}/{output_id}",
+            f"namespaces/{namespace}/compute_graphs/{graph}/invocations/{invocation_id}/fn/{fn_name}/output/{output_id}",
         )
         response.raise_for_status()
         data_dict = msgpack.unpackb(response.content)

indexify-0.2.14/indexify/remote_pipeline.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from indexify.functions_sdk.pipeline import Pipeline
+
+from .http_client import IndexifyClient
+from .remote_graph import RemoteGraph
+
+
+class RemotePipeline(RemoteGraph):
+    @classmethod
+    def deploy(
+        cls,
+        p: Pipeline,
+        additional_modules=[],
+        server_url: Optional[str] = "http://localhost:8900",
+    ):
+        """
+        Create a new RemoteGraph from a local Graph object.
+        :param g: The local Graph object.
+        :param server_url: The URL of the server where the graph will be registered.
+        """
+        client = IndexifyClient(service_url=server_url)
+        client.register_compute_graph(p._graph, additional_modules)
+        return cls(name=p._graph.name, server_url=server_url)

{indexify-0.2.12 → indexify-0.2.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "indexify"
-version = "0.2.12"
+version = "0.2.14"
 description = "Python Client for Indexify"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -16,8 +16,8 @@ python = "^3.9"
 httpx = { version = "^0", extras = ["http2"] }
 pyyaml = "^6"
 pydantic = "^2.9.2"
-cloudpickle = "^3"
-rich = "^13"
+cloudpickle = "^3.1.0"
+rich = "^13.9.2"
 nanoid = "^2.0.0"
 docker = "^7.1.0"
 msgpack= "^1.1.0"