indexify 0.2.11__tar.gz → 0.2.13__tar.gz

{indexify-0.2.11 → indexify-0.2.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.11
+Version: 0.2.13
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -42,12 +42,12 @@ pip install indexify
 ```
 
 ## Examples 
-**[PDF Document Extraction](./examples/pdf_document_extraction/workflow.py)**
+**[PDF Document Extraction](../examples/pdf_document_extraction/workflow.py)**
 1. Extracts text, tables and images from an ingested PDF file
 2. Indexes the text using MiniLM-L6-v2, the images with CLIP
 3. Writes the results into a vector database.
 
-**[Youtube Transcription Summarizer](./examples/video_summarization/workflow.py)**
+**[Youtube Transcription Summarizer](../examples/video_summarization/workflow.py)**
 1. Downloads Youtube Video
 2. Extracts audio from the video and transcribes using `Faster Whisper` 
 3. Uses Llama 3.1 backed by `Llama.cpp` to understand and classify the nature of the video.

{indexify-0.2.11 → indexify-0.2.13}/README.md

@@ -14,12 +14,12 @@ pip install indexify
 ```
 
 ## Examples 
-**[PDF Document Extraction](./examples/pdf_document_extraction/workflow.py)**
+**[PDF Document Extraction](../examples/pdf_document_extraction/workflow.py)**
 1. Extracts text, tables and images from an ingested PDF file
 2. Indexes the text using MiniLM-L6-v2, the images with CLIP
 3. Writes the results into a vector database.
 
-**[Youtube Transcription Summarizer](./examples/video_summarization/workflow.py)**
+**[Youtube Transcription Summarizer](../examples/video_summarization/workflow.py)**
 1. Downloads Youtube Video
 2. Extracts audio from the video and transcribes using `Faster Whisper` 
 3. Uses Llama 3.1 backed by `Llama.cpp` to understand and classify the nature of the video.

{indexify-0.2.11 → indexify-0.2.13}/indexify/__init__.py

@@ -5,14 +5,18 @@ from .functions_sdk.indexify_functions import (
     indexify_function,
     indexify_router,
 )
+from .functions_sdk.pipeline import Pipeline
 from .http_client import IndexifyClient
 from .remote_graph import RemoteGraph
+from .remote_pipeline import RemotePipeline
 from .settings import DEFAULT_SERVICE_URL
 
 __all__ = [
     "data_loaders",
     "Graph",
     "RemoteGraph",
+    "Pipeline",
+    "RemotePipeline",
     "Image",
     "indexify_function",
     "indexify_router",

{indexify-0.2.11 → indexify-0.2.13}/indexify/cli.py

@@ -18,7 +18,11 @@ from rich.theme import Theme
 
 from indexify.executor.agent import ExtractorAgent
 from indexify.executor.function_worker import FunctionWorker
-from indexify.functions_sdk.image import DEFAULT_IMAGE, Image
+from indexify.functions_sdk.image import (
+    DEFAULT_IMAGE_3_10,
+    DEFAULT_IMAGE_3_11,
+    Image,
+)
 
 custom_theme = Theme(
     {
@@ -120,7 +124,7 @@ def server_dev_mode():
 @app.command(help="Build image for function names")
 def build_image(
     workflow_file_path: str,
-    func_names: List[str],
+    image_names: Optional[List[str]] = None,
     python_sdk_path: Optional[str] = None,
 ):
     globals_dict = {}
@@ -136,25 +140,16 @@ def build_image(
         raise Exception(
             f"Could not find workflow file to execute at: " f"`{workflow_file_path}`"
         )
-
-    found_funcs = []
-    for name, obj in globals_dict.items():
-        for func_name in func_names:
-            if name == func_name:
-                found_funcs.append(name)
-                _create_image_for_func(
-                    func_name=func_name, func_obj=obj, python_sdk_path=python_sdk_path
-                )
-
-    console.print(
-        Text(f"Processed functions: ", style="cyan"),
-        Text(f"{found_funcs}", style="green"),
-    )
+    for _, obj in globals_dict.items():
+        if type(obj) and isinstance(obj, Image):
+            if image_names is None or obj._image_name in image_names:
+                _create_image(obj, python_sdk_path)
 
 
 @app.command(help="Build default image for indexify")
 def build_default_image():
-    _build_image(image=DEFAULT_IMAGE)
+    _build_image(image=DEFAULT_IMAGE_3_10)
+    _build_image(image=DEFAULT_IMAGE_3_11)
 
     console.print(
         Text(f"Built default indexify image", style="cyan"),
@@ -210,12 +205,12 @@ def executor(
         console.print(Text(f"Exiting gracefully: {ex}", style="bold yellow"))
 
 
-def _create_image_for_func(func_name, func_obj, python_sdk_path):
+def _create_image(image: Image, python_sdk_path):
     console.print(
         Text("Creating container for ", style="cyan"),
-        Text(f"`{func_name}`", style="cyan bold"),
+        Text(f"`{image._image_name}`", style="cyan bold"),
     )
-    _build_image(image=func_obj.image, python_sdk_path=python_sdk_path)
+    _build_image(image=image, python_sdk_path=python_sdk_path)
 
 
 def _build_image(image: Image, python_sdk_path: Optional[str] = None):

{indexify-0.2.11 → indexify-0.2.13}/indexify/executor/downloader.py

@@ -75,7 +75,7 @@ class Downloader:
 
         reducer_url = None
         if task.reducer_output_id:
-            reducer_url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/fn/{task.compute_fn}/{task.reducer_output_id}"
+            reducer_url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/fn/{task.compute_fn}/output/{task.reducer_output_id}"
 
         console.print(
             Panel(

{indexify-0.2.11 → indexify-0.2.13}/indexify/functions_sdk/image.py

@@ -4,7 +4,7 @@ class Image:
 
         self._tag = "latest"
 
-        self._base_image = "python:3.10.15-slim-bookworm"
+        self._base_image = "python:3.11.10-slim-bookworm"
 
         self._run_strs = []
 
@@ -25,10 +25,18 @@ class Image:
         return self
 
 
-DEFAULT_IMAGE = (
+DEFAULT_IMAGE_3_10 = (
     Image()
     .name("tensorlake/indexify-executor-default")
     .base_image("python:3.10.15-slim-bookworm")
-    .tag("latest")
+    .tag("3.10")
+    .run("pip install indexify")
+)
+
+DEFAULT_IMAGE_3_11 = (
+    Image()
+    .name("tensorlake/indexify-executor-default")
+    .base_image("python:3.11.10-slim-bookworm")
+    .tag("3.11")
     .run("pip install indexify")
 )

{indexify-0.2.11 → indexify-0.2.13}/indexify/functions_sdk/indexify_functions.py

@@ -18,7 +18,7 @@ from pydantic import BaseModel
 from typing_extensions import get_type_hints
 
 from .data_objects import IndexifyData, RouterOutput
-from .image import DEFAULT_IMAGE, Image
+from .image import DEFAULT_IMAGE_3_10, Image
 from .object_serializer import CloudPickleSerializer, get_serializer
 
 
@@ -65,7 +65,7 @@ class PlacementConstraints(BaseModel):
 class IndexifyFunction(ABC):
     name: str = ""
     description: str = ""
-    image: Optional[Image] = DEFAULT_IMAGE
+    image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
     accumulate: Optional[Type[Any]] = None
     payload_encoder: Optional[str] = "cloudpickle"
@@ -83,7 +83,7 @@ class IndexifyFunction(ABC):
 class IndexifyRouter(ABC):
     name: str = ""
     description: str = ""
-    image: Optional[Image] = DEFAULT_IMAGE
+    image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
     payload_encoder: Optional[str] = "cloudpickle"
 
@@ -95,7 +95,7 @@ class IndexifyRouter(ABC):
 def indexify_router(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Optional[Image] = DEFAULT_IMAGE,
+    image: Optional[Image] = DEFAULT_IMAGE_3_10,
     placement_constraints: List[PlacementConstraints] = [],
     payload_encoder: Optional[str] = "cloudpickle",
 ):
@@ -129,7 +129,7 @@ def indexify_router(
 def indexify_function(
     name: Optional[str] = None,
     description: Optional[str] = "",
-    image: Optional[Image] = DEFAULT_IMAGE,
+    image: Optional[Image] = DEFAULT_IMAGE_3_10,
     accumulate: Optional[Type[BaseModel]] = None,
     payload_encoder: Optional[str] = "cloudpickle",
     placement_constraints: List[PlacementConstraints] = [],

indexify-0.2.13/indexify/functions_sdk/pipeline.py

@@ -0,0 +1,30 @@
+from typing import Union
+
+from indexify.functions_sdk.indexify_functions import (
+    IndexifyFunction,
+    IndexifyRouter,
+)
+
+from .graph import Graph
+
+
+class Pipeline:
+    def __init__(self, name: str, description: str):
+        self.name = name
+        self.description = description
+        self._graph: Graph = None
+        self._last_step = None
+
+    def add_step(self, function: Union[IndexifyFunction, IndexifyRouter]):
+        if self._graph is None:
+            self._graph = Graph(
+                name=self.name, description=self.description, start_node=function
+            )
+            self._last_step = function
+            return
+        self._graph.add_edge(self._last_step, function)
+        self._last_step = function
+
+    def run(self, **kwargs):
+        invocation_id = self._graph.run(**kwargs)
+        return invocation_id

{indexify-0.2.11 → indexify-0.2.13}/indexify/http_client.py

@@ -300,7 +300,7 @@ class IndexifyClient:
         output_id: str,
     ) -> IndexifyData:
         response = self._get(
-            f"namespaces/{namespace}/compute_graphs/{graph}/invocations/{invocation_id}/fn/{fn_name}/{output_id}",
+            f"namespaces/{namespace}/compute_graphs/{graph}/invocations/{invocation_id}/fn/{fn_name}/output/{output_id}",
         )
         response.raise_for_status()
         data_dict = msgpack.unpackb(response.content)

indexify-0.2.13/indexify/remote_pipeline.py

@@ -0,0 +1,24 @@
+from typing import Optional
+
+from indexify.functions_sdk.pipeline import Pipeline
+
+from .http_client import IndexifyClient
+from .remote_graph import RemoteGraph
+
+
+class RemotePipeline(RemoteGraph):
+    @classmethod
+    def deploy(
+        cls,
+        p: Pipeline,
+        additional_modules=[],
+        server_url: Optional[str] = "http://localhost:8900",
+    ):
+        """
+        Create a new RemoteGraph from a local Graph object.
+        :param g: The local Graph object.
+        :param server_url: The URL of the server where the graph will be registered.
+        """
+        client = IndexifyClient(service_url=server_url)
+        client.register_compute_graph(p._graph, additional_modules)
+        return cls(name=p._graph.name, server_url=server_url)

{indexify-0.2.11 → indexify-0.2.13}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "indexify"
-version = "0.2.11"
+version = "0.2.13"
 description = "Python Client for Indexify"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"