indexify 0.2.27__py3-none-any.whl → 0.2.29__py3-none-any.whl

indexify/common_util.py

@@ -0,0 +1,88 @@
+from typing import Optional, Union
+
+import httpx
+import yaml
+from httpx import AsyncClient, Client
+
+
+def get_httpx_client(
+    config_path: Optional[str] = None, make_async: Optional[bool] = False
+) -> AsyncClient | Client:
+    """
+    Creates and returns an httpx.Client instance, optionally configured with TLS settings from a YAML config file.
+
+    The function creates a basic httpx.Client by default. If a config path is provided and the config specifies
+    'use_tls' as True, it creates a TLS-enabled client with HTTP/2 support using the provided certificate settings.
+    To get https.AsyncClient, provide make_async as True.
+
+    Args:
+        config_path (Optional[str]): Path to a YAML configuration file. If provided, the file should contain TLS
+            configuration with the following structure:
+            {
+                "use_tls": bool,
+                "tls_config": {
+                    "cert_path": str,  # Path to client certificate
+                    "key_path": str,   # Path to client private key
+                    "ca_bundle_path": str  # Optional: Path to CA bundle for verification
+                }
+            }
+        make_async (Optional[bool]): Whether to make an asynchronous httpx client instance.
+
+    Returns:
+        httpx.Client: An initialized httpx client instance, either with basic settings or TLS configuration
+                     if specified in the config file.
+
+    Example:
+        # Basic client without TLS
+        client = get_httpx_client()
+
+        # Client with TLS configuration from config file
+        client = get_httpx_client("/path/to/config.yaml")
+    """
+    if config_path:
+        with open(config_path, "r") as file:
+            config = yaml.safe_load(file)
+        if config.get("use_tls", False):
+            print(f"Configuring client with TLS config: {config}")
+            tls_config = config["tls_config"]
+            return get_sync_or_async_client(make_async, **tls_config)
+    return get_sync_or_async_client(make_async)
+
+
+def get_sync_or_async_client(
+    make_async: Optional[bool] = False,
+    cert_path: Optional[str] = None,
+    key_path: Optional[str] = None,
+    ca_bundle_path: Optional[str] = None,
+) -> AsyncClient | Client:
+    """
+    Creates and returns either a synchronous or asynchronous httpx client with optional TLS configuration.
+
+    Args:
+        make_async (Optional[bool]): If True, returns an AsyncClient; if False, returns a synchronous Client.
+            Defaults to False.
+        cert_path (Optional[str]): Path to the client certificate file. Required for TLS configuration
+            when key_path is also provided.
+        key_path (Optional[str]): Path to the client private key file. Required for TLS configuration
+            when cert_path is also provided.
+        ca_bundle_path (Optional[str]): Path to the CA bundle file for certificate verification.
+            If not provided, defaults to system CA certificates.
+    """
+    if make_async:
+        if cert_path and key_path:
+            return httpx.AsyncClient(
+                http2=True,
+                cert=(cert_path, key_path),
+                verify=ca_bundle_path if ca_bundle_path else True,
+            )
+        else:
+            return httpx.AsyncClient()
+    else:
+        if cert_path and key_path:
+            return httpx.Client(
+                http2=True,
+                cert=(cert_path, key_path),
+                verify=ca_bundle_path if ca_bundle_path else True,
+            )
+        else:
+            return httpx.Client()

indexify/executor/agent.py

@@ -1,13 +1,11 @@
 import asyncio
 import json
-import ssl
 import traceback
 from concurrent.futures.process import BrokenProcessPool
 from importlib.metadata import version
+from pathlib import Path
 from typing import Dict, List, Optional
 
-import httpx
-import yaml
 from httpx_sse import aconnect_sse
 from pydantic import BaseModel
 from rich.console import Console
@@ -15,6 +13,7 @@ from rich.panel import Panel
 from rich.text import Text
 from rich.theme import Theme
 
+from indexify.common_util import get_httpx_client
 from indexify.functions_sdk.data_objects import (
     FunctionWorkerOutput,
     IndexifyData,
@@ -58,7 +57,7 @@ class ExtractorAgent:
         self,
         executor_id: str,
         num_workers,
-        code_path: str,
+        code_path: Path,
         server_addr: str = "localhost:8900",
         config_path: Optional[str] = None,
         name_alias: Optional[str] = None,
@@ -66,7 +65,7 @@ class ExtractorAgent:
     ):
         self.name_alias = name_alias
         self.image_version = image_version
-
+        self._config_path = config_path
         self._probe = RuntimeProbes()
 
         runtime_probe: ProbeInfo = self._probe.probe()
@@ -82,39 +81,19 @@ class ExtractorAgent:
         )
 
         self.num_workers = num_workers
-        self._use_tls = False
         if config_path:
-            with open(config_path, "r") as f:
-                config = yaml.safe_load(f)
-                self._config = config
-            if config.get("use_tls", False):
-                console.print(
-                    "Running the extractor with TLS enabled", style="cyan bold"
-                )
-                self._use_tls = True
-                tls_config = config["tls_config"]
-                self._ssl_context = ssl.create_default_context(
-                    ssl.Purpose.SERVER_AUTH, cafile=tls_config["ca_bundle_path"]
-                )
-                self._ssl_context.load_cert_chain(
-                    certfile=tls_config["cert_path"], keyfile=tls_config["key_path"]
-                )
-                self._protocol = "wss"
-                self._tls_config = tls_config
-            else:
-                self._ssl_context = None
-                self._protocol = "ws"
+            console.print("Running the extractor with TLS enabled", style="cyan bold")
+            self._protocol = "https"
         else:
-            self._ssl_context = None
             self._protocol = "http"
-            self._config = {}
 
         self._task_store: TaskStore = TaskStore()
         self._executor_id = executor_id
         self._function_worker = FunctionWorker(
             workers=num_workers,
             indexify_client=IndexifyClient(
-                service_url=f"{self._protocol}://{server_addr}"
+                service_url=f"{self._protocol}://{server_addr}",
+                config_path=config_path,
             ),
         )
         self._has_registered = False
@@ -124,7 +103,9 @@ class ExtractorAgent:
         self._downloader = Downloader(code_path=code_path, base_url=self._base_url)
         self._max_queued_tasks = 10
         self._task_reporter = TaskReporter(
-            base_url=self._base_url, executor_id=self._executor_id
+            base_url=self._base_url,
+            executor_id=self._executor_id,
+            config_path=self._config_path,
         )
 
     async def task_completion_reporter(self):
@@ -197,7 +178,7 @@ class ExtractorAgent:
                 if self._require_image_bootstrap:
                     try:
                         image_info = await _get_image_info_for_compute_graph(
-                            task, self._protocol, self._server_addr
+                            task, self._protocol, self._server_addr, self._config_path
                         )
                         image_dependency_installer.executor_image_builder(
                             image_info, self.name_alias, self.image_version
@@ -365,8 +346,8 @@ class ExtractorAgent:
         asyncio.create_task(self.task_completion_reporter())
         self._should_run = True
         while self._should_run:
-            self._protocol = "http"
             url = f"{self._protocol}://{self._server_addr}/internal/executors/{self._executor_id}/tasks"
+            print(f"calling url: {url}")
 
             def to_sentence_case(snake_str):
                 words = snake_str.split("_")
@@ -407,9 +388,8 @@ class ExtractorAgent:
                     border_style="cyan",
                 )
             )
-
             try:
-                async with httpx.AsyncClient() as client:
+                async with get_httpx_client(self._config_path, True) as client:
                     async with aconnect_sse(
                         client,
                         "POST",
@@ -453,14 +433,16 @@ class ExtractorAgent:
 
 
 async def _get_image_info_for_compute_graph(
-    task: Task, protocol, server_addr
+    task: Task, protocol, server_addr, config_path: str
 ) -> ImageInformation:
     namespace = task.namespace
     graph_name: str = task.compute_graph
     compute_fn_name: str = task.compute_fn
 
     http_client = IndexifyClient(
-        service_url=f"{protocol}://{server_addr}", namespace=namespace
+        service_url=f"{protocol}://{server_addr}",
+        namespace=namespace,
+        config_path=config_path,
     )
     compute_graph: ComputeGraphMetadata = http_client.graph(graph_name)
 

indexify/executor/downloader.py

@@ -8,8 +8,9 @@ from rich.panel import Panel
 from rich.theme import Theme
 
 from indexify.functions_sdk.data_objects import IndexifyData
-from indexify.functions_sdk.object_serializer import MsgPackSerializer
 
+from ..common_util import get_httpx_client
+from ..functions_sdk.object_serializer import JsonSerializer, get_serializer
 from .api_objects import Task
 
 custom_theme = Theme(
@@ -29,9 +30,12 @@ class DownloadedInputs(BaseModel):
 
 
 class Downloader:
-    def __init__(self, code_path: str, base_url: str):
+    def __init__(
+        self, code_path: str, base_url: str, config_path: Optional[str] = None
+    ):
         self.code_path = code_path
         self.base_url = base_url
+        self._client = get_httpx_client(config_path)
 
     async def download_graph(self, namespace: str, name: str, version: int) -> str:
         path = os.path.join(self.code_path, namespace, f"{name}.{version}")
@@ -46,7 +50,7 @@ class Downloader:
             )
         )
 
-        response = httpx.get(
+        response = self._client.get(
             f"{self.base_url}/internal/namespaces/{namespace}/compute_graphs/{name}/code"
         )
         try:
@@ -66,7 +70,7 @@ class Downloader:
             f.write(response.content)
         return path
 
-    async def download_input(self, task: Task) -> IndexifyData:
+    async def download_input(self, task: Task) -> DownloadedInputs:
         input_id = task.input_key.split("|")[-1]
         if task.invocation_id == input_id:
             url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/payload"
@@ -85,7 +89,8 @@ class Downloader:
             )
         )
 
-        response = httpx.get(url)
+        response = self._client.get(url)
+
         try:
             response.raise_for_status()
         except httpx.HTTPStatusError as e:
@@ -98,12 +103,22 @@ class Downloader:
             )
             raise
 
+        encoder = (
+            "json"
+            if response.headers["content-type"] == JsonSerializer.content_type
+            else "cloudpickle"
+        )
+        serializer = get_serializer(encoder)
+
         if task.invocation_id == input_id:
             return DownloadedInputs(
-                input=IndexifyData(payload=response.content, id=input_id)
+                input=IndexifyData(
+                    payload=response.content, id=input_id, encoder=encoder
+                ),
             )
 
-        init_value = None
+        deserialized_content = serializer.deserialize(response.content)
+
         if reducer_url:
             init_value = httpx.get(reducer_url)
             try:
@@ -117,8 +132,22 @@ class Downloader:
                     )
                 )
                 raise
-            init_value = MsgPackSerializer.deserialize(init_value.content)
+            init_value = serializer.deserialize(init_value.content)
+            return DownloadedInputs(
+                input=IndexifyData(
+                    input_id=task.invocation_id,
+                    payload=deserialized_content,
+                    encoder=encoder,
+                ),
+                init_value=IndexifyData(
+                    input_id=task.invocation_id, payload=init_value, encoder=encoder
+                ),
+            )
 
         return DownloadedInputs(
-            input=MsgPackSerializer.deserialize(response.content), init_value=init_value
+            input=IndexifyData(
+                input_id=task.invocation_id,
+                payload=deserialized_content,
+                encoder=encoder,
+            )
         )

indexify/executor/function_worker.py

@@ -15,10 +15,10 @@ from indexify.functions_sdk.data_objects import (
 from indexify.functions_sdk.indexify_functions import (
     FunctionCallResult,
     GraphInvocationContext,
+    IndexifyFunction,
     IndexifyFunctionWrapper,
-    RouterCallResult,
     IndexifyRouter,
-    IndexifyFunction,
+    RouterCallResult,
 )
 
 function_wrapper_map: Dict[str, IndexifyFunctionWrapper] = {}

indexify/executor/task_reporter.py

@@ -1,15 +1,14 @@
 import io
-from typing import List, Optional
+from typing import Optional
 
-import httpx
 import nanoid
 from rich import print
 
+from indexify.common_util import get_httpx_client
 from indexify.executor.api_objects import RouterOutput as ApiRouterOutput
-from indexify.executor.api_objects import Task, TaskResult
+from indexify.executor.api_objects import TaskResult
 from indexify.executor.task_store import CompletedTask
-from indexify.functions_sdk.data_objects import IndexifyData, RouterOutput
-from indexify.functions_sdk.object_serializer import MsgPackSerializer
+from indexify.functions_sdk.object_serializer import get_serializer
 
 
 # https://github.com/psf/requests/issues/1081#issuecomment-428504128
@@ -19,12 +18,16 @@ class ForceMultipartDict(dict):
 
 
 FORCE_MULTIPART = ForceMultipartDict()
+UTF_8_CONTENT_TYPE = "application/octet-stream"
 
 
 class TaskReporter:
-    def __init__(self, base_url: str, executor_id: str):
+    def __init__(
+        self, base_url: str, executor_id: str, config_path: Optional[str] = None
+    ):
         self._base_url = base_url
         self._executor_id = executor_id
+        self._client = get_httpx_client(config_path)
 
     def report_task_outcome(self, completed_task: CompletedTask):
         fn_outputs = []
@@ -32,9 +35,13 @@ class TaskReporter:
             print(
                 f"[bold]task-reporter[/bold] uploading output of size: {len(output.payload)} bytes"
             )
-            output_bytes = MsgPackSerializer.serialize(output)
+            serializer = get_serializer(output.encoder)
+            serialized_output = serializer.serialize(output.payload)
             fn_outputs.append(
-                ("node_outputs", (nanoid.generate(), io.BytesIO(output_bytes)))
+                (
+                    "node_outputs",
+                    (nanoid.generate(), serialized_output, serializer.content_type),
+                )
             )
 
         if completed_task.stdout:
@@ -44,7 +51,11 @@ class TaskReporter:
             fn_outputs.append(
                 (
                     "stdout",
-                    (nanoid.generate(), io.BytesIO(completed_task.stdout.encode())),
+                    (
+                        nanoid.generate(),
+                        completed_task.stdout.encode(),
+                        UTF_8_CONTENT_TYPE,
+                    ),
                 )
             )
 
@@ -55,7 +66,11 @@ class TaskReporter:
             fn_outputs.append(
                 (
                     "stderr",
-                    (nanoid.generate(), io.BytesIO(completed_task.stderr.encode())),
+                    (
+                        nanoid.generate(),
+                        completed_task.stderr.encode(),
+                        UTF_8_CONTENT_TYPE,
+                    ),
                 )
             )
 
@@ -84,7 +99,7 @@ class TaskReporter:
         else:
             kwargs["files"] = FORCE_MULTIPART
         try:
-            response = httpx.post(
+            response = self._client.post(
                 url=f"{self._base_url}/internal/ingest_files",
                 **kwargs,
             )

indexify/functions_sdk/data_objects.py

@@ -16,7 +16,7 @@ class RouterOutput(BaseModel):
 
 class IndexifyData(BaseModel):
     id: Optional[str] = None
-    payload: bytes
+    payload: Union[bytes, str]
     encoder: Literal["cloudpickle", "json"] = "cloudpickle"
 
 

indexify/functions_sdk/graph.py

@@ -167,7 +167,7 @@ class Graph:
             reducer=is_reducer,
             image_name=start_node.image._image_name,
             image_information=start_node.image.to_image_information(),
-            payload_encoder=start_node.encoder,
+            encoder=start_node.encoder,
         )
         metadata_edges = self.edges.copy()
         metadata_nodes = {}
@@ -179,7 +179,7 @@ class Graph:
                         description=node.description or "",
                         source_fn=node_name,
                         target_fns=self.routers[node_name],
-                        payload_encoder=node.encoder,
+                        encoder=node.encoder,
                         image_name=node.image._image_name,
                         image_information=node.image.to_image_information(),
                     )

indexify/functions_sdk/graph_definition.py

@@ -14,7 +14,7 @@ class FunctionMetadata(BaseModel):
     reducer: bool = False
     image_name: str
     image_information: ImageInformation
-    payload_encoder: str = "cloudpickle"
+    encoder: str = "cloudpickle"
 
 
 class RouterMetadata(BaseModel):
@@ -24,7 +24,7 @@ class RouterMetadata(BaseModel):
     target_fns: List[str]
     image_name: str
     image_information: ImageInformation
-    payload_encoder: str = "cloudpickle"
+    encoder: str = "cloudpickle"
 
 
 class NodeMetadata(BaseModel):
@@ -49,3 +49,11 @@ class ComputeGraphMetadata(BaseModel):
 
     def get_input_payload_serializer(self):
         return get_serializer(self.start_node.compute_fn.encoder)
+
+    def get_input_encoder(self) -> str:
+        if self.start_node.compute_fn:
+            return self.start_node.compute_fn.encoder
+        elif self.start_node.dynamic_router:
+            return self.start_node.dynamic_router.encoder
+
+        raise ValueError("start node is not set on the graph")

indexify/functions_sdk/indexify_functions.py

@@ -1,8 +1,5 @@
 import inspect
-import re
-import sys
 import traceback
-from functools import update_wrapper
 from typing import (
     Any,
     Callable,
@@ -16,9 +13,7 @@ from typing import (
     get_origin,
 )
 
-import msgpack
-from pydantic import BaseModel, Field, PrivateAttr, model_validator
-from typing_extensions import get_type_hints
+from pydantic import BaseModel, Field, PrivateAttr
 
 from .data_objects import IndexifyData
 from .image import DEFAULT_IMAGE_3_10, Image

indexify/functions_sdk/object_serializer.py

@@ -2,23 +2,24 @@ from typing import Any, List
 
 import cloudpickle
 import jsonpickle
-import msgpack
-from pydantic import BaseModel
-
-from .data_objects import IndexifyData
 
 
 def get_serializer(serializer_type: str) -> Any:
     if serializer_type == "cloudpickle":
         return CloudPickleSerializer()
-    elif serializer_type == "msgpack":
-        return MsgPackSerializer()
     elif serializer_type == "json":
         return JsonSerializer()
+    elif serializer_type == JsonSerializer.content_type:
+        return JsonSerializer()
+    elif serializer_type == CloudPickleSerializer.content_type:
+        return CloudPickleSerializer()
     raise ValueError(f"Unknown serializer type: {serializer_type}")
 
 
 class JsonSerializer:
+    content_type = "application/json"
+    encoding_type = "json"
+
     @staticmethod
     def serialize(data: Any) -> str:
         return jsonpickle.encode(data)
@@ -37,6 +38,9 @@ class JsonSerializer:
 
 
 class CloudPickleSerializer:
+    content_type = "application/octet-stream"
+    encoding_type = "cloudpickle"
+
     @staticmethod
     def serialize(data: Any) -> bytes:
         return cloudpickle.dumps(data)
@@ -52,29 +56,3 @@ class CloudPickleSerializer:
     @staticmethod
     def deserialize_list(data: bytes) -> List[Any]:
         return cloudpickle.loads(data)
-
-
-class MsgPackSerializer:
-    @staticmethod
-    def serialize(data: Any) -> bytes:
-        if (
-            isinstance(data, type)
-            and issubclass(data, BaseModel)
-            or isinstance(data, BaseModel)
-        ):
-            return msgpack.packb(data.model_dump())
-        return msgpack.packb(data)
-
-    @staticmethod
-    def deserialize(data: bytes) -> IndexifyData:
-        cached_output = msgpack.unpackb(data)
-        return IndexifyData(**cached_output)
-
-    @staticmethod
-    def serialize_list(data: List[IndexifyData]) -> bytes:
-        data = [item.model_dump() for item in data]
-        return msgpack.packb(data)
-
-    @staticmethod
-    def deserialize_list(data: bytes) -> List[IndexifyData]:
-        return [IndexifyData(**item) for item in msgpack.unpackb(data)]

indexify/http_client.py

@@ -4,16 +4,16 @@ from typing import Any, Dict, List, Optional
 
 import cloudpickle
 import httpx
-import msgpack
-import yaml
 from httpx_sse import connect_sse
 from pydantic import BaseModel, Json
 from rich import print
 
+from indexify.common_util import get_httpx_client, get_sync_or_async_client
 from indexify.error import ApiException, GraphStillProcessing
 from indexify.functions_sdk.data_objects import IndexifyData
 from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
 from indexify.functions_sdk.indexify_functions import IndexifyFunction
+from indexify.functions_sdk.object_serializer import get_serializer
 from indexify.settings import DEFAULT_SERVICE_URL
 
 
@@ -55,18 +55,8 @@ class IndexifyClient:
             service_url = os.environ["INDEXIFY_URL"]
 
         self.service_url = service_url
-        self._client = httpx.Client()
-        if config_path:
-            with open(config_path, "r") as file:
-                config = yaml.safe_load(file)
-
-            if config.get("use_tls", False):
-                tls_config = config["tls_config"]
-                self._client = httpx.Client(
-                    http2=True,
-                    cert=(tls_config["cert_path"], tls_config["key_path"]),
-                    verify=tls_config.get("ca_bundle_path", True),
-                )
+        self._config_path = config_path
+        self._client = get_httpx_client(config_path)
 
         self.namespace: str = namespace
         self.compute_graphs: List[Graph] = []
@@ -135,17 +125,13 @@ class IndexifyClient:
         if not (cert_path and key_path):
             raise ValueError("Both cert and key must be provided for mTLS")
 
-        client_certs = (cert_path, key_path)
-        verify_option = ca_bundle_path if ca_bundle_path else True
-        client = IndexifyClient(
-            *args,
-            **kwargs,
-            service_url=service_url,
-            http2=True,
-            cert=client_certs,
-            verify=verify_option,
+        client = get_sync_or_async_client(
+            cert_path=cert_path, key_path=key_path, ca_bundle_path=ca_bundle_path
         )
-        return client
+
+        indexify_client = IndexifyClient(service_url, *args, **kwargs)
+        indexify_client._client = client
+        return indexify_client
 
     def _add_api_key(self, kwargs):
         if self._api_key:
@@ -189,6 +175,20 @@ class IndexifyClient:
         for fn_name, fn in graph.nodes.items():
             self._fns[f"{graph.name}/{fn_name}"] = fn
 
+    def delete_compute_graph(
+        self,
+        graph_name: str,
+    ) -> None:
+        """
+        Deletes a graph and all of its invocations from the namespace.
+        :param graph_name The name of the graph to delete.
+        WARNING: This operation is irreversible.
+        """
+        response = self._delete(
+            f"namespaces/{self.namespace}/compute_graphs/{graph_name}",
+        )
+        response.raise_for_status()
+
     def graphs(self) -> List[str]:
         response = self._get(f"graphs")
         return response.json()["graphs"]
@@ -265,21 +265,26 @@ class IndexifyClient:
             print(f"failed to fetch logs: {e}")
             return None
 
-    def rerun_graph(self, graph: str):
-        self._post(f"namespaces/{self.namespace}/compute_graphs/{graph}/rerun")
+    def replay_invocations(self, graph: str):
+        self._post(f"namespaces/{self.namespace}/compute_graphs/{graph}/replay")
 
     def invoke_graph_with_object(
-        self, graph: str, block_until_done: bool = False, **kwargs
+        self,
+        graph: str,
+        block_until_done: bool = False,
+        serializer: str = "cloudpickle",
+        **kwargs,
     ) -> str:
-        ser_input = cloudpickle.dumps(kwargs)
+        serializer = get_serializer(serializer)
+        ser_input = serializer.serialize(kwargs)
         params = {"block_until_finish": block_until_done}
         kwargs = {
-            "headers": {"Content-Type": "application/cbor"},
+            "headers": {"Content-Type": serializer.content_type},
             "data": ser_input,
             "params": params,
         }
         self._add_api_key(kwargs)
-        with httpx.Client() as client:
+        with get_httpx_client(self._config_path) as client:
             with connect_sse(
                 client,
                 "POST",
@@ -343,8 +348,12 @@ class IndexifyClient:
             f"namespaces/{namespace}/compute_graphs/{graph}/invocations/{invocation_id}/fn/{fn_name}/output/{output_id}",
         )
         response.raise_for_status()
-        data_dict = msgpack.unpackb(response.content)
-        return IndexifyData.model_validate(data_dict)
+        content_type = response.headers.get("Content-Type")
+        serializer = get_serializer(content_type)
+        decoded_response = serializer.deserialize(response.content)
+        return IndexifyData(
+            id=output_id, payload=decoded_response, encoder=serializer.encoding_type
+        )
 
     def graph_outputs(
         self,

indexify/remote_graph.py

@@ -1,6 +1,6 @@
 from typing import Any, List, Optional
 
-from indexify.functions_sdk.graph import Graph
+from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
 
 from .http_client import IndexifyClient
 from .settings import DEFAULT_SERVICE_URL
@@ -30,6 +30,8 @@ class RemoteGraph:
         else:
             self._client = IndexifyClient(service_url=server_url)
 
+        self._graph_definition: ComputeGraphMetadata = self._client.graph(self._name)
+
     def run(self, block_until_done: bool = False, **kwargs) -> str:
         """
         Run the graph with the given inputs. The input is for the start function of the graph.
@@ -39,7 +41,6 @@ class RemoteGraph:
         :return: The invocation ID of the graph execution.
 
         Example:
-
             @indexify_function()
             def foo(x: int) -> int:
                 return x + 1
@@ -48,16 +49,20 @@ class RemoteGraph:
             invocation_id = remote_graph.run(x=1)
         """
         return self._client.invoke_graph_with_object(
-            self._name, block_until_done, **kwargs
+            self._name,
+            block_until_done,
+            self._graph_definition.get_input_encoder(),
+            **kwargs
         )
 
-    def rerun(self):
+    def replay_invocations(self):
         """
-        Rerun the graph with the given invocation ID.
+        Replay all the graph previous runs/invocations on the latest version of the graph.
 
-        :param invocation_id: The invocation ID of the graph execution.
+        This is useful to make all the previous invocations go through
+        an updated graph to take advantage of graph improvements.
         """
-        self._client.rerun_graph(self._name)
+        self._client.replay_invocations(self._name)
 
     @classmethod
     def deploy(

indexify/remote_pipeline.py

@@ -19,6 +19,7 @@ class RemotePipeline(RemoteGraph):
         :param g: The local Graph object.
         :param server_url: The URL of the server where the graph will be registered.
         """
+        cls.graph = p._graph
         client = IndexifyClient(service_url=server_url)
         client.register_compute_graph(p._graph, additional_modules)
         return cls(name=p._graph.name, server_url=server_url)

{indexify-0.2.27.dist-info → indexify-0.2.29.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.27
+Version: 0.2.29
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -18,13 +18,12 @@ Requires-Dist: cloudpickle (>=3.1.0,<4.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
 Requires-Dist: httpx[http2] (>=0,<1)
-Requires-Dist: jsonpickle (>=3.3.0,<4.0.0)
-Requires-Dist: msgpack (>=1.1.0,<2.0.0)
+Requires-Dist: jsonpickle (>=4.0.0,<5.0.0)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: pydantic (>=2.9.2,<3.0.0)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
-Requires-Dist: typer (>=0.12.5,<0.13.0)
+Requires-Dist: typer (>=0.13.0,<0.14.0)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.2.27.dist-info → indexify-0.2.29.dist-info}/RECORD

@@ -1,34 +1,35 @@
 indexify/__init__.py,sha256=P0mvM8sbkeS2CjYzRYyzb42CnXGhyJXdz4FdmTBMSWM,697
 indexify/cli.py,sha256=TBycogrXcTK7q8IIcJxcpPPoZYve_rp6FYipWbkiTUI,8528
+indexify/common_util.py,sha256=LKS6yZ3yv8nF2J-KzisGIjqjTvCn7tLFifQJLT4gHRg,3529
 indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
 indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
 indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
 indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
-indexify/executor/agent.py,sha256=5TPCbeqjGg_3_EsBY8o3HmtZl8GSNEKYDiUA4SKJg0Y,18692
+indexify/executor/agent.py,sha256=mqEDcRdDfoUjw5XoFfVlOlJ-C_ODzyCQQMW90vDbWk4,18114
 indexify/executor/api_objects.py,sha256=mvmwGbK4paJNQGFvbtNHMPpiH_LpVhrlRnCcrqS6HOQ,859
-indexify/executor/downloader.py,sha256=3mEDdluTzspsLGAZtFHZOVuyKOzT3CSema2kIK6Z1yU,4005
+indexify/executor/downloader.py,sha256=_SQ-6-0Kj3Zg0Dqp_pMwTYxvBP3xvNUOREyDucCJp8M,4944
 indexify/executor/executor_tasks.py,sha256=A0UIEZ5VaB6zSkFQG81UmTW0E57MTYhGlaXuAbRV8lQ,1884
-indexify/executor/function_worker.py,sha256=mFvDwbPiX6QHTkZALqBOAYix4kHSNSAF8KDImB8OTWM,6462
+indexify/executor/function_worker.py,sha256=wRW2-X9dNI80KhwTD1vD-pcyetsVKVs6vVdg7L7JjcQ,6462
 indexify/executor/image_dependency_installer.py,sha256=ct8GmzgkaPi6NAblk68IJJWo5MecIUubELotmSrgoRQ,1759
 indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
 indexify/executor/runtime_probes.py,sha256=mjw2_mGQ622wRT_39WPGGgPEZQTgtrf3-ICcUUZOeyg,2126
-indexify/executor/task_reporter.py,sha256=7M2fDzLDkSNOSr42EBtkIbGvBzeZlTCqxQuDfdoFT4Y,3349
+indexify/executor/task_reporter.py,sha256=Wr8caDtjHBHvK5C-wwghJ5TNxIK_moxjQokGRcEs2rw,3798
 indexify/executor/task_store.py,sha256=u48FdRKAh_KH7WOMQOArdOY5CawlyW5MJx8V0W79JM0,3951
-indexify/functions_sdk/data_objects.py,sha256=G08Nrfe2nMUXEhMSedIZwNcZLEsEnj0NykmLPaMZmFs,842
-indexify/functions_sdk/graph.py,sha256=a28o07g0MJoBq2rQWfMGK_dtiSyryoADeF7tEuhI8gc,11817
-indexify/functions_sdk/graph_definition.py,sha256=s8NbqZRaVw1mhGvaaUZ2cJ0NitdlbLj1tnkKjsc71U4,1309
+indexify/functions_sdk/data_objects.py,sha256=wXbUa9hjU6rsXmmk19vQ5Kixf3FsI59VBWPNmHasAX0,854
+indexify/functions_sdk/graph.py,sha256=TbHtIcAzRcEn3BWewVhNsUGMNfRduI1aSAvK6Vyx-fk,11801
+indexify/functions_sdk/graph_definition.py,sha256=5jUzjfk4gxxQCmQeOlNS9VndZK7KQBsZzABcDpQKgrs,1594
 indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
 indexify/functions_sdk/image.py,sha256=QK0H6KxLWriB_z4M0kunKzzHdHxYLWL670RPYgYuf_8,1762
-indexify/functions_sdk/indexify_functions.py,sha256=UspEi8wYWf4lZ0f2Ln19RbQigocSPr23lgMvD9E-PS0,10475
+indexify/functions_sdk/indexify_functions.py,sha256=EbNC5gZktRc-G9ribQrpMG2X_G-8cujsxTgot734iGQ,10340
 indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
-indexify/functions_sdk/object_serializer.py,sha256=Nw7PqhSpkw1nCw5jo0XDBumzO2crLMIohSuIMKLaEOk,2161
+indexify/functions_sdk/object_serializer.py,sha256=pOgUOWbRNRix9uZT0aQn0LTCnJCeMNGO1nAE0jAybmg,1546
 indexify/functions_sdk/pipeline.py,sha256=KmxZE8eBFAQ4bbEcYURXXR26HSyoAT3O6iu9H38-OXE,974
-indexify/http_client.py,sha256=mGSSZSMJOShoZGDq42f0kiCQ_aIg2ZjWKeaOW35_r8g,15400
-indexify/remote_graph.py,sha256=_04bdraDaeEmkvEcsxIZ8M37BCAjfbfQccZpAgnZJ2c,4435
-indexify/remote_pipeline.py,sha256=FW7IAv3r24OOpiqlprw3kuFrpdkqi6Ic4_tT26FThjA,761
+indexify/http_client.py,sha256=rFYjv0rRMo4sXp-TjWoflQeJaxNZK-Di6Gk3G78xKQI,15838
+indexify/remote_graph.py,sha256=1JogFFcWCKc_Dq4Rh1s6IwAnxB-qrFnKuDQIbpJso1Y,4745
+indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
 indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
-indexify-0.2.27.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.27.dist-info/METADATA,sha256=Ok21hFzjW-T6uEn4fNrZC1ioUeDvmbQdUvlZcYNg7KU,6242
-indexify-0.2.27.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-indexify-0.2.27.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.27.dist-info/RECORD,,
+indexify-0.2.29.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.29.dist-info/METADATA,sha256=3lC5Cqd9Uym3Bk0AvQVTT8JTlHsodWXyktGjEAU9Y7s,6202
+indexify-0.2.29.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+indexify-0.2.29.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.29.dist-info/RECORD,,