indexify 0.2.28__tar.gz → 0.2.30__tar.gz

{indexify-0.2.28 → indexify-0.2.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.28
+Version: 0.2.30
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.2.28 → indexify-0.2.30}/indexify/executor/agent.py

@@ -100,7 +100,9 @@ class ExtractorAgent:
         self._server_addr = server_addr
         self._base_url = f"{self._protocol}://{self._server_addr}"
         self._code_path = code_path
-        self._downloader = Downloader(code_path=code_path, base_url=self._base_url)
+        self._downloader = Downloader(
+            code_path=code_path, base_url=self._base_url, config_path=self._config_path
+        )
         self._max_queued_tasks = 10
         self._task_reporter = TaskReporter(
             base_url=self._base_url,
@@ -112,9 +114,15 @@ class ExtractorAgent:
         console.print(Text("Starting task completion reporter", style="bold cyan"))
         # We should copy only the keys and not the values
         url = f"{self._protocol}://{self._server_addr}/write_content"
+
         while True:
             outcomes = await self._task_store.task_outcomes()
             for task_outcome in outcomes:
+                retryStr = (
+                    f"\nRetries: {task_outcome.reporting_retries}"
+                    if task_outcome.reporting_retries > 0
+                    else ""
+                )
                 outcome = task_outcome.task_outcome
                 style_outcome = (
                     f"[bold red] {outcome} [/]"
@@ -125,7 +133,9 @@ class ExtractorAgent:
                     Panel(
                         f"Reporting outcome of task: {task_outcome.task.id}, function: {task_outcome.task.compute_fn}\n"
                         f"Outcome: {style_outcome}\n"
-                        f"Num Fn Outputs: {len(task_outcome.outputs or [])}  Router Output: {task_outcome.router_output}",
+                        f"Num Fn Outputs: {len(task_outcome.outputs or [])}\n"
+                        f"Router Output: {task_outcome.router_output}\n"
+                        f"Retries: {task_outcome.reporting_retries}",
                         title="Task Completion",
                         border_style="info",
                     )
@@ -139,11 +149,14 @@ class ExtractorAgent:
                     console.print(
                         Panel(
                             f"Failed to report task {task_outcome.task.id}\n"
-                            f"Exception: {e}\nRetrying...",
+                            f"Exception: {type(e).__name__}({e})\n"
+                            f"Retries: {task_outcome.reporting_retries}\n"
+                            "Retrying...",
                             title="Reporting Error",
                             border_style="error",
                         )
                     )
+                    task_outcome.reporting_retries += 1
                     await asyncio.sleep(5)
                     continue
 

{indexify-0.2.28 → indexify-0.2.30}/indexify/executor/task_reporter.py

@@ -2,6 +2,8 @@ import io
 from typing import Optional
 
 import nanoid
+from httpx import Timeout
+from pydantic import BaseModel
 from rich import print
 
 from indexify.common_util import get_httpx_client
@@ -21,6 +23,15 @@ FORCE_MULTIPART = ForceMultipartDict()
 UTF_8_CONTENT_TYPE = "application/octet-stream"
 
 
+class ReportingData(BaseModel):
+    output_count: int = 0
+    output_total_bytes: int = 0
+    stdout_count: int = 0
+    stdout_total_bytes: int = 0
+    stderr_count: int = 0
+    stderr_total_bytes: int = 0
+
+
 class TaskReporter:
     def __init__(
         self, base_url: str, executor_id: str, config_path: Optional[str] = None
@@ -30,11 +41,10 @@ class TaskReporter:
         self._client = get_httpx_client(config_path)
 
     def report_task_outcome(self, completed_task: CompletedTask):
+
+        report = ReportingData()
         fn_outputs = []
         for output in completed_task.outputs or []:
-            print(
-                f"[bold]task-reporter[/bold] uploading output of size: {len(output.payload)} bytes"
-            )
             serializer = get_serializer(output.encoder)
             serialized_output = serializer.serialize(output.payload)
             fn_outputs.append(
@@ -43,11 +53,10 @@ class TaskReporter:
                     (nanoid.generate(), serialized_output, serializer.content_type),
                 )
             )
+            report.output_count += 1
+            report.output_total_bytes += len(serialized_output)
 
         if completed_task.stdout:
-            print(
-                f"[bold]task-reporter[/bold] uploading stdout of size: {len(completed_task.stdout)}"
-            )
             fn_outputs.append(
                 (
                     "stdout",
@@ -58,11 +67,10 @@ class TaskReporter:
                     ),
                 )
             )
+            report.stdout_count += 1
+            report.stdout_total_bytes += len(completed_task.stdout)
 
         if completed_task.stderr:
-            print(
-                f"[bold]task-reporter[/bold] uploading stderr of size: {len(completed_task.stderr)}"
-            )
             fn_outputs.append(
                 (
                     "stderr",
@@ -73,6 +81,8 @@ class TaskReporter:
                     ),
                 )
             )
+            report.stderr_count += 1
+            report.stderr_total_bytes += len(completed_task.stderr)
 
         router_output = (
             ApiRouterOutput(edges=completed_task.router_output.edges)
@@ -93,7 +103,30 @@ class TaskReporter:
         )
         task_result_data = task_result.model_dump_json(exclude_none=True)
 
-        kwargs = {"data": {"task_result": task_result_data}}
+        total_bytes = (
+            report.output_total_bytes
+            + report.stdout_total_bytes
+            + report.stderr_total_bytes
+        )
+
+        print(
+            f"[bold]task-reporter[/bold] reporting task outcome "
+            f"task_id={completed_task.task.id} retries={completed_task.reporting_retries} "
+            f"total_bytes={total_bytes} total_files={report.output_count + report.stdout_count + report.stderr_count} "
+            f"output_files={report.output_count} output_bytes={total_bytes} "
+            f"stdout_bytes={report.stdout_total_bytes} stderr_bytes={report.stderr_total_bytes} "
+        )
+
+        #
+        kwargs = {
+            "data": {"task_result": task_result_data},
+            # Use httpx default timeout of 5s for all timeout types.
+            # For read timeouts, use 5 minutes to allow for large file uploads.
+            "timeout": Timeout(
+                5.0,
+                read=5.0 * 60,
+            ),
+        }
         if fn_outputs and len(fn_outputs) > 0:
             kwargs["files"] = fn_outputs
         else:
@@ -104,11 +137,15 @@ class TaskReporter:
                 **kwargs,
             )
         except Exception as e:
-            print(f"failed to report task outcome {e}")
+            print(
+                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {type(e).__name__}({e})"
+            )
             raise e
 
         try:
             response.raise_for_status()
         except Exception as e:
-            print(f"failed to report task outcome {response.text}")
+            print(
+                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {response.text}"
+            )
             raise e

{indexify-0.2.28 → indexify-0.2.30}/indexify/executor/task_store.py

@@ -17,6 +17,7 @@ class CompletedTask(BaseModel):
     stdout: Optional[str] = None
     stderr: Optional[str] = None
     reducer: bool = False
+    reporting_retries: int = 0
 
 
 class TaskStore:

{indexify-0.2.28 → indexify-0.2.30}/indexify/functions_sdk/graph_definition.py

@@ -46,6 +46,15 @@ class ComputeGraphMetadata(BaseModel):
     edges: Dict[str, List[str]]
     accumulator_zero_values: Dict[str, bytes] = {}
     runtime_information: RuntimeInformation
+    replaying: bool = False
 
     def get_input_payload_serializer(self):
         return get_serializer(self.start_node.compute_fn.encoder)
+
+    def get_input_encoder(self) -> str:
+        if self.start_node.compute_fn:
+            return self.start_node.compute_fn.encoder
+        elif self.start_node.dynamic_router:
+            return self.start_node.dynamic_router.encoder
+
+        raise ValueError("start node is not set on the graph")

{indexify-0.2.28 → indexify-0.2.30}/indexify/http_client.py

@@ -1,6 +1,6 @@
 import json
 import os
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 import cloudpickle
 import httpx
@@ -13,11 +13,7 @@ from indexify.error import ApiException, GraphStillProcessing
 from indexify.functions_sdk.data_objects import IndexifyData
 from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
 from indexify.functions_sdk.indexify_functions import IndexifyFunction
-from indexify.functions_sdk.object_serializer import (
-    CloudPickleSerializer,
-    JsonSerializer,
-    get_serializer,
-)
+from indexify.functions_sdk.object_serializer import get_serializer
 from indexify.settings import DEFAULT_SERVICE_URL
 
 
@@ -139,7 +135,9 @@ class IndexifyClient:
 
     def _add_api_key(self, kwargs):
         if self._api_key:
-            kwargs["headers"] = {"Authorization": f"Bearer {self._api_key}"}
+            if "headers" not in kwargs:
+                kwargs["headers"] = {}
+            kwargs["headers"]["Authorization"] = f"Bearer {self._api_key}"
 
     def _get(self, endpoint: str, **kwargs) -> httpx.Response:
         self._add_api_key(kwargs)
@@ -276,11 +274,10 @@ class IndexifyClient:
         self,
         graph: str,
         block_until_done: bool = False,
-        serializer: Union[
-            CloudPickleSerializer, JsonSerializer
-        ] = CloudPickleSerializer,
+        serializer: str = "cloudpickle",
         **kwargs,
     ) -> str:
+        serializer = get_serializer(serializer)
         ser_input = serializer.serialize(kwargs)
         params = {"block_until_finish": block_until_done}
         kwargs = {

{indexify-0.2.28 → indexify-0.2.30}/indexify/remote_graph.py

@@ -1,6 +1,7 @@
 from typing import Any, List, Optional
 
-from indexify.functions_sdk.graph import Graph
+from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
+from indexify.functions_sdk.graph_definition import ComputeGraphMetadata
 
 from .http_client import IndexifyClient
 from .settings import DEFAULT_SERVICE_URL
@@ -30,6 +31,8 @@ class RemoteGraph:
         else:
             self._client = IndexifyClient(service_url=server_url)
 
+        self._graph_definition: ComputeGraphMetadata = self._client.graph(self._name)
+
     def run(self, block_until_done: bool = False, **kwargs) -> str:
         """
         Run the graph with the given inputs. The input is for the start function of the graph.
@@ -49,10 +52,16 @@ class RemoteGraph:
         return self._client.invoke_graph_with_object(
             self._name,
             block_until_done,
-            self.graph.definition().get_input_payload_serializer(),
+            self._graph_definition.get_input_encoder(),
             **kwargs
         )
 
+    def metadata(self) -> ComputeGraphMetadata:
+        """
+        Get the metadata of the graph.
+        """
+        return self._client.graph(self._name)
+
     def replay_invocations(self):
         """
         Replay all the graph previous runs/invocations on the latest version of the graph.
@@ -81,7 +90,6 @@ class RemoteGraph:
         :param client: The IndexifyClient used to communicate with the server.
             Prefered over server_url.
         """
-        cls.graph = g
         if not client:
             client = IndexifyClient(service_url=server_url)
         client.register_compute_graph(g, additional_modules)

{indexify-0.2.28 → indexify-0.2.30}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "indexify"
-version = "0.2.28"
+version = "0.2.30"
 description = "Python Client for Indexify"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"