indexify 0.2.29__tar.gz → 0.2.30__tar.gz

{indexify-0.2.29 → indexify-0.2.30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.29
+Version: 0.2.30
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.2.29 → indexify-0.2.30}/indexify/executor/agent.py

@@ -100,7 +100,9 @@ class ExtractorAgent:
         self._server_addr = server_addr
         self._base_url = f"{self._protocol}://{self._server_addr}"
         self._code_path = code_path
-        self._downloader = Downloader(code_path=code_path, base_url=self._base_url)
+        self._downloader = Downloader(
+            code_path=code_path, base_url=self._base_url, config_path=self._config_path
+        )
         self._max_queued_tasks = 10
         self._task_reporter = TaskReporter(
             base_url=self._base_url,
@@ -112,9 +114,15 @@ class ExtractorAgent:
         console.print(Text("Starting task completion reporter", style="bold cyan"))
         # We should copy only the keys and not the values
         url = f"{self._protocol}://{self._server_addr}/write_content"
+
         while True:
             outcomes = await self._task_store.task_outcomes()
             for task_outcome in outcomes:
+                retryStr = (
+                    f"\nRetries: {task_outcome.reporting_retries}"
+                    if task_outcome.reporting_retries > 0
+                    else ""
+                )
                 outcome = task_outcome.task_outcome
                 style_outcome = (
                     f"[bold red] {outcome} [/]"
@@ -125,7 +133,9 @@ class ExtractorAgent:
                     Panel(
                         f"Reporting outcome of task: {task_outcome.task.id}, function: {task_outcome.task.compute_fn}\n"
                         f"Outcome: {style_outcome}\n"
-                        f"Num Fn Outputs: {len(task_outcome.outputs or [])}  Router Output: {task_outcome.router_output}",
+                        f"Num Fn Outputs: {len(task_outcome.outputs or [])}\n"
+                        f"Router Output: {task_outcome.router_output}\n"
+                        f"Retries: {task_outcome.reporting_retries}",
                         title="Task Completion",
                         border_style="info",
                     )
@@ -139,11 +149,14 @@ class ExtractorAgent:
                     console.print(
                         Panel(
                             f"Failed to report task {task_outcome.task.id}\n"
-                            f"Exception: {e}\nRetrying...",
+                            f"Exception: {type(e).__name__}({e})\n"
+                            f"Retries: {task_outcome.reporting_retries}\n"
+                            "Retrying...",
                             title="Reporting Error",
                             border_style="error",
                         )
                     )
+                    task_outcome.reporting_retries += 1
                     await asyncio.sleep(5)
                     continue
 

{indexify-0.2.29 → indexify-0.2.30}/indexify/executor/task_reporter.py

@@ -2,6 +2,8 @@ import io
 from typing import Optional
 
 import nanoid
+from httpx import Timeout
+from pydantic import BaseModel
 from rich import print
 
 from indexify.common_util import get_httpx_client
@@ -21,6 +23,15 @@ FORCE_MULTIPART = ForceMultipartDict()
 UTF_8_CONTENT_TYPE = "application/octet-stream"
 
 
+class ReportingData(BaseModel):
+    output_count: int = 0
+    output_total_bytes: int = 0
+    stdout_count: int = 0
+    stdout_total_bytes: int = 0
+    stderr_count: int = 0
+    stderr_total_bytes: int = 0
+
+
 class TaskReporter:
     def __init__(
         self, base_url: str, executor_id: str, config_path: Optional[str] = None
@@ -30,11 +41,10 @@ class TaskReporter:
         self._client = get_httpx_client(config_path)
 
     def report_task_outcome(self, completed_task: CompletedTask):
+
+        report = ReportingData()
         fn_outputs = []
         for output in completed_task.outputs or []:
-            print(
-                f"[bold]task-reporter[/bold] uploading output of size: {len(output.payload)} bytes"
-            )
             serializer = get_serializer(output.encoder)
             serialized_output = serializer.serialize(output.payload)
             fn_outputs.append(
@@ -43,11 +53,10 @@ class TaskReporter:
                     (nanoid.generate(), serialized_output, serializer.content_type),
                 )
             )
+            report.output_count += 1
+            report.output_total_bytes += len(serialized_output)
 
         if completed_task.stdout:
-            print(
-                f"[bold]task-reporter[/bold] uploading stdout of size: {len(completed_task.stdout)}"
-            )
             fn_outputs.append(
                 (
                     "stdout",
@@ -58,11 +67,10 @@ class TaskReporter:
                     ),
                 )
             )
+            report.stdout_count += 1
+            report.stdout_total_bytes += len(completed_task.stdout)
 
         if completed_task.stderr:
-            print(
-                f"[bold]task-reporter[/bold] uploading stderr of size: {len(completed_task.stderr)}"
-            )
             fn_outputs.append(
                 (
                     "stderr",
@@ -73,6 +81,8 @@ class TaskReporter:
                     ),
                 )
             )
+            report.stderr_count += 1
+            report.stderr_total_bytes += len(completed_task.stderr)
 
         router_output = (
             ApiRouterOutput(edges=completed_task.router_output.edges)
@@ -93,7 +103,30 @@ class TaskReporter:
         )
         task_result_data = task_result.model_dump_json(exclude_none=True)
 
-        kwargs = {"data": {"task_result": task_result_data}}
+        total_bytes = (
+            report.output_total_bytes
+            + report.stdout_total_bytes
+            + report.stderr_total_bytes
+        )
+
+        print(
+            f"[bold]task-reporter[/bold] reporting task outcome "
+            f"task_id={completed_task.task.id} retries={completed_task.reporting_retries} "
+            f"total_bytes={total_bytes} total_files={report.output_count + report.stdout_count + report.stderr_count} "
+            f"output_files={report.output_count} output_bytes={total_bytes} "
+            f"stdout_bytes={report.stdout_total_bytes} stderr_bytes={report.stderr_total_bytes} "
+        )
+
+        #
+        kwargs = {
+            "data": {"task_result": task_result_data},
+            # Use httpx default timeout of 5s for all timeout types.
+            # For read timeouts, use 5 minutes to allow for large file uploads.
+            "timeout": Timeout(
+                5.0,
+                read=5.0 * 60,
+            ),
+        }
         if fn_outputs and len(fn_outputs) > 0:
             kwargs["files"] = fn_outputs
         else:
@@ -104,11 +137,15 @@ class TaskReporter:
                 **kwargs,
             )
         except Exception as e:
-            print(f"failed to report task outcome {e}")
+            print(
+                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {type(e).__name__}({e})"
+            )
             raise e
 
         try:
             response.raise_for_status()
         except Exception as e:
-            print(f"failed to report task outcome {response.text}")
+            print(
+                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {response.text}"
+            )
             raise e

{indexify-0.2.29 → indexify-0.2.30}/indexify/executor/task_store.py

@@ -17,6 +17,7 @@ class CompletedTask(BaseModel):
     stdout: Optional[str] = None
     stderr: Optional[str] = None
     reducer: bool = False
+    reporting_retries: int = 0
 
 
 class TaskStore:

{indexify-0.2.29 → indexify-0.2.30}/indexify/functions_sdk/graph_definition.py

@@ -46,6 +46,7 @@ class ComputeGraphMetadata(BaseModel):
     edges: Dict[str, List[str]]
     accumulator_zero_values: Dict[str, bytes] = {}
     runtime_information: RuntimeInformation
+    replaying: bool = False
 
     def get_input_payload_serializer(self):
         return get_serializer(self.start_node.compute_fn.encoder)

{indexify-0.2.29 → indexify-0.2.30}/indexify/http_client.py

@@ -135,7 +135,9 @@ class IndexifyClient:
 
     def _add_api_key(self, kwargs):
         if self._api_key:
-            kwargs["headers"] = {"Authorization": f"Bearer {self._api_key}"}
+            if "headers" not in kwargs:
+                kwargs["headers"] = {}
+            kwargs["headers"]["Authorization"] = f"Bearer {self._api_key}"
 
     def _get(self, endpoint: str, **kwargs) -> httpx.Response:
         self._add_api_key(kwargs)

{indexify-0.2.29 → indexify-0.2.30}/indexify/remote_graph.py

@@ -1,6 +1,7 @@
 from typing import Any, List, Optional
 
 from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
+from indexify.functions_sdk.graph_definition import ComputeGraphMetadata
 
 from .http_client import IndexifyClient
 from .settings import DEFAULT_SERVICE_URL
@@ -55,6 +56,12 @@ class RemoteGraph:
             **kwargs
         )
 
+    def metadata(self) -> ComputeGraphMetadata:
+        """
+        Get the metadata of the graph.
+        """
+        return self._client.graph(self._name)
+
     def replay_invocations(self):
         """
         Replay all the graph previous runs/invocations on the latest version of the graph.

{indexify-0.2.29 → indexify-0.2.30}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "indexify"
-version = "0.2.29"
+version = "0.2.30"
 description = "Python Client for Indexify"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"