indexify 0.2.29__tar.gz → 0.2.31__tar.gz

{indexify-0.2.29 → indexify-0.2.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.29
+Version: 0.2.31
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.2.29 → indexify-0.2.31}/indexify/executor/agent.py

@@ -100,7 +100,9 @@ class ExtractorAgent:
         self._server_addr = server_addr
         self._base_url = f"{self._protocol}://{self._server_addr}"
         self._code_path = code_path
-        self._downloader = Downloader(code_path=code_path, base_url=self._base_url)
+        self._downloader = Downloader(
+            code_path=code_path, base_url=self._base_url, config_path=self._config_path
+        )
         self._max_queued_tasks = 10
         self._task_reporter = TaskReporter(
             base_url=self._base_url,
@@ -112,9 +114,15 @@ class ExtractorAgent:
         console.print(Text("Starting task completion reporter", style="bold cyan"))
         # We should copy only the keys and not the values
         url = f"{self._protocol}://{self._server_addr}/write_content"
+
         while True:
             outcomes = await self._task_store.task_outcomes()
             for task_outcome in outcomes:
+                retryStr = (
+                    f"\nRetries: {task_outcome.reporting_retries}"
+                    if task_outcome.reporting_retries > 0
+                    else ""
+                )
                 outcome = task_outcome.task_outcome
                 style_outcome = (
                     f"[bold red] {outcome} [/]"
@@ -125,7 +133,9 @@ class ExtractorAgent:
                     Panel(
                         f"Reporting outcome of task: {task_outcome.task.id}, function: {task_outcome.task.compute_fn}\n"
                         f"Outcome: {style_outcome}\n"
-                        f"Num Fn Outputs: {len(task_outcome.outputs or [])}  Router Output: {task_outcome.router_output}",
+                        f"Num Fn Outputs: {len(task_outcome.outputs or [])}\n"
+                        f"Router Output: {task_outcome.router_output}\n"
+                        f"Retries: {task_outcome.reporting_retries}",
                         title="Task Completion",
                         border_style="info",
                     )
@@ -139,11 +149,14 @@ class ExtractorAgent:
                     console.print(
                         Panel(
                             f"Failed to report task {task_outcome.task.id}\n"
-                            f"Exception: {e}\nRetrying...",
+                            f"Exception: {type(e).__name__}({e})\n"
+                            f"Retries: {task_outcome.reporting_retries}\n"
+                            "Retrying...",
                             title="Reporting Error",
                             border_style="error",
                         )
                     )
+                    task_outcome.reporting_retries += 1
                     await asyncio.sleep(5)
                     continue
 

{indexify-0.2.29 → indexify-0.2.31}/indexify/executor/downloader.py

@@ -120,7 +120,7 @@ class Downloader:
         deserialized_content = serializer.deserialize(response.content)
 
         if reducer_url:
-            init_value = httpx.get(reducer_url)
+            init_value = self._client.get(reducer_url)
             try:
                 init_value.raise_for_status()
             except httpx.HTTPStatusError as e:

{indexify-0.2.29 → indexify-0.2.31}/indexify/executor/task_reporter.py

@@ -2,6 +2,8 @@ import io
 from typing import Optional
 
 import nanoid
+from httpx import Timeout
+from pydantic import BaseModel
 from rich import print
 
 from indexify.common_util import get_httpx_client
@@ -21,6 +23,15 @@ FORCE_MULTIPART = ForceMultipartDict()
 UTF_8_CONTENT_TYPE = "application/octet-stream"
 
 
+class ReportingData(BaseModel):
+    output_count: int = 0
+    output_total_bytes: int = 0
+    stdout_count: int = 0
+    stdout_total_bytes: int = 0
+    stderr_count: int = 0
+    stderr_total_bytes: int = 0
+
+
 class TaskReporter:
     def __init__(
         self, base_url: str, executor_id: str, config_path: Optional[str] = None
@@ -30,11 +41,10 @@ class TaskReporter:
         self._client = get_httpx_client(config_path)
 
     def report_task_outcome(self, completed_task: CompletedTask):
+
+        report = ReportingData()
         fn_outputs = []
         for output in completed_task.outputs or []:
-            print(
-                f"[bold]task-reporter[/bold] uploading output of size: {len(output.payload)} bytes"
-            )
             serializer = get_serializer(output.encoder)
             serialized_output = serializer.serialize(output.payload)
             fn_outputs.append(
@@ -43,11 +53,10 @@ class TaskReporter:
                     (nanoid.generate(), serialized_output, serializer.content_type),
                 )
             )
+            report.output_count += 1
+            report.output_total_bytes += len(serialized_output)
 
         if completed_task.stdout:
-            print(
-                f"[bold]task-reporter[/bold] uploading stdout of size: {len(completed_task.stdout)}"
-            )
             fn_outputs.append(
                 (
                     "stdout",
@@ -58,11 +67,10 @@ class TaskReporter:
                     ),
                 )
             )
+            report.stdout_count += 1
+            report.stdout_total_bytes += len(completed_task.stdout)
 
         if completed_task.stderr:
-            print(
-                f"[bold]task-reporter[/bold] uploading stderr of size: {len(completed_task.stderr)}"
-            )
             fn_outputs.append(
                 (
                     "stderr",
@@ -73,6 +81,8 @@ class TaskReporter:
                     ),
                 )
             )
+            report.stderr_count += 1
+            report.stderr_total_bytes += len(completed_task.stderr)
 
         router_output = (
             ApiRouterOutput(edges=completed_task.router_output.edges)
@@ -93,7 +103,30 @@ class TaskReporter:
         )
         task_result_data = task_result.model_dump_json(exclude_none=True)
 
-        kwargs = {"data": {"task_result": task_result_data}}
+        total_bytes = (
+            report.output_total_bytes
+            + report.stdout_total_bytes
+            + report.stderr_total_bytes
+        )
+
+        print(
+            f"[bold]task-reporter[/bold] reporting task outcome "
+            f"task_id={completed_task.task.id} retries={completed_task.reporting_retries} "
+            f"total_bytes={total_bytes} total_files={report.output_count + report.stdout_count + report.stderr_count} "
+            f"output_files={report.output_count} output_bytes={total_bytes} "
+            f"stdout_bytes={report.stdout_total_bytes} stderr_bytes={report.stderr_total_bytes} "
+        )
+
+        #
+        kwargs = {
+            "data": {"task_result": task_result_data},
+            # Use httpx default timeout of 5s for all timeout types.
+            # For read timeouts, use 5 minutes to allow for large file uploads.
+            "timeout": Timeout(
+                5.0,
+                read=5.0 * 60,
+            ),
+        }
         if fn_outputs and len(fn_outputs) > 0:
             kwargs["files"] = fn_outputs
         else:
@@ -104,11 +137,15 @@ class TaskReporter:
                 **kwargs,
             )
         except Exception as e:
-            print(f"failed to report task outcome {e}")
+            print(
+                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {type(e).__name__}({e})"
+            )
             raise e
 
         try:
             response.raise_for_status()
         except Exception as e:
-            print(f"failed to report task outcome {response.text}")
+            print(
+                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {response.text}"
+            )
             raise e

{indexify-0.2.29 → indexify-0.2.31}/indexify/executor/task_store.py

@@ -17,6 +17,7 @@ class CompletedTask(BaseModel):
     stdout: Optional[str] = None
     stderr: Optional[str] = None
     reducer: bool = False
+    reporting_retries: int = 0
 
 
 class TaskStore:

{indexify-0.2.29 → indexify-0.2.31}/indexify/functions_sdk/graph.py

@@ -101,9 +101,9 @@ class Graph:
             return self
 
         if issubclass(indexify_fn, IndexifyFunction) and indexify_fn.accumulate:
-            self.accumulator_zero_values[
-                indexify_fn.name
-            ] = indexify_fn.accumulate().model_dump()
+            self.accumulator_zero_values[indexify_fn.name] = (
+                indexify_fn.accumulate().model_dump()
+            )
 
         self.nodes[indexify_fn.name] = indexify_fn
         return self
@@ -244,9 +244,9 @@ class Graph:
         queue = deque([(self._start_node, initial_input)])
         while queue:
             node_name, input = queue.popleft()
-            function_outputs: Union[
-                FunctionCallResult, RouterCallResult
-            ] = self._invoke_fn(node_name, input)
+            function_outputs: Union[FunctionCallResult, RouterCallResult] = (
+                self._invoke_fn(node_name, input)
+            )
             self._log_local_exec_tracebacks(function_outputs)
             if isinstance(function_outputs, RouterCallResult):
                 for edge in function_outputs.edges:

{indexify-0.2.29 → indexify-0.2.31}/indexify/functions_sdk/graph_definition.py

@@ -46,6 +46,7 @@ class ComputeGraphMetadata(BaseModel):
     edges: Dict[str, List[str]]
     accumulator_zero_values: Dict[str, bytes] = {}
     runtime_information: RuntimeInformation
+    replaying: bool = False
 
     def get_input_payload_serializer(self):
         return get_serializer(self.start_node.compute_fn.encoder)

{indexify-0.2.29 → indexify-0.2.31}/indexify/functions_sdk/indexify_functions.py

@@ -137,9 +137,11 @@ def indexify_router(
 
         attrs = {
             "name": name if name else fn.__name__,
-            "description": description
-            if description
-            else (fn.__doc__ or "").strip().replace("\n", ""),
+            "description": (
+                description
+                if description
+                else (fn.__doc__ or "").strip().replace("\n", "")
+            ),
             "image": image,
             "placement_constraints": placement_constraints,
             "encoder": encoder,
@@ -174,9 +176,11 @@ def indexify_function(
 
         attrs = {
             "name": name if name else fn.__name__,
-            "description": description
-            if description
-            else (fn.__doc__ or "").strip().replace("\n", ""),
+            "description": (
+                description
+                if description
+                else (fn.__doc__ or "").strip().replace("\n", "")
+            ),
             "image": image,
             "placement_constraints": placement_constraints,
             "accumulate": accumulate,
@@ -205,9 +209,9 @@ class IndexifyFunctionWrapper:
         indexify_function: Union[IndexifyFunction, IndexifyRouter],
         context: GraphInvocationContext,
     ):
-        self.indexify_function: Union[
-            IndexifyFunction, IndexifyRouter
-        ] = indexify_function()
+        self.indexify_function: Union[IndexifyFunction, IndexifyRouter] = (
+            indexify_function()
+        )
         self.indexify_function._ctx = context
 
     def get_output_model(self) -> Any:

{indexify-0.2.29 → indexify-0.2.31}/indexify/http_client.py

@@ -135,7 +135,9 @@ class IndexifyClient:
 
     def _add_api_key(self, kwargs):
         if self._api_key:
-            kwargs["headers"] = {"Authorization": f"Bearer {self._api_key}"}
+            if "headers" not in kwargs:
+                kwargs["headers"] = {}
+            kwargs["headers"]["Authorization"] = f"Bearer {self._api_key}"
 
     def _get(self, endpoint: str, **kwargs) -> httpx.Response:
         self._add_api_key(kwargs)

{indexify-0.2.29 → indexify-0.2.31}/indexify/remote_graph.py

@@ -1,6 +1,7 @@
 from typing import Any, List, Optional
 
 from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
+from indexify.functions_sdk.graph_definition import ComputeGraphMetadata
 
 from .http_client import IndexifyClient
 from .settings import DEFAULT_SERVICE_URL
@@ -55,6 +56,12 @@ class RemoteGraph:
             **kwargs
         )
 
+    def metadata(self) -> ComputeGraphMetadata:
+        """
+        Get the metadata of the graph.
+        """
+        return self._client.graph(self._name)
+
     def replay_invocations(self):
         """
         Replay all the graph previous runs/invocations on the latest version of the graph.

{indexify-0.2.29 → indexify-0.2.31}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "indexify"
-version = "0.2.29"
+version = "0.2.31"
 description = "Python Client for Indexify"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -27,10 +27,6 @@ jsonpickle = "^4.0.0"
 [tool.poetry.dev-dependencies]
 black = "^22.3.0"
 pylint = "^2.4.0"
-pytest = "^7.1.1"
-pytest-cov = "^4.0.0"
-pytest-runner = "^6.0.0"
-pytest-watch = "^4.2.0"
 parameterized = "^0.9.0"
 
 [build-system]