indexify 0.2.29__py3-none-any.whl → 0.2.31__py3-none-any.whl

indexify/executor/agent.py

@@ -100,7 +100,9 @@ class ExtractorAgent:
         self._server_addr = server_addr
         self._base_url = f"{self._protocol}://{self._server_addr}"
         self._code_path = code_path
-        self._downloader = Downloader(code_path=code_path, base_url=self._base_url)
+        self._downloader = Downloader(
+            code_path=code_path, base_url=self._base_url, config_path=self._config_path
+        )
         self._max_queued_tasks = 10
         self._task_reporter = TaskReporter(
             base_url=self._base_url,
@@ -112,9 +114,15 @@ class ExtractorAgent:
         console.print(Text("Starting task completion reporter", style="bold cyan"))
         # We should copy only the keys and not the values
         url = f"{self._protocol}://{self._server_addr}/write_content"
+
         while True:
             outcomes = await self._task_store.task_outcomes()
             for task_outcome in outcomes:
+                retryStr = (
+                    f"\nRetries: {task_outcome.reporting_retries}"
+                    if task_outcome.reporting_retries > 0
+                    else ""
+                )
                 outcome = task_outcome.task_outcome
                 style_outcome = (
                     f"[bold red] {outcome} [/]"
@@ -125,7 +133,9 @@ class ExtractorAgent:
                     Panel(
                         f"Reporting outcome of task: {task_outcome.task.id}, function: {task_outcome.task.compute_fn}\n"
                         f"Outcome: {style_outcome}\n"
-                        f"Num Fn Outputs: {len(task_outcome.outputs or [])}  Router Output: {task_outcome.router_output}",
+                        f"Num Fn Outputs: {len(task_outcome.outputs or [])}\n"
+                        f"Router Output: {task_outcome.router_output}\n"
+                        f"Retries: {task_outcome.reporting_retries}",
                         title="Task Completion",
                         border_style="info",
                     )
@@ -139,11 +149,14 @@ class ExtractorAgent:
                     console.print(
                         Panel(
                             f"Failed to report task {task_outcome.task.id}\n"
-                            f"Exception: {e}\nRetrying...",
+                            f"Exception: {type(e).__name__}({e})\n"
+                            f"Retries: {task_outcome.reporting_retries}\n"
+                            "Retrying...",
                             title="Reporting Error",
                             border_style="error",
                         )
                     )
+                    task_outcome.reporting_retries += 1
                     await asyncio.sleep(5)
                     continue
 

indexify/executor/downloader.py

@@ -120,7 +120,7 @@ class Downloader:
         deserialized_content = serializer.deserialize(response.content)
 
         if reducer_url:
-            init_value = httpx.get(reducer_url)
+            init_value = self._client.get(reducer_url)
             try:
                 init_value.raise_for_status()
             except httpx.HTTPStatusError as e:

indexify/executor/task_reporter.py

@@ -2,6 +2,8 @@ import io
 from typing import Optional
 
 import nanoid
+from httpx import Timeout
+from pydantic import BaseModel
 from rich import print
 
 from indexify.common_util import get_httpx_client
@@ -21,6 +23,15 @@ FORCE_MULTIPART = ForceMultipartDict()
 UTF_8_CONTENT_TYPE = "application/octet-stream"
 
 
+class ReportingData(BaseModel):
+    output_count: int = 0
+    output_total_bytes: int = 0
+    stdout_count: int = 0
+    stdout_total_bytes: int = 0
+    stderr_count: int = 0
+    stderr_total_bytes: int = 0
+
+
 class TaskReporter:
     def __init__(
         self, base_url: str, executor_id: str, config_path: Optional[str] = None
@@ -30,11 +41,10 @@ class TaskReporter:
         self._client = get_httpx_client(config_path)
 
     def report_task_outcome(self, completed_task: CompletedTask):
+
+        report = ReportingData()
         fn_outputs = []
         for output in completed_task.outputs or []:
-            print(
-                f"[bold]task-reporter[/bold] uploading output of size: {len(output.payload)} bytes"
-            )
             serializer = get_serializer(output.encoder)
             serialized_output = serializer.serialize(output.payload)
             fn_outputs.append(
@@ -43,11 +53,10 @@ class TaskReporter:
                     (nanoid.generate(), serialized_output, serializer.content_type),
                 )
             )
+            report.output_count += 1
+            report.output_total_bytes += len(serialized_output)
 
         if completed_task.stdout:
-            print(
-                f"[bold]task-reporter[/bold] uploading stdout of size: {len(completed_task.stdout)}"
-            )
             fn_outputs.append(
                 (
                     "stdout",
@@ -58,11 +67,10 @@ class TaskReporter:
                     ),
                 )
             )
+            report.stdout_count += 1
+            report.stdout_total_bytes += len(completed_task.stdout)
 
         if completed_task.stderr:
-            print(
-                f"[bold]task-reporter[/bold] uploading stderr of size: {len(completed_task.stderr)}"
-            )
             fn_outputs.append(
                 (
                     "stderr",
@@ -73,6 +81,8 @@ class TaskReporter:
                     ),
                 )
             )
+            report.stderr_count += 1
+            report.stderr_total_bytes += len(completed_task.stderr)
 
         router_output = (
             ApiRouterOutput(edges=completed_task.router_output.edges)
@@ -93,7 +103,30 @@ class TaskReporter:
         )
         task_result_data = task_result.model_dump_json(exclude_none=True)
 
-        kwargs = {"data": {"task_result": task_result_data}}
+        total_bytes = (
+            report.output_total_bytes
+            + report.stdout_total_bytes
+            + report.stderr_total_bytes
+        )
+
+        print(
+            f"[bold]task-reporter[/bold] reporting task outcome "
+            f"task_id={completed_task.task.id} retries={completed_task.reporting_retries} "
+            f"total_bytes={total_bytes} total_files={report.output_count + report.stdout_count + report.stderr_count} "
+            f"output_files={report.output_count} output_bytes={total_bytes} "
+            f"stdout_bytes={report.stdout_total_bytes} stderr_bytes={report.stderr_total_bytes} "
+        )
+
+        #
+        kwargs = {
+            "data": {"task_result": task_result_data},
+            # Use httpx default timeout of 5s for all timeout types.
+            # For read timeouts, use 5 minutes to allow for large file uploads.
+            "timeout": Timeout(
+                5.0,
+                read=5.0 * 60,
+            ),
+        }
         if fn_outputs and len(fn_outputs) > 0:
             kwargs["files"] = fn_outputs
         else:
@@ -104,11 +137,15 @@ class TaskReporter:
                 **kwargs,
             )
         except Exception as e:
-            print(f"failed to report task outcome {e}")
+            print(
+                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {type(e).__name__}({e})"
+            )
             raise e
 
         try:
             response.raise_for_status()
         except Exception as e:
-            print(f"failed to report task outcome {response.text}")
+            print(
+                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {response.text}"
+            )
             raise e

indexify/executor/task_store.py

@@ -17,6 +17,7 @@ class CompletedTask(BaseModel):
     stdout: Optional[str] = None
     stderr: Optional[str] = None
     reducer: bool = False
+    reporting_retries: int = 0
 
 
 class TaskStore:

indexify/functions_sdk/graph.py

@@ -101,9 +101,9 @@ class Graph:
             return self
 
         if issubclass(indexify_fn, IndexifyFunction) and indexify_fn.accumulate:
-            self.accumulator_zero_values[
-                indexify_fn.name
-            ] = indexify_fn.accumulate().model_dump()
+            self.accumulator_zero_values[indexify_fn.name] = (
+                indexify_fn.accumulate().model_dump()
+            )
 
         self.nodes[indexify_fn.name] = indexify_fn
         return self
@@ -244,9 +244,9 @@ class Graph:
         queue = deque([(self._start_node, initial_input)])
         while queue:
             node_name, input = queue.popleft()
-            function_outputs: Union[
-                FunctionCallResult, RouterCallResult
-            ] = self._invoke_fn(node_name, input)
+            function_outputs: Union[FunctionCallResult, RouterCallResult] = (
+                self._invoke_fn(node_name, input)
+            )
             self._log_local_exec_tracebacks(function_outputs)
             if isinstance(function_outputs, RouterCallResult):
                 for edge in function_outputs.edges:

indexify/functions_sdk/graph_definition.py

@@ -46,6 +46,7 @@ class ComputeGraphMetadata(BaseModel):
     edges: Dict[str, List[str]]
     accumulator_zero_values: Dict[str, bytes] = {}
     runtime_information: RuntimeInformation
+    replaying: bool = False
 
     def get_input_payload_serializer(self):
         return get_serializer(self.start_node.compute_fn.encoder)

indexify/functions_sdk/indexify_functions.py

@@ -137,9 +137,11 @@ def indexify_router(
 
         attrs = {
             "name": name if name else fn.__name__,
-            "description": description
-            if description
-            else (fn.__doc__ or "").strip().replace("\n", ""),
+            "description": (
+                description
+                if description
+                else (fn.__doc__ or "").strip().replace("\n", "")
+            ),
             "image": image,
             "placement_constraints": placement_constraints,
             "encoder": encoder,
@@ -174,9 +176,11 @@ def indexify_function(
 
         attrs = {
             "name": name if name else fn.__name__,
-            "description": description
-            if description
-            else (fn.__doc__ or "").strip().replace("\n", ""),
+            "description": (
+                description
+                if description
+                else (fn.__doc__ or "").strip().replace("\n", "")
+            ),
             "image": image,
             "placement_constraints": placement_constraints,
             "accumulate": accumulate,
@@ -205,9 +209,9 @@ class IndexifyFunctionWrapper:
         indexify_function: Union[IndexifyFunction, IndexifyRouter],
         context: GraphInvocationContext,
     ):
-        self.indexify_function: Union[
-            IndexifyFunction, IndexifyRouter
-        ] = indexify_function()
+        self.indexify_function: Union[IndexifyFunction, IndexifyRouter] = (
+            indexify_function()
+        )
         self.indexify_function._ctx = context
 
     def get_output_model(self) -> Any:

indexify/http_client.py

@@ -135,7 +135,9 @@ class IndexifyClient:
 
     def _add_api_key(self, kwargs):
         if self._api_key:
-            kwargs["headers"] = {"Authorization": f"Bearer {self._api_key}"}
+            if "headers" not in kwargs:
+                kwargs["headers"] = {}
+            kwargs["headers"]["Authorization"] = f"Bearer {self._api_key}"
 
     def _get(self, endpoint: str, **kwargs) -> httpx.Response:
         self._add_api_key(kwargs)

indexify/remote_graph.py

@@ -1,6 +1,7 @@
 from typing import Any, List, Optional
 
 from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
+from indexify.functions_sdk.graph_definition import ComputeGraphMetadata
 
 from .http_client import IndexifyClient
 from .settings import DEFAULT_SERVICE_URL
@@ -55,6 +56,12 @@ class RemoteGraph:
             **kwargs
         )
 
+    def metadata(self) -> ComputeGraphMetadata:
+        """
+        Get the metadata of the graph.
+        """
+        return self._client.graph(self._name)
+
     def replay_invocations(self):
         """
         Replay all the graph previous runs/invocations on the latest version of the graph.

{indexify-0.2.29.dist-info → indexify-0.2.31.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.29
+Version: 0.2.31
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.2.29.dist-info → indexify-0.2.31.dist-info}/RECORD

@@ -5,31 +5,31 @@ indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-
 indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
 indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
 indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
-indexify/executor/agent.py,sha256=mqEDcRdDfoUjw5XoFfVlOlJ-C_ODzyCQQMW90vDbWk4,18114
+indexify/executor/agent.py,sha256=Zt-KU2yrOuFuHu8CiOg-QmAzkNU1idcv5UZ3YN2YSDU,18649
 indexify/executor/api_objects.py,sha256=mvmwGbK4paJNQGFvbtNHMPpiH_LpVhrlRnCcrqS6HOQ,859
-indexify/executor/downloader.py,sha256=_SQ-6-0Kj3Zg0Dqp_pMwTYxvBP3xvNUOREyDucCJp8M,4944
+indexify/executor/downloader.py,sha256=wwkTdKRrDBHwKQOb_3uUb8pVl1tvg4QzOcYFnGQwNtA,4951
 indexify/executor/executor_tasks.py,sha256=A0UIEZ5VaB6zSkFQG81UmTW0E57MTYhGlaXuAbRV8lQ,1884
 indexify/executor/function_worker.py,sha256=wRW2-X9dNI80KhwTD1vD-pcyetsVKVs6vVdg7L7JjcQ,6462
 indexify/executor/image_dependency_installer.py,sha256=ct8GmzgkaPi6NAblk68IJJWo5MecIUubELotmSrgoRQ,1759
 indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
 indexify/executor/runtime_probes.py,sha256=mjw2_mGQ622wRT_39WPGGgPEZQTgtrf3-ICcUUZOeyg,2126
-indexify/executor/task_reporter.py,sha256=Wr8caDtjHBHvK5C-wwghJ5TNxIK_moxjQokGRcEs2rw,3798
-indexify/executor/task_store.py,sha256=u48FdRKAh_KH7WOMQOArdOY5CawlyW5MJx8V0W79JM0,3951
+indexify/executor/task_reporter.py,sha256=4unHxLUHedKwIoqO3e5YdDJpUe_pJng-vHReoqcPNNU,5141
+indexify/executor/task_store.py,sha256=JlRlWwAm4YjFRkTNRx-6GsUcmOzcyvzb5Csa5XDpRTI,3982
 indexify/functions_sdk/data_objects.py,sha256=wXbUa9hjU6rsXmmk19vQ5Kixf3FsI59VBWPNmHasAX0,854
-indexify/functions_sdk/graph.py,sha256=TbHtIcAzRcEn3BWewVhNsUGMNfRduI1aSAvK6Vyx-fk,11801
-indexify/functions_sdk/graph_definition.py,sha256=5jUzjfk4gxxQCmQeOlNS9VndZK7KQBsZzABcDpQKgrs,1594
+indexify/functions_sdk/graph.py,sha256=_XsHfkQaxTnBNVcDlq4KsnyZ0HQ_DZOAuwygDzpsCNo,11805
+indexify/functions_sdk/graph_definition.py,sha256=UeC0PMchgZgo2T0Goog0SW1lIRdFf6ug4iHOS9ItRvI,1622
 indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
 indexify/functions_sdk/image.py,sha256=QK0H6KxLWriB_z4M0kunKzzHdHxYLWL670RPYgYuf_8,1762
-indexify/functions_sdk/indexify_functions.py,sha256=EbNC5gZktRc-G9ribQrpMG2X_G-8cujsxTgot734iGQ,10340
+indexify/functions_sdk/indexify_functions.py,sha256=cPUN7Zlgkwl6YUOuhgKogPdoBakQuf-VspQvU3IDmfU,10422
 indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
 indexify/functions_sdk/object_serializer.py,sha256=pOgUOWbRNRix9uZT0aQn0LTCnJCeMNGO1nAE0jAybmg,1546
 indexify/functions_sdk/pipeline.py,sha256=KmxZE8eBFAQ4bbEcYURXXR26HSyoAT3O6iu9H38-OXE,974
-indexify/http_client.py,sha256=rFYjv0rRMo4sXp-TjWoflQeJaxNZK-Di6Gk3G78xKQI,15838
-indexify/remote_graph.py,sha256=1JogFFcWCKc_Dq4Rh1s6IwAnxB-qrFnKuDQIbpJso1Y,4745
+indexify/http_client.py,sha256=5d70A6j2vJBn3OfUVjrQvnPml07p7fxOXWBUAUfAkiU,15915
+indexify/remote_graph.py,sha256=CWfFxB0jlx8Zgp_xUjvjgrIwv-yNAfEbQwlXt3jfRNM,4976
 indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
 indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
-indexify-0.2.29.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.29.dist-info/METADATA,sha256=3lC5Cqd9Uym3Bk0AvQVTT8JTlHsodWXyktGjEAU9Y7s,6202
-indexify-0.2.29.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-indexify-0.2.29.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.29.dist-info/RECORD,,
+indexify-0.2.31.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.31.dist-info/METADATA,sha256=Jrl1EVAJoI1uYMYKaA1pofIao2iV8FhfytvxEHXZvCE,6202
+indexify-0.2.31.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+indexify-0.2.31.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.31.dist-info/RECORD,,