indexify 0.2.41__tar.gz → 0.2.43__tar.gz

{indexify-0.2.41 → indexify-0.2.43}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.41
+Version: 0.2.43
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -25,7 +25,7 @@ Requires-Dist: pydantic (==2.10.2)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
 Requires-Dist: structlog (>=24.4.0,<25.0.0)
-Requires-Dist: typer (>=0.13.0,<0.14.0)
+Requires-Dist: typer (>=0.12,<0.13)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown
 

{indexify-0.2.41 → indexify-0.2.43}/indexify/cli.py

@@ -22,7 +22,7 @@ from rich.panel import Panel
 from rich.text import Text
 from rich.theme import Theme
 
-from indexify.executor.agent import ExtractorAgent
+from indexify.executor.executor import Executor
 from indexify.function_executor.function_executor_service import (
     FunctionExecutorService,
 )
@@ -221,7 +221,7 @@ def executor(
         shutil.rmtree(executor_cache)
     Path(executor_cache).mkdir(parents=True, exist_ok=True)
 
-    agent = ExtractorAgent(
+    executor = Executor(
         id,
         server_addr=server_addr,
         config_path=config_path,
@@ -232,7 +232,7 @@ def executor(
     )
 
     try:
-        asyncio.get_event_loop().run_until_complete(agent.run())
+        asyncio.get_event_loop().run_until_complete(executor.run())
     except asyncio.CancelledError:
         logger.info("graceful shutdown")
 

indexify-0.2.43/indexify/executor/executor.py

@@ -0,0 +1,155 @@
+import asyncio
+import signal
+from pathlib import Path
+from typing import Any, Optional
+
+import structlog
+
+from indexify.function_executor.proto.function_executor_pb2 import (
+    SerializedObject,
+)
+
+from .api_objects import Task
+from .downloader import DownloadedInputs, Downloader
+from .function_executor.process_function_executor_factory import (
+    ProcessFunctionExecutorFactory,
+)
+from .function_worker import (
+    FunctionWorker,
+    FunctionWorkerInput,
+    FunctionWorkerOutput,
+)
+from .task_fetcher import TaskFetcher
+from .task_reporter import TaskReporter
+
+
+class Executor:
+    def __init__(
+        self,
+        executor_id: str,
+        code_path: Path,
+        server_addr: str = "localhost:8900",
+        development_mode: bool = False,
+        config_path: Optional[str] = None,
+        name_alias: Optional[str] = None,
+        image_hash: Optional[str] = None,
+    ):
+        self._logger = structlog.get_logger(module=__name__)
+        self._should_run = True
+        self._config_path = config_path
+        protocol: str = "http"
+        if config_path:
+            self._logger.info("running the extractor with TLS enabled")
+            protocol = "https"
+
+        self._function_worker = FunctionWorker(
+            function_executor_factory=ProcessFunctionExecutorFactory(
+                indexify_server_address=server_addr,
+                development_mode=development_mode,
+                config_path=config_path,
+            )
+        )
+        self._server_addr = server_addr
+        self._base_url = f"{protocol}://{self._server_addr}"
+        self._code_path = code_path
+        self._downloader = Downloader(
+            code_path=code_path, base_url=self._base_url, config_path=config_path
+        )
+        self._task_fetcher = TaskFetcher(
+            protocol=protocol,
+            indexify_server_addr=self._server_addr,
+            executor_id=executor_id,
+            name_alias=name_alias,
+            image_hash=image_hash,
+            config_path=config_path,
+        )
+        self._task_reporter = TaskReporter(
+            base_url=self._base_url,
+            executor_id=executor_id,
+            config_path=self._config_path,
+        )
+
+    async def run(self):
+        asyncio.get_event_loop().add_signal_handler(
+            signal.SIGINT, self.shutdown, asyncio.get_event_loop()
+        )
+        asyncio.get_event_loop().add_signal_handler(
+            signal.SIGTERM, self.shutdown, asyncio.get_event_loop()
+        )
+
+        while self._should_run:
+            try:
+                async for task in self._task_fetcher.run():
+                    asyncio.create_task(self._run_task(task))
+            except Exception as e:
+                self._logger.error(
+                    "failed fetching tasks, retrying in 5 seconds", exc_info=e
+                )
+                await asyncio.sleep(5)
+
+    async def _run_task(self, task: Task) -> None:
+        """Runs the supplied task.
+
+        Doesn't raise any Exceptions. All errors are reported to the server."""
+        logger = self._task_logger(task)
+        output: Optional[FunctionWorkerOutput] = None
+
+        try:
+            graph: SerializedObject = await self._downloader.download_graph(task)
+            input: DownloadedInputs = await self._downloader.download_inputs(task)
+            output = await self._function_worker.run(
+                input=FunctionWorkerInput(
+                    task=task,
+                    graph=graph,
+                    function_input=input,
+                )
+            )
+            logger.info("task_execution_finished", success=output.success)
+        except Exception as e:
+            logger.error("failed running the task", exc_info=e)
+
+        await self._report_task_outcome(task=task, output=output, logger=logger)
+
+    async def _report_task_outcome(
+        self, task: Task, output: Optional[FunctionWorkerOutput], logger: Any
+    ) -> None:
+        """Reports the task with the given output to the server.
+
+        None output means that the task execution didn't finish due to an internal error.
+        Doesn't raise any exceptions."""
+        reporting_retries: int = 0
+
+        while True:
+            logger = logger.bind(retries=reporting_retries)
+            try:
+                await self._task_reporter.report(
+                    task=task, output=output, logger=logger
+                )
+                break
+            except Exception as e:
+                logger.error(
+                    "failed_to_report_task",
+                    exc_info=e,
+                )
+                reporting_retries += 1
+                await asyncio.sleep(5)
+
+    async def _shutdown(self, loop):
+        self._logger.info("shutting_down")
+        self._should_run = False
+        await self._function_worker.shutdown()
+        for task in asyncio.all_tasks(loop):
+            task.cancel()
+
+    def shutdown(self, loop):
+        loop.create_task(self._shutdown(loop))
+
+    def _task_logger(self, task: Task) -> Any:
+        return self._logger.bind(
+            namespace=task.namespace,
+            graph=task.compute_graph,
+            graph_version=task.graph_version,
+            invocation_id=task.invocation_id,
+            function_name=task.compute_fn,
+            task_id=task.id,
+        )

{indexify-0.2.41 → indexify-0.2.43}/indexify/executor/function_worker.py

@@ -31,13 +31,11 @@ class FunctionWorkerInput:
     def __init__(
         self,
         task: Task,
-        graph: Optional[SerializedObject] = None,
-        function_input: Optional[DownloadedInputs] = None,
+        graph: SerializedObject,
+        function_input: DownloadedInputs,
     ):
         self.task = task
-        # Must not be None when running the task.
         self.graph = graph
-        # Must not be None when running the task.
         self.function_input = function_input
 
 

{indexify-0.2.41 → indexify-0.2.43}/indexify/executor/task_fetcher.py

@@ -2,7 +2,6 @@ import json
 from importlib.metadata import version
 from typing import AsyncGenerator, Optional
 
-import httpx
 import structlog
 from httpx_sse import aconnect_sse
 
@@ -66,7 +65,7 @@ class TaskFetcher:
                 except Exception as e:
                     await event_source.response.aread()
                     raise Exception(
-                        "Failed to register at server. "
+                        "failed to register at server. "
                         f"Response code: {event_source.response.status_code}. "
                         f"Response text: '{event_source.response.text}'."
                     ) from e

indexify-0.2.43/indexify/executor/task_reporter.py

@@ -0,0 +1,215 @@
+import asyncio
+from typing import Any, List, Optional, Tuple
+
+import nanoid
+from httpx import Timeout
+
+from indexify.common_util import get_httpx_client
+from indexify.executor.api_objects import RouterOutput, Task, TaskResult
+from indexify.function_executor.proto.function_executor_pb2 import (
+    FunctionOutput,
+)
+
+from .function_worker import FunctionWorkerOutput
+
+
+# https://github.com/psf/requests/issues/1081#issuecomment-428504128
+class ForceMultipartDict(dict):
+    def __bool__(self):
+        return True
+
+
+FORCE_MULTIPART = ForceMultipartDict()
+UTF_8_CONTENT_TYPE = "application/octet-stream"
+
+
+class TaskOutputSummary:
+    def __init__(self):
+        self.output_count: int = 0
+        self.output_total_bytes: int = 0
+        self.router_output_count: int = 0
+        self.stdout_count: int = 0
+        self.stdout_total_bytes: int = 0
+        self.stderr_count: int = 0
+        self.stderr_total_bytes: int = 0
+        self.total_bytes: int = 0
+
+
+class TaskReporter:
+    def __init__(
+        self, base_url: str, executor_id: str, config_path: Optional[str] = None
+    ):
+        self._base_url = base_url
+        self._executor_id = executor_id
+        # Use thread-safe sync client due to issues with async client.
+        # Async client attempts to use connections it already closed.
+        # See e.g. https://github.com/encode/httpx/issues/2337.
+        # Creating a new async client for each request fixes this but it
+        # results in not reusing established TCP connections to server.
+        self._client = get_httpx_client(config_path, make_async=False)
+
+    async def report(
+        self, task: Task, output: Optional[FunctionWorkerOutput], logger: Any
+    ):
+        """Reports result of the supplied task.
+
+        If FunctionWorkerOutput is None this means that the task didn't finish and failed with internal error.
+        """
+        logger = logger.bind(module=__name__)
+        task_result, output_files, output_summary = self._process_task_output(
+            task, output
+        )
+        task_result_data = task_result.model_dump_json(exclude_none=True)
+
+        logger.info(
+            "reporting task outcome",
+            total_bytes=output_summary.total_bytes,
+            total_files=output_summary.output_count
+            + output_summary.stdout_count
+            + output_summary.stderr_count,
+            output_files=output_summary.output_count,
+            output_bytes=output_summary.total_bytes,
+            router_output_count=output_summary.router_output_count,
+            stdout_bytes=output_summary.stdout_total_bytes,
+            stderr_bytes=output_summary.stderr_total_bytes,
+        )
+
+        kwargs = {
+            "data": {"task_result": task_result_data},
+            # Use httpx default timeout of 5s for all timeout types.
+            # For read timeouts, use 5 minutes to allow for large file uploads.
+            "timeout": Timeout(
+                5.0,
+                read=5.0 * 60,
+            ),
+            "files": output_files if len(output_files) > 0 else FORCE_MULTIPART,
+        }
+        # Run in a separate thread to not block the main event loop.
+        response = await asyncio.to_thread(
+            self._client.post, url=f"{self._base_url}/internal/ingest_files", **kwargs
+        )
+
+        try:
+            response.raise_for_status()
+        except Exception as e:
+            # Caller catches and logs the exception.
+            raise Exception(
+                "failed to report task outcome. "
+                f"Response code: {response.status_code}. "
+                f"Response text: '{response.text}'."
+            ) from e
+
+    def _process_task_output(
+        self, task: Task, output: Optional[FunctionWorkerOutput]
+    ) -> Tuple[TaskResult, List[Any], TaskOutputSummary]:
+        task_result = TaskResult(
+            outcome="failure",
+            namespace=task.namespace,
+            compute_graph=task.compute_graph,
+            compute_fn=task.compute_fn,
+            invocation_id=task.invocation_id,
+            executor_id=self._executor_id,
+            task_id=task.id,
+        )
+        output_files: List[Any] = []
+        summary: TaskOutputSummary = TaskOutputSummary()
+        if output is None:
+            return task_result, output_files, summary
+
+        task_result.outcome = "success" if output.success else "failure"
+        task_result.reducer = output.reducer
+
+        _process_function_output(
+            function_output=output.function_output,
+            output_files=output_files,
+            summary=summary,
+        )
+        _process_router_output(
+            router_output=output.router_output, task_result=task_result, summary=summary
+        )
+        _process_stdout(
+            stdout=output.stdout, output_files=output_files, summary=summary
+        )
+        _process_stderr(
+            stderr=output.stderr, output_files=output_files, summary=summary
+        )
+
+        summary.total_bytes = (
+            summary.output_total_bytes
+            + summary.stdout_total_bytes
+            + summary.stderr_total_bytes
+        )
+
+        return task_result, output_files, summary
+
+
+def _process_function_output(
+    function_output: Optional[FunctionOutput],
+    output_files: List[Any],
+    summary: TaskOutputSummary,
+) -> None:
+    if function_output is None:
+        return
+
+    for output in function_output.outputs or []:
+        payload = output.bytes if output.HasField("bytes") else output.string
+        output_files.append(
+            (
+                "node_outputs",
+                (nanoid.generate(), payload, output.content_type),
+            )
+        )
+        summary.output_count += 1
+        summary.output_total_bytes += len(payload)
+
+
+def _process_router_output(
+    router_output: Optional[RouterOutput],
+    task_result: TaskResult,
+    summary: TaskOutputSummary,
+) -> None:
+    if router_output is None:
+        return
+
+    task_result.router_output = RouterOutput(edges=router_output.edges)
+    summary.router_output_count += 1
+
+
+def _process_stdout(
+    stdout: Optional[str], output_files: List[Any], summary: TaskOutputSummary
+) -> None:
+    if stdout is None:
+        return
+
+    output_files.append(
+        (
+            "stdout",
+            (
+                nanoid.generate(),
+                stdout.encode(),
+                UTF_8_CONTENT_TYPE,
+            ),
+        )
+    )
+    summary.stdout_count += 1
+    summary.stdout_total_bytes += len(stdout)
+
+
+def _process_stderr(
+    stderr: Optional[str], output_files: List[Any], summary: TaskOutputSummary
+) -> None:
+    if stderr is None:
+        return
+
+    output_files.append(
+        (
+            "stderr",
+            (
+                nanoid.generate(),
+                stderr.encode(),
+                UTF_8_CONTENT_TYPE,
+            ),
+        )
+    )
+    summary.stderr_count += 1
+    summary.stderr_total_bytes += len(stderr)

{indexify-0.2.41 → indexify-0.2.43}/indexify/function_executor/proto/message_validator.py

@@ -1,4 +1,4 @@
-from typing import Any, Self
+from typing import Any
 
 from .function_executor_pb2 import SerializedObject
 
@@ -7,21 +7,21 @@ class MessageValidator:
     def __init__(self, message: Any):
         self._message = message
 
-    def required_field(self, field_name: str) -> Self:
+    def required_field(self, field_name: str) -> "MessageValidator":
         if not self._message.HasField(field_name):
             raise ValueError(
                 f"Field '{field_name}' is required in {type(self._message).__name__}"
             )
         return self
 
-    def required_serialized_object(self, field_name: str) -> Self:
+    def required_serialized_object(self, field_name: str) -> "MessageValidator":
         """Validates the SerializedObject.
 
         Raises: ValueError: If the SerializedObject is invalid or not present."""
         self.required_field(field_name)
         return self.optional_serialized_object(field_name)
 
-    def optional_serialized_object(self, field_name: str) -> Self:
+    def optional_serialized_object(self, field_name: str) -> "MessageValidator":
         """Validates the SerializedObject.
 
         Raises: ValueError: If the SerializedObject is invalid."""
@@ -32,7 +32,7 @@ class MessageValidator:
         if not serializedObject.HasField("string") and not serializedObject.HasField(
             "bytes"
         ):
-            raise ValueError("oneof 'data' is requred in SerializedObject")
+            raise ValueError("oneof 'data' is required in SerializedObject")
         if not serializedObject.HasField("content_type"):
-            raise ValueError("Field 'content_type' is requred in SerializedObject")
+            raise ValueError("Field 'content_type' is required in SerializedObject")
         return self

{indexify-0.2.41 → indexify-0.2.43}/indexify/functions_sdk/graph.py

@@ -37,7 +37,6 @@ from .indexify_functions import (
     IndexifyRouter,
     RouterCallResult,
 )
-from .local_cache import CacheAwareFunctionWrapper
 from .object_serializer import get_serializer
 
 RouterFn = Annotated[
@@ -86,7 +85,6 @@ class Graph:
 
         # Storage for local execution
         self._results: Dict[str, Dict[str, List[IndexifyData]]] = {}
-        self._cache = CacheAwareFunctionWrapper("./indexify_local_runner_cache")
         self._accumulator_values: Dict[str, IndexifyData] = {}
         self._local_graph_ctx: Optional[GraphInvocationContext] = None
 

{indexify-0.2.41 → indexify-0.2.43}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "indexify"
-version = "0.2.41"
+version = "0.2.43"
 description = "Python Client for Indexify"
 authors = ["Tensorlake Inc. <support@tensorlake.ai>"]
 license = "Apache 2.0"
@@ -20,7 +20,7 @@ cloudpickle = "^3.1.0"
 rich = "^13.9.2"
 nanoid = "^2.0.0"
 docker = "^7.1.0"
-typer = "^0.13.0"
+typer = "^0.12"
 httpx-sse = "^0.4.0"
 structlog = "^24.4.0"
 grpcio = "1.68.1"