indexify 0.2.32__py3-none-any.whl → 0.2.34__py3-none-any.whl

indexify/executor/agent.py

@@ -1,28 +1,21 @@
 import asyncio
 import json
-import traceback
 from concurrent.futures.process import BrokenProcessPool
 from importlib.metadata import version
 from pathlib import Path
 from typing import Dict, List, Optional
 
+import structlog
 from httpx_sse import aconnect_sse
 from pydantic import BaseModel
-from rich.console import Console
-from rich.panel import Panel
-from rich.text import Text
-from rich.theme import Theme
 
 from indexify.common_util import get_httpx_client
 from indexify.functions_sdk.data_objects import (
     FunctionWorkerOutput,
     IndexifyData,
 )
-from indexify.functions_sdk.graph_definition import ComputeGraphMetadata
 from indexify.http_client import IndexifyClient
 
-from ..functions_sdk.image import ImageInformation
-from . import image_dependency_installer
 from .api_objects import ExecutorMetadata, Task
 from .downloader import DownloadedInputs, Downloader
 from .executor_tasks import DownloadGraphTask, DownloadInputTask, ExtractTask
@@ -31,16 +24,7 @@ from .runtime_probes import ProbeInfo, RuntimeProbes
 from .task_reporter import TaskReporter
 from .task_store import CompletedTask, TaskStore
 
-custom_theme = Theme(
-    {
-        "info": "cyan",
-        "warning": "yellow",
-        "error": "red",
-        "success": "green",
-    }
-)
-
-console = Console(theme=custom_theme)
+logging = structlog.get_logger(module=__name__)
 
 
 class FunctionInput(BaseModel):
@@ -68,21 +52,9 @@ class ExtractorAgent:
         self._config_path = config_path
         self._probe = RuntimeProbes()
 
-        runtime_probe: ProbeInfo = self._probe.probe()
-        self._require_image_bootstrap = (
-            True
-            if (runtime_probe.is_default_executor and self.name_alias is not None)
-            else False
-        )
-        self._executor_bootstrap_failed = False
-
-        console.print(
-            f"Require Bootstrap? {self._require_image_bootstrap}", style="cyan bold"
-        )
-
         self.num_workers = num_workers
         if config_path:
-            console.print("Running the extractor with TLS enabled", style="cyan bold")
+            logging.info("running the extractor with TLS enabled")
             self._protocol = "https"
         else:
             self._protocol = "http"
@@ -111,10 +83,8 @@ class ExtractorAgent:
         )
 
     async def task_completion_reporter(self):
-        console.print(Text("Starting task completion reporter", style="bold cyan"))
+        logging.info("starting task completion reporter")
         # We should copy only the keys and not the values
-        url = f"{self._protocol}://{self._server_addr}/write_content"
-
         while True:
             outcomes = await self._task_store.task_outcomes()
             for task_outcome in outcomes:
@@ -129,16 +99,14 @@ class ExtractorAgent:
                     if "fail" in outcome
                     else f"[bold green] {outcome} [/]"
                 )
-                console.print(
-                    Panel(
-                        f"Reporting outcome of task: {task_outcome.task.id}, function: {task_outcome.task.compute_fn}\n"
-                        f"Outcome: {style_outcome}\n"
-                        f"Num Fn Outputs: {len(task_outcome.outputs or [])}\n"
-                        f"Router Output: {task_outcome.router_output}\n"
-                        f"Retries: {task_outcome.reporting_retries}",
-                        title="Task Completion",
-                        border_style="info",
-                    )
+                logging.info(
+                    "reporting_task_outcome",
+                    task_id=task_outcome.task.id,
+                    fn_name=task_outcome.task.compute_fn,
+                    num_outputs=len(task_outcome.outputs or []),
+                    router_output=task_outcome.router_output,
+                    outcome=task_outcome.task_outcome,
+                    retries=task_outcome.reporting_retries,
                 )
 
                 try:
@@ -146,15 +114,11 @@ class ExtractorAgent:
                     self._task_reporter.report_task_outcome(completed_task=task_outcome)
                 except Exception as e:
                     # The connection was dropped in the middle of the reporting, process, retry
-                    console.print(
-                        Panel(
-                            f"Failed to report task {task_outcome.task.id}\n"
-                            f"Exception: {type(e).__name__}({e})\n"
-                            f"Retries: {task_outcome.reporting_retries}\n"
-                            "Retrying...",
-                            title="Reporting Error",
-                            border_style="error",
-                        )
+                    logging.error(
+                        "failed_to_report_task",
+                        task_id=task_outcome.task.id,
+                        exception=f"exception: {type(e).__name__}({e})",
+                        retries=task_outcome.reporting_retries,
                     )
                     task_outcome.reporting_retries += 1
                     await asyncio.sleep(5)
@@ -176,44 +140,6 @@ class ExtractorAgent:
             fn: FunctionInput
             for fn in fn_queue:
                 task: Task = self._task_store.get_task(fn.task_id)
-
-                if self._executor_bootstrap_failed:
-                    completed_task = CompletedTask(
-                        task=task,
-                        outputs=[],
-                        task_outcome="failure",
-                    )
-                    self._task_store.complete(outcome=completed_task)
-
-                    continue
-
-                # Bootstrap this executor. Fail the task if we can't.
-                if self._require_image_bootstrap:
-                    try:
-                        image_info = await _get_image_info_for_compute_graph(
-                            task, self._protocol, self._server_addr, self._config_path
-                        )
-                        image_dependency_installer.executor_image_builder(
-                            image_info, self.name_alias, self.image_version
-                        )
-                        self._require_image_bootstrap = False
-                    except Exception as e:
-                        console.print(
-                            Text("Failed to bootstrap the executor ", style="red bold")
-                            + Text(f"Exception: {traceback.format_exc()}", style="red")
-                        )
-
-                        self._executor_bootstrap_failed = True
-
-                        completed_task = CompletedTask(
-                            task=task,
-                            outputs=[],
-                            task_outcome="failure",
-                        )
-                        self._task_store.complete(outcome=completed_task)
-
-                        continue
-
                 async_tasks.append(
                     ExtractTask(
                         function_worker=self._function_worker,
@@ -233,12 +159,9 @@ class ExtractorAgent:
             for async_task in done:
                 if async_task.get_name() == "get_runnable_tasks":
                     if async_task.exception():
-                        console.print(
-                            Text("Task Launcher Error: ", style="red bold")
-                            + Text(
-                                f"Failed to get runnable tasks: {async_task.exception()}",
-                                style="red",
-                            )
+                        logging.error(
+                            "task_launcher_error, failed to get runnable tasks",
+                            exception=async_task.exception(),
                         )
                         continue
                     result: Dict[str, Task] = await async_task
@@ -255,12 +178,9 @@ class ExtractorAgent:
                     )
                 elif async_task.get_name() == "download_graph":
                     if async_task.exception():
-                        console.print(
-                            Text(
-                                f"Failed to download graph for task {async_task.task.id}\n",
-                                style="red bold",
-                            )
-                            + Text(f"Exception: {async_task.exception()}", style="red")
+                        logging.error(
+                            "task_launcher_error, failed to download graph",
+                            exception=async_task.exception(),
                         )
                         completed_task = CompletedTask(
                             task=async_task.task,
@@ -276,12 +196,9 @@ class ExtractorAgent:
                     )
                 elif async_task.get_name() == "download_input":
                     if async_task.exception():
-                        console.print(
-                            Text(
-                                f"Failed to download input for task {async_task.task.id}\n",
-                                style="red bold",
-                            )
-                            + Text(f"Exception: {async_task.exception()}", style="red")
+                        logging.error(
+                            "task_launcher_error, failed to download input",
+                            exception=str(async_task.exception()),
                         )
                         completed_task = CompletedTask(
                             task=async_task.task,
@@ -334,12 +251,10 @@ class ExtractorAgent:
                         self._task_store.retriable_failure(async_task.task.id)
                         continue
                     except Exception as e:
-                        console.print(
-                            Text(
-                                f"Failed to execute task {async_task.task.id}\n",
-                                style="red bold",
-                            )
-                            + Text(f"Exception: {e}", style="red")
+                        logging.error(
+                            "failed to execute task",
+                            task_id=async_task.task.id,
+                            exception=str(e),
                         )
                         completed_task = CompletedTask(
                             task=async_task.task,
@@ -360,12 +275,6 @@ class ExtractorAgent:
         self._should_run = True
         while self._should_run:
             url = f"{self._protocol}://{self._server_addr}/internal/executors/{self._executor_id}/tasks"
-            print(f"calling url: {url}")
-
-            def to_sentence_case(snake_str):
-                words = snake_str.split("_")
-                return words[0].capitalize() + "" + " ".join(words[1:])
-
             runtime_probe: ProbeInfo = self._probe.probe()
 
             executor_version = version("indexify")
@@ -390,17 +299,7 @@ class ExtractorAgent:
                 image_version=image_version,
                 labels=runtime_probe.labels,
             ).model_dump()
-
-            panel_content = "\n".join(
-                [f"{to_sentence_case(key)}: {value}" for key, value in data.items()]
-            )
-            console.print(
-                Panel(
-                    panel_content,
-                    title="attempting to Register Executor",
-                    border_style="cyan",
-                )
-            )
+            logging.info("registering_executor", executor_id=self._executor_id)
             try:
                 async with get_httpx_client(self._config_path, True) as client:
                     async with aconnect_sse(
@@ -411,12 +310,17 @@ class ExtractorAgent:
                         headers={"Content-Type": "application/json"},
                     ) as event_source:
                         if not event_source.response.is_success:
-                            resp = await event_source.response.aread().decode("utf-8")
-                            console.print(f"failed to register: {str(resp)}")
+                            resp = await event_source.response
+                            resp_content = resp.aread()
+                            logging.error(
+                                f"failed to register",
+                                resp=str(resp_content),
+                                status_code=event_source.response.status_code,
+                            )
                             await asyncio.sleep(5)
                             continue
-                        console.print(
-                            Text("executor registered successfully", style="bold green")
+                        logging.info(
+                            "executor_registered", executor_id=self._executor_id
                         )
                         async for sse in event_source.aiter_sse():
                             data = json.loads(sse.data)
@@ -427,15 +331,12 @@ class ExtractorAgent:
                                 )
                             self._task_store.add_tasks(tasks)
             except Exception as e:
-                console.print(
-                    Text("registration Error: ", style="red bold")
-                    + Text(f"failed to register: {e}", style="red")
-                )
+                logging.error(f"failed to register: {e}")
                 await asyncio.sleep(5)
                 continue
 
     async def _shutdown(self, loop):
-        console.print(Text("shutting down agent...", style="bold yellow"))
+        logging.info("shutting_down")
         self._should_run = False
         for task in asyncio.all_tasks(loop):
             task.cancel()
@@ -443,27 +344,3 @@ class ExtractorAgent:
     def shutdown(self, loop):
         self._function_worker.shutdown()
         loop.create_task(self._shutdown(loop))
-
-
-async def _get_image_info_for_compute_graph(
-    task: Task, protocol, server_addr, config_path: str
-) -> ImageInformation:
-    namespace = task.namespace
-    graph_name: str = task.compute_graph
-    compute_fn_name: str = task.compute_fn
-
-    http_client = IndexifyClient(
-        service_url=f"{protocol}://{server_addr}",
-        namespace=namespace,
-        config_path=config_path,
-    )
-    compute_graph: ComputeGraphMetadata = http_client.graph(graph_name)
-
-    console.print(
-        Text(
-            f"Compute_fn name {compute_fn_name}, ComputeGraph {compute_graph} \n",
-            style="red yellow",
-        )
-    )
-
-    return compute_graph.nodes[compute_fn_name].compute_fn.image_information

indexify/executor/downloader.py

@@ -2,10 +2,8 @@ import os
 from typing import Optional
 
 import httpx
+import structlog
 from pydantic import BaseModel
-from rich.console import Console
-from rich.panel import Panel
-from rich.theme import Theme
 
 from indexify.functions_sdk.data_objects import IndexifyData
 
@@ -13,15 +11,7 @@ from ..common_util import get_httpx_client
 from ..functions_sdk.object_serializer import JsonSerializer, get_serializer
 from .api_objects import Task
 
-custom_theme = Theme(
-    {
-        "info": "cyan",
-        "warning": "yellow",
-        "error": "red",
-    }
-)
-
-console = Console(theme=custom_theme)
+logger = structlog.get_logger(module=__name__)
 
 
 class DownloadedInputs(BaseModel):
@@ -42,26 +32,21 @@ class Downloader:
         if os.path.exists(path):
             return path
 
-        console.print(
-            Panel(
-                f"Downloading graph: {name}\nPath: {path}",
-                title="downloader",
-                border_style="cyan",
-            )
+        logger.info(
+            "downloading graph", namespace=namespace, name=name, version=version
         )
-
         response = self._client.get(
             f"{self.base_url}/internal/namespaces/{namespace}/compute_graphs/{name}/code"
         )
         try:
             response.raise_for_status()
         except httpx.HTTPStatusError as e:
-            console.print(
-                Panel(
-                    f"Failed to download graph: {name}\nError: {response.text}",
-                    title="downloader error",
-                    border_style="error",
-                )
+            logger.error(
+                "failed to download graph",
+                namespace=namespace,
+                name=name,
+                version=version,
+                error=response.text,
             )
             raise
 
@@ -81,25 +66,17 @@ class Downloader:
         if task.reducer_output_id:
             reducer_url = f"{self.base_url}/namespaces/{task.namespace}/compute_graphs/{task.compute_graph}/invocations/{task.invocation_id}/fn/{task.compute_fn}/output/{task.reducer_output_id}"
 
-        console.print(
-            Panel(
-                f"downloading input\nURL: {url} \n reducer input URL: {reducer_url}",
-                title="downloader",
-                border_style="cyan",
-            )
-        )
-
+        logger.info("downloading input", url=url, reducer_url=reducer_url)
         response = self._client.get(url)
 
         try:
             response.raise_for_status()
         except httpx.HTTPStatusError as e:
-            console.print(
-                Panel(
-                    f"failed to download input: {task.input_key}\nError: {response.text}",
-                    title="downloader error",
-                    border_style="error",
-                )
+            logger.error(
+                "failed to download input",
+                url=url,
+                reducer_url=reducer_url,
+                error=response.text,
             )
             raise
 
@@ -108,8 +85,6 @@ class Downloader:
             if response.headers["content-type"] == JsonSerializer.content_type
             else "cloudpickle"
         )
-        serializer = get_serializer(encoder)
-
         if task.invocation_id == input_id:
             return DownloadedInputs(
                 input=IndexifyData(
@@ -117,26 +92,24 @@ class Downloader:
                 ),
             )
 
-        deserialized_content = serializer.deserialize(response.content)
+        input_payload = response.content
 
         if reducer_url:
-            init_value = self._client.get(reducer_url)
+            response = self._client.get(reducer_url)
             try:
-                init_value.raise_for_status()
+                response.raise_for_status()
+                init_value = response.content
             except httpx.HTTPStatusError as e:
-                console.print(
-                    Panel(
-                        f"failed to download reducer output: {task.reducer_output_id}\nError: {init_value.text}",
-                        title="downloader error",
-                        border_style="error",
-                    )
+                logger.error(
+                    "failed to download reducer output",
+                    url=reducer_url,
+                    error=response.text,
                 )
                 raise
-            init_value = serializer.deserialize(init_value.content)
             return DownloadedInputs(
                 input=IndexifyData(
                     input_id=task.invocation_id,
-                    payload=deserialized_content,
+                    payload=input_payload,
                     encoder=encoder,
                 ),
                 init_value=IndexifyData(
@@ -147,7 +120,7 @@ class Downloader:
         return DownloadedInputs(
             input=IndexifyData(
                 input_id=task.invocation_id,
-                payload=deserialized_content,
+                payload=input_payload,
                 encoder=encoder,
             )
         )

indexify/executor/task_reporter.py

@@ -1,10 +1,9 @@
-import io
 from typing import Optional
 
 import nanoid
+import structlog
 from httpx import Timeout
 from pydantic import BaseModel
-from rich import print
 
 from indexify.common_util import get_httpx_client
 from indexify.executor.api_objects import RouterOutput as ApiRouterOutput
@@ -12,6 +11,8 @@ from indexify.executor.api_objects import TaskResult
 from indexify.executor.task_store import CompletedTask
 from indexify.functions_sdk.object_serializer import get_serializer
 
+logger = structlog.get_logger(__name__)
+
 
 # https://github.com/psf/requests/issues/1081#issuecomment-428504128
 class ForceMultipartDict(dict):
@@ -46,15 +47,14 @@ class TaskReporter:
         fn_outputs = []
         for output in completed_task.outputs or []:
             serializer = get_serializer(output.encoder)
-            serialized_output = serializer.serialize(output.payload)
             fn_outputs.append(
                 (
                     "node_outputs",
-                    (nanoid.generate(), serialized_output, serializer.content_type),
+                    (nanoid.generate(), output.payload, serializer.content_type),
                 )
             )
             report.output_count += 1
-            report.output_total_bytes += len(serialized_output)
+            report.output_total_bytes += len(output.payload)
 
         if completed_task.stdout:
             fn_outputs.append(
@@ -109,14 +109,17 @@ class TaskReporter:
             + report.stderr_total_bytes
         )
 
-        print(
-            f"[bold]task-reporter[/bold] reporting task outcome "
-            f"task_id={completed_task.task.id} retries={completed_task.reporting_retries} "
-            f"total_bytes={total_bytes} total_files={report.output_count + report.stdout_count + report.stderr_count} "
-            f"output_files={report.output_count} output_bytes={total_bytes} "
-            f"stdout_bytes={report.stdout_total_bytes} stderr_bytes={report.stderr_total_bytes} "
+        logger.info(
+            "reporting task outcome",
+            task_id=completed_task.task.id,
+            retries=completed_task.reporting_retries,
+            total_bytes=total_bytes,
+            total_files=report.output_count + report.stdout_count + report.stderr_count,
+            output_files=report.output_count,
+            output_bytes=total_bytes,
+            stdout_bytes=report.stdout_total_bytes,
+            stderr_bytes=report.stderr_total_bytes,
         )
-
         #
         kwargs = {
             "data": {"task_result": task_result_data},
@@ -137,15 +140,23 @@ class TaskReporter:
                 **kwargs,
             )
         except Exception as e:
-            print(
-                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {type(e).__name__}({e})"
+            logger.error(
+                "failed to report task outcome",
+                task_id=completed_task.task.id,
+                retries=completed_task.reporting_retries,
+                error=type(e).__name__,
+                message=str(e),
             )
             raise e
 
         try:
             response.raise_for_status()
         except Exception as e:
-            print(
-                f"[bold]task-reporter[/bold] failed to report task outcome retries={completed_task.reporting_retries} {response.text}"
+            logger.error(
+                "failed to report task outcome",
+                task_id=completed_task.task.id,
+                retries=completed_task.reporting_retries,
+                status_code=response.status_code,
+                response_text=response.text,
             )
             raise e

indexify/functions_sdk/graph.py

@@ -1,3 +1,4 @@
+import json
 import sys
 from collections import defaultdict
 from queue import deque
@@ -101,9 +102,7 @@ class Graph:
             return self
 
         if issubclass(indexify_fn, IndexifyFunction) and indexify_fn.accumulate:
-            self.accumulator_zero_values[indexify_fn.name] = (
-                indexify_fn.accumulate().model_dump()
-            )
+            self.accumulator_zero_values[indexify_fn.name] = indexify_fn.accumulate()
 
         self.nodes[indexify_fn.name] = indexify_fn
         return self
@@ -167,7 +166,8 @@ class Graph:
             reducer=is_reducer,
             image_name=start_node.image._image_name,
             image_information=start_node.image.to_image_information(),
-            encoder=start_node.encoder,
+            input_encoder=start_node.input_encoder,
+            output_encoder=start_node.output_encoder,
         )
         metadata_edges = self.edges.copy()
         metadata_nodes = {}
@@ -179,7 +179,8 @@ class Graph:
                         description=node.description or "",
                         source_fn=node_name,
                         target_fns=self.routers[node_name],
-                        encoder=node.encoder,
+                        input_encoder=node.input_encoder,
+                        output_encoder=node.output_encoder,
                         image_name=node.image._image_name,
                         image_information=node.image.to_image_information(),
                     )
@@ -193,7 +194,8 @@ class Graph:
                         reducer=node.accumulate is not None,
                         image_name=node.image._image_name,
                         image_information=node.image.to_image_information(),
-                        encoder=node.encoder,
+                        input_encoder=node.input_encoder,
+                        output_encoder=node.output_encoder,
                     )
                 )
 
@@ -212,19 +214,19 @@ class Graph:
     def run(self, block_until_done: bool = False, **kwargs) -> str:
         self.validate_graph()
         start_node = self.nodes[self._start_node]
-        serializer = get_serializer(start_node.encoder)
+        serializer = get_serializer(start_node.input_encoder)
         input = IndexifyData(
             id=generate(),
             payload=serializer.serialize(kwargs),
-            encoder=start_node.encoder,
+            encoder=start_node.input_encoder,
         )
         print(f"[bold] Invoking {self._start_node}[/bold]")
         outputs = defaultdict(list)
         for k, v in self.accumulator_zero_values.items():
             node = self.nodes[k]
-            serializer = get_serializer(node.encoder)
+            serializer = get_serializer(node.input_encoder)
             self._accumulator_values[k] = IndexifyData(
-                payload=serializer.serialize(v), encoder=node.encoder
+                payload=serializer.serialize(v), encoder=node.input_encoder
             )
         self._results[input.id] = outputs
         ctx = GraphInvocationContext(
@@ -287,7 +289,8 @@ class Graph:
             fn_outputs = function_outputs.ser_outputs
             print(f"ran {node_name}: num outputs: {len(fn_outputs)}")
             if self._accumulator_values.get(node_name, None) is not None:
-                self._accumulator_values[node_name] = fn_outputs[-1].model_copy()
+                acc_output = fn_outputs[-1].copy()
+                self._accumulator_values[node_name] = acc_output
                 outputs[node_name] = []
             if fn_outputs:
                 outputs[node_name].extend(fn_outputs)
@@ -339,7 +342,7 @@ class Graph:
             raise ValueError(f"no results found for fn {fn_name} on graph {self.name}")
         fn = self.nodes[fn_name]
         fn_model = self.get_function(fn_name).get_output_model()
-        serializer = get_serializer(fn.encoder)
+        serializer = get_serializer(fn.output_encoder)
         outputs = []
         for result in results[fn_name]:
             payload_dict = serializer.deserialize(result.payload)

indexify/functions_sdk/graph_definition.py

@@ -14,7 +14,8 @@ class FunctionMetadata(BaseModel):
     reducer: bool = False
     image_name: str
     image_information: ImageInformation
-    encoder: str = "cloudpickle"
+    input_encoder: str = "cloudpickle"
+    output_encoder: str = "cloudpickle"
 
 
 class RouterMetadata(BaseModel):
@@ -24,7 +25,8 @@ class RouterMetadata(BaseModel):
     target_fns: List[str]
     image_name: str
     image_information: ImageInformation
-    encoder: str = "cloudpickle"
+    input_encoder: str = "cloudpickle"
+    output_encoder: str = "cloudpickle"
 
 
 class NodeMetadata(BaseModel):
@@ -49,12 +51,12 @@ class ComputeGraphMetadata(BaseModel):
     replaying: bool = False
 
     def get_input_payload_serializer(self):
-        return get_serializer(self.start_node.compute_fn.encoder)
+        return get_serializer(self.start_node.compute_fn.input_encoder)
 
     def get_input_encoder(self) -> str:
         if self.start_node.compute_fn:
-            return self.start_node.compute_fn.encoder
+            return self.start_node.compute_fn.input_encoder
         elif self.start_node.dynamic_router:
-            return self.start_node.dynamic_router.encoder
+            return self.start_node.dynamic_router.input_encoder
 
         raise ValueError("start node is not set on the graph")

indexify/functions_sdk/indexify_functions.py

@@ -83,7 +83,8 @@ class IndexifyFunction:
     image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
     accumulate: Optional[Type[Any]] = None
-    encoder: Optional[str] = "cloudpickle"
+    input_encoder: Optional[str] = "cloudpickle"
+    output_encoder: Optional[str] = "cloudpickle"
 
     def run(self, *args, **kwargs) -> Union[List[Any], Any]:
         pass
@@ -95,7 +96,7 @@ class IndexifyFunction:
 
     @classmethod
     def deserialize_output(cls, output: IndexifyData) -> Any:
-        serializer = get_serializer(cls.encoder)
+        serializer = get_serializer(cls.output_encoder)
         return serializer.deserialize(output.payload)
 
 
@@ -104,7 +105,8 @@ class IndexifyRouter:
     description: str = ""
     image: Optional[Image] = DEFAULT_IMAGE_3_10
     placement_constraints: List[PlacementConstraints] = []
-    encoder: Optional[str] = "cloudpickle"
+    input_encoder: Optional[str] = "cloudpickle"
+    output_encoder: Optional[str] = "cloudpickle"
 
     def run(self, *args, **kwargs) -> Optional[List[IndexifyFunction]]:
         pass
@@ -120,7 +122,8 @@ def indexify_router(
     description: Optional[str] = "",
     image: Optional[Image] = DEFAULT_IMAGE_3_10,
     placement_constraints: List[PlacementConstraints] = [],
-    encoder: Optional[str] = "cloudpickle",
+    input_encoder: Optional[str] = "cloudpickle",
+    output_encoder: Optional[str] = "cloudpickle",
 ):
     def construct(fn):
         # Get function signature using inspect.signature
@@ -144,7 +147,8 @@ def indexify_router(
             ),
             "image": image,
             "placement_constraints": placement_constraints,
-            "encoder": encoder,
+            "input_encoder": input_encoder,
+            "output_encoder": output_encoder,
             "run": run,
         }
 
@@ -158,7 +162,8 @@ def indexify_function(
     description: Optional[str] = "",
     image: Optional[Image] = DEFAULT_IMAGE_3_10,
     accumulate: Optional[Type[BaseModel]] = None,
-    encoder: Optional[str] = "cloudpickle",
+    input_encoder: Optional[str] = "cloudpickle",
+    output_encoder: Optional[str] = "cloudpickle",
     placement_constraints: List[PlacementConstraints] = [],
 ):
     def construct(fn):
@@ -184,7 +189,8 @@ def indexify_function(
             "image": image,
             "placement_constraints": placement_constraints,
             "accumulate": accumulate,
-            "encoder": encoder,
+            "input_encoder": input_encoder,
+            "output_encoder": output_encoder,
             "run": run,
         }
 
@@ -231,6 +237,18 @@ class IndexifyFunctionWrapper:
                 )
         return return_type
 
+    def get_input_types(self) -> Dict[str, Any]:
+        if not isinstance(self.indexify_function, IndexifyFunction):
+            raise TypeError("Input must be an instance of IndexifyFunction")
+
+        extract_method = self.indexify_function.run
+        type_hints = get_type_hints(extract_method)
+        return {
+            k: v
+            for k, v in type_hints.items()
+            if k != "return" and not is_pydantic_model_from_annotation(v)
+        }
+
     def run_router(
         self, input: Union[Dict, Type[BaseModel]]
     ) -> Tuple[List[str], Optional[str]]:
@@ -280,20 +298,17 @@ class IndexifyFunctionWrapper:
         self, name: str, input: IndexifyData, acc: Optional[Any] = None
     ) -> FunctionCallResult:
         input = self.deserialize_input(name, input)
-        serializer = get_serializer(self.indexify_function.encoder)
+        input_serializer = get_serializer(self.indexify_function.input_encoder)
+        output_serializer = get_serializer(self.indexify_function.output_encoder)
         if acc is not None:
-            acc = self.indexify_function.accumulate.model_validate(
-                serializer.deserialize(acc.payload)
-            )
+            acc = input_serializer.deserialize(acc.payload)
         if acc is None and self.indexify_function.accumulate is not None:
-            acc = self.indexify_function.accumulate.model_validate(
-                self.indexify_function.accumulate()
-            )
+            acc = self.indexify_function.accumulate()
         outputs, err = self.run_fn(input, acc=acc)
         ser_outputs = [
             IndexifyData(
-                payload=serializer.serialize(output),
-                encoder=self.indexify_function.encoder,
+                payload=output_serializer.serialize(output),
+                encoder=self.indexify_function.output_encoder,
             )
             for output in outputs
         ]

indexify/functions_sdk/object_serializer.py

@@ -1,7 +1,7 @@
-from typing import Any, List
+import json
+from typing import Any, List, Type
 
 import cloudpickle
-import jsonpickle
 
 
 def get_serializer(serializer_type: str) -> Any:
@@ -22,19 +22,29 @@ class JsonSerializer:
 
     @staticmethod
     def serialize(data: Any) -> str:
-        return jsonpickle.encode(data)
+        try:
+            return json.dumps(data)
+        except Exception as e:
+            raise ValueError(f"failed to serialize data with json: {e}")
 
     @staticmethod
     def deserialize(data: str) -> Any:
-        return jsonpickle.decode(data)
+        try:
+            if isinstance(data, bytes):
+                data = data.decode("utf-8")
+            return json.loads(data)
+        except Exception as e:
+            raise ValueError(f"failed to deserialize data with json: {e}")
 
     @staticmethod
     def serialize_list(data: List[Any]) -> str:
-        return jsonpickle.encode(data)
+        return json.dumps(data)
 
     @staticmethod
-    def deserialize_list(data: str) -> List[Any]:
-        return jsonpickle.decode(data)
+    def deserialize_list(data: str, t: Type) -> List[Any]:
+        if isinstance(data, bytes):
+            data = data.decode("utf-8")
+        return json.loads(data)
 
 
 class CloudPickleSerializer:

indexify/http_client.py

@@ -274,10 +274,10 @@ class IndexifyClient:
         self,
         graph: str,
         block_until_done: bool = False,
-        serializer: str = "cloudpickle",
+        input_encoding: str = "cloudpickle",
         **kwargs,
     ) -> str:
-        serializer = get_serializer(serializer)
+        serializer = get_serializer(input_encoding)
         ser_input = serializer.serialize(kwargs)
         params = {"block_until_finish": block_until_done}
         kwargs = {
@@ -351,11 +351,11 @@ class IndexifyClient:
         )
         response.raise_for_status()
         content_type = response.headers.get("Content-Type")
-        serializer = get_serializer(content_type)
-        decoded_response = serializer.deserialize(response.content)
-        return IndexifyData(
-            id=output_id, payload=decoded_response, encoder=serializer.encoding_type
-        )
+        if content_type == "application/octet-stream":
+            encoding = "cloudpickle"
+        else:
+            encoding = "json"
+        return IndexifyData(id=output_id, payload=response.content, encoder=encoding)
 
     def graph_outputs(
         self,

{indexify-0.2.32.dist-info → indexify-0.2.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.32
+Version: 0.2.34
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -18,11 +18,11 @@ Requires-Dist: cloudpickle (>=3.1.0,<4.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
 Requires-Dist: httpx[http2] (>=0,<1)
-Requires-Dist: jsonpickle (>=4.0.0,<5.0.0)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)
 Requires-Dist: pydantic (==2.10.2)
 Requires-Dist: pyyaml (>=6,<7)
 Requires-Dist: rich (>=13.9.2,<14.0.0)
+Requires-Dist: structlog (>=24.4.0,<25.0.0)
 Requires-Dist: typer (>=0.13.0,<0.14.0)
 Project-URL: Repository, https://github.com/tensorlakeai/indexify
 Description-Content-Type: text/markdown

{indexify-0.2.32.dist-info → indexify-0.2.34.dist-info}/RECORD

@@ -5,31 +5,31 @@ indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-
 indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
 indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
 indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
-indexify/executor/agent.py,sha256=Zt-KU2yrOuFuHu8CiOg-QmAzkNU1idcv5UZ3YN2YSDU,18649
+indexify/executor/agent.py,sha256=FxKEoabt9b3YR7cl26uEcGtOuusTJyCQjoA7zOy2aX8,14100
 indexify/executor/api_objects.py,sha256=mvmwGbK4paJNQGFvbtNHMPpiH_LpVhrlRnCcrqS6HOQ,859
-indexify/executor/downloader.py,sha256=wwkTdKRrDBHwKQOb_3uUb8pVl1tvg4QzOcYFnGQwNtA,4951
+indexify/executor/downloader.py,sha256=dHLxoBnX8-Bh4yZtFDYptZNF6rlVtmTk_70JK8Ect5w,4184
 indexify/executor/executor_tasks.py,sha256=A0UIEZ5VaB6zSkFQG81UmTW0E57MTYhGlaXuAbRV8lQ,1884
 indexify/executor/function_worker.py,sha256=wRW2-X9dNI80KhwTD1vD-pcyetsVKVs6vVdg7L7JjcQ,6462
 indexify/executor/image_dependency_installer.py,sha256=ct8GmzgkaPi6NAblk68IJJWo5MecIUubELotmSrgoRQ,1759
 indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
 indexify/executor/runtime_probes.py,sha256=mjw2_mGQ622wRT_39WPGGgPEZQTgtrf3-ICcUUZOeyg,2126
-indexify/executor/task_reporter.py,sha256=4unHxLUHedKwIoqO3e5YdDJpUe_pJng-vHReoqcPNNU,5141
+indexify/executor/task_reporter.py,sha256=XlEhNf_ScNnzG67zbtVwL7_9Bo8MvPZiHLI5UHymUnM,5305
 indexify/executor/task_store.py,sha256=JlRlWwAm4YjFRkTNRx-6GsUcmOzcyvzb5Csa5XDpRTI,3982
 indexify/functions_sdk/data_objects.py,sha256=wXbUa9hjU6rsXmmk19vQ5Kixf3FsI59VBWPNmHasAX0,854
-indexify/functions_sdk/graph.py,sha256=ewm-XUlqIXA0RSl1JxU91yT2XXK3WRqrD9_YSvlK_44,12884
-indexify/functions_sdk/graph_definition.py,sha256=UeC0PMchgZgo2T0Goog0SW1lIRdFf6ug4iHOS9ItRvI,1622
+indexify/functions_sdk/graph.py,sha256=pca6LKbPAfYT-BMGTVL5sbPuuOhvvr_Yve6u3H0NWVk,13126
+indexify/functions_sdk/graph_definition.py,sha256=rJmGcy9u5A_Sme6Ol33NsCnSKQVjyUfeN9LnH3bU88Y,1732
 indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
 indexify/functions_sdk/image.py,sha256=QK0H6KxLWriB_z4M0kunKzzHdHxYLWL670RPYgYuf_8,1762
-indexify/functions_sdk/indexify_functions.py,sha256=cPUN7Zlgkwl6YUOuhgKogPdoBakQuf-VspQvU3IDmfU,10422
+indexify/functions_sdk/indexify_functions.py,sha256=J-etsuC_IGNCsUfbsJMrolyZVHZsurIMC5IaMQGiZnM,11187
 indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
-indexify/functions_sdk/object_serializer.py,sha256=pOgUOWbRNRix9uZT0aQn0LTCnJCeMNGO1nAE0jAybmg,1546
+indexify/functions_sdk/object_serializer.py,sha256=R58ALsl2Lb87ii6km4D6hBBsqRs_CHNISxhUICE2d9o,1931
 indexify/functions_sdk/pipeline.py,sha256=KmxZE8eBFAQ4bbEcYURXXR26HSyoAT3O6iu9H38-OXE,974
-indexify/http_client.py,sha256=5d70A6j2vJBn3OfUVjrQvnPml07p7fxOXWBUAUfAkiU,15915
+indexify/http_client.py,sha256=iLafZagCFnlTS6uHfOjInogjg0uXW_zXEspIN7ttB5I,15903
 indexify/remote_graph.py,sha256=aox9NibZIU8YDiP92syerFSjiZc2jVPkF6g2kfqOvCA,5003
 indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
 indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
-indexify-0.2.32.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.32.dist-info/METADATA,sha256=3-sKMgJvXaZByfEAUE6h2fBtymF3pw7MXhXvv9gwhnU,6196
-indexify-0.2.32.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-indexify-0.2.32.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.32.dist-info/RECORD,,
+indexify-0.2.34.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.34.dist-info/METADATA,sha256=GMGPVYUyKI8b02nA7b5xkG7d8gvyyqlxEOw75uCTkYI,6197
+indexify-0.2.34.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+indexify-0.2.34.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.34.dist-info/RECORD,,