indexify 0.2.48__py3-none-any.whl → 0.3.1__py3-none-any.whl

indexify/function_executor/proto/function_executor_pb2.py

@@ -24,7 +24,7 @@ _sym_db = _symbol_database.Default()
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n8indexify/function_executor/proto/function_executor.proto\x12\x19\x66unction_executor_service"i\n\x10SerializedObject\x12\x0f\n\x05\x62ytes\x18\x01 \x01(\x0cH\x00\x12\x10\n\x06string\x18\x02 \x01(\tH\x00\x12\x19\n\x0c\x63ontent_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x06\n\x04\x64\x61taB\x0f\n\r_content_type"\x88\x02\n\x11InitializeRequest\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x03\x88\x01\x01\x12?\n\x05graph\x18\x07 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x04\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x08\n\x06_graph"6\n\x12InitializeResponse\x12\x14\n\x07success\x18\x01 \x01(\x08H\x00\x88\x01\x01\x42\n\n\x08_success"\x80\x01\n\x19SetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\x1c\n\x1aSetInvocationStateResponse"5\n\x19GetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_key"\x81\x01\n\x1aGetInvocationStateResponse\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\xf7\x01\n\x16InvocationStateRequest\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07task_id\x18\x02 \x01(\tH\x02\x88\x01\x01\x12\x43\n\x03set\x18\x03 \x01(\x0b\x32\x34.function_executor_service.SetInvocationStateRequestH\x00\x12\x43\n\x03get\x18\x04 \x01(\x0b\x32\x34.function_executor_service.GetInvocationStateRequestH\x00\x42\t\n\x07requestB\r\n\x0b_request_idB\n\n\x08_task_id"\xfb\x01\n\x17InvocationStateResponse\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07success\x18\x02 \x01(\x08H\x02\x88\x01\x01\x12\x44\n\x03set\x18\x03 \x01(\x0b\x32\x35.function_executor_service.SetInvocationStateResponseH\x00\x12\x44\n\x03get\x18\x04 \x01(\x0b\x32\x35.function_executor_service.GetInvocationStateResponseH\x00\x42\n\n\x08responseB\r\n\x0b_request_idB\n\n\x08_success"N\n\x0e\x46unctionOutput\x12<\n\x07outputs\x18\x01 \x03(\x0b\x32+.function_executor_service.SerializedObject"\x1d\n\x0cRouterOutput\x12\r\n\x05\x65\x64ges\x18\x01 \x03(\t"\xb0\x02\n\x0eRunTaskRequest\x12 \n\x13graph_invocation_id\x18\x04 \x01(\tH\x00\x88\x01\x01\x12\x14\n\x07task_id\x18\x06 \x01(\tH\x01\x88\x01\x01\x12H\n\x0e\x66unction_input\x18\t \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x02\x88\x01\x01\x12M\n\x13\x66unction_init_value\x18\n \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x03\x88\x01\x01\x42\x16\n\x14_graph_invocation_idB\n\n\x08_task_idB\x11\n\x0f_function_inputB\x16\n\x14_function_init_value"\xf1\x02\n\x0fRunTaskResponse\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12G\n\x0f\x66unction_output\x18\x02 \x01(\x0b\x32).function_executor_service.FunctionOutputH\x01\x88\x01\x01\x12\x43\n\rrouter_output\x18\x03 \x01(\x0b\x32\'.function_executor_service.RouterOutputH\x02\x88\x01\x01\x12\x13\n\x06stdout\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06stderr\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x17\n\nis_reducer\x18\x06 \x01(\x08H\x05\x88\x01\x01\x12\x14\n\x07success\x18\x07 \x01(\x08H\x06\x88\x01\x01\x42\n\n\x08_task_idB\x12\n\x10_function_outputB\x10\n\x0e_router_outputB\t\n\x07_stdoutB\t\n\x07_stderrB\r\n\x0b_is_reducerB\n\n\x08_success2\xf2\x02\n\x10\x46unctionExecutor\x12i\n\ninitialize\x12,.function_executor_service.InitializeRequest\x1a-.function_executor_service.InitializeResponse\x12\x8f\x01\n"initialize_invocation_state_server\x12\x32.function_executor_service.InvocationStateResponse\x1a\x31.function_executor_service.InvocationStateRequest(\x01\x30\x01\x12\x61\n\x08run_task\x12).function_executor_service.RunTaskRequest\x1a*.function_executor_service.RunTaskResponseb\x06proto3'
+    b'\n8indexify/function_executor/proto/function_executor.proto\x12\x19\x66unction_executor_service"i\n\x10SerializedObject\x12\x0f\n\x05\x62ytes\x18\x01 \x01(\x0cH\x00\x12\x10\n\x06string\x18\x02 \x01(\tH\x00\x12\x19\n\x0c\x63ontent_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x42\x06\n\x04\x64\x61taB\x0f\n\r_content_type"\x88\x02\n\x11InitializeRequest\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x03\x88\x01\x01\x12?\n\x05graph\x18\x07 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x04\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x08\n\x06_graph"f\n\x12InitializeResponse\x12\x14\n\x07success\x18\x01 \x01(\x08H\x00\x88\x01\x01\x12\x1b\n\x0e\x63ustomer_error\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\n\n\x08_successB\x11\n\x0f_customer_error"\x80\x01\n\x19SetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\x1c\n\x1aSetInvocationStateResponse"5\n\x19GetInvocationStateRequest\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x06\n\x04_key"\x81\x01\n\x1aGetInvocationStateResponse\x12\x10\n\x03key\x18\x01 \x01(\tH\x00\x88\x01\x01\x12?\n\x05value\x18\x02 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x01\x88\x01\x01\x42\x06\n\x04_keyB\x08\n\x06_value"\xf7\x01\n\x16InvocationStateRequest\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07task_id\x18\x02 \x01(\tH\x02\x88\x01\x01\x12\x43\n\x03set\x18\x03 \x01(\x0b\x32\x34.function_executor_service.SetInvocationStateRequestH\x00\x12\x43\n\x03get\x18\x04 \x01(\x0b\x32\x34.function_executor_service.GetInvocationStateRequestH\x00\x42\t\n\x07requestB\r\n\x0b_request_idB\n\n\x08_task_id"\xfb\x01\n\x17InvocationStateResponse\x12\x17\n\nrequest_id\x18\x01 \x01(\tH\x01\x88\x01\x01\x12\x14\n\x07success\x18\x02 \x01(\x08H\x02\x88\x01\x01\x12\x44\n\x03set\x18\x03 \x01(\x0b\x32\x35.function_executor_service.SetInvocationStateResponseH\x00\x12\x44\n\x03get\x18\x04 \x01(\x0b\x32\x35.function_executor_service.GetInvocationStateResponseH\x00\x42\n\n\x08responseB\r\n\x0b_request_idB\n\n\x08_success"N\n\x0e\x46unctionOutput\x12<\n\x07outputs\x18\x01 \x03(\x0b\x32+.function_executor_service.SerializedObject"\x1d\n\x0cRouterOutput\x12\r\n\x05\x65\x64ges\x18\x01 \x03(\t"\xda\x03\n\x0eRunTaskRequest\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x04 \x01(\tH\x03\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x14\n\x07task_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12H\n\x0e\x66unction_input\x18\x07 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x06\x88\x01\x01\x12M\n\x13\x66unction_init_value\x18\x08 \x01(\x0b\x32+.function_executor_service.SerializedObjectH\x07\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\n\n\x08_task_idB\x11\n\x0f_function_inputB\x16\n\x14_function_init_value"\xf1\x02\n\x0fRunTaskResponse\x12\x14\n\x07task_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12G\n\x0f\x66unction_output\x18\x02 \x01(\x0b\x32).function_executor_service.FunctionOutputH\x01\x88\x01\x01\x12\x43\n\rrouter_output\x18\x03 \x01(\x0b\x32\'.function_executor_service.RouterOutputH\x02\x88\x01\x01\x12\x13\n\x06stdout\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x13\n\x06stderr\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x17\n\nis_reducer\x18\x06 \x01(\x08H\x05\x88\x01\x01\x12\x14\n\x07success\x18\x07 \x01(\x08H\x06\x88\x01\x01\x42\n\n\x08_task_idB\x12\n\x10_function_outputB\x10\n\x0e_router_outputB\t\n\x07_stdoutB\t\n\x07_stderrB\r\n\x0b_is_reducerB\n\n\x08_success2\xf2\x02\n\x10\x46unctionExecutor\x12i\n\ninitialize\x12,.function_executor_service.InitializeRequest\x1a-.function_executor_service.InitializeResponse\x12\x8f\x01\n"initialize_invocation_state_server\x12\x32.function_executor_service.InvocationStateResponse\x1a\x31.function_executor_service.InvocationStateRequest(\x01\x30\x01\x12\x61\n\x08run_task\x12).function_executor_service.RunTaskRequest\x1a*.function_executor_service.RunTaskResponseb\x06proto3'
 )
 
 _globals = globals()
@@ -39,27 +39,27 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_INITIALIZEREQUEST"]._serialized_start = 195
     _globals["_INITIALIZEREQUEST"]._serialized_end = 459
     _globals["_INITIALIZERESPONSE"]._serialized_start = 461
-    _globals["_INITIALIZERESPONSE"]._serialized_end = 515
-    _globals["_SETINVOCATIONSTATEREQUEST"]._serialized_start = 518
-    _globals["_SETINVOCATIONSTATEREQUEST"]._serialized_end = 646
-    _globals["_SETINVOCATIONSTATERESPONSE"]._serialized_start = 648
-    _globals["_SETINVOCATIONSTATERESPONSE"]._serialized_end = 676
-    _globals["_GETINVOCATIONSTATEREQUEST"]._serialized_start = 678
-    _globals["_GETINVOCATIONSTATEREQUEST"]._serialized_end = 731
-    _globals["_GETINVOCATIONSTATERESPONSE"]._serialized_start = 734
-    _globals["_GETINVOCATIONSTATERESPONSE"]._serialized_end = 863
-    _globals["_INVOCATIONSTATEREQUEST"]._serialized_start = 866
-    _globals["_INVOCATIONSTATEREQUEST"]._serialized_end = 1113
-    _globals["_INVOCATIONSTATERESPONSE"]._serialized_start = 1116
-    _globals["_INVOCATIONSTATERESPONSE"]._serialized_end = 1367
-    _globals["_FUNCTIONOUTPUT"]._serialized_start = 1369
-    _globals["_FUNCTIONOUTPUT"]._serialized_end = 1447
-    _globals["_ROUTEROUTPUT"]._serialized_start = 1449
-    _globals["_ROUTEROUTPUT"]._serialized_end = 1478
-    _globals["_RUNTASKREQUEST"]._serialized_start = 1481
-    _globals["_RUNTASKREQUEST"]._serialized_end = 1785
-    _globals["_RUNTASKRESPONSE"]._serialized_start = 1788
-    _globals["_RUNTASKRESPONSE"]._serialized_end = 2157
-    _globals["_FUNCTIONEXECUTOR"]._serialized_start = 2160
-    _globals["_FUNCTIONEXECUTOR"]._serialized_end = 2530
+    _globals["_INITIALIZERESPONSE"]._serialized_end = 563
+    _globals["_SETINVOCATIONSTATEREQUEST"]._serialized_start = 566
+    _globals["_SETINVOCATIONSTATEREQUEST"]._serialized_end = 694
+    _globals["_SETINVOCATIONSTATERESPONSE"]._serialized_start = 696
+    _globals["_SETINVOCATIONSTATERESPONSE"]._serialized_end = 724
+    _globals["_GETINVOCATIONSTATEREQUEST"]._serialized_start = 726
+    _globals["_GETINVOCATIONSTATEREQUEST"]._serialized_end = 779
+    _globals["_GETINVOCATIONSTATERESPONSE"]._serialized_start = 782
+    _globals["_GETINVOCATIONSTATERESPONSE"]._serialized_end = 911
+    _globals["_INVOCATIONSTATEREQUEST"]._serialized_start = 914
+    _globals["_INVOCATIONSTATEREQUEST"]._serialized_end = 1161
+    _globals["_INVOCATIONSTATERESPONSE"]._serialized_start = 1164
+    _globals["_INVOCATIONSTATERESPONSE"]._serialized_end = 1415
+    _globals["_FUNCTIONOUTPUT"]._serialized_start = 1417
+    _globals["_FUNCTIONOUTPUT"]._serialized_end = 1495
+    _globals["_ROUTEROUTPUT"]._serialized_start = 1497
+    _globals["_ROUTEROUTPUT"]._serialized_end = 1526
+    _globals["_RUNTASKREQUEST"]._serialized_start = 1529
+    _globals["_RUNTASKREQUEST"]._serialized_end = 2003
+    _globals["_RUNTASKRESPONSE"]._serialized_start = 2006
+    _globals["_RUNTASKRESPONSE"]._serialized_end = 2375
+    _globals["_FUNCTIONEXECUTOR"]._serialized_start = 2378
+    _globals["_FUNCTIONEXECUTOR"]._serialized_end = 2748
 # @@protoc_insertion_point(module_scope)

indexify/function_executor/proto/function_executor_pb2.pyi

@@ -34,23 +34,27 @@ class InitializeRequest(_message.Message):
     GRAPH_FIELD_NUMBER: _ClassVar[int]
     namespace: str
     graph_name: str
-    graph_version: int
+    graph_version: str
     function_name: str
     graph: SerializedObject
     def __init__(
         self,
         namespace: _Optional[str] = ...,
         graph_name: _Optional[str] = ...,
-        graph_version: _Optional[int] = ...,
+        graph_version: _Optional[str] = ...,
         function_name: _Optional[str] = ...,
         graph: _Optional[_Union[SerializedObject, _Mapping]] = ...,
     ) -> None: ...
 
 class InitializeResponse(_message.Message):
-    __slots__ = ("success",)
+    __slots__ = ("success", "customer_error")
     SUCCESS_FIELD_NUMBER: _ClassVar[int]
+    CUSTOMER_ERROR_FIELD_NUMBER: _ClassVar[int]
     success: bool
-    def __init__(self, success: bool = ...) -> None: ...
+    customer_error: str
+    def __init__(
+        self, success: bool = ..., customer_error: _Optional[str] = ...
+    ) -> None: ...
 
 class SetInvocationStateRequest(_message.Message):
     __slots__ = ("key", "value")
@@ -138,21 +142,37 @@ class RouterOutput(_message.Message):
 
 class RunTaskRequest(_message.Message):
     __slots__ = (
+        "namespace",
+        "graph_name",
+        "graph_version",
+        "function_name",
         "graph_invocation_id",
         "task_id",
         "function_input",
         "function_init_value",
     )
+    NAMESPACE_FIELD_NUMBER: _ClassVar[int]
+    GRAPH_NAME_FIELD_NUMBER: _ClassVar[int]
+    GRAPH_VERSION_FIELD_NUMBER: _ClassVar[int]
+    FUNCTION_NAME_FIELD_NUMBER: _ClassVar[int]
     GRAPH_INVOCATION_ID_FIELD_NUMBER: _ClassVar[int]
     TASK_ID_FIELD_NUMBER: _ClassVar[int]
     FUNCTION_INPUT_FIELD_NUMBER: _ClassVar[int]
     FUNCTION_INIT_VALUE_FIELD_NUMBER: _ClassVar[int]
+    namespace: str
+    graph_name: str
+    graph_version: str
+    function_name: str
     graph_invocation_id: str
     task_id: str
     function_input: SerializedObject
     function_init_value: SerializedObject
     def __init__(
         self,
+        namespace: _Optional[str] = ...,
+        graph_name: _Optional[str] = ...,
+        graph_version: _Optional[str] = ...,
+        function_name: _Optional[str] = ...,
         graph_invocation_id: _Optional[str] = ...,
         task_id: _Optional[str] = ...,
         function_input: _Optional[_Union[SerializedObject, _Mapping]] = ...,

indexify/function_executor/server.py

@@ -2,11 +2,9 @@ from concurrent.futures import ThreadPoolExecutor
 
 import grpc
 
-from .function_executor_service import FunctionExecutorService
 from .proto.configuration import GRPC_SERVER_OPTIONS
-from .proto.function_executor_pb2_grpc import (
-    add_FunctionExecutorServicer_to_server,
-)
+from .proto.function_executor_pb2_grpc import add_FunctionExecutorServicer_to_server
+from .service import Service
 
 # Temporary limit until we have a better way to control this.
 # This limits the number of concurrent tasks that Function Executor can run.
@@ -14,9 +12,9 @@ MAX_RPC_CONCURRENCY = 100
 
 
 class Server:
-    def __init__(self, server_address: str, service: FunctionExecutorService):
+    def __init__(self, server_address: str, service: Service):
         self._server_address: str = server_address
-        self._service: FunctionExecutorService = service
+        self._service: Service = service
 
     def run(self):
         """Runs Function Executor Service at the configured address."""

indexify/function_executor/{function_executor_service.py → service.py}

@@ -2,21 +2,15 @@ from typing import Iterator, Optional, Union
 
 import grpc
 import structlog
-
-from indexify.functions_sdk.indexify_functions import (
-    IndexifyFunction,
-    IndexifyRouter,
-)
-from indexify.functions_sdk.object_serializer import get_serializer
+from tensorlake.functions_sdk.functions import TensorlakeCompute
+from tensorlake.functions_sdk.object_serializer import get_serializer
 
 from .handlers.run_function.handler import Handler as RunTaskHandler
 from .handlers.run_function.request_validator import (
     RequestValidator as RunTaskRequestValidator,
 )
 from .initialize_request_validator import InitializeRequestValidator
-from .invocation_state.invocation_state_proxy_server import (
-    InvocationStateProxyServer,
-)
+from .invocation_state.invocation_state_proxy_server import InvocationStateProxyServer
 from .invocation_state.proxied_invocation_state import ProxiedInvocationState
 from .proto.function_executor_pb2 import (
     InitializeRequest,
@@ -28,14 +22,14 @@ from .proto.function_executor_pb2 import (
 from .proto.function_executor_pb2_grpc import FunctionExecutorServicer
 
 
-class FunctionExecutorService(FunctionExecutorServicer):
+class Service(FunctionExecutorServicer):
     def __init__(self):
         self._logger = structlog.get_logger(module=__name__)
         self._namespace: Optional[str] = None
         self._graph_name: Optional[str] = None
-        self._graph_version: Optional[int] = None
+        self._graph_version: Optional[str] = None
         self._function_name: Optional[str] = None
-        self._function: Optional[Union[IndexifyFunction, IndexifyRouter]] = None
+        self._function: Optional[Union[TensorlakeCompute, TensorlakeCompute]] = None
         self._invocation_state_proxy_server: Optional[InvocationStateProxyServer] = None
 
     def initialize(
@@ -55,24 +49,22 @@ class FunctionExecutorService(FunctionExecutorServicer):
         # implementing smart caching in customer code. E.g. load a model into GPU only once and
         # share the model's file descriptor between all tasks or download function configuration
         # only once.
-        graph_serializer = get_serializer(request.graph.content_type)
-
-        try:
-            graph = graph_serializer.deserialize(request.graph.bytes)
-        except Exception as e:
-            self._logger.error(f"Caught exception {e}")
-            return InitializeResponse(success=False)
-
-        self._function = graph_serializer.deserialize(graph[request.function_name])
-
         self._logger = self._logger.bind(
             namespace=request.namespace,
             graph_name=request.graph_name,
-            graph_version=str(request.graph_version),
+            graph_version=request.graph_version,
             function_name=request.function_name,
         )
-        self._logger.info("initialized function executor service")
+        graph_serializer = get_serializer(request.graph.content_type)
+        try:
+            # Process user controlled input in a try-except block to not treat errors here as our
+            # internal platform errors.
+            graph = graph_serializer.deserialize(request.graph.bytes)
+            self._function = graph_serializer.deserialize(graph[request.function_name])
+        except Exception as e:
+            return InitializeResponse(success=False, customer_error=str(e))
 
+        self._logger.info("initialized function executor service")
         return InitializeResponse(success=True)
 
     def initialize_invocation_state_server(
@@ -94,6 +86,25 @@ class FunctionExecutorService(FunctionExecutorServicer):
         # If our code raises an exception the grpc framework converts it into GRPC_STATUS_UNKNOWN
         # error with the exception message. Differentiating errors is not needed for now.
         RunTaskRequestValidator(request=request).check()
+
+        # Fail with internal error as this happened due to wrong task routing to this Server.
+        if request.namespace != self._namespace:
+            raise ValueError(
+                f"This Function Executor is not initialized for this namespace {request.namespace}"
+            )
+        if request.graph_name != self._graph_name:
+            raise ValueError(
+                f"This Function Executor is not initialized for this graph {request.graph_name}"
+            )
+        if request.graph_version != self._graph_version:
+            raise ValueError(
+                f"This Function Executor is not initialized for this graph version {request.graph_version}"
+            )
+        if request.function_name != self._function_name:
+            raise ValueError(
+                f"This Function Executor is not initialized for this function {request.function_name}"
+            )
+
         return RunTaskHandler(
             request=request,
             graph_name=self._graph_name,

indexify/utils/README.md

@@ -0,0 +1,3 @@
+## Overview
+
+Put here common functionality that is useful for code written in Python.

indexify/{common_util.py → utils/http_client.py}

@@ -7,7 +7,7 @@ from httpx import AsyncClient, Client
 
 def get_httpx_client(
     config_path: Optional[str] = None, make_async: Optional[bool] = False
-) -> AsyncClient | Client:
+) -> Union[AsyncClient, Client]:
     """
     Creates and returns an httpx.Client instance, optionally configured with TLS settings from a YAML config file.
 
@@ -54,7 +54,7 @@ def get_sync_or_async_client(
     cert_path: Optional[str] = None,
     key_path: Optional[str] = None,
     ca_bundle_path: Optional[str] = None,
-) -> AsyncClient | Client:
+) -> Union[AsyncClient, Client]:
     """
     Creates and returns either a synchronous or asynchronous httpx client with optional TLS configuration.
 

indexify/{logging.py → utils/logging.py}

@@ -2,6 +2,9 @@ import logging
 import sys
 
 import structlog
+from structlog.contextvars import merge_contextvars
+from structlog.dev import ConsoleRenderer, set_exc_info
+from structlog.processors import StackInfoRenderer, TimeStamper, add_log_level
 
 # Using this module allows us to be consistent with the logging configuration across all Python programs.
 
@@ -24,9 +27,40 @@ def configure_logging_early():
     )
 
 
-def configure_production_logging():
+def configure_development_mode_logging():
     processors = [
-        structlog.processors.dict_tracebacks,
+        structlog_suppressor,
+        merge_contextvars,
+        add_log_level,
+        StackInfoRenderer(),
+        set_exc_info,
+        TimeStamper(fmt="%Y-%m-%d %H:%M:%S", utc=False),
+        ConsoleRenderer(),
+    ]
+    structlog.configure(
+        processors=processors,
+    )
+
+
+def configure_production_mode_logging():
+    processors = [
+        structlog_suppressor,
+        structlog.processors.format_exc_info,
         structlog.processors.JSONRenderer(),
     ]
     structlog.configure(processors=processors)
+
+
+_suppress_logging = False
+
+
+def structlog_suppressor(logger, name, event_dict):
+    global _suppress_logging
+    return None if _suppress_logging else event_dict
+
+
+def suppress():
+    """Sets the log level for the root logger to the new level."""
+    global _suppress_logging
+    _suppress_logging = True
+    logging.getLogger().setLevel(logging.CRITICAL)

indexify-0.3.1.dist-info/METADATA

@@ -0,0 +1,38 @@
+Metadata-Version: 2.3
+Name: indexify
+Version: 0.3.1
+Summary: Open Source Indexify components and helper tools
+Home-page: https://github.com/tensorlakeai/indexify
+License: Apache 2.0
+Author: Tensorlake Inc.
+Author-email: support@tensorlake.ai
+Requires-Python: >=3.9,<4.0
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: grpcio (==1.68.1)
+Requires-Dist: grpcio-tools (==1.68.1)
+Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
+Requires-Dist: httpx[http2] (>=0.28.1,<0.29.0)
+Requires-Dist: nanoid (>=2.0.0,<3.0.0)
+Requires-Dist: pydantic (==2.10.4)
+Requires-Dist: pyyaml (>=6,<7)
+Requires-Dist: rich (>=13.9.2,<14.0.0)
+Requires-Dist: structlog (>=24.4.0,<25.0.0)
+Requires-Dist: tensorlake (>=0.1.7,<0.2.0)
+Requires-Dist: typer (>=0.12,<0.13)
+Project-URL: Repository, https://github.com/tensorlakeai/indexify
+Description-Content-Type: text/markdown
+
+## Overview
+
+This a package with all Open Source Indexify components and helper tools
+available via a CLI.
+
+The CLI allows to:
+* Setup a local or a distributed Indexify cluster.
+* Build container images for Indexify functions.

indexify-0.3.1.dist-info/RECORD

@@ -0,0 +1,44 @@
+indexify/cli/cli.py,sha256=II73vJlVeEy51pPLnrnBcYJbZQxCPItkFRFEN8MD37c,9122
+indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
+indexify/executor/api_objects.py,sha256=QeXOZpS7nkVA_dsRQFlS3EuMvqFvLGaw1Spcr0c-5lA,1015
+indexify/executor/downloader.py,sha256=Vrq1dAW4BifG62tlqFnImiMxdezIgOZcByTRnhDsnnw,6457
+indexify/executor/executor.py,sha256=lln0p0cVN2aNY3bsHFp7Phbaa2q9gnQABi1xEPxo1Nc,5859
+indexify/executor/function_executor/function_executor.py,sha256=a9pAWQWDLJL-i5bp1PEMD2eMYOTvXAh_1jQsXcL_JCQ,5226
+indexify/executor/function_executor/function_executor_state.py,sha256=_85dpaudYM0sekOqwjMxKGdK7MNQdTGUhHi67sqVHyY,2853
+indexify/executor/function_executor/invocation_state_client.py,sha256=jxElxnYFfdwCxKfKvQsjLIE-Z_1386wxednJ2pGKSL0,8581
+indexify/executor/function_executor/server/function_executor_server.py,sha256=_DLivLDikupZusRk8gVWDk7fWPT9XjZ4un1yWSlOObs,883
+indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=pGbJMQfC5TNvyWOs6VDKdqd2PK5OHQh5_wSDP-E7DbI,1677
+indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=DxQMwAvQi03OGo2sFruUrxwcMPG8In4C62L3sped2FI,694
+indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=cVr2wXQL5zVlq-Ep44ZBXBFCQbrbZSMscjRmzVDakvk,4295
+indexify/executor/function_executor/single_task_runner.py,sha256=NGji5iIoA9vxRzSvyUTL3Gs5g4HMq5b1LYN5ZsENcls,6244
+indexify/executor/function_executor/task_input.py,sha256=8pBRuaQbBBPrDWwByOKZlOaXu6UDmJUO5w-6XxAYSgY,582
+indexify/executor/function_executor/task_output.py,sha256=3-qcT1aigmOFEp6QSJtUcQfqeJfrxvo265qgk9WAUVg,1064
+indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
+indexify/executor/task_fetcher.py,sha256=Cpm-LRZz1MSfW-xHJbIaKc1wxzRhIrud9GTOIxWODt0,2883
+indexify/executor/task_reporter.py,sha256=eueJNHYJIwAm8lY8SQJrUOWC-Nk-DPesDis1YTR4k7c,6643
+indexify/executor/task_runner.py,sha256=RSFeJYhQ_agXXPBm8u13HErSUPZGsCGwV1stgS7g258,5035
+indexify/function_executor/README.md,sha256=TAcbYYe6noUMriaQabJAaZ8a2SAYzlXl7hIdTfULqYY,950
+indexify/function_executor/handlers/run_function/function_inputs_loader.py,sha256=StSJCp-kkOWUuUIzONnoqtcB37nzBYGcIVy2oTEwOIQ,1588
+indexify/function_executor/handlers/run_function/handler.py,sha256=DY_aIR8MZsa1cnFxbVEHhcxQSqBwW4cYRN9NhzuDDFo,5090
+indexify/function_executor/handlers/run_function/request_validator.py,sha256=GS7_zxAlyNg0vpUrhot-DpVWFyI9Y9P8jvb7UuO4b1A,856
+indexify/function_executor/handlers/run_function/response_helper.py,sha256=FJVl-LyoRBx9HRaOo_umveiWzuh4anO7aXUmEMEki20,3074
+indexify/function_executor/initialize_request_validator.py,sha256=agYWGRHUC-7IuYtm9NPjcVo5SVOWM4PxewA93dqdkJM,682
+indexify/function_executor/invocation_state/invocation_state_proxy_server.py,sha256=YEnxvWV8NFVK_CxHI_C-BNizuagalY3ieN62IOxNttU,7023
+indexify/function_executor/invocation_state/proxied_invocation_state.py,sha256=rjZ2192bSXonNKkaDaByeOL-wVx6x6Qrtio0UVzGSz8,944
+indexify/function_executor/invocation_state/response_validator.py,sha256=05Y9_5s010owaDb6OUia9W9Alx1r1zJ179HboEkT-Xk,896
+indexify/function_executor/main.py,sha256=uOzVUhl4LzGJ9Cw3kyyFddaEIMxRETEKFHPnvJVtc50,1152
+indexify/function_executor/proto/configuration.py,sha256=oHp_Hh1_XdL4v7M2Ok-Z9r7ClTxiWdSP1Ip-NK4Ee6E,1096
+indexify/function_executor/proto/function_executor.proto,sha256=Ae6vMCNnacHWmrF6aZVPzIVVzG5sJgP7LXV4vIUBKZM,3978
+indexify/function_executor/proto/function_executor_pb2.py,sha256=X2G2xZQy8VRWqlDKV2dUgD3CkEecFx6Jp2EuTuXrx9o,7358
+indexify/function_executor/proto/function_executor_pb2.pyi,sha256=TQEOl3W1s4fxtp1g8Hi5S-keMF25DY5-6miyDVHhBkE,7168
+indexify/function_executor/proto/function_executor_pb2_grpc.py,sha256=xrqIDrcl1PTyQTGAAGrgSDKEtZilC445Op0tJ7LmS5Q,8716
+indexify/function_executor/proto/message_validator.py,sha256=OKXPYgy5L9c-spnV9Zjv7PA_yxwzvykfhbYylYx8cwQ,1456
+indexify/function_executor/server.py,sha256=tJzUy_v4BT8Le9G3hgtiuDJo9YVFkAU2dVISSsX36II,1061
+indexify/function_executor/service.py,sha256=3JYcMcWiCgvxXhRPReUUBjTBNLIqUbhhsGZIsqIcVw4,5343
+indexify/utils/README.md,sha256=2g8-H9GopacOW4YrViZc0QsaJPtK-Fox7GyfX01kcDk,86
+indexify/utils/http_client.py,sha256=deMlmAu4E_ZXV3blCdWNag3uO_cyD-GsMZjFFmO5r7s,3541
+indexify/utils/logging.py,sha256=c6NwzY7uVHMRJc8f2w2KF36rNkeZVoQfGdq7suIg9s8,2025
+indexify-0.3.1.dist-info/METADATA,sha256=4UcqDjQRLNcVGFezaIROqWdtJxCsPP0hLq8-hlFvDn0,1383
+indexify-0.3.1.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
+indexify-0.3.1.dist-info/entry_points.txt,sha256=pJG0YRnypesbiNJHuObfHEkjk0p_ZvEDTyyTH0kGVTY,108
+indexify-0.3.1.dist-info/RECORD,,

{indexify-0.2.48.dist-info → indexify-0.3.1.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: poetry-core 2.0.0
 Root-Is-Purelib: true
 Tag: py3-none-any

indexify-0.3.1.dist-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+function-executor=indexify.function_executor.main:main
+indexify-cli=indexify.cli.cli:app
+

indexify/__init__.py

@@ -1,31 +0,0 @@
-from . import data_loaders
-from .functions_sdk.graph import Graph
-from .functions_sdk.image import Image
-from .functions_sdk.indexify_functions import (
-    IndexifyFunction,
-    IndexifyRouter,
-    get_ctx,
-    indexify_function,
-    indexify_router,
-)
-from .functions_sdk.pipeline import Pipeline
-from .http_client import IndexifyClient
-from .remote_graph import RemoteGraph
-from .remote_pipeline import RemotePipeline
-from .settings import DEFAULT_SERVICE_URL
-
-__all__ = [
-    "data_loaders",
-    "Graph",
-    "RemoteGraph",
-    "Pipeline",
-    "RemotePipeline",
-    "Image",
-    "indexify_function",
-    "get_ctx",
-    "IndexifyFunction",
-    "IndexifyRouter",
-    "indexify_router",
-    "DEFAULT_SERVICE_URL",
-    "IndexifyClient",
-]

indexify/data_loaders/__init__.py

@@ -1,58 +0,0 @@
-import hashlib
-import mimetypes
-import os
-from abc import ABC, abstractmethod
-from typing import List
-
-from pydantic import BaseModel
-
-
-class FileMetadata(BaseModel):
-    path: str
-    file_size: int
-    mime_type: str
-    md5_hash: str
-    created_at: int
-    updated_at: int
-
-    @classmethod
-    def from_path(cls, path: str):
-        file_size = os.path.getsize(path)
-        mime_type = mimetypes.guess_type(path)[0]
-
-        # Compute MD5 hash
-        hash_md5 = hashlib.md5()
-        with open(path, "rb") as f:
-            for chunk in iter(lambda: f.read(4096), b""):
-                hash_md5.update(chunk)
-        md5_hash = hash_md5.hexdigest()
-
-        created_at = int(os.path.getctime(path))
-        updated_at = int(os.path.getmtime(path))
-
-        return cls(
-            path=path,
-            file_size=file_size,
-            mime_type=str(mime_type),
-            md5_hash=md5_hash,
-            created_at=created_at,
-            updated_at=updated_at,
-        )
-
-
-class DataLoader(ABC):
-    @abstractmethod
-    def load(self) -> List[FileMetadata]:
-        pass
-
-    @abstractmethod
-    def read_all_bytes(self, file_metadata: FileMetadata) -> bytes:
-        pass
-
-    @abstractmethod
-    def state(self) -> dict:
-        pass
-
-
-from .local_directory_loader import LocalDirectoryLoader
-from .url_loader import UrlLoader

indexify/data_loaders/local_directory_loader.py

@@ -1,37 +0,0 @@
-import os
-from typing import List, Optional
-
-from . import DataLoader, FileMetadata
-
-
-class LocalDirectoryLoader(DataLoader):
-    def __init__(
-        self,
-        directory: str,
-        file_extensions: Optional[List[str]] = None,
-        state: dict = {},
-    ):
-        self.directory = directory
-        self.file_extensions = file_extensions
-        self.processed_files = set(state.get("processed_files", []))
-
-    def load(self) -> List[FileMetadata]:
-        file_metadata_list = []
-        for root, _, files in os.walk(self.directory):
-            for file in files:
-                if self.file_extensions is None or any(
-                    file.endswith(ext) for ext in self.file_extensions
-                ):
-                    file_path = os.path.join(root, file)
-                    if file_path not in self.processed_files:
-                        file_metadata_list.append(FileMetadata.from_path(file_path))
-                        self.processed_files.add(file_path)
-
-        return file_metadata_list
-
-    def read_all_bytes(self, file: FileMetadata) -> bytes:
-        with open(file.path, "rb") as f:
-            return f.read()
-
-    def state(self) -> dict:
-        return {"processed_files": list(self.processed_files)}

indexify/data_loaders/url_loader.py

@@ -1,52 +0,0 @@
-import email.utils
-from typing import List
-
-import httpx
-
-from . import DataLoader, FileMetadata
-
-
-def convert_date_to_epoch(date_str: str) -> int:
-    """
-    Convert a date string from URL header to Unix epoch time.
-
-    Args:
-        date_str (str): The date string from the URL header.
-
-    Returns:
-        int: The Unix epoch time.
-    """
-    if not date_str:
-        return 0
-    parsed_date = email.utils.parsedate_to_datetime(date_str)
-    return int(parsed_date.timestamp())
-
-
-class UrlLoader(DataLoader):
-    def __init__(self, urls: List[str], state: dict = {}):
-        self.urls = urls
-
-    def load(self) -> List[FileMetadata]:
-        file_metadata_list = []
-        for url in self.urls:
-            response = httpx.head(url, follow_redirects=True)
-            file_metadata_list.append(
-                FileMetadata(
-                    path=url,
-                    file_size=response.headers.get("content-length", 0),
-                    mime_type=response.headers.get("content-type"),
-                    md5_hash="",
-                    created_at=convert_date_to_epoch(response.headers.get("date")),
-                    updated_at=convert_date_to_epoch(
-                        response.headers.get("last-modified")
-                    ),
-                )
-            )
-        return file_metadata_list
-
-    def read_all_bytes(self, file: FileMetadata) -> bytes:
-        response = httpx.get(file.path, follow_redirects=True)
-        return response.content
-
-    def state(self) -> dict:
-        return {}

indexify/error.py

@@ -1,8 +0,0 @@
-class ApiException(Exception):
-    def __init__(self, message: str) -> None:
-        super().__init__(message)
-
-
-class GraphStillProcessing(Exception):
-    def __init__(self) -> None:
-        super().__init__("graph is still processing")