indexify 0.2.10__py3-none-any.whl → 0.2.12__py3-none-any.whl

indexify/cli.py

@@ -119,7 +119,9 @@ def server_dev_mode():
 
 @app.command(help="Build image for function names")
 def build_image(
-    workflow_file_path: str, func_names: List[str], python_sdk_path: Optional[str] = None
+    workflow_file_path: str,
+    image_names: List[str],
+    python_sdk_path: Optional[str] = None,
 ):
     globals_dict = {}
 
@@ -134,20 +136,9 @@ def build_image(
         raise Exception(
             f"Could not find workflow file to execute at: " f"`{workflow_file_path}`"
         )
-
-    found_funcs = []
     for name, obj in globals_dict.items():
-        for func_name in func_names:
-            if name == func_name:
-                found_funcs.append(name)
-                _create_image_for_func(
-                    func_name=func_name, func_obj=obj, python_sdk_path=python_sdk_path
-                )
-
-    console.print(
-        Text(f"Processed functions: ", style="cyan"),
-        Text(f"{found_funcs}", style="green"),
-    )
+        if type(obj) and isinstance(obj, Image) and obj._image_name in image_names:
+            _create_image(obj, python_sdk_path)
 
 
 @app.command(help="Build default image for indexify")
@@ -208,12 +199,12 @@ def executor(
         console.print(Text(f"Exiting gracefully: {ex}", style="bold yellow"))
 
 
-def _create_image_for_func(func_name, func_obj, python_sdk_path):
+def _create_image(image: Image, python_sdk_path):
     console.print(
         Text("Creating container for ", style="cyan"),
-        Text(f"`{func_name}`", style="cyan bold"),
+        Text(f"`{image._image_name}`", style="cyan bold"),
     )
-    _build_image(image=func_obj.image, python_sdk_path=python_sdk_path)
+    _build_image(image=image, python_sdk_path=python_sdk_path)
 
 
 def _build_image(image: Image, python_sdk_path: Optional[str] = None):

indexify/executor/function_worker.py

@@ -18,6 +18,7 @@ function_wrapper_map: Dict[str, IndexifyFunctionWrapper] = {}
 
 import concurrent.futures
 
+
 class FunctionRunException(Exception):
     def __init__(
         self, exception: Exception, stdout: str, stderr: str, is_reducer: bool
@@ -131,7 +132,9 @@ def _run_function(
     fn_output = None
     has_failed = False
     exception_msg = None
-    print(f"[bold] function_worker: [/bold] invoking function {fn_name} in graph {graph_name}")
+    print(
+        f"[bold] function_worker: [/bold] invoking function {fn_name} in graph {graph_name}"
+    )
     with redirect_stdout(stdout_capture), redirect_stderr(stderr_capture):
         try:
             key = f"{namespace}/{graph_name}/{version}/{fn_name}"
@@ -139,7 +142,10 @@ def _run_function(
                 _load_function(namespace, graph_name, fn_name, code_path, version)
 
             fn = function_wrapper_map[key]
-            if str(type(fn.indexify_function)) == "<class 'indexify.functions_sdk.indexify_functions.IndexifyRo'>":
+            if (
+                str(type(fn.indexify_function))
+                == "<class 'indexify.functions_sdk.indexify_functions.IndexifyRo'>"
+            ):
                 router_output = fn.invoke_router(fn_name, input)
             else:
                 fn_output = fn.invoke_fn_ser(fn_name, input, init_value)
@@ -147,6 +153,7 @@ def _run_function(
                 is_reducer = fn.indexify_function.accumulate is not None
         except Exception as e:
             import sys
+
             print(traceback.format_exc(), file=sys.stderr)
             has_failed = True
             exception_msg = str(e)

indexify/functions_sdk/graph.py

@@ -117,13 +117,16 @@ class Graph:
             self.routers[from_node.name].append(node.name)
         return self
 
-    def serialize(self):
+    def serialize(self, additional_modules):
         # Get all unique modules from nodes and edges
         pickled_functions = {}
+        for module in additional_modules:
+            cloudpickle.register_pickle_by_value(module)
         for node in self.nodes.values():
             cloudpickle.register_pickle_by_value(sys.modules[node.__module__])
             pickled_functions[node.name] = cloudpickle.dumps(node)
-            cloudpickle.unregister_pickle_by_value(sys.modules[node.__module__])
+            if not sys.modules[node.__module__] in additional_modules:
+                cloudpickle.unregister_pickle_by_value(sys.modules[node.__module__])
         return pickled_functions
 
     def add_edge(

indexify/functions_sdk/image.py

@@ -4,7 +4,7 @@ class Image:
 
         self._tag = "latest"
 
-        self._base_image = "python:3.10.15-slim-bookworm"
+        self._base_image = "python:3.11.10-slim-bookworm"
 
         self._run_strs = []
 
@@ -29,6 +29,14 @@ DEFAULT_IMAGE = (
     Image()
     .name("tensorlake/indexify-executor-default")
     .base_image("python:3.10.15-slim-bookworm")
-    .tag("latest")
+    .tag("3:10")
+    .run("pip install indexify")
+)
+
+DEFAULT_IMAGE_3_11 = (
+    Image()
+    .name("tensorlake/indexify-executor-default")
+    .base_image("python:3.11.10-slim-bookworm")
+    .tag("3:11")
     .run("pip install indexify")
 )

indexify/http_client.py

@@ -160,9 +160,9 @@ class IndexifyClient:
     def __exit__(self, exc_type, exc_value, traceback):
         self.close()
 
-    def register_compute_graph(self, graph: Graph):
+    def register_compute_graph(self, graph: Graph, additional_modules):
         graph_metadata = graph.definition()
-        serialized_code = cloudpickle.dumps(graph.serialize())
+        serialized_code = cloudpickle.dumps(graph.serialize(additional_modules))
         response = self._post(
             f"namespaces/{self.namespace}/compute_graphs",
             files={"code": serialized_code},
@@ -197,9 +197,11 @@ class IndexifyClient:
         for item in namespaces_dict:
             namespaces.append(item["name"])
         return namespaces
-    
+
     @classmethod
-    def new_namespace(cls, namespace: str, server_addr: Optional[str] = "http://localhost:8900"):
+    def new_namespace(
+        cls, namespace: str, server_addr: Optional[str] = "http://localhost:8900"
+    ):
         # Create a new client instance with the specified server address
         client = cls(service_url=server_addr)
 
@@ -212,11 +214,10 @@ class IndexifyClient:
 
         # Set the namespace for the newly created client
         client.namespace = namespace
-        
+
         # Return the client instance with the new namespace
         return client
 
-
     def create_namespace(self, namespace: str):
         self._post("namespaces", json={"name": namespace})
 
@@ -259,7 +260,9 @@ class IndexifyClient:
                             return v["id"]
                         if k == "DiagnosticMessage":
                             message = v.get("message", None)
-                            print(f"[bold red]scheduler diagnostic: [/bold red]{message}")
+                            print(
+                                f"[bold red]scheduler diagnostic: [/bold red]{message}"
+                            )
                             continue
                         event_payload = InvocationEventPayload.model_validate(v)
                         event = InvocationEvent(event_name=k, payload=event_payload)

indexify/remote_graph.py

@@ -32,7 +32,7 @@ class RemoteGraph:
         return self._client.invoke_graph_with_object(
             self._name, block_until_done, **kwargs
         )
-    
+
     def rerun(self):
         """
         Rerun the graph with the given invocation ID.
@@ -41,17 +41,19 @@ class RemoteGraph:
         self._client.rerun_graph(self._name)
 
     @classmethod
-    def deploy(cls, g: Graph, additional_modules=[], server_url: Optional[str] = "http://localhost:8900"):
+    def deploy(
+        cls,
+        g: Graph,
+        additional_modules=[],
+        server_url: Optional[str] = "http://localhost:8900",
+    ):
         """
         Create a new RemoteGraph from a local Graph object.
         :param g: The local Graph object.
         :param server_url: The URL of the server where the graph will be registered.
         """
-        import cloudpickle
-        for module in additional_modules:
-            cloudpickle.register_pickle_by_value(module)
         client = IndexifyClient(service_url=server_url)
-        client.register_compute_graph(g)
+        client.register_compute_graph(g, additional_modules)
         return cls(name=g.name, server_url=server_url)
 
     @classmethod

{indexify-0.2.10.dist-info → indexify-0.2.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.10
+Version: 0.2.12
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0

{indexify-0.2.10.dist-info → indexify-0.2.12.dist-info}/RECORD

@@ -1,5 +1,5 @@
 indexify/__init__.py,sha256=qgMBKVrM_tI-tFeWpE8ktlC5rcExk05nbWyFqxxqeEU,496
-indexify/cli.py,sha256=pE_RYmqkF35uRD18kbOQ1k9WIXIt2KQIzxyxRmJUxnI,8382
+indexify/cli.py,sha256=SR4AWCxObIwHZzKb2iKMwtOQ3BcJUJIZ3SRiDvhv1sQ,8101
 indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
 indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
 indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
@@ -8,24 +8,24 @@ indexify/executor/agent.py,sha256=I08CiOWeJ_mz8OHr9_iJfp07Ma1VMQirZ2MsDp8lDZw,14
 indexify/executor/api_objects.py,sha256=SysjlGYu4JtYdqfexZHHN1IW4TtaDdFUF3hYZ5mpUJU,810
 indexify/executor/downloader.py,sha256=0MPiKw0AWs3Z7ReC9l2z-3515yqq85ghPzdh485dnuw,3998
 indexify/executor/executor_tasks.py,sha256=gAZ2pvza1YwGlaR1o_tJW4SXtdCgK7sLJgp4W7rOjR0,1834
-indexify/executor/function_worker.py,sha256=QafSvTiniiQUHIMbSRYcEpmFuy7--Cgk3s0ISCViggE,5372
+indexify/executor/function_worker.py,sha256=RQh-lJD2UtUuYXyCDkGohe8rVpHEriJg-AjBKSgD5KI,5436
 indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
 indexify/executor/runtime_probes.py,sha256=L-nB4D2zBZ287xxPwErv5R_gxFOmTEvg61FMe3rUB_A,1460
 indexify/executor/task_reporter.py,sha256=gnnse0v6rjjni8lNzeb-ZYq6iF2DgafKoT7dcGUZhQ4,3716
 indexify/executor/task_store.py,sha256=q8s2gImsFffWeXQR0mk1Xlo1Aj_2GfclNPjQ2EA_YBo,3984
 indexify/functions_sdk/data_objects.py,sha256=CQZMzYiV7l6dyzFkYquWQHqdte6JnC7XA3i2ZyvPvgQ,844
-indexify/functions_sdk/graph.py,sha256=2KyEggNTzB56PKnuKcO4g5lOl_XVzuVb7gfQKituDUc,11633
+indexify/functions_sdk/graph.py,sha256=t35z26jRCBgoz7fncdlAL7MVTOYjeJuoPpz-xI7WjBw,11827
 indexify/functions_sdk/graph_definition.py,sha256=EJfC0MdKEbFF1CBaU0htrveSlcAQJCH96DLSNfZ02V4,1178
 indexify/functions_sdk/graph_validation.py,sha256=XLHiC9PAtZungJLysU3hIUOPNDkO5TXUDZ_jiZ0H4hg,2508
-indexify/functions_sdk/image.py,sha256=RcFhVjmQv83S2wuf7Yn9RMhdpT-WfRqr1-krCF5UjNM,716
+indexify/functions_sdk/image.py,sha256=0-QrvMZSFT0_RGb8aeUfSYpSlh6QQkssFHLibGP9ywI,900
 indexify/functions_sdk/indexify_functions.py,sha256=oUssAADTt3byhz3Azl_GiYxV1BmhwuGE1CjmN-oiBpg,9575
 indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
 indexify/functions_sdk/object_serializer.py,sha256=Zz4GobW3ZamBBtFDF76QxU3TP6oJNdWnhsfKd0OUFoc,1660
-indexify/http_client.py,sha256=HMhNASth9B-sPdxo69v7kRVcqoCfU3ounur3Gxb37uk,13535
-indexify/remote_graph.py,sha256=Qy_mxyC00N4zBOV-ibuTak8ht2GlMck3nh0syq0VqOE,3165
+indexify/http_client.py,sha256=0nB-2yQPoVfvTRGvRru_pW6SWjqeT8jtXJPm-MC71pg,13636
+indexify/remote_graph.py,sha256=ILg6IY6EFgyvnfz1DSzicBZhqvPkg2-UUwgI6lxp6sA,3094
 indexify/settings.py,sha256=LSaWZ0ADIVmUv6o6dHWRC3-Ry5uLbCw2sBSg1e_U7UM,99
-indexify-0.2.10.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.10.dist-info/METADATA,sha256=zfFaEEl3NCpcX0lidfSePRdLIixeaefwfseOqIFfxlw,6130
-indexify-0.2.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-indexify-0.2.10.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.10.dist-info/RECORD,,
+indexify-0.2.12.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.12.dist-info/METADATA,sha256=ryNoVKX3M3qm7MjZ_briF2E_BBF9kCx0eaKgDgmPpiA,6130
+indexify-0.2.12.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+indexify-0.2.12.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.12.dist-info/RECORD,,