indexify 0.2.40__py3-none-any.whl → 0.2.41__py3-none-any.whl

indexify/http_client.py

@@ -64,7 +64,6 @@ class IndexifyClient:
         self._service_url = service_url
         self._timeout = kwargs.get("timeout")
         self._graphs: Dict[str, Graph] = {}
-        self._fns: Dict[str, IndexifyFunction] = {}
         self._api_key = api_key
         if not self._api_key:
             print(
@@ -174,8 +173,6 @@ class IndexifyClient:
         )
         response.raise_for_status()
         self._graphs[graph.name] = graph
-        for fn_name, fn in graph.nodes.items():
-            self._fns[f"{graph.name}/{fn_name}"] = fn
 
     def delete_compute_graph(
         self,
@@ -191,21 +188,18 @@ class IndexifyClient:
         )
         response.raise_for_status()
 
-    def graphs(self) -> List[str]:
-        response = self._get(f"graphs")
-        return response.json()["graphs"]
+    def graphs(self, namespace="default") -> List[ComputeGraphMetadata]:
+        response = self._get(f"namespaces/{namespace}/compute_graphs")
+        graphs = []
+        for graph in response.json()["compute_graphs"]:
+            graphs.append(ComputeGraphMetadata(**graph))
+
+        return graphs
 
     def graph(self, name: str) -> ComputeGraphMetadata:
         response = self._get(f"namespaces/{self.namespace}/compute_graphs/{name}")
         return ComputeGraphMetadata(**response.json())
 
-    def load_fn(self, name: str, fn_name: str) -> IndexifyFunction:
-        response = self._get(
-            f"internal/namespaces/{self.namespace}/compute_graphs/{name}/code"
-        )
-        pickled_functions_by_name = cloudpickle.loads(response.content)
-        return cloudpickle.loads(pickled_functions_by_name[fn_name])
-
     def namespaces(self) -> List[str]:
         response = self._get(f"namespaces")
         namespaces_dict = response.json()["namespaces"]
@@ -351,30 +345,27 @@ class IndexifyClient:
         )
         response.raise_for_status()
         content_type = response.headers.get("Content-Type")
-        if content_type == "application/octet-stream":
-            encoding = "cloudpickle"
-        else:
+        if content_type == "application/json":
             encoding = "json"
+        else:
+            encoding = "cloudpickle"
         return IndexifyData(id=output_id, payload=response.content, encoder=encoding)
 
     def graph_outputs(
         self,
         graph: str,
         invocation_id: str,
-        fn_name: Optional[str],
+        fn_name: str,
     ) -> List[Any]:
         """
         Returns the extracted objects by a graph for an ingested object. If the extractor name is provided, only the objects extracted by that extractor are returned.
         If the extractor name is not provided, all the extracted objects are returned for the input object.
         graph: str: The name of the graph
-        invocation_id: str: The ID of the ingested object
-        extractor_name: Optional[str]: The name of the extractor whose output is to be returned if provided
-        block_until_done: bool = True: If True, the method will block until the extraction is done. If False, the method will return immediately.
+        invocation_id: str: The ID of the invocation.
+        fn_name: Optional[str]: The name of the function whose output is to be returned if provided
         return: Union[Dict[str, List[Any]], List[Any]]: The extracted objects. If the extractor name is provided, the output is a list of extracted objects by the extractor. If the extractor name is not provided, the output is a dictionary with the extractor name as the key and the extracted objects as the value. If no objects are found, an empty list is returned.
         """
         fn_key = f"{graph}/{fn_name}"
-        if fn_key not in self._fns:
-            self._fns[fn_key] = self.load_fn(graph, fn_name)
         response = self._get(
             f"namespaces/{self.namespace}/compute_graphs/{graph}/invocations/{invocation_id}/outputs",
         )
@@ -388,7 +379,8 @@ class IndexifyClient:
                 indexify_data = self._download_output(
                     self.namespace, graph, invocation_id, fn_name, output.id
                 )
-                output = self._fns[fn_key].deserialize_output(indexify_data)
+                serializer = get_serializer(indexify_data.encoder)
+                output = serializer.deserialize(indexify_data.payload)
                 outputs.append(output)
         return outputs
 

indexify/logging.py

@@ -0,0 +1,32 @@
+import logging
+import sys
+
+import structlog
+
+# Using this module allows us to be consistent with the logging configuration across all Python programs.
+
+
+def configure_logging_early():
+    """Configures standard Python logging module.
+
+    By default 3rd party modules that are using standard Python logging module
+    (logging.getLogger()) have their log lines dropped unless the module gets configured.
+
+    Not dropping log lines from 3rd party modules is useful for debugging. E.g. this helps
+    debugging errors in grpc servers if exceptions happen inside grpc module code.
+    """
+    logging.basicConfig(
+        level=logging.WARNING,
+        # This log message format is a bit similar to the default structlog format.
+        format="%(asctime)s [%(levelname)s] %(message)s logger=%(name)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        handlers=[logging.StreamHandler(sys.stdout)],
+    )
+
+
+def configure_production_logging():
+    processors = [
+        structlog.processors.dict_tracebacks,
+        structlog.processors.JSONRenderer(),
+    ]
+    structlog.configure(processors=processors)

{indexify-0.2.40.dist-info → indexify-0.2.41.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: indexify
-Version: 0.2.40
+Version: 0.2.41
 Summary: Python Client for Indexify
 Home-page: https://github.com/tensorlakeai/indexify
 License: Apache 2.0
@@ -16,6 +16,8 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: cloudpickle (>=3.1.0,<4.0.0)
 Requires-Dist: docker (>=7.1.0,<8.0.0)
+Requires-Dist: grpcio (==1.68.1)
+Requires-Dist: grpcio-tools (==1.68.1)
 Requires-Dist: httpx-sse (>=0.4.0,<0.5.0)
 Requires-Dist: httpx[http2] (==0.27.2)
 Requires-Dist: nanoid (>=2.0.0,<3.0.0)

indexify-0.2.41.dist-info/RECORD

@@ -0,0 +1,53 @@
+indexify/__init__.py,sha256=P0mvM8sbkeS2CjYzRYyzb42CnXGhyJXdz4FdmTBMSWM,697
+indexify/cli.py,sha256=a6KPM7uWE3kcbREP1IkkY3Z30vJ-9B_70r6j9-1msSk,10385
+indexify/common_util.py,sha256=LKS6yZ3yv8nF2J-KzisGIjqjTvCn7tLFifQJLT4gHRg,3529
+indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
+indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
+indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
+indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
+indexify/executor/agent.py,sha256=vPbGUN2E9N6fa7WDCpzUD4CzNwSnDMQ0_vs1d5R61GU,10902
+indexify/executor/api_objects.py,sha256=vp7aEjvfWL2j4nYDV0xLgLVZlGsbStMEu0nzrgJVq1A,741
+indexify/executor/downloader.py,sha256=oeiSGxIIn7uBTKT6hzyPUa-AlOUlqy7WQkinm7qpV0I,6442
+indexify/executor/executor_tasks.py,sha256=vxXK2pCSqu5_HaEPKj2VF9k8vgA1mUD8v4bx0TXGR-I,1570
+indexify/executor/function_executor/function_executor.py,sha256=0W6vCT-DBVPUMPoOXLVRVHZjVGGUeTDV_NWb-XLP2gM,1218
+indexify/executor/function_executor/function_executor_factory.py,sha256=23BU_i47Tz6xd9fAwsKH29lm261bZXfzYGRhSqeFtwY,900
+indexify/executor/function_executor/function_executor_map.py,sha256=HXeCjkRusumJisU74Iy6M4e13k8jeHk4Yq8PyODSsUc,3922
+indexify/executor/function_executor/process_function_executor.py,sha256=CdqFtvlvS_VIzVdbE3Wnzj_Kqfzhzf4X05avHIielEE,2150
+indexify/executor/function_executor/process_function_executor_factory.py,sha256=ognYRd5l10kVD2KAShfkd8vg-JElYctG77Pcx77-grM,3850
+indexify/executor/function_worker.py,sha256=JcVrDZPj6XE0c-SDyXMlS5ZMfr5FuByZxG2XoACHsoU,9803
+indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
+indexify/executor/task_fetcher.py,sha256=481Ta7UAlytDpf99wNGzcqaCpB7NGJitb1nn_rAE9Do,3017
+indexify/executor/task_reporter.py,sha256=jThbGHte8nBikBIiSFgjRUQ6o3wE3Ms2zyz1NaZWIUQ,5113
+indexify/executor/task_store.py,sha256=Gpsr7S6UuLhGiB4Vh-9A0-iU4Qgji5CYEzjwKWeETl4,4430
+indexify/function_executor/function_executor_service.py,sha256=EwwHTPX-AVqbSl-BONWWy3Z6gowXnYj5tigGoFs1u0s,3694
+indexify/function_executor/handlers/run_function/function_inputs_loader.py,sha256=x2Lrzb6CsEWDWW5qG9etS7TyV7KVYfuFKEbzKVhYizo,1553
+indexify/function_executor/handlers/run_function/handler.py,sha256=eklicxgyExd9ScdDahQO9LaLNXPNBPAAI0KBt3Wb9-w,5636
+indexify/function_executor/handlers/run_function/request_validator.py,sha256=tVmbOS2g4AeV48CZCSDRycnoQjjGy12XD_vbaOLX_Z4,740
+indexify/function_executor/handlers/run_function/response_helper.py,sha256=KDSadee08g6JRFwKRTdcOwbP6SmRJCGVHh1FOLdt_ws,3109
+indexify/function_executor/initialize_request_validator.py,sha256=AtyKgIjlGHB-Fh3Vhso86TC0o65DfuVEY8GFfRSfylo,709
+indexify/function_executor/proto/configuration.py,sha256=P0xtxwf-XaCxi0kvtCbm4X8y0YOuHhR-uh9xUuLdFoM,595
+indexify/function_executor/proto/function_executor.proto,sha256=PiftSSIDjHkxUbD7rNVmgfTCP12bFJ7oKJXuRx2sM28,2188
+indexify/function_executor/proto/function_executor_pb2.py,sha256=tEDJJUywM6-j4dBwvBEHXnGCv9spGWVuhtHuiTDxw7I,4582
+indexify/function_executor/proto/function_executor_pb2.pyi,sha256=E-UnFIFbej-QPhaX1X4oYGeUW6_HuEi13qhCYkN1tzI,4182
+indexify/function_executor/proto/function_executor_pb2_grpc.py,sha256=OIErF2wYWaRI2TyF8EbWGjj5K9aD7OIJwAp1BRMQ0ic,6265
+indexify/function_executor/proto/message_validator.py,sha256=IkIHryanhFMRM6IU9F0nsGIlcEktXgcbomLFTWtPWYU,1418
+indexify/function_executor/server.py,sha256=YEeKHR6qjNJyLFb8KNjcCYClBgVzdJNmKwdwmyJ-ybE,1136
+indexify/functions_sdk/data_objects.py,sha256=ZJ7B9b5OI7aieCWJFx18pV6tqScssIFtmth84i7nKTg,623
+indexify/functions_sdk/graph.py,sha256=WNaS5yGNKNLa55gH3LPIlp4fH9VpAsb7VAVkELozrZ0,13413
+indexify/functions_sdk/graph_definition.py,sha256=yQ7c7jpA8RB8Yt81D1i5R3LPT8fmLvN8bLuEHvS4P_o,1775
+indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
+indexify/functions_sdk/image.py,sha256=wVDs5O5yshjSSZ1d-keyMp9zUrck2TazJlkVq94jMZk,1965
+indexify/functions_sdk/indexify_functions.py,sha256=USuEeZnbQTd6uUjv-ZAwdBrbR8gGrTD78oXjyW6DBbw,13376
+indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
+indexify/functions_sdk/object_serializer.py,sha256=R58ALsl2Lb87ii6km4D6hBBsqRs_CHNISxhUICE2d9o,1931
+indexify/functions_sdk/pipeline.py,sha256=KmxZE8eBFAQ4bbEcYURXXR26HSyoAT3O6iu9H38-OXE,974
+indexify/http_client.py,sha256=XvIi7z7h7PZomMKlL8WEhFIWAAPzFe2hrLQUz4QC_Zc,15392
+indexify/logging.py,sha256=_TLnWT0RzABvExwTixjdpuxoSNUev4zeNV0K1VlEUmA,1099
+indexify/remote_graph.py,sha256=OzBucU4buR5UdTViNJvh1RfnOTmYA34Uel5qXnRsQsA,5006
+indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
+indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
+indexify-0.2.41.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+indexify-0.2.41.dist-info/METADATA,sha256=j9wkgdgqWKprrfh0sXEhs4yVTWnEs54M9bWWnVEFBmo,6271
+indexify-0.2.41.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+indexify-0.2.41.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
+indexify-0.2.41.dist-info/RECORD,,

indexify/executor/indexify_executor.py

@@ -1,32 +0,0 @@
-import asyncio
-from typing import List, Optional
-
-import nanoid
-
-from .agent import ExtractorAgent
-from .function_worker import FunctionWorker
-
-
-def join(
-    workers: int,
-    server_addr: str = "localhost:8900",
-    config_path: Optional[str] = None,
-):
-    print(f"receiving tasks from server addr: {server_addr}")
-    id = nanoid.generate()
-    print(f"executor id: {id}")
-
-    function_worker = FunctionWorker(workers=workers)
-
-    agent = ExtractorAgent(
-        id,
-        num_workers=workers,
-        function_worker=function_worker,
-        server_addr=server_addr,
-        config_path=config_path,
-    )
-
-    try:
-        asyncio.get_event_loop().run_until_complete(agent.run())
-    except asyncio.CancelledError as ex:
-        print("exiting gracefully", ex)

indexify-0.2.40.dist-info/RECORD

@@ -1,34 +0,0 @@
-indexify/__init__.py,sha256=P0mvM8sbkeS2CjYzRYyzb42CnXGhyJXdz4FdmTBMSWM,697
-indexify/cli.py,sha256=QqaRpm-XywJQ3zh-2pgABi15Jl0IkwA_sz6PRNvQ4wc,8872
-indexify/common_util.py,sha256=LKS6yZ3yv8nF2J-KzisGIjqjTvCn7tLFifQJLT4gHRg,3529
-indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
-indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
-indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
-indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
-indexify/executor/agent.py,sha256=6xcGDZLwFHlFDiqW9EOI_Dfxh8ahdhGZq4PZUoyPLEY,14167
-indexify/executor/api_objects.py,sha256=mvmwGbK4paJNQGFvbtNHMPpiH_LpVhrlRnCcrqS6HOQ,859
-indexify/executor/downloader.py,sha256=dHLxoBnX8-Bh4yZtFDYptZNF6rlVtmTk_70JK8Ect5w,4184
-indexify/executor/executor_tasks.py,sha256=A0UIEZ5VaB6zSkFQG81UmTW0E57MTYhGlaXuAbRV8lQ,1884
-indexify/executor/function_worker.py,sha256=wRW2-X9dNI80KhwTD1vD-pcyetsVKVs6vVdg7L7JjcQ,6462
-indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
-indexify/executor/runtime_probes.py,sha256=mjw2_mGQ622wRT_39WPGGgPEZQTgtrf3-ICcUUZOeyg,2126
-indexify/executor/task_reporter.py,sha256=XlEhNf_ScNnzG67zbtVwL7_9Bo8MvPZiHLI5UHymUnM,5305
-indexify/executor/task_store.py,sha256=JlRlWwAm4YjFRkTNRx-6GsUcmOzcyvzb5Csa5XDpRTI,3982
-indexify/functions_sdk/data_objects.py,sha256=wXbUa9hjU6rsXmmk19vQ5Kixf3FsI59VBWPNmHasAX0,854
-indexify/functions_sdk/graph.py,sha256=jZSOVmlQB8Eb8MBz0TeRADl_4y5rnrjdTTy6VdRLISY,13448
-indexify/functions_sdk/graph_definition.py,sha256=W3zRERTOC-UbiRms-NqvtlLDpSZM-33TocKeZXb5o_A,1762
-indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
-indexify/functions_sdk/image.py,sha256=QK0H6KxLWriB_z4M0kunKzzHdHxYLWL670RPYgYuf_8,1762
-indexify/functions_sdk/indexify_functions.py,sha256=kjGmkRYnA_KO5ypkyaIhqrz0-3UJOFTPGldgOrHSRHY,13401
-indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
-indexify/functions_sdk/object_serializer.py,sha256=R58ALsl2Lb87ii6km4D6hBBsqRs_CHNISxhUICE2d9o,1931
-indexify/functions_sdk/pipeline.py,sha256=KmxZE8eBFAQ4bbEcYURXXR26HSyoAT3O6iu9H38-OXE,974
-indexify/http_client.py,sha256=iLafZagCFnlTS6uHfOjInogjg0uXW_zXEspIN7ttB5I,15903
-indexify/remote_graph.py,sha256=OzBucU4buR5UdTViNJvh1RfnOTmYA34Uel5qXnRsQsA,5006
-indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
-indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
-indexify-0.2.40.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-indexify-0.2.40.dist-info/METADATA,sha256=x-8XRtgGLpvgQ5fEXUxamZwgoNtGCWPcbyvAgciO2HE,6199
-indexify-0.2.40.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-indexify-0.2.40.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
-indexify-0.2.40.dist-info/RECORD,,