indexify 0.2.6__tar.gz → 0.2.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {indexify-0.2.6 → indexify-0.2.8}/PKG-INFO +1 -1
- {indexify-0.2.6 → indexify-0.2.8}/indexify/cli.py +35 -9
- {indexify-0.2.6 → indexify-0.2.8}/indexify/executor/function_worker.py +15 -20
- {indexify-0.2.6 → indexify-0.2.8}/indexify/functions_sdk/data_objects.py +1 -1
- {indexify-0.2.6 → indexify-0.2.8}/indexify/functions_sdk/graph.py +14 -99
- {indexify-0.2.6 → indexify-0.2.8}/indexify/functions_sdk/image.py +1 -1
- {indexify-0.2.6 → indexify-0.2.8}/indexify/functions_sdk/indexify_functions.py +83 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/http_client.py +36 -11
- {indexify-0.2.6 → indexify-0.2.8}/indexify/remote_graph.py +8 -1
- {indexify-0.2.6 → indexify-0.2.8}/pyproject.toml +1 -1
- {indexify-0.2.6 → indexify-0.2.8}/LICENSE.txt +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/README.md +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/__init__.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/data_loaders/__init__.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/data_loaders/local_directory_loader.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/data_loaders/url_loader.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/error.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/executor/agent.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/executor/api_objects.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/executor/downloader.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/executor/executor_tasks.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/executor/indexify_executor.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/executor/runtime_probes.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/executor/task_reporter.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/executor/task_store.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/functions_sdk/graph_definition.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/functions_sdk/graph_validation.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/functions_sdk/local_cache.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/functions_sdk/object_serializer.py +0 -0
- {indexify-0.2.6 → indexify-0.2.8}/indexify/settings.py +0 -0
@@ -9,7 +9,6 @@ import threading
|
|
9
9
|
import time
|
10
10
|
from typing import Annotated, List, Optional
|
11
11
|
|
12
|
-
import docker
|
13
12
|
import nanoid
|
14
13
|
import typer
|
15
14
|
from rich.console import Console
|
@@ -119,7 +118,9 @@ def server_dev_mode():
|
|
119
118
|
|
120
119
|
|
121
120
|
@app.command(help="Build image for function names")
|
122
|
-
def build_image(
|
121
|
+
def build_image(
|
122
|
+
workflow_file_path: str, func_names: List[str], python_sdk_path: Optional[str] = None
|
123
|
+
):
|
123
124
|
globals_dict = {}
|
124
125
|
|
125
126
|
# Add the folder in the workflow file path to the current Python path
|
@@ -139,7 +140,9 @@ def build_image(workflow_file_path: str, func_names: List[str]):
|
|
139
140
|
for func_name in func_names:
|
140
141
|
if name == func_name:
|
141
142
|
found_funcs.append(name)
|
142
|
-
_create_image_for_func(
|
143
|
+
_create_image_for_func(
|
144
|
+
func_name=func_name, func_obj=obj, python_sdk_path=python_sdk_path
|
145
|
+
)
|
143
146
|
|
144
147
|
console.print(
|
145
148
|
Text(f"Processed functions: ", style="cyan"),
|
@@ -205,16 +208,18 @@ def executor(
|
|
205
208
|
console.print(Text(f"Exiting gracefully: {ex}", style="bold yellow"))
|
206
209
|
|
207
210
|
|
208
|
-
def _create_image_for_func(func_name, func_obj):
|
211
|
+
def _create_image_for_func(func_name, func_obj, python_sdk_path):
|
209
212
|
console.print(
|
210
213
|
Text("Creating container for ", style="cyan"),
|
211
214
|
Text(f"`{func_name}`", style="cyan bold"),
|
212
215
|
)
|
213
|
-
_build_image(image=func_obj.image,
|
216
|
+
_build_image(image=func_obj.image, python_sdk_path=python_sdk_path)
|
214
217
|
|
215
218
|
|
216
|
-
def _build_image(image: Image,
|
219
|
+
def _build_image(image: Image, python_sdk_path: Optional[str] = None):
|
217
220
|
try:
|
221
|
+
import docker
|
222
|
+
|
218
223
|
client = docker.from_env()
|
219
224
|
client.ping()
|
220
225
|
except Exception as e:
|
@@ -240,15 +245,36 @@ WORKDIR /app
|
|
240
245
|
run_strs = ["RUN " + i for i in image._run_strs]
|
241
246
|
|
242
247
|
docker_file += "\n".join(run_strs)
|
248
|
+
print(os.getcwd())
|
249
|
+
import docker
|
250
|
+
import docker.api.build
|
251
|
+
|
252
|
+
docker.api.build.process_dockerfile = lambda dockerfile, path: (
|
253
|
+
"Dockerfile",
|
254
|
+
dockerfile,
|
255
|
+
)
|
256
|
+
|
257
|
+
if python_sdk_path is not None:
|
258
|
+
if not os.path.exists(python_sdk_path):
|
259
|
+
print(f"error: {python_sdk_path} does not exist")
|
260
|
+
os.exit(1)
|
261
|
+
docker_file += f"\nCOPY {python_sdk_path} /app/python-sdk"
|
262
|
+
docker_file += f"\nRUN (cd /app/python-sdk && pip install .)"
|
263
|
+
else:
|
264
|
+
docker_file += f"\nRUN pip install indexify"
|
243
265
|
|
244
266
|
console.print("Creating image using Dockerfile contents:", style="cyan bold")
|
245
267
|
console.print(f"{docker_file}", style="magenta")
|
246
268
|
|
247
269
|
client = docker.from_env()
|
248
|
-
image_name =f"{image._image_name}:{image._tag}"
|
249
|
-
client.images.build(
|
250
|
-
|
270
|
+
image_name = f"{image._image_name}:{image._tag}"
|
271
|
+
(_image, generator) = client.images.build(
|
272
|
+
path=".",
|
273
|
+
dockerfile=docker_file,
|
251
274
|
tag=image_name,
|
252
275
|
rm=True,
|
253
276
|
)
|
277
|
+
for result in generator:
|
278
|
+
print(result)
|
279
|
+
|
254
280
|
print(f"built image: {image_name}")
|
@@ -1,8 +1,9 @@
|
|
1
1
|
import asyncio
|
2
2
|
import traceback
|
3
3
|
from concurrent.futures.process import BrokenProcessPool
|
4
|
-
from typing import Dict, List, Optional
|
4
|
+
from typing import Dict, List, Optional
|
5
5
|
|
6
|
+
import cloudpickle
|
6
7
|
from pydantic import BaseModel
|
7
8
|
from rich import print
|
8
9
|
|
@@ -11,16 +12,11 @@ from indexify.functions_sdk.data_objects import (
|
|
11
12
|
IndexifyData,
|
12
13
|
RouterOutput,
|
13
14
|
)
|
14
|
-
from indexify.functions_sdk.graph import Graph
|
15
15
|
from indexify.functions_sdk.indexify_functions import IndexifyFunctionWrapper
|
16
16
|
|
17
|
-
graphs: Dict[str, Graph] = {}
|
18
17
|
function_wrapper_map: Dict[str, IndexifyFunctionWrapper] = {}
|
19
18
|
|
20
19
|
import concurrent.futures
|
21
|
-
import io
|
22
|
-
from contextlib import redirect_stderr, redirect_stdout
|
23
|
-
|
24
20
|
|
25
21
|
class FunctionRunException(Exception):
|
26
22
|
def __init__(
|
@@ -51,11 +47,13 @@ def _load_function(
|
|
51
47
|
key = f"{namespace}/{graph_name}/{version}/{fn_name}"
|
52
48
|
if key in function_wrapper_map:
|
53
49
|
return
|
54
|
-
|
55
|
-
|
50
|
+
with open(code_path, "rb") as f:
|
51
|
+
code = f.read()
|
52
|
+
pickled_functions = cloudpickle.loads(code)
|
53
|
+
function_wrapper = IndexifyFunctionWrapper(
|
54
|
+
cloudpickle.loads(pickled_functions[fn_name])
|
55
|
+
)
|
56
56
|
function_wrapper_map[key] = function_wrapper
|
57
|
-
graph_key = f"{namespace}/{graph_name}/{version}"
|
58
|
-
graphs[graph_key] = graph
|
59
57
|
|
60
58
|
|
61
59
|
class FunctionWorker:
|
@@ -91,8 +89,6 @@ class FunctionWorker:
|
|
91
89
|
traceback.print_exc()
|
92
90
|
raise mp
|
93
91
|
except FunctionRunException as e:
|
94
|
-
print(e)
|
95
|
-
print(traceback.format_exc())
|
96
92
|
return FunctionWorkerOutput(
|
97
93
|
exception=str(e),
|
98
94
|
stdout=e.stdout,
|
@@ -142,17 +138,16 @@ def _run_function(
|
|
142
138
|
if key not in function_wrapper_map:
|
143
139
|
_load_function(namespace, graph_name, fn_name, code_path, version)
|
144
140
|
|
145
|
-
|
146
|
-
if
|
147
|
-
router_output =
|
141
|
+
fn = function_wrapper_map[key]
|
142
|
+
if str(type(fn.indexify_function)) == "<class 'indexify.functions_sdk.indexify_functions.IndexifyRo'>":
|
143
|
+
router_output = fn.invoke_router(fn_name, input)
|
148
144
|
else:
|
149
|
-
fn_output =
|
145
|
+
fn_output = fn.invoke_fn_ser(fn_name, input, init_value)
|
150
146
|
|
151
|
-
is_reducer =
|
152
|
-
graph.get_function(fn_name).indexify_function.accumulate is not None
|
153
|
-
)
|
147
|
+
is_reducer = fn.indexify_function.accumulate is not None
|
154
148
|
except Exception as e:
|
155
|
-
|
149
|
+
import sys
|
150
|
+
print(traceback.format_exc(), file=sys.stderr)
|
156
151
|
has_failed = True
|
157
152
|
exception_msg = str(e)
|
158
153
|
|
@@ -1,4 +1,3 @@
|
|
1
|
-
import inspect
|
2
1
|
import sys
|
3
2
|
from collections import defaultdict
|
4
3
|
from queue import deque
|
@@ -16,7 +15,6 @@ from typing import (
|
|
16
15
|
)
|
17
16
|
|
18
17
|
import cloudpickle
|
19
|
-
import msgpack
|
20
18
|
from nanoid import generate
|
21
19
|
from pydantic import BaseModel
|
22
20
|
from typing_extensions import get_args, get_origin
|
@@ -36,7 +34,7 @@ from .indexify_functions import (
|
|
36
34
|
IndexifyRouter,
|
37
35
|
)
|
38
36
|
from .local_cache import CacheAwareFunctionWrapper
|
39
|
-
from .object_serializer import
|
37
|
+
from .object_serializer import get_serializer
|
40
38
|
|
41
39
|
RouterFn = Annotated[
|
42
40
|
Callable[[IndexifyData], Optional[List[IndexifyFunction]]], "RouterFn"
|
@@ -45,26 +43,16 @@ GraphNode = Annotated[Union[IndexifyFunctionWrapper, RouterFn], "GraphNode"]
|
|
45
43
|
|
46
44
|
|
47
45
|
def is_pydantic_model_from_annotation(type_annotation):
|
48
|
-
# If it's a string representation
|
49
46
|
if isinstance(type_annotation, str):
|
50
|
-
# Extract the class name from the string
|
51
47
|
class_name = type_annotation.split("'")[-2].split(".")[-1]
|
52
|
-
# This part is tricky and might require additional context or imports
|
53
|
-
# You might need to import the actual class or module where it's defined
|
54
|
-
# For example:
|
55
|
-
# from indexify.functions_sdk.data_objects import File
|
56
|
-
# return issubclass(eval(class_name), BaseModel)
|
57
48
|
return False # Default to False if we can't evaluate
|
58
49
|
|
59
|
-
# If it's a Type object
|
60
50
|
origin = get_origin(type_annotation)
|
61
51
|
if origin is not None:
|
62
|
-
# Handle generic types like List[File], Optional[File], etc.
|
63
52
|
args = get_args(type_annotation)
|
64
53
|
if args:
|
65
54
|
return is_pydantic_model_from_annotation(args[0])
|
66
55
|
|
67
|
-
# If it's a direct class reference
|
68
56
|
if isinstance(type_annotation, type):
|
69
57
|
return issubclass(type_annotation, BaseModel)
|
70
58
|
|
@@ -98,10 +86,6 @@ class Graph:
|
|
98
86
|
def get_accumulators(self) -> Dict[str, Any]:
|
99
87
|
return self.accumulator_zero_values
|
100
88
|
|
101
|
-
def deserialize_fn_output(self, name: str, output: IndexifyData) -> Any:
|
102
|
-
serializer = get_serializer(self.nodes[name].payload_encoder)
|
103
|
-
return serializer.deserialize(output.payload)
|
104
|
-
|
105
89
|
def add_node(
|
106
90
|
self, indexify_fn: Union[Type[IndexifyFunction], Type[IndexifyRouter]]
|
107
91
|
) -> "Graph":
|
@@ -132,33 +116,15 @@ class Graph:
|
|
132
116
|
self.add_node(node)
|
133
117
|
self.routers[from_node.name].append(node.name)
|
134
118
|
return self
|
135
|
-
|
136
|
-
|
137
|
-
def _register_cloudpickle(self):
|
138
|
-
# Get all unique modules from nodes and edges
|
139
|
-
modules = set()
|
140
|
-
for node in self.nodes.values():
|
141
|
-
modules.add(node.__module__)
|
142
|
-
|
143
|
-
# Register each module with cloudpickle
|
144
|
-
for module_name in modules:
|
145
|
-
module = sys.modules[module_name]
|
146
|
-
print(f"registering module {module_name} with cloudpickle")
|
147
|
-
cloudpickle.register_pickle_by_value(module)
|
148
|
-
|
149
119
|
|
150
120
|
def serialize(self):
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
@staticmethod
|
159
|
-
def from_path(path: str) -> "Graph":
|
160
|
-
with open(path, "rb") as f:
|
161
|
-
return cloudpickle.load(f)
|
121
|
+
# Get all unique modules from nodes and edges
|
122
|
+
pickled_functions = {}
|
123
|
+
for node in self.nodes.values():
|
124
|
+
cloudpickle.register_pickle_by_value(sys.modules[node.__module__])
|
125
|
+
pickled_functions[node.name] = cloudpickle.dumps(node)
|
126
|
+
cloudpickle.unregister_pickle_by_value(sys.modules[node.__module__])
|
127
|
+
return pickled_functions
|
162
128
|
|
163
129
|
def add_edge(
|
164
130
|
self,
|
@@ -168,60 +134,6 @@ class Graph:
|
|
168
134
|
self.add_edges(from_node, [to_node])
|
169
135
|
return self
|
170
136
|
|
171
|
-
def invoke_fn_ser(
|
172
|
-
self, name: str, input: IndexifyData, acc: Optional[Any] = None
|
173
|
-
) -> List[IndexifyData]:
|
174
|
-
fn_wrapper = self.get_function(name)
|
175
|
-
input = self.deserialize_input(name, input)
|
176
|
-
serializer = get_serializer(fn_wrapper.indexify_function.payload_encoder)
|
177
|
-
if acc is not None:
|
178
|
-
acc = fn_wrapper.indexify_function.accumulate.model_validate(
|
179
|
-
serializer.deserialize(acc.payload)
|
180
|
-
)
|
181
|
-
if acc is None and fn_wrapper.indexify_function.accumulate is not None:
|
182
|
-
acc = fn_wrapper.indexify_function.accumulate.model_validate(
|
183
|
-
self.accumulator_zero_values[name]
|
184
|
-
)
|
185
|
-
outputs: List[Any] = fn_wrapper.run_fn(input, acc=acc)
|
186
|
-
return [
|
187
|
-
IndexifyData(payload=serializer.serialize(output)) for output in outputs
|
188
|
-
]
|
189
|
-
|
190
|
-
def invoke_router(self, name: str, input: IndexifyData) -> Optional[RouterOutput]:
|
191
|
-
fn_wrapper = self.get_function(name)
|
192
|
-
input = self.deserialize_input(name, input)
|
193
|
-
return RouterOutput(edges=fn_wrapper.run_router(input))
|
194
|
-
|
195
|
-
def deserialize_input(self, compute_fn: str, indexify_data: IndexifyData) -> Any:
|
196
|
-
compute_fn = self.nodes[compute_fn]
|
197
|
-
if not compute_fn:
|
198
|
-
raise ValueError(f"Compute function {compute_fn} not found in graph")
|
199
|
-
if compute_fn.payload_encoder == "cloudpickle":
|
200
|
-
return CloudPickleSerializer.deserialize(indexify_data.payload)
|
201
|
-
payload = msgpack.unpackb(indexify_data.payload)
|
202
|
-
signature = inspect.signature(compute_fn.run)
|
203
|
-
arg_types = {}
|
204
|
-
for name, param in signature.parameters.items():
|
205
|
-
if (
|
206
|
-
param.annotation != inspect.Parameter.empty
|
207
|
-
and param.annotation != getattr(compute_fn, "accumulate", None)
|
208
|
-
):
|
209
|
-
arg_types[name] = param.annotation
|
210
|
-
if len(arg_types) > 1:
|
211
|
-
raise ValueError(
|
212
|
-
f"Compute function {compute_fn} has multiple arguments, but only one is supported"
|
213
|
-
)
|
214
|
-
elif len(arg_types) == 0:
|
215
|
-
raise ValueError(f"Compute function {compute_fn} has no arguments")
|
216
|
-
arg_name, arg_type = next(iter(arg_types.items()))
|
217
|
-
if arg_type is None:
|
218
|
-
raise ValueError(f"Argument {arg_name} has no type annotation")
|
219
|
-
if is_pydantic_model_from_annotation(arg_type):
|
220
|
-
if len(payload.keys()) == 1 and isinstance(list(payload.values())[0], dict):
|
221
|
-
payload = list(payload.values())[0]
|
222
|
-
return arg_type.model_validate(payload)
|
223
|
-
return payload
|
224
|
-
|
225
137
|
def add_edges(
|
226
138
|
self,
|
227
139
|
from_node: Union[Type[IndexifyFunction], Type[IndexifyRouter]],
|
@@ -324,7 +236,9 @@ class Graph:
|
|
324
236
|
function_outputs.extend(cached_output_list)
|
325
237
|
outputs[node_name].extend(cached_output_list)
|
326
238
|
else:
|
327
|
-
function_outputs: List[IndexifyData] =
|
239
|
+
function_outputs: List[IndexifyData] = IndexifyFunctionWrapper(
|
240
|
+
node
|
241
|
+
).invoke_fn_ser(
|
328
242
|
node_name, input, accumulator_values.get(node_name, None)
|
329
243
|
)
|
330
244
|
print(f"ran {node_name}: num outputs: {len(function_outputs)}")
|
@@ -365,9 +279,10 @@ class Graph:
|
|
365
279
|
queue.append((out_edge, output))
|
366
280
|
|
367
281
|
def _route(self, node_name: str, input: IndexifyData) -> Optional[RouterOutput]:
|
368
|
-
|
282
|
+
router = self.nodes[node_name]
|
283
|
+
return IndexifyFunctionWrapper(router).invoke_router(node_name, input)
|
369
284
|
|
370
|
-
def
|
285
|
+
def output(
|
371
286
|
self,
|
372
287
|
invocation_id: str,
|
373
288
|
fn_name: str,
|
@@ -1,3 +1,4 @@
|
|
1
|
+
import inspect
|
1
2
|
from abc import ABC, abstractmethod
|
2
3
|
from functools import update_wrapper
|
3
4
|
from typing import (
|
@@ -12,11 +13,40 @@ from typing import (
|
|
12
13
|
get_origin,
|
13
14
|
)
|
14
15
|
|
16
|
+
import msgpack
|
15
17
|
from pydantic import BaseModel
|
16
18
|
from typing_extensions import get_type_hints
|
17
19
|
|
18
20
|
from .data_objects import IndexifyData, RouterOutput
|
19
21
|
from .image import DEFAULT_IMAGE, Image
|
22
|
+
from .object_serializer import CloudPickleSerializer, get_serializer
|
23
|
+
|
24
|
+
|
25
|
+
def is_pydantic_model_from_annotation(type_annotation):
|
26
|
+
# If it's a string representation
|
27
|
+
if isinstance(type_annotation, str):
|
28
|
+
# Extract the class name from the string
|
29
|
+
class_name = type_annotation.split("'")[-2].split(".")[-1]
|
30
|
+
# This part is tricky and might require additional context or imports
|
31
|
+
# You might need to import the actual class or module where it's defined
|
32
|
+
# For example:
|
33
|
+
# from indexify.functions_sdk.data_objects import File
|
34
|
+
# return issubclass(eval(class_name), BaseModel)
|
35
|
+
return False # Default to False if we can't evaluate
|
36
|
+
|
37
|
+
# If it's a Type object
|
38
|
+
origin = get_origin(type_annotation)
|
39
|
+
if origin is not None:
|
40
|
+
# Handle generic types like List[File], Optional[File], etc.
|
41
|
+
args = get_args(type_annotation)
|
42
|
+
if args:
|
43
|
+
return is_pydantic_model_from_annotation(args[0])
|
44
|
+
|
45
|
+
# If it's a direct class reference
|
46
|
+
if isinstance(type_annotation, type):
|
47
|
+
return issubclass(type_annotation, BaseModel)
|
48
|
+
|
49
|
+
return False
|
20
50
|
|
21
51
|
|
22
52
|
class EmbeddingIndexes(BaseModel):
|
@@ -186,3 +216,56 @@ class IndexifyFunctionWrapper:
|
|
186
216
|
extracted_data = self.indexify_function.run(*args, **kwargs)
|
187
217
|
|
188
218
|
return extracted_data if isinstance(extracted_data, list) else [extracted_data]
|
219
|
+
|
220
|
+
def invoke_fn_ser(
|
221
|
+
self, name: str, input: IndexifyData, acc: Optional[Any] = None
|
222
|
+
) -> List[IndexifyData]:
|
223
|
+
input = self.deserialize_input(name, input)
|
224
|
+
serializer = get_serializer(self.indexify_function.payload_encoder)
|
225
|
+
if acc is not None:
|
226
|
+
acc = self.indexify_function.accumulate.model_validate(
|
227
|
+
serializer.deserialize(acc.payload)
|
228
|
+
)
|
229
|
+
if acc is None and self.indexify_function.accumulate is not None:
|
230
|
+
acc = self.indexify_function.accumulate.model_validate(
|
231
|
+
self.indexify_function.accumulate()
|
232
|
+
)
|
233
|
+
outputs: List[Any] = self.run_fn(input, acc=acc)
|
234
|
+
return [
|
235
|
+
IndexifyData(payload=serializer.serialize(output)) for output in outputs
|
236
|
+
]
|
237
|
+
|
238
|
+
def invoke_router(self, name: str, input: IndexifyData) -> Optional[RouterOutput]:
|
239
|
+
input = self.deserialize_input(name, input)
|
240
|
+
return RouterOutput(edges=self.run_router(input))
|
241
|
+
|
242
|
+
def deserialize_input(self, compute_fn: str, indexify_data: IndexifyData) -> Any:
|
243
|
+
if self.indexify_function.payload_encoder == "cloudpickle":
|
244
|
+
return CloudPickleSerializer.deserialize(indexify_data.payload)
|
245
|
+
payload = msgpack.unpackb(indexify_data.payload)
|
246
|
+
signature = inspect.signature(self.indexify_function.run)
|
247
|
+
arg_types = {}
|
248
|
+
for name, param in signature.parameters.items():
|
249
|
+
if (
|
250
|
+
param.annotation != inspect.Parameter.empty
|
251
|
+
and param.annotation != getattr(compute_fn, "accumulate", None)
|
252
|
+
):
|
253
|
+
arg_types[name] = param.annotation
|
254
|
+
if len(arg_types) > 1:
|
255
|
+
raise ValueError(
|
256
|
+
f"Compute function {compute_fn} has multiple arguments, but only one is supported"
|
257
|
+
)
|
258
|
+
elif len(arg_types) == 0:
|
259
|
+
raise ValueError(f"Compute function {compute_fn} has no arguments")
|
260
|
+
arg_name, arg_type = next(iter(arg_types.items()))
|
261
|
+
if arg_type is None:
|
262
|
+
raise ValueError(f"Argument {arg_name} has no type annotation")
|
263
|
+
if is_pydantic_model_from_annotation(arg_type):
|
264
|
+
if len(payload.keys()) == 1 and isinstance(list(payload.values())[0], dict):
|
265
|
+
payload = list(payload.values())[0]
|
266
|
+
return arg_type.model_validate(payload)
|
267
|
+
return payload
|
268
|
+
|
269
|
+
def deserialize_fn_output(self, output: IndexifyData) -> Any:
|
270
|
+
serializer = get_serializer(self.indexify_function.payload_encoder)
|
271
|
+
return serializer.deserialize(output.payload)
|
@@ -13,6 +13,7 @@ from rich import print
|
|
13
13
|
from indexify.error import ApiException
|
14
14
|
from indexify.functions_sdk.data_objects import IndexifyData
|
15
15
|
from indexify.functions_sdk.graph import ComputeGraphMetadata, Graph
|
16
|
+
from indexify.functions_sdk.indexify_functions import IndexifyFunctionWrapper
|
16
17
|
from indexify.settings import DEFAULT_SERVICE_URL, DEFAULT_SERVICE_URL_HTTPS
|
17
18
|
|
18
19
|
|
@@ -70,6 +71,7 @@ class IndexifyClient:
|
|
70
71
|
self._service_url = service_url
|
71
72
|
self._timeout = kwargs.get("timeout")
|
72
73
|
self._graphs: Dict[str, Graph] = {}
|
74
|
+
self._fns = {}
|
73
75
|
|
74
76
|
def _request(self, method: str, **kwargs) -> httpx.Response:
|
75
77
|
try:
|
@@ -99,7 +101,7 @@ class IndexifyClient:
|
|
99
101
|
service_url: str = DEFAULT_SERVICE_URL_HTTPS,
|
100
102
|
*args,
|
101
103
|
**kwargs,
|
102
|
-
) -> "
|
104
|
+
) -> "IndexifyClient":
|
103
105
|
"""
|
104
106
|
Create a client with mutual TLS authentication. Also enables HTTP/2,
|
105
107
|
which is required for mTLS.
|
@@ -127,7 +129,7 @@ class IndexifyClient:
|
|
127
129
|
|
128
130
|
client_certs = (cert_path, key_path)
|
129
131
|
verify_option = ca_bundle_path if ca_bundle_path else True
|
130
|
-
client =
|
132
|
+
client = IndexifyClient(
|
131
133
|
*args,
|
132
134
|
**kwargs,
|
133
135
|
service_url=service_url,
|
@@ -160,7 +162,7 @@ class IndexifyClient:
|
|
160
162
|
|
161
163
|
def register_compute_graph(self, graph: Graph):
|
162
164
|
graph_metadata = graph.definition()
|
163
|
-
serialized_code = graph.serialize()
|
165
|
+
serialized_code = cloudpickle.dumps(graph.serialize())
|
164
166
|
response = self._post(
|
165
167
|
f"namespaces/{self.namespace}/compute_graphs",
|
166
168
|
files={"code": serialized_code},
|
@@ -168,6 +170,8 @@ class IndexifyClient:
|
|
168
170
|
)
|
169
171
|
response.raise_for_status()
|
170
172
|
self._graphs[graph.name] = graph
|
173
|
+
for fn_name, fn in graph.nodes.items():
|
174
|
+
self._fns[f"{graph.name}/{fn_name}"] = fn
|
171
175
|
|
172
176
|
def graphs(self) -> List[str]:
|
173
177
|
response = self._get(f"graphs")
|
@@ -177,11 +181,14 @@ class IndexifyClient:
|
|
177
181
|
response = self._get(f"namespaces/{self.namespace}/compute_graphs/{name}")
|
178
182
|
return ComputeGraphMetadata(**response.json())
|
179
183
|
|
180
|
-
def
|
184
|
+
def load_fn_wrapper(self, name: str, fn_name: str) -> IndexifyFunctionWrapper:
|
181
185
|
response = self._get(
|
182
186
|
f"internal/namespaces/{self.namespace}/compute_graphs/{name}/code"
|
183
187
|
)
|
184
|
-
|
188
|
+
pickled_functions_by_name = cloudpickle.loads(response.content)
|
189
|
+
return IndexifyFunctionWrapper(
|
190
|
+
cloudpickle.loads(pickled_functions_by_name[fn_name])
|
191
|
+
)
|
185
192
|
|
186
193
|
def namespaces(self) -> List[str]:
|
187
194
|
response = self._get(f"namespaces")
|
@@ -190,9 +197,28 @@ class IndexifyClient:
|
|
190
197
|
for item in namespaces_dict:
|
191
198
|
namespaces.append(item["name"])
|
192
199
|
return namespaces
|
200
|
+
|
201
|
+
@classmethod
|
202
|
+
def new_namespace(cls, namespace: str, server_addr: Optional[str] = "http://localhost:8900"):
|
203
|
+
# Create a new client instance with the specified server address
|
204
|
+
client = cls(service_url=server_addr)
|
205
|
+
|
206
|
+
try:
|
207
|
+
# Create the new namespace using the client
|
208
|
+
client.create_namespace(namespace)
|
209
|
+
except ApiException as e:
|
210
|
+
print(f"Failed to create namespace '{namespace}': {e}")
|
211
|
+
raise
|
212
|
+
|
213
|
+
# Set the namespace for the newly created client
|
214
|
+
client.namespace = namespace
|
215
|
+
|
216
|
+
# Return the client instance with the new namespace
|
217
|
+
return client
|
218
|
+
|
193
219
|
|
194
220
|
def create_namespace(self, namespace: str):
|
195
|
-
self._post("namespaces", json={"
|
221
|
+
self._post("namespaces", json={"name": namespace})
|
196
222
|
|
197
223
|
def logs(
|
198
224
|
self, invocation_id: str, cg_name: str, fn_name: str, file: str
|
@@ -286,8 +312,9 @@ class IndexifyClient:
|
|
286
312
|
block_until_done: bool = True: If True, the method will block until the extraction is done. If False, the method will return immediately.
|
287
313
|
return: Union[Dict[str, List[Any]], List[Any]]: The extracted objects. If the extractor name is provided, the output is a list of extracted objects by the extractor. If the extractor name is not provided, the output is a dictionary with the extractor name as the key and the extracted objects as the value. If no objects are found, an empty list is returned.
|
288
314
|
"""
|
289
|
-
|
290
|
-
|
315
|
+
fn_key = f"{graph}/{fn_name}"
|
316
|
+
if fn_key not in self._fns:
|
317
|
+
self._fns[fn_key] = self.load_fn_wrapper(graph, fn_name)
|
291
318
|
response = self._get(
|
292
319
|
f"namespaces/{self.namespace}/compute_graphs/{graph}/invocations/{invocation_id}/outputs",
|
293
320
|
)
|
@@ -299,9 +326,7 @@ class IndexifyClient:
|
|
299
326
|
indexify_data = self._download_output(
|
300
327
|
self.namespace, graph, invocation_id, fn_name, output.id
|
301
328
|
)
|
302
|
-
output = self.
|
303
|
-
fn_name, indexify_data
|
304
|
-
)
|
329
|
+
output = self._fns[fn_key].deserialize_fn_output(indexify_data)
|
305
330
|
outputs.append(output)
|
306
331
|
return outputs
|
307
332
|
|
@@ -32,6 +32,13 @@ class RemoteGraph:
|
|
32
32
|
return self._client.invoke_graph_with_object(
|
33
33
|
self._name, block_until_done, **kwargs
|
34
34
|
)
|
35
|
+
|
36
|
+
def rerun(self):
|
37
|
+
"""
|
38
|
+
Rerun the graph with the given invocation ID.
|
39
|
+
:param invocation_id: The invocation ID of the graph execution.
|
40
|
+
"""
|
41
|
+
self._client.rerun_graph(self._name)
|
35
42
|
|
36
43
|
@classmethod
|
37
44
|
def deploy(cls, g: Graph, additional_modules=[], server_url: Optional[str] = "http://localhost:8900"):
|
@@ -57,7 +64,7 @@ class RemoteGraph:
|
|
57
64
|
"""
|
58
65
|
return cls(name=name, server_url=server_url)
|
59
66
|
|
60
|
-
def
|
67
|
+
def output(
|
61
68
|
self,
|
62
69
|
invocation_id: str,
|
63
70
|
fn_name: str,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|