indexify 0.2.16__py3-none-any.whl → 0.2.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indexify/cli.py +1 -1
- indexify/executor/agent.py +7 -0
- indexify/executor/function_worker.py +25 -24
- indexify/executor/runtime_probes.py +9 -1
- indexify/functions_sdk/graph.py +25 -41
- indexify/functions_sdk/graph_definition.py +4 -0
- indexify/functions_sdk/image.py +21 -0
- indexify/functions_sdk/indexify_functions.py +74 -23
- indexify/http_client.py +4 -2
- {indexify-0.2.16.dist-info → indexify-0.2.18.dist-info}/METADATA +1 -1
- {indexify-0.2.16.dist-info → indexify-0.2.18.dist-info}/RECORD +14 -14
- {indexify-0.2.16.dist-info → indexify-0.2.18.dist-info}/LICENSE.txt +0 -0
- {indexify-0.2.16.dist-info → indexify-0.2.18.dist-info}/WHEEL +0 -0
- {indexify-0.2.16.dist-info → indexify-0.2.18.dist-info}/entry_points.txt +0 -0
indexify/cli.py
CHANGED
@@ -261,7 +261,7 @@ WORKDIR /app
|
|
261
261
|
docker_file += f"\nRUN pip install indexify"
|
262
262
|
|
263
263
|
console.print("Creating image using Dockerfile contents:", style="cyan bold")
|
264
|
-
|
264
|
+
print(f"{docker_file}")
|
265
265
|
|
266
266
|
client = docker.from_env()
|
267
267
|
image_name = f"{image._image_name}:{image._tag}"
|
indexify/executor/agent.py
CHANGED
@@ -307,6 +307,13 @@ class ExtractorAgent:
|
|
307
307
|
return words[0].capitalize() + "" + " ".join(words[1:])
|
308
308
|
|
309
309
|
runtime_probe: ProbeInfo = self._probe.probe()
|
310
|
+
|
311
|
+
# Inspect the image
|
312
|
+
if runtime_probe.is_default_executor:
|
313
|
+
# install dependencies
|
314
|
+
# rewrite the image name
|
315
|
+
pass
|
316
|
+
|
310
317
|
data = ExecutorMetadata(
|
311
318
|
id=self._executor_id,
|
312
319
|
addr="",
|
@@ -1,6 +1,5 @@
|
|
1
|
-
import
|
1
|
+
import sys
|
2
2
|
import traceback
|
3
|
-
from concurrent.futures.process import BrokenProcessPool
|
4
3
|
from typing import Dict, List, Optional
|
5
4
|
|
6
5
|
import cloudpickle
|
@@ -12,7 +11,11 @@ from indexify.functions_sdk.data_objects import (
|
|
12
11
|
IndexifyData,
|
13
12
|
RouterOutput,
|
14
13
|
)
|
15
|
-
from indexify.functions_sdk.indexify_functions import
|
14
|
+
from indexify.functions_sdk.indexify_functions import (
|
15
|
+
FunctionCallResult,
|
16
|
+
IndexifyFunctionWrapper,
|
17
|
+
RouterCallResult,
|
18
|
+
)
|
16
19
|
|
17
20
|
function_wrapper_map: Dict[str, IndexifyFunctionWrapper] = {}
|
18
21
|
|
@@ -62,6 +65,7 @@ class FunctionWorker:
|
|
62
65
|
self._executor: concurrent.futures.ProcessPoolExecutor = (
|
63
66
|
concurrent.futures.ProcessPoolExecutor(max_workers=workers)
|
64
67
|
)
|
68
|
+
self._workers = workers
|
65
69
|
|
66
70
|
async def async_submit(
|
67
71
|
self,
|
@@ -74,22 +78,11 @@ class FunctionWorker:
|
|
74
78
|
init_value: Optional[IndexifyData] = None,
|
75
79
|
) -> FunctionWorkerOutput:
|
76
80
|
try:
|
77
|
-
result =
|
78
|
-
|
79
|
-
_run_function,
|
80
|
-
namespace,
|
81
|
-
graph_name,
|
82
|
-
fn_name,
|
83
|
-
input,
|
84
|
-
code_path,
|
85
|
-
version,
|
86
|
-
init_value,
|
81
|
+
result = _run_function(
|
82
|
+
namespace, graph_name, fn_name, input, code_path, version, init_value
|
87
83
|
)
|
88
|
-
|
89
|
-
|
90
|
-
traceback.print_exc()
|
91
|
-
raise mp
|
92
|
-
except FunctionRunException as e:
|
84
|
+
# TODO - bring back running in a separate process
|
85
|
+
except Exception as e:
|
93
86
|
return FunctionWorkerOutput(
|
94
87
|
exception=str(e),
|
95
88
|
stdout=e.stdout,
|
@@ -122,7 +115,6 @@ def _run_function(
|
|
122
115
|
init_value: Optional[IndexifyData] = None,
|
123
116
|
) -> FunctionOutput:
|
124
117
|
import io
|
125
|
-
import traceback
|
126
118
|
from contextlib import redirect_stderr, redirect_stdout
|
127
119
|
|
128
120
|
stdout_capture = io.StringIO()
|
@@ -146,14 +138,23 @@ def _run_function(
|
|
146
138
|
str(type(fn.indexify_function))
|
147
139
|
== "<class 'indexify.functions_sdk.indexify_functions.IndexifyRo'>"
|
148
140
|
):
|
149
|
-
|
141
|
+
router_call_result: RouterCallResult = fn.invoke_router(fn_name, input)
|
142
|
+
router_output = RouterOutput(edges=router_call_result.edges)
|
143
|
+
if router_call_result.traceback_msg is not None:
|
144
|
+
print(router_call_result.traceback_msg, file=sys.stderr)
|
145
|
+
has_failed = True
|
146
|
+
exception_msg = router_call_result.traceback_msg
|
150
147
|
else:
|
151
|
-
|
152
|
-
|
148
|
+
fn_call_result: FunctionCallResult = fn.invoke_fn_ser(
|
149
|
+
fn_name, input, init_value
|
150
|
+
)
|
153
151
|
is_reducer = fn.indexify_function.accumulate is not None
|
152
|
+
fn_output = fn_call_result.ser_outputs
|
153
|
+
if fn_call_result.traceback_msg is not None:
|
154
|
+
print(fn_call_result.traceback_msg, file=sys.stderr)
|
155
|
+
has_failed = True
|
156
|
+
exception_msg = fn_call_result.traceback_msg
|
154
157
|
except Exception as e:
|
155
|
-
import sys
|
156
|
-
|
157
158
|
print(traceback.format_exc(), file=sys.stderr)
|
158
159
|
has_failed = True
|
159
160
|
exception_msg = str(e)
|
@@ -5,11 +5,14 @@ from typing import Any, Dict, Tuple
|
|
5
5
|
|
6
6
|
from pydantic import BaseModel
|
7
7
|
|
8
|
+
DEFAULT_EXECUTOR = "tensorlake/indexify-executor-default"
|
9
|
+
|
8
10
|
|
9
11
|
class ProbeInfo(BaseModel):
|
10
12
|
image_name: str
|
11
13
|
python_major_version: int
|
12
14
|
labels: Dict[str, Any] = {}
|
15
|
+
is_default_executor: bool
|
13
16
|
|
14
17
|
|
15
18
|
class RuntimeProbes:
|
@@ -27,12 +30,15 @@ class RuntimeProbes:
|
|
27
30
|
if os.path.exists(file_path):
|
28
31
|
with open(file_path, "r") as file:
|
29
32
|
return file.read().strip()
|
30
|
-
return
|
33
|
+
return DEFAULT_EXECUTOR
|
31
34
|
|
32
35
|
def _get_python_version(self) -> Tuple[int, int]:
|
33
36
|
version_info = sys.version_info
|
34
37
|
return version_info.major, version_info.minor
|
35
38
|
|
39
|
+
def _is_default_executor(self):
|
40
|
+
return True if self._read_image_name() == DEFAULT_EXECUTOR else False
|
41
|
+
|
36
42
|
def probe(self) -> ProbeInfo:
|
37
43
|
labels = {
|
38
44
|
"os": self._os_name,
|
@@ -41,8 +47,10 @@ class RuntimeProbes:
|
|
41
47
|
"python_major_version": self._python_version_major,
|
42
48
|
"python_minor_version": self._python_version_minor,
|
43
49
|
}
|
50
|
+
|
44
51
|
return ProbeInfo(
|
45
52
|
image_name=self._image_name,
|
46
53
|
python_major_version=self._python_version_major,
|
47
54
|
labels=labels,
|
55
|
+
is_default_executor=self._is_default_executor(),
|
48
56
|
)
|
indexify/functions_sdk/graph.py
CHANGED
@@ -29,6 +29,7 @@ from .graph_definition import (
|
|
29
29
|
)
|
30
30
|
from .graph_validation import validate_node, validate_route
|
31
31
|
from .indexify_functions import (
|
32
|
+
FunctionCallResult,
|
32
33
|
IndexifyFunction,
|
33
34
|
IndexifyFunctionWrapper,
|
34
35
|
IndexifyRouter,
|
@@ -157,6 +158,7 @@ class Graph:
|
|
157
158
|
description=start_node.description,
|
158
159
|
reducer=start_node.accumulate is not None,
|
159
160
|
image_name=start_node.image._image_name,
|
161
|
+
image_information=start_node.image.to_image_information(),
|
160
162
|
)
|
161
163
|
metadata_edges = self.edges.copy()
|
162
164
|
metadata_nodes = {}
|
@@ -170,6 +172,7 @@ class Graph:
|
|
170
172
|
target_fns=self.routers[node_name],
|
171
173
|
payload_encoder=node.payload_encoder,
|
172
174
|
image_name=node.image._image_name,
|
175
|
+
image_information=node.image.to_image_information(),
|
173
176
|
)
|
174
177
|
)
|
175
178
|
else:
|
@@ -180,6 +183,7 @@ class Graph:
|
|
180
183
|
description=node.description,
|
181
184
|
reducer=node.accumulate is not None,
|
182
185
|
image_name=node.image._image_name,
|
186
|
+
image_information=node.image.to_image_information(),
|
183
187
|
)
|
184
188
|
)
|
185
189
|
|
@@ -209,57 +213,37 @@ class Graph:
|
|
209
213
|
k: IndexifyData(payload=serializer.serialize(v))
|
210
214
|
}
|
211
215
|
self._results[input.id] = outputs
|
212
|
-
enable_cache = kwargs.get(
|
213
|
-
self._run(input, outputs,enable_cache)
|
216
|
+
enable_cache = kwargs.get("enable_cache", True)
|
217
|
+
self._run(input, outputs, enable_cache)
|
214
218
|
return input.id
|
215
219
|
|
216
220
|
def _run(
|
217
221
|
self,
|
218
222
|
initial_input: IndexifyData,
|
219
223
|
outputs: Dict[str, List[bytes]],
|
220
|
-
enable_cache: bool
|
224
|
+
enable_cache: bool,
|
221
225
|
):
|
222
226
|
accumulator_values = self._accumulator_values[initial_input.id]
|
223
227
|
queue = deque([(self._start_node, initial_input)])
|
224
228
|
while queue:
|
225
229
|
node_name, input = queue.popleft()
|
226
230
|
node = self.nodes[node_name]
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
)
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
outputs[node_name].extend(
|
243
|
-
else:
|
244
|
-
function_outputs: List[IndexifyData] = IndexifyFunctionWrapper(
|
245
|
-
node
|
246
|
-
).invoke_fn_ser(
|
247
|
-
node_name, input, accumulator_values.get(node_name, None)
|
248
|
-
)
|
249
|
-
print(f"ran {node_name}: num outputs: {len(function_outputs)}")
|
250
|
-
if accumulator_values.get(node_name, None) is not None:
|
251
|
-
accumulator_values[node_name] = function_outputs[-1].model_copy()
|
252
|
-
outputs[node_name] = []
|
253
|
-
outputs[node_name].extend(function_outputs)
|
254
|
-
function_outputs_bytes: List[bytes] = [
|
255
|
-
serializer.serialize_list(function_outputs)
|
256
|
-
]
|
257
|
-
self._cache.set(
|
258
|
-
self.name,
|
259
|
-
node_name,
|
260
|
-
input_bytes,
|
261
|
-
function_outputs_bytes,
|
262
|
-
)
|
231
|
+
function_outputs: FunctionCallResult = IndexifyFunctionWrapper(
|
232
|
+
node
|
233
|
+
).invoke_fn_ser(node_name, input, accumulator_values.get(node_name, None))
|
234
|
+
if function_outputs.traceback_msg is not None:
|
235
|
+
print(function_outputs.traceback_msg)
|
236
|
+
import os
|
237
|
+
|
238
|
+
print("exiting local execution due to error")
|
239
|
+
os._exit(1)
|
240
|
+
fn_outputs = function_outputs.ser_outputs
|
241
|
+
print(f"ran {node_name}: num outputs: {len(fn_outputs)}")
|
242
|
+
if accumulator_values.get(node_name, None) is not None:
|
243
|
+
accumulator_values[node_name] = fn_outputs[-1].model_copy()
|
244
|
+
outputs[node_name] = []
|
245
|
+
if fn_outputs:
|
246
|
+
outputs[node_name].extend(fn_outputs)
|
263
247
|
if accumulator_values.get(node_name, None) is not None and queue:
|
264
248
|
print(
|
265
249
|
f"accumulator not none for {node_name}, continuing, len queue: {len(queue)}"
|
@@ -271,7 +255,7 @@ class Graph:
|
|
271
255
|
for i, edge in enumerate(out_edges):
|
272
256
|
if edge in self.routers:
|
273
257
|
out_edges.remove(edge)
|
274
|
-
for output in
|
258
|
+
for output in fn_outputs:
|
275
259
|
dynamic_edges = self._route(edge, output) or []
|
276
260
|
for dynamic_edge in dynamic_edges.edges:
|
277
261
|
if dynamic_edge in self.nodes:
|
@@ -280,7 +264,7 @@ class Graph:
|
|
280
264
|
)
|
281
265
|
out_edges.append(dynamic_edge)
|
282
266
|
for out_edge in out_edges:
|
283
|
-
for output in
|
267
|
+
for output in fn_outputs:
|
284
268
|
queue.append((out_edge, output))
|
285
269
|
|
286
270
|
def _route(self, node_name: str, input: IndexifyData) -> Optional[RouterOutput]:
|
@@ -2,6 +2,8 @@ from typing import Dict, List, Optional
|
|
2
2
|
|
3
3
|
from pydantic import BaseModel
|
4
4
|
|
5
|
+
from indexify.functions_sdk.image import ImageInformation
|
6
|
+
|
5
7
|
from .object_serializer import get_serializer
|
6
8
|
|
7
9
|
|
@@ -11,6 +13,7 @@ class FunctionMetadata(BaseModel):
|
|
11
13
|
description: str
|
12
14
|
reducer: bool = False
|
13
15
|
image_name: str
|
16
|
+
image_information: ImageInformation
|
14
17
|
payload_encoder: str = "cloudpickle"
|
15
18
|
|
16
19
|
|
@@ -20,6 +23,7 @@ class RouterMetadata(BaseModel):
|
|
20
23
|
source_fn: str
|
21
24
|
target_fns: List[str]
|
22
25
|
image_name: str
|
26
|
+
image_information: ImageInformation
|
23
27
|
payload_encoder: str = "cloudpickle"
|
24
28
|
|
25
29
|
|
indexify/functions_sdk/image.py
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
from typing import List
|
2
|
+
|
3
|
+
from pydantic import BaseModel
|
4
|
+
|
5
|
+
|
1
6
|
def python_version_to_image(python_version):
|
2
7
|
if python_version.startswith("3.9"):
|
3
8
|
return "python:3.9.20-bookworm"
|
@@ -9,6 +14,14 @@ def python_version_to_image(python_version):
|
|
9
14
|
raise ValueError(f"unsupported Python version: {python_version}")
|
10
15
|
|
11
16
|
|
17
|
+
# Pydantic object for API
|
18
|
+
class ImageInformation(BaseModel):
|
19
|
+
image_name: str
|
20
|
+
tag: str
|
21
|
+
base_image: str
|
22
|
+
run_strs: List[str]
|
23
|
+
|
24
|
+
|
12
25
|
class Image:
|
13
26
|
def __init__(self, python="3.10"):
|
14
27
|
self._image_name = None
|
@@ -33,6 +46,14 @@ class Image:
|
|
33
46
|
self._run_strs.append(run_str)
|
34
47
|
return self
|
35
48
|
|
49
|
+
def to_image_information(self):
|
50
|
+
return ImageInformation(
|
51
|
+
image_name=self._image_name,
|
52
|
+
tag=self._tag,
|
53
|
+
base_image=self._base_image,
|
54
|
+
run_strs=self._run_strs,
|
55
|
+
)
|
56
|
+
|
36
57
|
|
37
58
|
DEFAULT_IMAGE_3_10 = (
|
38
59
|
Image()
|
@@ -1,4 +1,7 @@
|
|
1
1
|
import inspect
|
2
|
+
import re
|
3
|
+
import sys
|
4
|
+
import traceback
|
2
5
|
from abc import ABC, abstractmethod
|
3
6
|
from functools import update_wrapper
|
4
7
|
from typing import (
|
@@ -7,6 +10,7 @@ from typing import (
|
|
7
10
|
Dict,
|
8
11
|
List,
|
9
12
|
Optional,
|
13
|
+
Tuple,
|
10
14
|
Type,
|
11
15
|
Union,
|
12
16
|
get_args,
|
@@ -22,6 +26,37 @@ from .image import DEFAULT_IMAGE_3_10, Image
|
|
22
26
|
from .object_serializer import CloudPickleSerializer, get_serializer
|
23
27
|
|
24
28
|
|
29
|
+
def format_filtered_traceback(exc_info=None):
|
30
|
+
"""
|
31
|
+
Format a traceback excluding indexify_functions.py lines.
|
32
|
+
Can be used in exception handlers to replace traceback.format_exc()
|
33
|
+
"""
|
34
|
+
if exc_info is None:
|
35
|
+
exc_info = sys.exc_info()
|
36
|
+
|
37
|
+
# Get the full traceback as a string
|
38
|
+
full_traceback = traceback.format_exception(*exc_info)
|
39
|
+
|
40
|
+
# Filter out lines containing indexify_functions.py
|
41
|
+
filtered_lines = []
|
42
|
+
skip_next = False
|
43
|
+
|
44
|
+
for line in full_traceback:
|
45
|
+
if "indexify_functions.py" in line:
|
46
|
+
skip_next = True
|
47
|
+
continue
|
48
|
+
if skip_next:
|
49
|
+
if line.strip().startswith("File "):
|
50
|
+
skip_next = False
|
51
|
+
else:
|
52
|
+
continue
|
53
|
+
filtered_lines.append(line)
|
54
|
+
|
55
|
+
# Clean up any double blank lines that might have been created
|
56
|
+
cleaned = re.sub(r"\n\s*\n\s*\n", "\n\n", "".join(filtered_lines))
|
57
|
+
return cleaned
|
58
|
+
|
59
|
+
|
25
60
|
def is_pydantic_model_from_annotation(type_annotation):
|
26
61
|
# If it's a string representation
|
27
62
|
if isinstance(type_annotation, str):
|
@@ -49,12 +84,6 @@ def is_pydantic_model_from_annotation(type_annotation):
|
|
49
84
|
return False
|
50
85
|
|
51
86
|
|
52
|
-
class EmbeddingIndexes(BaseModel):
|
53
|
-
dim: int
|
54
|
-
distance: Optional[str] = "cosine"
|
55
|
-
database_url: Optional[str] = None
|
56
|
-
|
57
|
-
|
58
87
|
class PlacementConstraints(BaseModel):
|
59
88
|
min_python_version: Optional[str] = "3.9"
|
60
89
|
max_python_version: Optional[str] = None
|
@@ -62,7 +91,7 @@ class PlacementConstraints(BaseModel):
|
|
62
91
|
image_name: Optional[str] = None
|
63
92
|
|
64
93
|
|
65
|
-
class IndexifyFunction
|
94
|
+
class IndexifyFunction:
|
66
95
|
name: str = ""
|
67
96
|
description: str = ""
|
68
97
|
image: Optional[Image] = DEFAULT_IMAGE_3_10
|
@@ -70,7 +99,6 @@ class IndexifyFunction(ABC):
|
|
70
99
|
accumulate: Optional[Type[Any]] = None
|
71
100
|
payload_encoder: Optional[str] = "cloudpickle"
|
72
101
|
|
73
|
-
@abstractmethod
|
74
102
|
def run(self, *args, **kwargs) -> Union[List[Any], Any]:
|
75
103
|
pass
|
76
104
|
|
@@ -84,14 +112,14 @@ class IndexifyFunction(ABC):
|
|
84
112
|
serializer = get_serializer(cls.payload_encoder)
|
85
113
|
return serializer.deserialize(output.payload)
|
86
114
|
|
87
|
-
|
115
|
+
|
116
|
+
class IndexifyRouter:
|
88
117
|
name: str = ""
|
89
118
|
description: str = ""
|
90
119
|
image: Optional[Image] = DEFAULT_IMAGE_3_10
|
91
120
|
placement_constraints: List[PlacementConstraints] = []
|
92
121
|
payload_encoder: Optional[str] = "cloudpickle"
|
93
122
|
|
94
|
-
@abstractmethod
|
95
123
|
def run(self, *args, **kwargs) -> Optional[List[IndexifyFunction]]:
|
96
124
|
pass
|
97
125
|
|
@@ -166,6 +194,16 @@ def indexify_function(
|
|
166
194
|
return construct
|
167
195
|
|
168
196
|
|
197
|
+
class FunctionCallResult(BaseModel):
|
198
|
+
ser_outputs: List[IndexifyData]
|
199
|
+
traceback_msg: Optional[str] = None
|
200
|
+
|
201
|
+
|
202
|
+
class RouterCallResult(BaseModel):
|
203
|
+
edges: List[str]
|
204
|
+
traceback_msg: Optional[str] = None
|
205
|
+
|
206
|
+
|
169
207
|
class IndexifyFunctionWrapper:
|
170
208
|
def __init__(self, indexify_function: Union[IndexifyFunction, IndexifyRouter]):
|
171
209
|
self.indexify_function: Union[
|
@@ -189,7 +227,9 @@ class IndexifyFunctionWrapper:
|
|
189
227
|
)
|
190
228
|
return return_type
|
191
229
|
|
192
|
-
def run_router(
|
230
|
+
def run_router(
|
231
|
+
self, input: Union[Dict, Type[BaseModel]]
|
232
|
+
) -> Tuple[List[str], Optional[str]]:
|
193
233
|
kwargs = input if isinstance(input, dict) else {"input": input}
|
194
234
|
args = []
|
195
235
|
kwargs = {}
|
@@ -197,17 +237,20 @@ class IndexifyFunctionWrapper:
|
|
197
237
|
kwargs = input
|
198
238
|
else:
|
199
239
|
args.append(input)
|
200
|
-
|
240
|
+
try:
|
241
|
+
extracted_data = self.indexify_function.run(*args, **kwargs)
|
242
|
+
except Exception as e:
|
243
|
+
return [], format_filtered_traceback()
|
201
244
|
if not isinstance(extracted_data, list) and extracted_data is not None:
|
202
|
-
return [extracted_data.name]
|
245
|
+
return [extracted_data.name], None
|
203
246
|
edges = []
|
204
247
|
for fn in extracted_data or []:
|
205
248
|
edges.append(fn.name)
|
206
|
-
return edges
|
249
|
+
return edges, None
|
207
250
|
|
208
251
|
def run_fn(
|
209
252
|
self, input: Union[Dict, Type[BaseModel]], acc: Type[Any] = None
|
210
|
-
) -> List[
|
253
|
+
) -> Tuple[List[Any], Optional[str]]:
|
211
254
|
args = []
|
212
255
|
kwargs = {}
|
213
256
|
if acc is not None:
|
@@ -217,15 +260,21 @@ class IndexifyFunctionWrapper:
|
|
217
260
|
else:
|
218
261
|
args.append(input)
|
219
262
|
|
220
|
-
|
263
|
+
try:
|
264
|
+
extracted_data = self.indexify_function.run(*args, **kwargs)
|
265
|
+
except Exception as e:
|
266
|
+
return [], format_filtered_traceback()
|
221
267
|
if extracted_data is None:
|
222
|
-
return []
|
268
|
+
return [], None
|
223
269
|
|
224
|
-
|
270
|
+
output = (
|
271
|
+
extracted_data if isinstance(extracted_data, list) else [extracted_data]
|
272
|
+
)
|
273
|
+
return output, None
|
225
274
|
|
226
275
|
def invoke_fn_ser(
|
227
276
|
self, name: str, input: IndexifyData, acc: Optional[Any] = None
|
228
|
-
) ->
|
277
|
+
) -> FunctionCallResult:
|
229
278
|
input = self.deserialize_input(name, input)
|
230
279
|
serializer = get_serializer(self.indexify_function.payload_encoder)
|
231
280
|
if acc is not None:
|
@@ -236,14 +285,16 @@ class IndexifyFunctionWrapper:
|
|
236
285
|
acc = self.indexify_function.accumulate.model_validate(
|
237
286
|
self.indexify_function.accumulate()
|
238
287
|
)
|
239
|
-
outputs
|
240
|
-
|
288
|
+
outputs, err = self.run_fn(input, acc=acc)
|
289
|
+
ser_outputs = [
|
241
290
|
IndexifyData(payload=serializer.serialize(output)) for output in outputs
|
242
291
|
]
|
292
|
+
return FunctionCallResult(ser_outputs=ser_outputs, traceback_msg=err)
|
243
293
|
|
244
|
-
def invoke_router(self, name: str, input: IndexifyData) ->
|
294
|
+
def invoke_router(self, name: str, input: IndexifyData) -> RouterCallResult:
|
245
295
|
input = self.deserialize_input(name, input)
|
246
|
-
|
296
|
+
edges, err = self.run_router(input)
|
297
|
+
return RouterCallResult(edges=edges, traceback_msg=err)
|
247
298
|
|
248
299
|
def deserialize_input(self, compute_fn: str, indexify_data: IndexifyData) -> Any:
|
249
300
|
if self.indexify_function.payload_encoder == "cloudpickle":
|
indexify/http_client.py
CHANGED
@@ -220,11 +220,11 @@ class IndexifyClient:
|
|
220
220
|
self._post("namespaces", json={"name": namespace})
|
221
221
|
|
222
222
|
def logs(
|
223
|
-
self, invocation_id: str, cg_name: str, fn_name: str, file: str
|
223
|
+
self, invocation_id: str, cg_name: str, fn_name: str, task_id: str, file: str
|
224
224
|
) -> Optional[str]:
|
225
225
|
try:
|
226
226
|
response = self._get(
|
227
|
-
f"namespaces/{self.namespace}/compute_graphs/{cg_name}/invocations/{invocation_id}/fn/{fn_name}/logs/{file}"
|
227
|
+
f"namespaces/{self.namespace}/compute_graphs/{cg_name}/invocations/{invocation_id}/fn/{fn_name}/tasks/{task_id}/logs/{file}"
|
228
228
|
)
|
229
229
|
response.raise_for_status()
|
230
230
|
return response.content.decode("utf-8")
|
@@ -272,12 +272,14 @@ class IndexifyClient:
|
|
272
272
|
event.payload.invocation_id,
|
273
273
|
graph,
|
274
274
|
event.payload.fn_name,
|
275
|
+
event.payload.task_id,
|
275
276
|
"stdout",
|
276
277
|
)
|
277
278
|
stderr = self.logs(
|
278
279
|
event.payload.invocation_id,
|
279
280
|
graph,
|
280
281
|
event.payload.fn_name,
|
282
|
+
event.payload.task_id,
|
281
283
|
"stderr",
|
282
284
|
)
|
283
285
|
if stdout:
|
@@ -1,33 +1,33 @@
|
|
1
1
|
indexify/__init__.py,sha256=xYjdqZQ9CEtV-s4PJhj_6Vx_dfg_qknWHkLqOSVom8g,623
|
2
|
-
indexify/cli.py,sha256=
|
2
|
+
indexify/cli.py,sha256=E7NIWMAJcGHeOFG0rE2mCgT_V0e1l2M6tOPDSjKzrzg,8219
|
3
3
|
indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
|
4
4
|
indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
|
5
5
|
indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
|
6
6
|
indexify/error.py,sha256=vjd5SPPNFIEW35GorSIodsqvm9RKHQm9kdp8t9gv-WM,111
|
7
|
-
indexify/executor/agent.py,sha256=
|
7
|
+
indexify/executor/agent.py,sha256=IvHexGFtapPfAB3Ays2KBY1Z0BVknTAj3lbqSzCj7PE,14908
|
8
8
|
indexify/executor/api_objects.py,sha256=SysjlGYu4JtYdqfexZHHN1IW4TtaDdFUF3hYZ5mpUJU,810
|
9
9
|
indexify/executor/downloader.py,sha256=3mEDdluTzspsLGAZtFHZOVuyKOzT3CSema2kIK6Z1yU,4005
|
10
10
|
indexify/executor/executor_tasks.py,sha256=gAZ2pvza1YwGlaR1o_tJW4SXtdCgK7sLJgp4W7rOjR0,1834
|
11
|
-
indexify/executor/function_worker.py,sha256=
|
11
|
+
indexify/executor/function_worker.py,sha256=pCGn13rg4dEykzmwYNyCTkewrpyQXQR1cH6n2Hx5Lfc,5813
|
12
12
|
indexify/executor/indexify_executor.py,sha256=2Ut_VX-Su_lm4b4aEROyRJ3gXx-uFHA-V7EN0sWiARE,771
|
13
|
-
indexify/executor/runtime_probes.py,sha256=
|
13
|
+
indexify/executor/runtime_probes.py,sha256=JY0FoxtlQ9sgsE8gBKWM5h3R1TWkYENhNF0HR2KkV4Q,1704
|
14
14
|
indexify/executor/task_reporter.py,sha256=gnnse0v6rjjni8lNzeb-ZYq6iF2DgafKoT7dcGUZhQ4,3716
|
15
15
|
indexify/executor/task_store.py,sha256=q8s2gImsFffWeXQR0mk1Xlo1Aj_2GfclNPjQ2EA_YBo,3984
|
16
16
|
indexify/functions_sdk/data_objects.py,sha256=CQZMzYiV7l6dyzFkYquWQHqdte6JnC7XA3i2ZyvPvgQ,844
|
17
|
-
indexify/functions_sdk/graph.py,sha256=
|
18
|
-
indexify/functions_sdk/graph_definition.py,sha256=
|
17
|
+
indexify/functions_sdk/graph.py,sha256=AaNU0D3YHFZorEkXKLiaftE8MncO6oChG5AKXD0rd1o,11141
|
18
|
+
indexify/functions_sdk/graph_definition.py,sha256=fwv63wkuKeVQxJQk9ofu8ZL3B0NzJzF9bJhFdVZZvMQ,1317
|
19
19
|
indexify/functions_sdk/graph_validation.py,sha256=XLHiC9PAtZungJLysU3hIUOPNDkO5TXUDZ_jiZ0H4hg,2508
|
20
|
-
indexify/functions_sdk/image.py,sha256=
|
21
|
-
indexify/functions_sdk/indexify_functions.py,sha256=
|
20
|
+
indexify/functions_sdk/image.py,sha256=QK0H6KxLWriB_z4M0kunKzzHdHxYLWL670RPYgYuf_8,1762
|
21
|
+
indexify/functions_sdk/indexify_functions.py,sha256=XRsa4IR9rQXdvF2X7Tbudj0rTCMOV8HJqezgaPgH9sU,11107
|
22
22
|
indexify/functions_sdk/local_cache.py,sha256=cNWF67zbhbTJe3g86hyLBy3Rqzs6dNvp2SjLazGZWvw,1348
|
23
23
|
indexify/functions_sdk/object_serializer.py,sha256=Zz4GobW3ZamBBtFDF76QxU3TP6oJNdWnhsfKd0OUFoc,1660
|
24
24
|
indexify/functions_sdk/pipeline.py,sha256=7hDatRK-SCHYvttf2Vj5YFyiJEVU0OOXEZBOIQenSmk,847
|
25
|
-
indexify/http_client.py,sha256=
|
25
|
+
indexify/http_client.py,sha256=MflqHbkkzYWw64nnmZCeJE76Q9p-o5hFnlI3h7kbJFI,13729
|
26
26
|
indexify/remote_graph.py,sha256=ILg6IY6EFgyvnfz1DSzicBZhqvPkg2-UUwgI6lxp6sA,3094
|
27
27
|
indexify/remote_pipeline.py,sha256=FW7IAv3r24OOpiqlprw3kuFrpdkqi6Ic4_tT26FThjA,761
|
28
28
|
indexify/settings.py,sha256=LSaWZ0ADIVmUv6o6dHWRC3-Ry5uLbCw2sBSg1e_U7UM,99
|
29
|
-
indexify-0.2.
|
30
|
-
indexify-0.2.
|
31
|
-
indexify-0.2.
|
32
|
-
indexify-0.2.
|
33
|
-
indexify-0.2.
|
29
|
+
indexify-0.2.18.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
30
|
+
indexify-0.2.18.dist-info/METADATA,sha256=NqazD7Kypzfm19nWd4N8bzAVymbNLKUV3jz1YO9LBMw,6199
|
31
|
+
indexify-0.2.18.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
32
|
+
indexify-0.2.18.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
|
33
|
+
indexify-0.2.18.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|