indexify 0.2.44__py3-none-any.whl → 0.2.45__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indexify/__init__.py +2 -0
- indexify/cli.py +41 -80
- indexify/executor/api_objects.py +2 -0
- indexify/executor/downloader.py +23 -25
- indexify/executor/executor.py +29 -35
- indexify/executor/function_executor/function_executor.py +120 -19
- indexify/executor/function_executor/function_executor_state.py +75 -0
- indexify/executor/function_executor/invocation_state_client.py +232 -0
- indexify/executor/function_executor/server/function_executor_server.py +24 -0
- indexify/executor/function_executor/server/function_executor_server_factory.py +43 -0
- indexify/executor/function_executor/server/subprocess_function_executor_server.py +25 -0
- indexify/executor/function_executor/{process_function_executor_factory.py → server/subprocess_function_executor_server_factory.py} +21 -21
- indexify/executor/function_executor/single_task_runner.py +160 -0
- indexify/executor/function_executor/task_input.py +23 -0
- indexify/executor/function_executor/task_output.py +36 -0
- indexify/executor/task_reporter.py +10 -17
- indexify/executor/task_runner.py +104 -0
- indexify/function_executor/function_executor_service.py +22 -7
- indexify/function_executor/handlers/run_function/handler.py +13 -12
- indexify/function_executor/invocation_state/invocation_state_proxy_server.py +170 -0
- indexify/function_executor/invocation_state/proxied_invocation_state.py +24 -0
- indexify/function_executor/invocation_state/response_validator.py +29 -0
- indexify/function_executor/proto/function_executor.proto +47 -0
- indexify/function_executor/proto/function_executor_pb2.py +23 -11
- indexify/function_executor/proto/function_executor_pb2.pyi +70 -0
- indexify/function_executor/proto/function_executor_pb2_grpc.py +50 -0
- indexify/functions_sdk/graph.py +3 -3
- indexify/functions_sdk/image.py +142 -9
- indexify/functions_sdk/indexify_functions.py +45 -79
- indexify/functions_sdk/invocation_state/invocation_state.py +22 -0
- indexify/functions_sdk/invocation_state/local_invocation_state.py +30 -0
- indexify/http_client.py +0 -17
- {indexify-0.2.44.dist-info → indexify-0.2.45.dist-info}/METADATA +1 -1
- indexify-0.2.45.dist-info/RECORD +60 -0
- indexify/executor/function_executor/function_executor_factory.py +0 -26
- indexify/executor/function_executor/function_executor_map.py +0 -91
- indexify/executor/function_executor/process_function_executor.py +0 -64
- indexify/executor/function_worker.py +0 -253
- indexify-0.2.44.dist-info/RECORD +0 -50
- {indexify-0.2.44.dist-info → indexify-0.2.45.dist-info}/LICENSE.txt +0 -0
- {indexify-0.2.44.dist-info → indexify-0.2.45.dist-info}/WHEEL +0 -0
- {indexify-0.2.44.dist-info → indexify-0.2.45.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,60 @@
|
|
1
|
+
indexify/__init__.py,sha256=ERgPjxxLiyvtT1y6aapzKspJQuEb_eBBUIN8idKmey0,739
|
2
|
+
indexify/cli.py,sha256=BWR3L7-WovzMvoZbwMbv3CyIlBt3DcT11dqz6nYlaQA,9473
|
3
|
+
indexify/common_util.py,sha256=LKS6yZ3yv8nF2J-KzisGIjqjTvCn7tLFifQJLT4gHRg,3529
|
4
|
+
indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
|
5
|
+
indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
|
6
|
+
indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
|
7
|
+
indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
|
8
|
+
indexify/executor/api_objects.py,sha256=CJ1RUZWAw5WC1dO6IPx36s9PPlY_ONNHFCAhV7Ql4s0,885
|
9
|
+
indexify/executor/downloader.py,sha256=9zzV1febq5xXlvDjsDPms8rvBTIE0ulXKqeWpvvfaGQ,6368
|
10
|
+
indexify/executor/executor.py,sha256=cnTzqqokZj2YxAwV4R9ihMTw5DpanWaF-ceNRm5Y3qk,5209
|
11
|
+
indexify/executor/function_executor/function_executor.py,sha256=u9hzRAH8_rcDpIWOjT0CA6yNOo8qlyBYqLg9VESqBvo,4863
|
12
|
+
indexify/executor/function_executor/function_executor_state.py,sha256=CKU3ujetHf6_8zEXtY5eSxcaFB6PfPQSuPlL1AJDJf8,2758
|
13
|
+
indexify/executor/function_executor/invocation_state_client.py,sha256=LgVFRaklo201AogAM-hLrfTfUeQZxmwYRZv0X2u-64Y,8596
|
14
|
+
indexify/executor/function_executor/server/function_executor_server.py,sha256=_DLivLDikupZusRk8gVWDk7fWPT9XjZ4un1yWSlOObs,883
|
15
|
+
indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=pGbJMQfC5TNvyWOs6VDKdqd2PK5OHQh5_wSDP-E7DbI,1677
|
16
|
+
indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=DxQMwAvQi03OGo2sFruUrxwcMPG8In4C62L3sped2FI,694
|
17
|
+
indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=MrhlcH5UZXJn9mAuyCJVVcJTihx8SDDuaraXFvkWjsU,3628
|
18
|
+
indexify/executor/function_executor/single_task_runner.py,sha256=mzYSUswpN-q98L1VALnt246AwmyNlQ-BcKZR96wF6gw,5722
|
19
|
+
indexify/executor/function_executor/task_input.py,sha256=7Mkd-OCt1s9HdbyhrxKQkhxsVR9yeT6jC6zd2knt5-8,591
|
20
|
+
indexify/executor/function_executor/task_output.py,sha256=3-qcT1aigmOFEp6QSJtUcQfqeJfrxvo265qgk9WAUVg,1064
|
21
|
+
indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
|
22
|
+
indexify/executor/task_fetcher.py,sha256=qBuVxgjzjkTi_1_hyQqtBxetBxdL7TGMgI4FBwyTANE,3004
|
23
|
+
indexify/executor/task_reporter.py,sha256=xHEk_TVeXYo6rcuDWImFmjLjfwRvpohP1-MS4grCASM,6663
|
24
|
+
indexify/executor/task_runner.py,sha256=yO_5o8-oA-TeSyxqS-DX6XfUb0947jrPijJeGI4wqQk,4807
|
25
|
+
indexify/function_executor/function_executor_service.py,sha256=DVP19TUo5iPAki5dh320kT2JjO69bLszE1rWkWg2MiY,4244
|
26
|
+
indexify/function_executor/handlers/run_function/function_inputs_loader.py,sha256=x2Lrzb6CsEWDWW5qG9etS7TyV7KVYfuFKEbzKVhYizo,1553
|
27
|
+
indexify/function_executor/handlers/run_function/handler.py,sha256=g6JZna852tQleASn9lA651fZh_chVYTKbyWDvgXi0NA,5695
|
28
|
+
indexify/function_executor/handlers/run_function/request_validator.py,sha256=tVmbOS2g4AeV48CZCSDRycnoQjjGy12XD_vbaOLX_Z4,740
|
29
|
+
indexify/function_executor/handlers/run_function/response_helper.py,sha256=KDSadee08g6JRFwKRTdcOwbP6SmRJCGVHh1FOLdt_ws,3109
|
30
|
+
indexify/function_executor/initialize_request_validator.py,sha256=AtyKgIjlGHB-Fh3Vhso86TC0o65DfuVEY8GFfRSfylo,709
|
31
|
+
indexify/function_executor/invocation_state/invocation_state_proxy_server.py,sha256=JFan-Ku9aFy72_54j0tyg5e8LdpvEIcbWL0pbNfcFSs,7021
|
32
|
+
indexify/function_executor/invocation_state/proxied_invocation_state.py,sha256=AOztc2uSTaPQEqwc8Fh5zLHgJlgv6u68kvP0IolecOA,951
|
33
|
+
indexify/function_executor/invocation_state/response_validator.py,sha256=05Y9_5s010owaDb6OUia9W9Alx1r1zJ179HboEkT-Xk,896
|
34
|
+
indexify/function_executor/proto/configuration.py,sha256=P0xtxwf-XaCxi0kvtCbm4X8y0YOuHhR-uh9xUuLdFoM,595
|
35
|
+
indexify/function_executor/proto/function_executor.proto,sha256=S1hnn0WLOKGuQAOOSTBxnPTOuDTxhqcGg7rHegSzl5M,3789
|
36
|
+
indexify/function_executor/proto/function_executor_pb2.py,sha256=lwBmDF2gTwcMYgp_4fLIm3SpS1rBoleAeXSJWSUU7BY,6952
|
37
|
+
indexify/function_executor/proto/function_executor_pb2.pyi,sha256=rgOBthcwNli9FUBb6oDY0KfWWrgszQpGVvAWxxVTr24,6495
|
38
|
+
indexify/function_executor/proto/function_executor_pb2_grpc.py,sha256=xrqIDrcl1PTyQTGAAGrgSDKEtZilC445Op0tJ7LmS5Q,8716
|
39
|
+
indexify/function_executor/proto/message_validator.py,sha256=OKXPYgy5L9c-spnV9Zjv7PA_yxwzvykfhbYylYx8cwQ,1456
|
40
|
+
indexify/function_executor/server.py,sha256=YEeKHR6qjNJyLFb8KNjcCYClBgVzdJNmKwdwmyJ-ybE,1136
|
41
|
+
indexify/functions_sdk/data_objects.py,sha256=ZJ7B9b5OI7aieCWJFx18pV6tqScssIFtmth84i7nKTg,623
|
42
|
+
indexify/functions_sdk/graph.py,sha256=_54nK1xckZ8dYf6ApalTDmaQwa_QPf66e9rDXf5Ii_4,13356
|
43
|
+
indexify/functions_sdk/graph_definition.py,sha256=yQ7c7jpA8RB8Yt81D1i5R3LPT8fmLvN8bLuEHvS4P_o,1775
|
44
|
+
indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
|
45
|
+
indexify/functions_sdk/image.py,sha256=tXt_Ho5kIqH54MTCdK_Hp1eG6xnmYVnYiY70xlABZ0I,6245
|
46
|
+
indexify/functions_sdk/indexify_functions.py,sha256=6OzZZILH3xoWzPYFMo167N6p8KmbN_-cEJRJMEzL5YE,12284
|
47
|
+
indexify/functions_sdk/invocation_state/invocation_state.py,sha256=azSRnX-ryii2Ehqfn5v2vuoXd0pWLu4koRf6RPBNQJk,828
|
48
|
+
indexify/functions_sdk/invocation_state/local_invocation_state.py,sha256=f3m_xlfaGrep-YBqJFXUXpBykyKfEQtyaJ606htwGX4,1107
|
49
|
+
indexify/functions_sdk/object_serializer.py,sha256=R58ALsl2Lb87ii6km4D6hBBsqRs_CHNISxhUICE2d9o,1931
|
50
|
+
indexify/functions_sdk/pipeline.py,sha256=KmxZE8eBFAQ4bbEcYURXXR26HSyoAT3O6iu9H38-OXE,974
|
51
|
+
indexify/http_client.py,sha256=VKrEqffjkA5S2LRrs0O1ierjha7YILs6g2DhI0oFf1E,14682
|
52
|
+
indexify/logging.py,sha256=_TLnWT0RzABvExwTixjdpuxoSNUev4zeNV0K1VlEUmA,1099
|
53
|
+
indexify/remote_graph.py,sha256=OzBucU4buR5UdTViNJvh1RfnOTmYA34Uel5qXnRsQsA,5006
|
54
|
+
indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
|
55
|
+
indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
|
56
|
+
indexify-0.2.45.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
57
|
+
indexify-0.2.45.dist-info/METADATA,sha256=BiIGGhlP8PC8wElMimdvoj_8MKFIvFcP2DhwQQvc1yM,6274
|
58
|
+
indexify-0.2.45.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
59
|
+
indexify-0.2.45.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
|
60
|
+
indexify-0.2.45.dist-info/RECORD,,
|
@@ -1,26 +0,0 @@
|
|
1
|
-
from typing import Any, Optional
|
2
|
-
|
3
|
-
from .function_executor import FunctionExecutor
|
4
|
-
|
5
|
-
|
6
|
-
class FunctionExecutorFactory:
|
7
|
-
"""Abstract class for creating function executors."""
|
8
|
-
|
9
|
-
async def create(
|
10
|
-
self, logger: Any, state: Optional[Any] = None
|
11
|
-
) -> FunctionExecutor:
|
12
|
-
"""Creates a new FunctionExecutor.
|
13
|
-
|
14
|
-
Args:
|
15
|
-
logger: logger to be used during the function.
|
16
|
-
state: state to be stored in the FunctionExecutor."""
|
17
|
-
raise NotImplementedError()
|
18
|
-
|
19
|
-
async def destroy(self, executor: FunctionExecutor, logger: Any) -> None:
|
20
|
-
"""Destroys the FunctionExecutor and release all its resources.
|
21
|
-
|
22
|
-
Args:
|
23
|
-
logger: logger to be used during the function.
|
24
|
-
FunctionExecutor and customer code running inside of it are not notified about the destruction.
|
25
|
-
Never raises any Exceptions."""
|
26
|
-
raise NotImplementedError
|
@@ -1,91 +0,0 @@
|
|
1
|
-
import asyncio
|
2
|
-
from typing import Any, Dict, Optional
|
3
|
-
|
4
|
-
import grpc
|
5
|
-
|
6
|
-
from indexify.function_executor.proto.function_executor_pb2 import (
|
7
|
-
InitializeRequest,
|
8
|
-
InitializeResponse,
|
9
|
-
)
|
10
|
-
from indexify.function_executor.proto.function_executor_pb2_grpc import (
|
11
|
-
FunctionExecutorStub,
|
12
|
-
)
|
13
|
-
|
14
|
-
from .function_executor import FunctionExecutor
|
15
|
-
from .function_executor_factory import FunctionExecutorFactory
|
16
|
-
|
17
|
-
|
18
|
-
class FunctionExecutorMap:
|
19
|
-
"""A map of ID => FunctionExecutor.
|
20
|
-
|
21
|
-
The map is safe to use by multiple couroutines running in event loop on the same thread
|
22
|
-
but it's not thread safe (can't be used from different threads concurrently)."""
|
23
|
-
|
24
|
-
def __init__(self, factory: FunctionExecutorFactory):
|
25
|
-
self._factory = factory
|
26
|
-
# Map of initialized Function executors ready to run tasks.
|
27
|
-
# function ID -> FunctionExecutor
|
28
|
-
self._executors: Dict[str, FunctionExecutor] = {}
|
29
|
-
# We have to do all operations under this lock because we need to ensure
|
30
|
-
# that we don't create more Function Executors than required. This is important
|
31
|
-
# e.g. when a Function Executor is using the only available GPU on the machine.
|
32
|
-
# We can get rid of this locking in the future once we assing GPUs explicitly to Function Executors.
|
33
|
-
# Running the full test suite with all this locking removed doesn't make it run faster,
|
34
|
-
# so it looks like this full locking doesn't really result in any performance penalty so far.
|
35
|
-
self._executors_lock = asyncio.Lock()
|
36
|
-
|
37
|
-
async def get_or_create(
|
38
|
-
self,
|
39
|
-
id: str,
|
40
|
-
initialize_request: InitializeRequest,
|
41
|
-
initial_state: Any,
|
42
|
-
logger: Any,
|
43
|
-
) -> FunctionExecutor:
|
44
|
-
"""Returns a FunctionExecutor for the given ID.
|
45
|
-
|
46
|
-
If the FunctionExecutor for the given ID doesn't exist then it will be created and initialized.
|
47
|
-
Raises an exception if the FunctionExecutor creation or initialization failed.
|
48
|
-
"""
|
49
|
-
async with self._executors_lock:
|
50
|
-
# Use existing Function Executor if it's already initialized.
|
51
|
-
if id in self._executors:
|
52
|
-
return self._executors[id]
|
53
|
-
|
54
|
-
executor: Optional[FunctionExecutor] = None
|
55
|
-
try:
|
56
|
-
executor = await self._factory.create(logger, state=initial_state)
|
57
|
-
channel: grpc.aio.Channel = await executor.channel()
|
58
|
-
stub: FunctionExecutorStub = FunctionExecutorStub(channel)
|
59
|
-
initialize_response: InitializeResponse = await stub.initialize(
|
60
|
-
initialize_request
|
61
|
-
)
|
62
|
-
if not initialize_response.success:
|
63
|
-
raise Exception("initialize RPC failed at function executor")
|
64
|
-
except Exception:
|
65
|
-
if executor is not None:
|
66
|
-
await self._factory.destroy(executor=executor, logger=logger)
|
67
|
-
# Function Executor creation or initialization failed.
|
68
|
-
raise
|
69
|
-
|
70
|
-
self._executors[id] = executor
|
71
|
-
return executor
|
72
|
-
|
73
|
-
async def delete(
|
74
|
-
self, id: str, function_executor: FunctionExecutor, logger: Any
|
75
|
-
) -> None:
|
76
|
-
"""Deletes the FunctionExecutor for the given ID.
|
77
|
-
|
78
|
-
Does nothing if the FunctionExecutor for the given ID doesn't exist or was already deleted.
|
79
|
-
"""
|
80
|
-
async with self._executors_lock:
|
81
|
-
if self._executors[id] != function_executor:
|
82
|
-
# Function Executor was already deleted or replaced and the caller is not aware of this.
|
83
|
-
return
|
84
|
-
del self._executors[id]
|
85
|
-
await self._factory.destroy(executor=function_executor, logger=logger)
|
86
|
-
|
87
|
-
async def clear(self, logger):
|
88
|
-
async with self._executors_lock:
|
89
|
-
while self._executors:
|
90
|
-
id, function_executor = self._executors.popitem()
|
91
|
-
await self._factory.destroy(function_executor, logger)
|
@@ -1,64 +0,0 @@
|
|
1
|
-
import asyncio
|
2
|
-
from typing import Any, Optional
|
3
|
-
|
4
|
-
import grpc
|
5
|
-
|
6
|
-
from indexify.function_executor.proto.configuration import GRPC_CHANNEL_OPTIONS
|
7
|
-
|
8
|
-
from .function_executor import (
|
9
|
-
FUNCTION_EXECUTOR_READY_TIMEOUT_SEC,
|
10
|
-
FunctionExecutor,
|
11
|
-
)
|
12
|
-
|
13
|
-
|
14
|
-
class ProcessFunctionExecutor(FunctionExecutor):
|
15
|
-
"""A FunctionExecutor that runs in a separate host process."""
|
16
|
-
|
17
|
-
def __init__(
|
18
|
-
self,
|
19
|
-
process: asyncio.subprocess.Process,
|
20
|
-
port: int,
|
21
|
-
address: str,
|
22
|
-
logger: Any,
|
23
|
-
state: Optional[Any] = None,
|
24
|
-
):
|
25
|
-
self._proc = process
|
26
|
-
self._port = port
|
27
|
-
self._address = address
|
28
|
-
self._logger = logger.bind(module=__name__)
|
29
|
-
self._channel: Optional[grpc.aio.Channel] = None
|
30
|
-
self._state: Optional[Any] = state
|
31
|
-
|
32
|
-
async def channel(self) -> grpc.aio.Channel:
|
33
|
-
# Not thread safe but async safe because we don't await.
|
34
|
-
if self._channel is not None:
|
35
|
-
return self._channel
|
36
|
-
|
37
|
-
channel: Optional[grpc.aio.Channel] = None
|
38
|
-
try:
|
39
|
-
channel = grpc.aio.insecure_channel(
|
40
|
-
self._address, options=GRPC_CHANNEL_OPTIONS
|
41
|
-
)
|
42
|
-
await asyncio.wait_for(
|
43
|
-
channel.channel_ready(),
|
44
|
-
timeout=FUNCTION_EXECUTOR_READY_TIMEOUT_SEC,
|
45
|
-
)
|
46
|
-
# Check if another channel was created by a concurrent coroutine.
|
47
|
-
# Not thread safe but async safe because we never overwrite non-None self._channel.
|
48
|
-
if self._channel is not None:
|
49
|
-
# Don't close and overwrite existing channel because it might be used for RPCs already.
|
50
|
-
await channel.close()
|
51
|
-
return self._channel
|
52
|
-
else:
|
53
|
-
self._channel = channel
|
54
|
-
return channel
|
55
|
-
except Exception:
|
56
|
-
if channel is not None:
|
57
|
-
await channel.close()
|
58
|
-
self._logger.error(
|
59
|
-
f"failed to connect to the gRPC server at {self._address} within {FUNCTION_EXECUTOR_READY_TIMEOUT_SEC} seconds"
|
60
|
-
)
|
61
|
-
raise
|
62
|
-
|
63
|
-
def state(self) -> Optional[Any]:
|
64
|
-
return self._state
|
@@ -1,253 +0,0 @@
|
|
1
|
-
import asyncio
|
2
|
-
from typing import Any, Dict, Optional
|
3
|
-
|
4
|
-
import grpc
|
5
|
-
import structlog
|
6
|
-
|
7
|
-
from indexify.function_executor.proto.function_executor_pb2 import (
|
8
|
-
FunctionOutput,
|
9
|
-
InitializeRequest,
|
10
|
-
RouterOutput,
|
11
|
-
RunTaskRequest,
|
12
|
-
RunTaskResponse,
|
13
|
-
SerializedObject,
|
14
|
-
)
|
15
|
-
from indexify.function_executor.proto.function_executor_pb2_grpc import (
|
16
|
-
FunctionExecutorStub,
|
17
|
-
)
|
18
|
-
|
19
|
-
from .api_objects import Task
|
20
|
-
from .downloader import DownloadedInputs
|
21
|
-
from .function_executor.function_executor import FunctionExecutor
|
22
|
-
from .function_executor.function_executor_factory import (
|
23
|
-
FunctionExecutorFactory,
|
24
|
-
)
|
25
|
-
from .function_executor.function_executor_map import FunctionExecutorMap
|
26
|
-
|
27
|
-
|
28
|
-
class FunctionWorkerInput:
|
29
|
-
"""Task with all the resources required to run it."""
|
30
|
-
|
31
|
-
def __init__(
|
32
|
-
self,
|
33
|
-
task: Task,
|
34
|
-
graph: SerializedObject,
|
35
|
-
function_input: DownloadedInputs,
|
36
|
-
):
|
37
|
-
self.task = task
|
38
|
-
self.graph = graph
|
39
|
-
self.function_input = function_input
|
40
|
-
|
41
|
-
|
42
|
-
class FunctionWorkerOutput:
|
43
|
-
def __init__(
|
44
|
-
self,
|
45
|
-
function_output: Optional[FunctionOutput] = None,
|
46
|
-
router_output: Optional[RouterOutput] = None,
|
47
|
-
stdout: Optional[str] = None,
|
48
|
-
stderr: Optional[str] = None,
|
49
|
-
reducer: bool = False,
|
50
|
-
success: bool = False,
|
51
|
-
):
|
52
|
-
self.function_output = function_output
|
53
|
-
self.router_output = router_output
|
54
|
-
self.stdout = stdout
|
55
|
-
self.stderr = stderr
|
56
|
-
self.reducer = reducer
|
57
|
-
self.success = success
|
58
|
-
|
59
|
-
|
60
|
-
class FunctionExecutorState:
|
61
|
-
def __init__(
|
62
|
-
self,
|
63
|
-
function_id_with_version: str,
|
64
|
-
function_id_without_version: str,
|
65
|
-
ongoing_tasks_count: int,
|
66
|
-
):
|
67
|
-
self.function_id_with_version: str = function_id_with_version
|
68
|
-
self.function_id_without_version: str = function_id_without_version
|
69
|
-
self.ongoing_tasks_count: int = ongoing_tasks_count
|
70
|
-
|
71
|
-
|
72
|
-
class FunctionWorker:
|
73
|
-
def __init__(self, function_executor_factory: FunctionExecutorFactory):
|
74
|
-
self._function_executors = FunctionExecutorMap(function_executor_factory)
|
75
|
-
|
76
|
-
async def run(self, input: FunctionWorkerInput) -> FunctionWorkerOutput:
|
77
|
-
logger = _logger(input.task)
|
78
|
-
function_executor: Optional[FunctionExecutor] = None
|
79
|
-
try:
|
80
|
-
function_executor = await self._obtain_function_executor(input, logger)
|
81
|
-
return await self._run_in_executor(
|
82
|
-
function_executor=function_executor, input=input
|
83
|
-
)
|
84
|
-
except Exception as e:
|
85
|
-
logger.error(
|
86
|
-
"failed running the task",
|
87
|
-
exc_info=e,
|
88
|
-
)
|
89
|
-
if function_executor is not None:
|
90
|
-
# This will fail all the tasks concurrently running in this Function Executor. Not great.
|
91
|
-
await self._function_executors.delete(
|
92
|
-
id=_function_id_without_version(input.task),
|
93
|
-
function_executor=function_executor,
|
94
|
-
logger=logger,
|
95
|
-
)
|
96
|
-
return _internal_error_output()
|
97
|
-
|
98
|
-
async def _obtain_function_executor(
|
99
|
-
self, input: FunctionWorkerInput, logger: Any
|
100
|
-
) -> FunctionExecutor:
|
101
|
-
# Temporary policy for Function Executors lifecycle:
|
102
|
-
# There can only be a single Function Executor per function.
|
103
|
-
# If a Function Executor already exists for a different function version then wait until
|
104
|
-
# all the tasks finish in the existing Function Executor and then destroy it first.
|
105
|
-
initialize_request: InitializeRequest = InitializeRequest(
|
106
|
-
namespace=input.task.namespace,
|
107
|
-
graph_name=input.task.compute_graph,
|
108
|
-
graph_version=input.task.graph_version,
|
109
|
-
function_name=input.task.compute_fn,
|
110
|
-
graph=input.graph,
|
111
|
-
)
|
112
|
-
initial_function_executor_state: FunctionExecutorState = FunctionExecutorState(
|
113
|
-
function_id_with_version=_function_id_with_version(input.task),
|
114
|
-
function_id_without_version=_function_id_without_version(input.task),
|
115
|
-
ongoing_tasks_count=0,
|
116
|
-
)
|
117
|
-
|
118
|
-
while True:
|
119
|
-
function_executor = await self._function_executors.get_or_create(
|
120
|
-
id=_function_id_without_version(input.task),
|
121
|
-
initialize_request=initialize_request,
|
122
|
-
initial_state=initial_function_executor_state,
|
123
|
-
logger=logger,
|
124
|
-
)
|
125
|
-
|
126
|
-
# No need to lock Function Executor state as we are not awaiting.
|
127
|
-
function_executor_state: FunctionExecutorState = function_executor.state()
|
128
|
-
if (
|
129
|
-
function_executor_state.function_id_with_version
|
130
|
-
== _function_id_with_version(input.task)
|
131
|
-
):
|
132
|
-
# The existing Function Executor is for the same function version so we can run the task in it.
|
133
|
-
# Increment the ongoing tasks count before awaiting to prevent the Function Executor from being destroyed
|
134
|
-
# by another coroutine.
|
135
|
-
function_executor_state.ongoing_tasks_count += 1
|
136
|
-
return function_executor
|
137
|
-
|
138
|
-
# This loop implements the temporary policy so it's implemented using polling instead of a lower
|
139
|
-
# latency event based mechanism with a higher complexity.
|
140
|
-
if function_executor_state.ongoing_tasks_count == 0:
|
141
|
-
logger.info(
|
142
|
-
"destroying existing Function Executor for different function version",
|
143
|
-
function_id=_function_id_with_version(input.task),
|
144
|
-
executor_function_id=function_executor_state.function_id_with_version,
|
145
|
-
)
|
146
|
-
await self._function_executors.delete(
|
147
|
-
id=_function_id_without_version(input.task),
|
148
|
-
function_executor=function_executor,
|
149
|
-
logger=logger,
|
150
|
-
)
|
151
|
-
else:
|
152
|
-
logger.info(
|
153
|
-
"waiting for existing Function Executor to finish",
|
154
|
-
function_id=_function_id_with_version(input.task),
|
155
|
-
executor_function_id=function_executor_state.function_id_with_version,
|
156
|
-
)
|
157
|
-
await asyncio.sleep(
|
158
|
-
5
|
159
|
-
) # Wait for 5 secs before checking if all tasks for the existing Function Executor finished.
|
160
|
-
|
161
|
-
async def _run_in_executor(
|
162
|
-
self, function_executor: FunctionExecutor, input: FunctionWorkerInput
|
163
|
-
) -> FunctionWorkerOutput:
|
164
|
-
"""Runs the task in the Function Executor.
|
165
|
-
|
166
|
-
The Function Executor's ongoing_tasks_count must be incremented before calling this function.
|
167
|
-
"""
|
168
|
-
try:
|
169
|
-
run_task_request: RunTaskRequest = RunTaskRequest(
|
170
|
-
graph_invocation_id=input.task.invocation_id,
|
171
|
-
task_id=input.task.id,
|
172
|
-
function_input=input.function_input.input,
|
173
|
-
)
|
174
|
-
if input.function_input.init_value is not None:
|
175
|
-
run_task_request.function_init_value.CopyFrom(
|
176
|
-
input.function_input.init_value
|
177
|
-
)
|
178
|
-
channel: grpc.aio.Channel = await function_executor.channel()
|
179
|
-
run_task_response: RunTaskResponse = await FunctionExecutorStub(
|
180
|
-
channel
|
181
|
-
).run_task(run_task_request)
|
182
|
-
return _to_output(run_task_response)
|
183
|
-
finally:
|
184
|
-
# If this Function Executor was destroyed then it's not
|
185
|
-
# visible in the map but we still have a reference to it.
|
186
|
-
function_executor.state().ongoing_tasks_count -= 1
|
187
|
-
|
188
|
-
async def shutdown(self) -> None:
|
189
|
-
await self._function_executors.clear(
|
190
|
-
logger=structlog.get_logger(module=__name__, event="shutdown")
|
191
|
-
)
|
192
|
-
|
193
|
-
|
194
|
-
def _to_output(response: RunTaskResponse) -> FunctionWorkerOutput:
|
195
|
-
required_fields = [
|
196
|
-
"stdout",
|
197
|
-
"stderr",
|
198
|
-
"is_reducer",
|
199
|
-
"success",
|
200
|
-
]
|
201
|
-
|
202
|
-
for field in required_fields:
|
203
|
-
if not response.HasField(field):
|
204
|
-
raise ValueError(f"Response is missing required field: {field}")
|
205
|
-
|
206
|
-
output = FunctionWorkerOutput(
|
207
|
-
stdout=response.stdout,
|
208
|
-
stderr=response.stderr,
|
209
|
-
reducer=response.is_reducer,
|
210
|
-
success=response.success,
|
211
|
-
)
|
212
|
-
|
213
|
-
if response.HasField("function_output"):
|
214
|
-
output.function_output = response.function_output
|
215
|
-
if response.HasField("router_output"):
|
216
|
-
output.router_output = response.router_output
|
217
|
-
|
218
|
-
return output
|
219
|
-
|
220
|
-
|
221
|
-
def _internal_error_output() -> FunctionWorkerOutput:
|
222
|
-
return FunctionWorkerOutput(
|
223
|
-
function_output=None,
|
224
|
-
router_output=None,
|
225
|
-
stdout=None,
|
226
|
-
# We are not sharing internal error messages with the customer.
|
227
|
-
# If customer code failed then we won't get any exceptions here.
|
228
|
-
# All customer code errors are returned in the gRPC response.
|
229
|
-
stderr="Platform failed to execute the function.",
|
230
|
-
reducer=False,
|
231
|
-
success=False,
|
232
|
-
)
|
233
|
-
|
234
|
-
|
235
|
-
def _logger(task: Task) -> Any:
|
236
|
-
return structlog.get_logger(
|
237
|
-
module=__name__,
|
238
|
-
namespace=task.namespace,
|
239
|
-
graph_name=task.compute_graph,
|
240
|
-
graph_version=task.graph_version,
|
241
|
-
function_name=task.compute_fn,
|
242
|
-
graph_invocation_id=task.invocation_id,
|
243
|
-
task_id=task.id,
|
244
|
-
function_id=_function_id_with_version(task),
|
245
|
-
)
|
246
|
-
|
247
|
-
|
248
|
-
def _function_id_with_version(task: Task) -> str:
|
249
|
-
return f"versioned/{task.namespace}/{task.compute_graph}/{task.graph_version}/{task.compute_fn}"
|
250
|
-
|
251
|
-
|
252
|
-
def _function_id_without_version(task: Task) -> str:
|
253
|
-
return f"not_versioned/{task.namespace}/{task.compute_graph}/{task.compute_fn}"
|
indexify-0.2.44.dist-info/RECORD
DELETED
@@ -1,50 +0,0 @@
|
|
1
|
-
indexify/__init__.py,sha256=P0mvM8sbkeS2CjYzRYyzb42CnXGhyJXdz4FdmTBMSWM,697
|
2
|
-
indexify/cli.py,sha256=Y8mijSECm_s-ZLNcba-PIFrnAjB7KX7TVdw0q6KVD2I,10382
|
3
|
-
indexify/common_util.py,sha256=LKS6yZ3yv8nF2J-KzisGIjqjTvCn7tLFifQJLT4gHRg,3529
|
4
|
-
indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
|
5
|
-
indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
|
6
|
-
indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
|
7
|
-
indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
|
8
|
-
indexify/executor/api_objects.py,sha256=vp7aEjvfWL2j4nYDV0xLgLVZlGsbStMEu0nzrgJVq1A,741
|
9
|
-
indexify/executor/downloader.py,sha256=oeiSGxIIn7uBTKT6hzyPUa-AlOUlqy7WQkinm7qpV0I,6442
|
10
|
-
indexify/executor/executor.py,sha256=i9TnF899scLkUWCpomnDh_dwUDqtXgiPWvnwsF7DUEs,5272
|
11
|
-
indexify/executor/function_executor/function_executor.py,sha256=0W6vCT-DBVPUMPoOXLVRVHZjVGGUeTDV_NWb-XLP2gM,1218
|
12
|
-
indexify/executor/function_executor/function_executor_factory.py,sha256=23BU_i47Tz6xd9fAwsKH29lm261bZXfzYGRhSqeFtwY,900
|
13
|
-
indexify/executor/function_executor/function_executor_map.py,sha256=HXeCjkRusumJisU74Iy6M4e13k8jeHk4Yq8PyODSsUc,3922
|
14
|
-
indexify/executor/function_executor/process_function_executor.py,sha256=CdqFtvlvS_VIzVdbE3Wnzj_Kqfzhzf4X05avHIielEE,2150
|
15
|
-
indexify/executor/function_executor/process_function_executor_factory.py,sha256=ognYRd5l10kVD2KAShfkd8vg-JElYctG77Pcx77-grM,3850
|
16
|
-
indexify/executor/function_worker.py,sha256=YfNcDvwlX5dvKF-4x6XkvjX4E0S-5WKZkmsLFaesapg,9669
|
17
|
-
indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
|
18
|
-
indexify/executor/task_fetcher.py,sha256=qBuVxgjzjkTi_1_hyQqtBxetBxdL7TGMgI4FBwyTANE,3004
|
19
|
-
indexify/executor/task_reporter.py,sha256=soCB31JNi0IZDgd0VRTm90X1Prd3_tHMGPqxccgGLKc,6869
|
20
|
-
indexify/function_executor/function_executor_service.py,sha256=EwwHTPX-AVqbSl-BONWWy3Z6gowXnYj5tigGoFs1u0s,3694
|
21
|
-
indexify/function_executor/handlers/run_function/function_inputs_loader.py,sha256=x2Lrzb6CsEWDWW5qG9etS7TyV7KVYfuFKEbzKVhYizo,1553
|
22
|
-
indexify/function_executor/handlers/run_function/handler.py,sha256=eklicxgyExd9ScdDahQO9LaLNXPNBPAAI0KBt3Wb9-w,5636
|
23
|
-
indexify/function_executor/handlers/run_function/request_validator.py,sha256=tVmbOS2g4AeV48CZCSDRycnoQjjGy12XD_vbaOLX_Z4,740
|
24
|
-
indexify/function_executor/handlers/run_function/response_helper.py,sha256=KDSadee08g6JRFwKRTdcOwbP6SmRJCGVHh1FOLdt_ws,3109
|
25
|
-
indexify/function_executor/initialize_request_validator.py,sha256=AtyKgIjlGHB-Fh3Vhso86TC0o65DfuVEY8GFfRSfylo,709
|
26
|
-
indexify/function_executor/proto/configuration.py,sha256=P0xtxwf-XaCxi0kvtCbm4X8y0YOuHhR-uh9xUuLdFoM,595
|
27
|
-
indexify/function_executor/proto/function_executor.proto,sha256=PiftSSIDjHkxUbD7rNVmgfTCP12bFJ7oKJXuRx2sM28,2188
|
28
|
-
indexify/function_executor/proto/function_executor_pb2.py,sha256=tEDJJUywM6-j4dBwvBEHXnGCv9spGWVuhtHuiTDxw7I,4582
|
29
|
-
indexify/function_executor/proto/function_executor_pb2.pyi,sha256=E-UnFIFbej-QPhaX1X4oYGeUW6_HuEi13qhCYkN1tzI,4182
|
30
|
-
indexify/function_executor/proto/function_executor_pb2_grpc.py,sha256=OIErF2wYWaRI2TyF8EbWGjj5K9aD7OIJwAp1BRMQ0ic,6265
|
31
|
-
indexify/function_executor/proto/message_validator.py,sha256=OKXPYgy5L9c-spnV9Zjv7PA_yxwzvykfhbYylYx8cwQ,1456
|
32
|
-
indexify/function_executor/server.py,sha256=YEeKHR6qjNJyLFb8KNjcCYClBgVzdJNmKwdwmyJ-ybE,1136
|
33
|
-
indexify/functions_sdk/data_objects.py,sha256=ZJ7B9b5OI7aieCWJFx18pV6tqScssIFtmth84i7nKTg,623
|
34
|
-
indexify/functions_sdk/graph.py,sha256=cyaOHpyUVkUq5hyS85ojf6GN_tBGXQbwWlFQnktj1q0,13281
|
35
|
-
indexify/functions_sdk/graph_definition.py,sha256=yQ7c7jpA8RB8Yt81D1i5R3LPT8fmLvN8bLuEHvS4P_o,1775
|
36
|
-
indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
|
37
|
-
indexify/functions_sdk/image.py,sha256=wVDs5O5yshjSSZ1d-keyMp9zUrck2TazJlkVq94jMZk,1965
|
38
|
-
indexify/functions_sdk/indexify_functions.py,sha256=USuEeZnbQTd6uUjv-ZAwdBrbR8gGrTD78oXjyW6DBbw,13376
|
39
|
-
indexify/functions_sdk/object_serializer.py,sha256=R58ALsl2Lb87ii6km4D6hBBsqRs_CHNISxhUICE2d9o,1931
|
40
|
-
indexify/functions_sdk/pipeline.py,sha256=KmxZE8eBFAQ4bbEcYURXXR26HSyoAT3O6iu9H38-OXE,974
|
41
|
-
indexify/http_client.py,sha256=XvIi7z7h7PZomMKlL8WEhFIWAAPzFe2hrLQUz4QC_Zc,15392
|
42
|
-
indexify/logging.py,sha256=_TLnWT0RzABvExwTixjdpuxoSNUev4zeNV0K1VlEUmA,1099
|
43
|
-
indexify/remote_graph.py,sha256=OzBucU4buR5UdTViNJvh1RfnOTmYA34Uel5qXnRsQsA,5006
|
44
|
-
indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
|
45
|
-
indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
|
46
|
-
indexify-0.2.44.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
47
|
-
indexify-0.2.44.dist-info/METADATA,sha256=UKyA-H8cr6C0GdpIuOLot1ISFowFxFmVZJme0KV1iQY,6274
|
48
|
-
indexify-0.2.44.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
49
|
-
indexify-0.2.44.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
|
50
|
-
indexify-0.2.44.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|