indexify 0.2.43__py3-none-any.whl → 0.2.45__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. indexify/__init__.py +2 -0
  2. indexify/cli.py +41 -80
  3. indexify/executor/api_objects.py +2 -0
  4. indexify/executor/downloader.py +23 -25
  5. indexify/executor/executor.py +29 -35
  6. indexify/executor/function_executor/function_executor.py +120 -19
  7. indexify/executor/function_executor/function_executor_state.py +75 -0
  8. indexify/executor/function_executor/invocation_state_client.py +232 -0
  9. indexify/executor/function_executor/server/function_executor_server.py +24 -0
  10. indexify/executor/function_executor/server/function_executor_server_factory.py +43 -0
  11. indexify/executor/function_executor/server/subprocess_function_executor_server.py +25 -0
  12. indexify/executor/function_executor/{process_function_executor_factory.py → server/subprocess_function_executor_server_factory.py} +21 -21
  13. indexify/executor/function_executor/single_task_runner.py +160 -0
  14. indexify/executor/function_executor/task_input.py +23 -0
  15. indexify/executor/function_executor/task_output.py +36 -0
  16. indexify/executor/task_reporter.py +10 -17
  17. indexify/executor/task_runner.py +104 -0
  18. indexify/function_executor/function_executor_service.py +22 -7
  19. indexify/function_executor/handlers/run_function/handler.py +13 -12
  20. indexify/function_executor/invocation_state/invocation_state_proxy_server.py +170 -0
  21. indexify/function_executor/invocation_state/proxied_invocation_state.py +24 -0
  22. indexify/function_executor/invocation_state/response_validator.py +29 -0
  23. indexify/function_executor/proto/function_executor.proto +47 -0
  24. indexify/function_executor/proto/function_executor_pb2.py +23 -11
  25. indexify/function_executor/proto/function_executor_pb2.pyi +70 -0
  26. indexify/function_executor/proto/function_executor_pb2_grpc.py +50 -0
  27. indexify/functions_sdk/graph.py +3 -3
  28. indexify/functions_sdk/image.py +142 -9
  29. indexify/functions_sdk/indexify_functions.py +45 -79
  30. indexify/functions_sdk/invocation_state/invocation_state.py +22 -0
  31. indexify/functions_sdk/invocation_state/local_invocation_state.py +30 -0
  32. indexify/http_client.py +0 -17
  33. {indexify-0.2.43.dist-info → indexify-0.2.45.dist-info}/METADATA +2 -2
  34. indexify-0.2.45.dist-info/RECORD +60 -0
  35. indexify/executor/function_executor/function_executor_factory.py +0 -26
  36. indexify/executor/function_executor/function_executor_map.py +0 -91
  37. indexify/executor/function_executor/process_function_executor.py +0 -64
  38. indexify/executor/function_worker.py +0 -253
  39. indexify-0.2.43.dist-info/RECORD +0 -50
  40. {indexify-0.2.43.dist-info → indexify-0.2.45.dist-info}/LICENSE.txt +0 -0
  41. {indexify-0.2.43.dist-info → indexify-0.2.45.dist-info}/WHEEL +0 -0
  42. {indexify-0.2.43.dist-info → indexify-0.2.45.dist-info}/entry_points.txt +0 -0
@@ -0,0 +1,60 @@
1
+ indexify/__init__.py,sha256=ERgPjxxLiyvtT1y6aapzKspJQuEb_eBBUIN8idKmey0,739
2
+ indexify/cli.py,sha256=BWR3L7-WovzMvoZbwMbv3CyIlBt3DcT11dqz6nYlaQA,9473
3
+ indexify/common_util.py,sha256=LKS6yZ3yv8nF2J-KzisGIjqjTvCn7tLFifQJLT4gHRg,3529
4
+ indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
5
+ indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
6
+ indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
7
+ indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
8
+ indexify/executor/api_objects.py,sha256=CJ1RUZWAw5WC1dO6IPx36s9PPlY_ONNHFCAhV7Ql4s0,885
9
+ indexify/executor/downloader.py,sha256=9zzV1febq5xXlvDjsDPms8rvBTIE0ulXKqeWpvvfaGQ,6368
10
+ indexify/executor/executor.py,sha256=cnTzqqokZj2YxAwV4R9ihMTw5DpanWaF-ceNRm5Y3qk,5209
11
+ indexify/executor/function_executor/function_executor.py,sha256=u9hzRAH8_rcDpIWOjT0CA6yNOo8qlyBYqLg9VESqBvo,4863
12
+ indexify/executor/function_executor/function_executor_state.py,sha256=CKU3ujetHf6_8zEXtY5eSxcaFB6PfPQSuPlL1AJDJf8,2758
13
+ indexify/executor/function_executor/invocation_state_client.py,sha256=LgVFRaklo201AogAM-hLrfTfUeQZxmwYRZv0X2u-64Y,8596
14
+ indexify/executor/function_executor/server/function_executor_server.py,sha256=_DLivLDikupZusRk8gVWDk7fWPT9XjZ4un1yWSlOObs,883
15
+ indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=pGbJMQfC5TNvyWOs6VDKdqd2PK5OHQh5_wSDP-E7DbI,1677
16
+ indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=DxQMwAvQi03OGo2sFruUrxwcMPG8In4C62L3sped2FI,694
17
+ indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=MrhlcH5UZXJn9mAuyCJVVcJTihx8SDDuaraXFvkWjsU,3628
18
+ indexify/executor/function_executor/single_task_runner.py,sha256=mzYSUswpN-q98L1VALnt246AwmyNlQ-BcKZR96wF6gw,5722
19
+ indexify/executor/function_executor/task_input.py,sha256=7Mkd-OCt1s9HdbyhrxKQkhxsVR9yeT6jC6zd2knt5-8,591
20
+ indexify/executor/function_executor/task_output.py,sha256=3-qcT1aigmOFEp6QSJtUcQfqeJfrxvo265qgk9WAUVg,1064
21
+ indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
22
+ indexify/executor/task_fetcher.py,sha256=qBuVxgjzjkTi_1_hyQqtBxetBxdL7TGMgI4FBwyTANE,3004
23
+ indexify/executor/task_reporter.py,sha256=xHEk_TVeXYo6rcuDWImFmjLjfwRvpohP1-MS4grCASM,6663
24
+ indexify/executor/task_runner.py,sha256=yO_5o8-oA-TeSyxqS-DX6XfUb0947jrPijJeGI4wqQk,4807
25
+ indexify/function_executor/function_executor_service.py,sha256=DVP19TUo5iPAki5dh320kT2JjO69bLszE1rWkWg2MiY,4244
26
+ indexify/function_executor/handlers/run_function/function_inputs_loader.py,sha256=x2Lrzb6CsEWDWW5qG9etS7TyV7KVYfuFKEbzKVhYizo,1553
27
+ indexify/function_executor/handlers/run_function/handler.py,sha256=g6JZna852tQleASn9lA651fZh_chVYTKbyWDvgXi0NA,5695
28
+ indexify/function_executor/handlers/run_function/request_validator.py,sha256=tVmbOS2g4AeV48CZCSDRycnoQjjGy12XD_vbaOLX_Z4,740
29
+ indexify/function_executor/handlers/run_function/response_helper.py,sha256=KDSadee08g6JRFwKRTdcOwbP6SmRJCGVHh1FOLdt_ws,3109
30
+ indexify/function_executor/initialize_request_validator.py,sha256=AtyKgIjlGHB-Fh3Vhso86TC0o65DfuVEY8GFfRSfylo,709
31
+ indexify/function_executor/invocation_state/invocation_state_proxy_server.py,sha256=JFan-Ku9aFy72_54j0tyg5e8LdpvEIcbWL0pbNfcFSs,7021
32
+ indexify/function_executor/invocation_state/proxied_invocation_state.py,sha256=AOztc2uSTaPQEqwc8Fh5zLHgJlgv6u68kvP0IolecOA,951
33
+ indexify/function_executor/invocation_state/response_validator.py,sha256=05Y9_5s010owaDb6OUia9W9Alx1r1zJ179HboEkT-Xk,896
34
+ indexify/function_executor/proto/configuration.py,sha256=P0xtxwf-XaCxi0kvtCbm4X8y0YOuHhR-uh9xUuLdFoM,595
35
+ indexify/function_executor/proto/function_executor.proto,sha256=S1hnn0WLOKGuQAOOSTBxnPTOuDTxhqcGg7rHegSzl5M,3789
36
+ indexify/function_executor/proto/function_executor_pb2.py,sha256=lwBmDF2gTwcMYgp_4fLIm3SpS1rBoleAeXSJWSUU7BY,6952
37
+ indexify/function_executor/proto/function_executor_pb2.pyi,sha256=rgOBthcwNli9FUBb6oDY0KfWWrgszQpGVvAWxxVTr24,6495
38
+ indexify/function_executor/proto/function_executor_pb2_grpc.py,sha256=xrqIDrcl1PTyQTGAAGrgSDKEtZilC445Op0tJ7LmS5Q,8716
39
+ indexify/function_executor/proto/message_validator.py,sha256=OKXPYgy5L9c-spnV9Zjv7PA_yxwzvykfhbYylYx8cwQ,1456
40
+ indexify/function_executor/server.py,sha256=YEeKHR6qjNJyLFb8KNjcCYClBgVzdJNmKwdwmyJ-ybE,1136
41
+ indexify/functions_sdk/data_objects.py,sha256=ZJ7B9b5OI7aieCWJFx18pV6tqScssIFtmth84i7nKTg,623
42
+ indexify/functions_sdk/graph.py,sha256=_54nK1xckZ8dYf6ApalTDmaQwa_QPf66e9rDXf5Ii_4,13356
43
+ indexify/functions_sdk/graph_definition.py,sha256=yQ7c7jpA8RB8Yt81D1i5R3LPT8fmLvN8bLuEHvS4P_o,1775
44
+ indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
45
+ indexify/functions_sdk/image.py,sha256=tXt_Ho5kIqH54MTCdK_Hp1eG6xnmYVnYiY70xlABZ0I,6245
46
+ indexify/functions_sdk/indexify_functions.py,sha256=6OzZZILH3xoWzPYFMo167N6p8KmbN_-cEJRJMEzL5YE,12284
47
+ indexify/functions_sdk/invocation_state/invocation_state.py,sha256=azSRnX-ryii2Ehqfn5v2vuoXd0pWLu4koRf6RPBNQJk,828
48
+ indexify/functions_sdk/invocation_state/local_invocation_state.py,sha256=f3m_xlfaGrep-YBqJFXUXpBykyKfEQtyaJ606htwGX4,1107
49
+ indexify/functions_sdk/object_serializer.py,sha256=R58ALsl2Lb87ii6km4D6hBBsqRs_CHNISxhUICE2d9o,1931
50
+ indexify/functions_sdk/pipeline.py,sha256=KmxZE8eBFAQ4bbEcYURXXR26HSyoAT3O6iu9H38-OXE,974
51
+ indexify/http_client.py,sha256=VKrEqffjkA5S2LRrs0O1ierjha7YILs6g2DhI0oFf1E,14682
52
+ indexify/logging.py,sha256=_TLnWT0RzABvExwTixjdpuxoSNUev4zeNV0K1VlEUmA,1099
53
+ indexify/remote_graph.py,sha256=OzBucU4buR5UdTViNJvh1RfnOTmYA34Uel5qXnRsQsA,5006
54
+ indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
55
+ indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
56
+ indexify-0.2.45.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
57
+ indexify-0.2.45.dist-info/METADATA,sha256=BiIGGhlP8PC8wElMimdvoj_8MKFIvFcP2DhwQQvc1yM,6274
58
+ indexify-0.2.45.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
59
+ indexify-0.2.45.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
60
+ indexify-0.2.45.dist-info/RECORD,,
@@ -1,26 +0,0 @@
1
- from typing import Any, Optional
2
-
3
- from .function_executor import FunctionExecutor
4
-
5
-
6
- class FunctionExecutorFactory:
7
- """Abstract class for creating function executors."""
8
-
9
- async def create(
10
- self, logger: Any, state: Optional[Any] = None
11
- ) -> FunctionExecutor:
12
- """Creates a new FunctionExecutor.
13
-
14
- Args:
15
- logger: logger to be used during the function.
16
- state: state to be stored in the FunctionExecutor."""
17
- raise NotImplementedError()
18
-
19
- async def destroy(self, executor: FunctionExecutor, logger: Any) -> None:
20
- """Destroys the FunctionExecutor and release all its resources.
21
-
22
- Args:
23
- logger: logger to be used during the function.
24
- FunctionExecutor and customer code running inside of it are not notified about the destruction.
25
- Never raises any Exceptions."""
26
- raise NotImplementedError
@@ -1,91 +0,0 @@
1
- import asyncio
2
- from typing import Any, Dict, Optional
3
-
4
- import grpc
5
-
6
- from indexify.function_executor.proto.function_executor_pb2 import (
7
- InitializeRequest,
8
- InitializeResponse,
9
- )
10
- from indexify.function_executor.proto.function_executor_pb2_grpc import (
11
- FunctionExecutorStub,
12
- )
13
-
14
- from .function_executor import FunctionExecutor
15
- from .function_executor_factory import FunctionExecutorFactory
16
-
17
-
18
- class FunctionExecutorMap:
19
- """A map of ID => FunctionExecutor.
20
-
21
- The map is safe to use by multiple couroutines running in event loop on the same thread
22
- but it's not thread safe (can't be used from different threads concurrently)."""
23
-
24
- def __init__(self, factory: FunctionExecutorFactory):
25
- self._factory = factory
26
- # Map of initialized Function executors ready to run tasks.
27
- # function ID -> FunctionExecutor
28
- self._executors: Dict[str, FunctionExecutor] = {}
29
- # We have to do all operations under this lock because we need to ensure
30
- # that we don't create more Function Executors than required. This is important
31
- # e.g. when a Function Executor is using the only available GPU on the machine.
32
- # We can get rid of this locking in the future once we assing GPUs explicitly to Function Executors.
33
- # Running the full test suite with all this locking removed doesn't make it run faster,
34
- # so it looks like this full locking doesn't really result in any performance penalty so far.
35
- self._executors_lock = asyncio.Lock()
36
-
37
- async def get_or_create(
38
- self,
39
- id: str,
40
- initialize_request: InitializeRequest,
41
- initial_state: Any,
42
- logger: Any,
43
- ) -> FunctionExecutor:
44
- """Returns a FunctionExecutor for the given ID.
45
-
46
- If the FunctionExecutor for the given ID doesn't exist then it will be created and initialized.
47
- Raises an exception if the FunctionExecutor creation or initialization failed.
48
- """
49
- async with self._executors_lock:
50
- # Use existing Function Executor if it's already initialized.
51
- if id in self._executors:
52
- return self._executors[id]
53
-
54
- executor: Optional[FunctionExecutor] = None
55
- try:
56
- executor = await self._factory.create(logger, state=initial_state)
57
- channel: grpc.aio.Channel = await executor.channel()
58
- stub: FunctionExecutorStub = FunctionExecutorStub(channel)
59
- initialize_response: InitializeResponse = await stub.initialize(
60
- initialize_request
61
- )
62
- if not initialize_response.success:
63
- raise Exception("initialize RPC failed at function executor")
64
- except Exception:
65
- if executor is not None:
66
- await self._factory.destroy(executor=executor, logger=logger)
67
- # Function Executor creation or initialization failed.
68
- raise
69
-
70
- self._executors[id] = executor
71
- return executor
72
-
73
- async def delete(
74
- self, id: str, function_executor: FunctionExecutor, logger: Any
75
- ) -> None:
76
- """Deletes the FunctionExecutor for the given ID.
77
-
78
- Does nothing if the FunctionExecutor for the given ID doesn't exist or was already deleted.
79
- """
80
- async with self._executors_lock:
81
- if self._executors[id] != function_executor:
82
- # Function Executor was already deleted or replaced and the caller is not aware of this.
83
- return
84
- del self._executors[id]
85
- await self._factory.destroy(executor=function_executor, logger=logger)
86
-
87
- async def clear(self, logger):
88
- async with self._executors_lock:
89
- while self._executors:
90
- id, function_executor = self._executors.popitem()
91
- await self._factory.destroy(function_executor, logger)
@@ -1,64 +0,0 @@
1
- import asyncio
2
- from typing import Any, Optional
3
-
4
- import grpc
5
-
6
- from indexify.function_executor.proto.configuration import GRPC_CHANNEL_OPTIONS
7
-
8
- from .function_executor import (
9
- FUNCTION_EXECUTOR_READY_TIMEOUT_SEC,
10
- FunctionExecutor,
11
- )
12
-
13
-
14
- class ProcessFunctionExecutor(FunctionExecutor):
15
- """A FunctionExecutor that runs in a separate host process."""
16
-
17
- def __init__(
18
- self,
19
- process: asyncio.subprocess.Process,
20
- port: int,
21
- address: str,
22
- logger: Any,
23
- state: Optional[Any] = None,
24
- ):
25
- self._proc = process
26
- self._port = port
27
- self._address = address
28
- self._logger = logger.bind(module=__name__)
29
- self._channel: Optional[grpc.aio.Channel] = None
30
- self._state: Optional[Any] = state
31
-
32
- async def channel(self) -> grpc.aio.Channel:
33
- # Not thread safe but async safe because we don't await.
34
- if self._channel is not None:
35
- return self._channel
36
-
37
- channel: Optional[grpc.aio.Channel] = None
38
- try:
39
- channel = grpc.aio.insecure_channel(
40
- self._address, options=GRPC_CHANNEL_OPTIONS
41
- )
42
- await asyncio.wait_for(
43
- channel.channel_ready(),
44
- timeout=FUNCTION_EXECUTOR_READY_TIMEOUT_SEC,
45
- )
46
- # Check if another channel was created by a concurrent coroutine.
47
- # Not thread safe but async safe because we never overwrite non-None self._channel.
48
- if self._channel is not None:
49
- # Don't close and overwrite existing channel because it might be used for RPCs already.
50
- await channel.close()
51
- return self._channel
52
- else:
53
- self._channel = channel
54
- return channel
55
- except Exception:
56
- if channel is not None:
57
- await channel.close()
58
- self._logger.error(
59
- f"failed to connect to the gRPC server at {self._address} within {FUNCTION_EXECUTOR_READY_TIMEOUT_SEC} seconds"
60
- )
61
- raise
62
-
63
- def state(self) -> Optional[Any]:
64
- return self._state
@@ -1,253 +0,0 @@
1
- import asyncio
2
- from typing import Any, Dict, Optional
3
-
4
- import grpc
5
- import structlog
6
-
7
- from indexify.function_executor.proto.function_executor_pb2 import (
8
- FunctionOutput,
9
- InitializeRequest,
10
- RouterOutput,
11
- RunTaskRequest,
12
- RunTaskResponse,
13
- SerializedObject,
14
- )
15
- from indexify.function_executor.proto.function_executor_pb2_grpc import (
16
- FunctionExecutorStub,
17
- )
18
-
19
- from .api_objects import Task
20
- from .downloader import DownloadedInputs
21
- from .function_executor.function_executor import FunctionExecutor
22
- from .function_executor.function_executor_factory import (
23
- FunctionExecutorFactory,
24
- )
25
- from .function_executor.function_executor_map import FunctionExecutorMap
26
-
27
-
28
- class FunctionWorkerInput:
29
- """Task with all the resources required to run it."""
30
-
31
- def __init__(
32
- self,
33
- task: Task,
34
- graph: SerializedObject,
35
- function_input: DownloadedInputs,
36
- ):
37
- self.task = task
38
- self.graph = graph
39
- self.function_input = function_input
40
-
41
-
42
- class FunctionWorkerOutput:
43
- def __init__(
44
- self,
45
- function_output: Optional[FunctionOutput] = None,
46
- router_output: Optional[RouterOutput] = None,
47
- stdout: Optional[str] = None,
48
- stderr: Optional[str] = None,
49
- reducer: bool = False,
50
- success: bool = False,
51
- ):
52
- self.function_output = function_output
53
- self.router_output = router_output
54
- self.stdout = stdout
55
- self.stderr = stderr
56
- self.reducer = reducer
57
- self.success = success
58
-
59
-
60
- class FunctionExecutorState:
61
- def __init__(
62
- self,
63
- function_id_with_version: str,
64
- function_id_without_version: str,
65
- ongoing_tasks_count: int,
66
- ):
67
- self.function_id_with_version: str = function_id_with_version
68
- self.function_id_without_version: str = function_id_without_version
69
- self.ongoing_tasks_count: int = ongoing_tasks_count
70
-
71
-
72
- class FunctionWorker:
73
- def __init__(self, function_executor_factory: FunctionExecutorFactory):
74
- self._function_executors = FunctionExecutorMap(function_executor_factory)
75
-
76
- async def run(self, input: FunctionWorkerInput) -> FunctionWorkerOutput:
77
- logger = _logger(input.task)
78
- function_executor: Optional[FunctionExecutor] = None
79
- try:
80
- function_executor = await self._obtain_function_executor(input, logger)
81
- return await self._run_in_executor(
82
- function_executor=function_executor, input=input
83
- )
84
- except Exception as e:
85
- logger.error(
86
- "failed running the task",
87
- exc_info=e,
88
- )
89
- if function_executor is not None:
90
- # This will fail all the tasks concurrently running in this Function Executor. Not great.
91
- await self._function_executors.delete(
92
- id=_function_id_without_version(input.task),
93
- function_executor=function_executor,
94
- logger=logger,
95
- )
96
- return _internal_error_output()
97
-
98
- async def _obtain_function_executor(
99
- self, input: FunctionWorkerInput, logger: Any
100
- ) -> FunctionExecutor:
101
- # Temporary policy for Function Executors lifecycle:
102
- # There can only be a single Function Executor per function.
103
- # If a Function Executor already exists for a different function version then wait until
104
- # all the tasks finish in the existing Function Executor and then destroy it first.
105
- initialize_request: InitializeRequest = InitializeRequest(
106
- namespace=input.task.namespace,
107
- graph_name=input.task.compute_graph,
108
- graph_version=input.task.graph_version,
109
- function_name=input.task.compute_fn,
110
- graph=input.graph,
111
- )
112
- initial_function_executor_state: FunctionExecutorState = FunctionExecutorState(
113
- function_id_with_version=_function_id_with_version(input.task),
114
- function_id_without_version=_function_id_without_version(input.task),
115
- ongoing_tasks_count=0,
116
- )
117
-
118
- while True:
119
- function_executor = await self._function_executors.get_or_create(
120
- id=_function_id_without_version(input.task),
121
- initialize_request=initialize_request,
122
- initial_state=initial_function_executor_state,
123
- logger=logger,
124
- )
125
-
126
- # No need to lock Function Executor state as we are not awaiting.
127
- function_executor_state: FunctionExecutorState = function_executor.state()
128
- if (
129
- function_executor_state.function_id_with_version
130
- == _function_id_with_version(input.task)
131
- ):
132
- # The existing Function Executor is for the same function version so we can run the task in it.
133
- # Increment the ongoing tasks count before awaiting to prevent the Function Executor from being destroyed
134
- # by another coroutine.
135
- function_executor_state.ongoing_tasks_count += 1
136
- return function_executor
137
-
138
- # This loop implements the temporary policy so it's implemented using polling instead of a lower
139
- # latency event based mechanism with a higher complexity.
140
- if function_executor_state.ongoing_tasks_count == 0:
141
- logger.info(
142
- "destroying existing Function Executor for different function version",
143
- function_id=_function_id_with_version(input.task),
144
- executor_function_id=function_executor_state.function_id_with_version,
145
- )
146
- await self._function_executors.delete(
147
- id=_function_id_without_version(input.task),
148
- function_executor=function_executor,
149
- logger=logger,
150
- )
151
- else:
152
- logger.info(
153
- "waiting for existing Function Executor to finish",
154
- function_id=_function_id_with_version(input.task),
155
- executor_function_id=function_executor_state.function_id_with_version,
156
- )
157
- await asyncio.sleep(
158
- 5
159
- ) # Wait for 5 secs before checking if all tasks for the existing Function Executor finished.
160
-
161
- async def _run_in_executor(
162
- self, function_executor: FunctionExecutor, input: FunctionWorkerInput
163
- ) -> FunctionWorkerOutput:
164
- """Runs the task in the Function Executor.
165
-
166
- The Function Executor's ongoing_tasks_count must be incremented before calling this function.
167
- """
168
- try:
169
- run_task_request: RunTaskRequest = RunTaskRequest(
170
- graph_invocation_id=input.task.invocation_id,
171
- task_id=input.task.id,
172
- function_input=input.function_input.input,
173
- )
174
- if input.function_input.init_value is not None:
175
- run_task_request.function_init_value.CopyFrom(
176
- input.function_input.init_value
177
- )
178
- channel: grpc.aio.Channel = await function_executor.channel()
179
- run_task_response: RunTaskResponse = await FunctionExecutorStub(
180
- channel
181
- ).run_task(run_task_request)
182
- return _to_output(run_task_response)
183
- finally:
184
- # If this Function Executor was destroyed then it's not
185
- # visible in the map but we still have a reference to it.
186
- function_executor.state().ongoing_tasks_count -= 1
187
-
188
- async def shutdown(self) -> None:
189
- await self._function_executors.clear(
190
- logger=structlog.get_logger(module=__name__, event="shutdown")
191
- )
192
-
193
-
194
- def _to_output(response: RunTaskResponse) -> FunctionWorkerOutput:
195
- required_fields = [
196
- "stdout",
197
- "stderr",
198
- "is_reducer",
199
- "success",
200
- ]
201
-
202
- for field in required_fields:
203
- if not response.HasField(field):
204
- raise ValueError(f"Response is missing required field: {field}")
205
-
206
- output = FunctionWorkerOutput(
207
- stdout=response.stdout,
208
- stderr=response.stderr,
209
- reducer=response.is_reducer,
210
- success=response.success,
211
- )
212
-
213
- if response.HasField("function_output"):
214
- output.function_output = response.function_output
215
- if response.HasField("router_output"):
216
- output.router_output = response.router_output
217
-
218
- return output
219
-
220
-
221
- def _internal_error_output() -> FunctionWorkerOutput:
222
- return FunctionWorkerOutput(
223
- function_output=None,
224
- router_output=None,
225
- stdout=None,
226
- # We are not sharing internal error messages with the customer.
227
- # If customer code failed then we won't get any exceptions here.
228
- # All customer code errors are returned in the gRPC response.
229
- stderr="Platform failed to execute the function.",
230
- reducer=False,
231
- success=False,
232
- )
233
-
234
-
235
- def _logger(task: Task) -> Any:
236
- return structlog.get_logger(
237
- module=__name__,
238
- namespace=task.namespace,
239
- graph_name=task.compute_graph,
240
- graph_version=task.graph_version,
241
- function_name=task.compute_fn,
242
- graph_invocation_id=task.invocation_id,
243
- task_id=task.id,
244
- function_id=_function_id_with_version(task),
245
- )
246
-
247
-
248
- def _function_id_with_version(task: Task) -> str:
249
- return f"versioned/{task.namespace}/{task.compute_graph}/{task.graph_version}/{task.compute_fn}"
250
-
251
-
252
- def _function_id_without_version(task: Task) -> str:
253
- return f"not_versioned/{task.namespace}/{task.compute_graph}/{task.compute_fn}"
@@ -1,50 +0,0 @@
1
- indexify/__init__.py,sha256=P0mvM8sbkeS2CjYzRYyzb42CnXGhyJXdz4FdmTBMSWM,697
2
- indexify/cli.py,sha256=Y8mijSECm_s-ZLNcba-PIFrnAjB7KX7TVdw0q6KVD2I,10382
3
- indexify/common_util.py,sha256=LKS6yZ3yv8nF2J-KzisGIjqjTvCn7tLFifQJLT4gHRg,3529
4
- indexify/data_loaders/__init__.py,sha256=Y5NEuseTcYAICRiweYw5wBQ2m2YplbsY21I7df-rdi4,1339
5
- indexify/data_loaders/local_directory_loader.py,sha256=fCrgj5drnW71ZUdDDvcB1-VJjIs1w6Q8sEW0HSGSAiA,1247
6
- indexify/data_loaders/url_loader.py,sha256=32SERljcq1Xsi4RdLz2dgyk2TER5pQPTtXl3gUzwHbY,1533
7
- indexify/error.py,sha256=qAWr8R6AxPkjsxHSzXTc8zqYnNO_AjOqqYEPsQvF1Zs,238
8
- indexify/executor/api_objects.py,sha256=vp7aEjvfWL2j4nYDV0xLgLVZlGsbStMEu0nzrgJVq1A,741
9
- indexify/executor/downloader.py,sha256=oeiSGxIIn7uBTKT6hzyPUa-AlOUlqy7WQkinm7qpV0I,6442
10
- indexify/executor/executor.py,sha256=i9TnF899scLkUWCpomnDh_dwUDqtXgiPWvnwsF7DUEs,5272
11
- indexify/executor/function_executor/function_executor.py,sha256=0W6vCT-DBVPUMPoOXLVRVHZjVGGUeTDV_NWb-XLP2gM,1218
12
- indexify/executor/function_executor/function_executor_factory.py,sha256=23BU_i47Tz6xd9fAwsKH29lm261bZXfzYGRhSqeFtwY,900
13
- indexify/executor/function_executor/function_executor_map.py,sha256=HXeCjkRusumJisU74Iy6M4e13k8jeHk4Yq8PyODSsUc,3922
14
- indexify/executor/function_executor/process_function_executor.py,sha256=CdqFtvlvS_VIzVdbE3Wnzj_Kqfzhzf4X05avHIielEE,2150
15
- indexify/executor/function_executor/process_function_executor_factory.py,sha256=ognYRd5l10kVD2KAShfkd8vg-JElYctG77Pcx77-grM,3850
16
- indexify/executor/function_worker.py,sha256=YfNcDvwlX5dvKF-4x6XkvjX4E0S-5WKZkmsLFaesapg,9669
17
- indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
18
- indexify/executor/task_fetcher.py,sha256=qBuVxgjzjkTi_1_hyQqtBxetBxdL7TGMgI4FBwyTANE,3004
19
- indexify/executor/task_reporter.py,sha256=soCB31JNi0IZDgd0VRTm90X1Prd3_tHMGPqxccgGLKc,6869
20
- indexify/function_executor/function_executor_service.py,sha256=EwwHTPX-AVqbSl-BONWWy3Z6gowXnYj5tigGoFs1u0s,3694
21
- indexify/function_executor/handlers/run_function/function_inputs_loader.py,sha256=x2Lrzb6CsEWDWW5qG9etS7TyV7KVYfuFKEbzKVhYizo,1553
22
- indexify/function_executor/handlers/run_function/handler.py,sha256=eklicxgyExd9ScdDahQO9LaLNXPNBPAAI0KBt3Wb9-w,5636
23
- indexify/function_executor/handlers/run_function/request_validator.py,sha256=tVmbOS2g4AeV48CZCSDRycnoQjjGy12XD_vbaOLX_Z4,740
24
- indexify/function_executor/handlers/run_function/response_helper.py,sha256=KDSadee08g6JRFwKRTdcOwbP6SmRJCGVHh1FOLdt_ws,3109
25
- indexify/function_executor/initialize_request_validator.py,sha256=AtyKgIjlGHB-Fh3Vhso86TC0o65DfuVEY8GFfRSfylo,709
26
- indexify/function_executor/proto/configuration.py,sha256=P0xtxwf-XaCxi0kvtCbm4X8y0YOuHhR-uh9xUuLdFoM,595
27
- indexify/function_executor/proto/function_executor.proto,sha256=PiftSSIDjHkxUbD7rNVmgfTCP12bFJ7oKJXuRx2sM28,2188
28
- indexify/function_executor/proto/function_executor_pb2.py,sha256=tEDJJUywM6-j4dBwvBEHXnGCv9spGWVuhtHuiTDxw7I,4582
29
- indexify/function_executor/proto/function_executor_pb2.pyi,sha256=E-UnFIFbej-QPhaX1X4oYGeUW6_HuEi13qhCYkN1tzI,4182
30
- indexify/function_executor/proto/function_executor_pb2_grpc.py,sha256=OIErF2wYWaRI2TyF8EbWGjj5K9aD7OIJwAp1BRMQ0ic,6265
31
- indexify/function_executor/proto/message_validator.py,sha256=OKXPYgy5L9c-spnV9Zjv7PA_yxwzvykfhbYylYx8cwQ,1456
32
- indexify/function_executor/server.py,sha256=YEeKHR6qjNJyLFb8KNjcCYClBgVzdJNmKwdwmyJ-ybE,1136
33
- indexify/functions_sdk/data_objects.py,sha256=ZJ7B9b5OI7aieCWJFx18pV6tqScssIFtmth84i7nKTg,623
34
- indexify/functions_sdk/graph.py,sha256=cyaOHpyUVkUq5hyS85ojf6GN_tBGXQbwWlFQnktj1q0,13281
35
- indexify/functions_sdk/graph_definition.py,sha256=yQ7c7jpA8RB8Yt81D1i5R3LPT8fmLvN8bLuEHvS4P_o,1775
36
- indexify/functions_sdk/graph_validation.py,sha256=mN2Fcp91GIwFZEQP6z_qGqt4LkLM70SnI7AWBi4CmKQ,2509
37
- indexify/functions_sdk/image.py,sha256=wVDs5O5yshjSSZ1d-keyMp9zUrck2TazJlkVq94jMZk,1965
38
- indexify/functions_sdk/indexify_functions.py,sha256=USuEeZnbQTd6uUjv-ZAwdBrbR8gGrTD78oXjyW6DBbw,13376
39
- indexify/functions_sdk/object_serializer.py,sha256=R58ALsl2Lb87ii6km4D6hBBsqRs_CHNISxhUICE2d9o,1931
40
- indexify/functions_sdk/pipeline.py,sha256=KmxZE8eBFAQ4bbEcYURXXR26HSyoAT3O6iu9H38-OXE,974
41
- indexify/http_client.py,sha256=XvIi7z7h7PZomMKlL8WEhFIWAAPzFe2hrLQUz4QC_Zc,15392
42
- indexify/logging.py,sha256=_TLnWT0RzABvExwTixjdpuxoSNUev4zeNV0K1VlEUmA,1099
43
- indexify/remote_graph.py,sha256=OzBucU4buR5UdTViNJvh1RfnOTmYA34Uel5qXnRsQsA,5006
44
- indexify/remote_pipeline.py,sha256=oqx57rSPszNS3DToXO_nf-CKqkCZWptm1u_p3orV_gQ,790
45
- indexify/settings.py,sha256=Ny59mzYI4gbXoK8hjx66a_men6ndbd1J1zCTcKOoyzg,50
46
- indexify-0.2.43.dist-info/LICENSE.txt,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
47
- indexify-0.2.43.dist-info/METADATA,sha256=zMNCsHsP14BeoCqph91AkqHd5RBGMrczAL9Qdx9gknw,6267
48
- indexify-0.2.43.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
49
- indexify-0.2.43.dist-info/entry_points.txt,sha256=Pih7WV-XMpAzI5dEvROcpLr-ybVhd9Y-AtuzBKUdcDs,49
50
- indexify-0.2.43.dist-info/RECORD,,