indexify 0.3.14__py3-none-any.whl → 0.3.16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indexify/cli/cli.py +20 -91
- indexify/executor/api_objects.py +2 -0
- indexify/executor/executor.py +77 -86
- indexify/executor/function_executor/function_executor_state.py +43 -43
- indexify/executor/function_executor/function_executor_states_container.py +10 -4
- indexify/executor/function_executor/function_executor_status.py +91 -0
- indexify/executor/function_executor/metrics/function_executor.py +1 -1
- indexify/executor/function_executor/metrics/function_executor_state.py +36 -0
- indexify/executor/function_executor/server/function_executor_server_factory.py +8 -8
- indexify/executor/function_executor/single_task_runner.py +100 -37
- indexify/executor/grpc/channel_creator.py +53 -0
- indexify/executor/grpc/metrics/channel_creator.py +18 -0
- indexify/executor/grpc/metrics/state_reporter.py +17 -0
- indexify/executor/{state_reconciler.py → grpc/state_reconciler.py} +60 -31
- indexify/executor/grpc/state_reporter.py +199 -0
- indexify/executor/monitoring/health_checker/generic_health_checker.py +27 -12
- indexify/executor/task_runner.py +30 -6
- indexify/{task_scheduler/proto → proto}/task_scheduler.proto +23 -17
- indexify/proto/task_scheduler_pb2.py +64 -0
- indexify/{task_scheduler/proto → proto}/task_scheduler_pb2.pyi +28 -10
- indexify/{task_scheduler/proto → proto}/task_scheduler_pb2_grpc.py +16 -16
- {indexify-0.3.14.dist-info → indexify-0.3.16.dist-info}/METADATA +1 -1
- {indexify-0.3.14.dist-info → indexify-0.3.16.dist-info}/RECORD +25 -21
- indexify/executor/state_reporter.py +0 -127
- indexify/task_scheduler/proto/task_scheduler_pb2.py +0 -69
- {indexify-0.3.14.dist-info → indexify-0.3.16.dist-info}/WHEEL +0 -0
- {indexify-0.3.14.dist-info → indexify-0.3.16.dist-info}/entry_points.txt +0 -0
@@ -4,8 +4,8 @@ import warnings
|
|
4
4
|
|
5
5
|
import grpc
|
6
6
|
|
7
|
-
from indexify.
|
8
|
-
task_scheduler_pb2 as
|
7
|
+
from indexify.proto import (
|
8
|
+
task_scheduler_pb2 as indexify_dot_proto_dot_task__scheduler__pb2,
|
9
9
|
)
|
10
10
|
|
11
11
|
GRPC_GENERATED_VERSION = "1.70.0"
|
@@ -24,7 +24,7 @@ except ImportError:
|
|
24
24
|
if _version_not_supported:
|
25
25
|
raise RuntimeError(
|
26
26
|
f"The grpc package installed is at version {GRPC_VERSION},"
|
27
|
-
+ f" but the generated code in indexify/
|
27
|
+
+ f" but the generated code in indexify/proto/task_scheduler_pb2_grpc.py depends on"
|
28
28
|
+ f" grpcio>={GRPC_GENERATED_VERSION}."
|
29
29
|
+ f" Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}"
|
30
30
|
+ f" or downgrade your generated code using grpcio-tools<={GRPC_VERSION}."
|
@@ -44,14 +44,14 @@ class TaskSchedulerServiceStub(object):
|
|
44
44
|
"""
|
45
45
|
self.report_executor_state = channel.unary_unary(
|
46
46
|
"/task_scheduler_service.TaskSchedulerService/report_executor_state",
|
47
|
-
request_serializer=
|
48
|
-
response_deserializer=
|
47
|
+
request_serializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateRequest.SerializeToString,
|
48
|
+
response_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateResponse.FromString,
|
49
49
|
_registered_method=True,
|
50
50
|
)
|
51
51
|
self.get_desired_executor_states = channel.unary_stream(
|
52
52
|
"/task_scheduler_service.TaskSchedulerService/get_desired_executor_states",
|
53
|
-
request_serializer=
|
54
|
-
response_deserializer=
|
53
|
+
request_serializer=indexify_dot_proto_dot_task__scheduler__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
|
54
|
+
response_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.DesiredExecutorState.FromString,
|
55
55
|
_registered_method=True,
|
56
56
|
)
|
57
57
|
|
@@ -74,7 +74,7 @@ class TaskSchedulerServiceServicer(object):
|
|
74
74
|
"""Called by Executor to open a stream of its desired states. When Server wants Executor to change something
|
75
75
|
it puts a message on the stream with the new desired state of the Executor.
|
76
76
|
|
77
|
-
|
77
|
+
Deprecated HTTP API is used to download the serialized graph and task inputs.
|
78
78
|
"""
|
79
79
|
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
80
80
|
context.set_details("Method not implemented!")
|
@@ -85,13 +85,13 @@ def add_TaskSchedulerServiceServicer_to_server(servicer, server):
|
|
85
85
|
rpc_method_handlers = {
|
86
86
|
"report_executor_state": grpc.unary_unary_rpc_method_handler(
|
87
87
|
servicer.report_executor_state,
|
88
|
-
request_deserializer=
|
89
|
-
response_serializer=
|
88
|
+
request_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateRequest.FromString,
|
89
|
+
response_serializer=indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateResponse.SerializeToString,
|
90
90
|
),
|
91
91
|
"get_desired_executor_states": grpc.unary_stream_rpc_method_handler(
|
92
92
|
servicer.get_desired_executor_states,
|
93
|
-
request_deserializer=
|
94
|
-
response_serializer=
|
93
|
+
request_deserializer=indexify_dot_proto_dot_task__scheduler__pb2.GetDesiredExecutorStatesRequest.FromString,
|
94
|
+
response_serializer=indexify_dot_proto_dot_task__scheduler__pb2.DesiredExecutorState.SerializeToString,
|
95
95
|
),
|
96
96
|
}
|
97
97
|
generic_handler = grpc.method_handlers_generic_handler(
|
@@ -126,8 +126,8 @@ class TaskSchedulerService(object):
|
|
126
126
|
request,
|
127
127
|
target,
|
128
128
|
"/task_scheduler_service.TaskSchedulerService/report_executor_state",
|
129
|
-
|
130
|
-
|
129
|
+
indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateRequest.SerializeToString,
|
130
|
+
indexify_dot_proto_dot_task__scheduler__pb2.ReportExecutorStateResponse.FromString,
|
131
131
|
options,
|
132
132
|
channel_credentials,
|
133
133
|
insecure,
|
@@ -156,8 +156,8 @@ class TaskSchedulerService(object):
|
|
156
156
|
request,
|
157
157
|
target,
|
158
158
|
"/task_scheduler_service.TaskSchedulerService/get_desired_executor_states",
|
159
|
-
|
160
|
-
|
159
|
+
indexify_dot_proto_dot_task__scheduler__pb2.GetDesiredExecutorStatesRequest.SerializeToString,
|
160
|
+
indexify_dot_proto_dot_task__scheduler__pb2.DesiredExecutorState.FromString,
|
161
161
|
options,
|
162
162
|
channel_credentials,
|
163
163
|
insecure,
|
@@ -1,27 +1,33 @@
|
|
1
|
-
indexify/cli/cli.py,sha256
|
1
|
+
indexify/cli/cli.py,sha256=-ngLINNhZ2Req18_dQ1rHEnusY2feQ-9gYxOP8jcorA,8768
|
2
2
|
indexify/executor/README.md,sha256=ozC6_hMkhQQNVCMEpBxwiUALz6lwErPQxNxQfQDqnG4,2029
|
3
|
-
indexify/executor/api_objects.py,sha256=
|
3
|
+
indexify/executor/api_objects.py,sha256=oUlH-GQPuPmwgcBzMpI2HehXeElBTCULECk-oHiBHwU,1263
|
4
4
|
indexify/executor/downloader.py,sha256=LkvAXfKxddnDzgfmwHcpDB_n795-eVKzn-hLjq4nUEM,9412
|
5
|
-
indexify/executor/executor.py,sha256=
|
5
|
+
indexify/executor/executor.py,sha256=FTp05YxuKlMUbI99OV7NkL3KuFD12evKcqxzb-fXWBk,14641
|
6
6
|
indexify/executor/function_executor/function_executor.py,sha256=sQ5FOdrjybDDsjagghlfjV06IXTpWWDBSTHqQXI-w9M,11245
|
7
|
-
indexify/executor/function_executor/function_executor_state.py,sha256=
|
8
|
-
indexify/executor/function_executor/function_executor_states_container.py,sha256=
|
7
|
+
indexify/executor/function_executor/function_executor_state.py,sha256=b2taGClg0BUnlD_rYGkpom6syXBMUp7UWWrjLrUCwyo,3966
|
8
|
+
indexify/executor/function_executor/function_executor_states_container.py,sha256=RclJDJqIr8ywKipPBC6_idnPAqYi0dPa1d4QUAaXqbw,3460
|
9
|
+
indexify/executor/function_executor/function_executor_status.py,sha256=U4p1fcdVWlHr7uPY7e7ZSb2_WelUmPeH-WgboQQ9mw4,3336
|
9
10
|
indexify/executor/function_executor/health_checker.py,sha256=qUUpG4oeVsPLibiCspAiRm-2Ldg46ulnnpj9EBXr1NQ,3916
|
10
11
|
indexify/executor/function_executor/invocation_state_client.py,sha256=p-xgM4__cHR1ApvMV9hShrGWee_Je0VDhICZUGjpQY4,9644
|
11
|
-
indexify/executor/function_executor/metrics/function_executor.py,sha256=
|
12
|
-
indexify/executor/function_executor/metrics/function_executor_state.py,sha256=
|
12
|
+
indexify/executor/function_executor/metrics/function_executor.py,sha256=TDksxLRJr-P9ZKhF2Orsaxzzb4lVIBxFEjd_9Zv53Ng,6313
|
13
|
+
indexify/executor/function_executor/metrics/function_executor_state.py,sha256=qheMhnoiYLiZB7ky5EyegfDy4Mr0Zh83bOE0gJ38YmU,1607
|
13
14
|
indexify/executor/function_executor/metrics/function_executor_state_container.py,sha256=6rrAfml-TivjkHatCM4BLY7jmVs523Wzb6QIysncc-0,302
|
14
15
|
indexify/executor/function_executor/metrics/health_checker.py,sha256=EaeIYJPrQ-qqNMGZVGkvjPoeQSCl4FzPKXEv3Cly1NE,456
|
15
16
|
indexify/executor/function_executor/metrics/invocation_state_client.py,sha256=6FCW6rXHVZZSmwLquZdpjgQPSmE_99naDLke5rZiwMI,1867
|
16
17
|
indexify/executor/function_executor/metrics/single_task_runner.py,sha256=7BJlGkdPGKeufMs3zWNO_1GRVzjINRY5rW3Mp4oWWec,805
|
17
18
|
indexify/executor/function_executor/server/client_configuration.py,sha256=gOywMus0cotlX6NKIadEJwvOmBE-LbGE_wvoMi5-HzY,994
|
18
19
|
indexify/executor/function_executor/server/function_executor_server.py,sha256=_DLivLDikupZusRk8gVWDk7fWPT9XjZ4un1yWSlOObs,883
|
19
|
-
indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=
|
20
|
+
indexify/executor/function_executor/server/function_executor_server_factory.py,sha256=cP93a3t1AfGx8qwageNLVdTwG52UOwzYNbbyrPq2TUQ,1692
|
20
21
|
indexify/executor/function_executor/server/subprocess_function_executor_server.py,sha256=JekDOqF7oFD4J6zcN3xB0Dxd1cgpEXMOsb_rKZOeBlI,668
|
21
22
|
indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py,sha256=xbH0-73tBM-i0dh1x_k0fDrNAi8WgDKjqiQ9xveU4Zg,4214
|
22
|
-
indexify/executor/function_executor/single_task_runner.py,sha256=
|
23
|
+
indexify/executor/function_executor/single_task_runner.py,sha256=iWnJsB2BGqdgAkrlJHbOvSIhVXc88X0AYbB2_o-bB-E,13547
|
23
24
|
indexify/executor/function_executor/task_input.py,sha256=wSrHR4m0juiGClQyeVdhRC37QzDt6Rrjq-ZXJkfBi9k,584
|
24
25
|
indexify/executor/function_executor/task_output.py,sha256=SQJSlrknB7Ylf5IOeINfBEgiplS5hAPJh1hYulhyvfU,1962
|
26
|
+
indexify/executor/grpc/channel_creator.py,sha256=Z_DU212-wkaU_m-I14OBbWKVeHo3aG5vPmF_ebJaZGc,1849
|
27
|
+
indexify/executor/grpc/metrics/channel_creator.py,sha256=k-WArgklmP5WhjcmFmrgRblB7yc3XlaOXO8owRyV-mw,649
|
28
|
+
indexify/executor/grpc/metrics/state_reporter.py,sha256=GggBEjMzQUYIG95LtTS4fUg1u9jYowkaXoUXppAXucs,543
|
29
|
+
indexify/executor/grpc/state_reconciler.py,sha256=g7Qi6t79vuldh4y0Ue2mfnU2Jj8J-rsDtQzNZZlbLfE,12973
|
30
|
+
indexify/executor/grpc/state_reporter.py,sha256=kk1kqvuUfgNZQriWj2FWjkPyloLcWPq7cCuNCK7mb5I,7669
|
25
31
|
indexify/executor/metrics/downloader.py,sha256=lctPh8xjkXeLEFJnl1hNrD1yEhLhIl5sggsR4Yoe_Zc,2746
|
26
32
|
indexify/executor/metrics/executor.py,sha256=ua-Vv_k1CB4juJdF7tEBQbBMksqWAA3iXKKMKXZUCLk,2369
|
27
33
|
indexify/executor/metrics/task_fetcher.py,sha256=iJEwCLzYr2cuz7hRvNiqaa2nvQP4OrA0hm0iJY0YKG0,736
|
@@ -30,23 +36,21 @@ indexify/executor/metrics/task_runner.py,sha256=o5ERNePKPmVKknFoSZUr-r597dEOOWvW
|
|
30
36
|
indexify/executor/monitoring/function_allowlist.py,sha256=wUGeiv3aAGWMlQXzHXq9O6MVHby6Tu-zY4U0MyWiQu0,683
|
31
37
|
indexify/executor/monitoring/handler.py,sha256=Cj1cu_LcsAP0tdviqNhoEtGm4h0OJAxxzW9C2YdNXYU,240
|
32
38
|
indexify/executor/monitoring/health_check_handler.py,sha256=e1pEtWFKaVs6H57Z4YLejNECrJtC38PweZc7xTJeqVw,695
|
33
|
-
indexify/executor/monitoring/health_checker/generic_health_checker.py,sha256=
|
39
|
+
indexify/executor/monitoring/health_checker/generic_health_checker.py,sha256=ot-nan5kDMFwXanJu0_OKV65fIoqhvATIoFW4LOO6V8,3373
|
34
40
|
indexify/executor/monitoring/health_checker/health_checker.py,sha256=c6UooJUIKaj2dYwU3507nnOglU51TC4FB9npCnLHjbY,838
|
35
41
|
indexify/executor/monitoring/metrics.py,sha256=Dx2wPcTKvbd5Y5rGOfeyscFtAQ2DZ16_s5BX6d4nhI8,6660
|
36
42
|
indexify/executor/monitoring/prometheus_metrics_handler.py,sha256=KiGqSf7rkXTfbDwThyXFpFe2jnuZD5q-5SBP_0GDo8Y,591
|
37
43
|
indexify/executor/monitoring/server.py,sha256=yzdYhcxnmY6uTQUMt3vatF5jilN52ZtfFseOmHyQpTo,1254
|
38
44
|
indexify/executor/monitoring/startup_probe_handler.py,sha256=zXXsBU15SMlBx1bSFpxWDfed1VHtKKnwvLQ8-frpG98,425
|
39
45
|
indexify/executor/runtime_probes.py,sha256=bo6Dq6AGZpJH099j0DHtVSDEH80tv3j9MXf3VXSx_p8,2182
|
40
|
-
indexify/executor/state_reconciler.py,sha256=hIO3BDjXJKicvS7Sgdd5eht9nTsZtleZi7beBPI8d8E,11764
|
41
|
-
indexify/executor/state_reporter.py,sha256=83ImGX4b78bEDBJzTDsiEhSd8780a0uob5v6RgA2kzA,4514
|
42
46
|
indexify/executor/task_fetcher.py,sha256=NpFfHgaY99bSL-K2D5kcDAMNUG2FArq0-qF_mgF-LBQ,3375
|
43
47
|
indexify/executor/task_reporter.py,sha256=mYgwozUO95PEwYMmeeIS0-HfMrO4z3Nhy6IduMsMahM,7367
|
44
|
-
indexify/executor/task_runner.py,sha256=
|
45
|
-
indexify/
|
46
|
-
indexify/
|
47
|
-
indexify/
|
48
|
-
indexify/
|
49
|
-
indexify-0.3.
|
50
|
-
indexify-0.3.
|
51
|
-
indexify-0.3.
|
52
|
-
indexify-0.3.
|
48
|
+
indexify/executor/task_runner.py,sha256=TGiTNE68HCm38HcpPRN5-hwNiTFw_gwBRkCcrw7sz2Q,6847
|
49
|
+
indexify/proto/task_scheduler.proto,sha256=kxMIJCj1pXG-fHeJGHXlthZTsB1dy_yvshQLt0UJRTM,5672
|
50
|
+
indexify/proto/task_scheduler_pb2.py,sha256=X97JBJZ2n6ToDtUlDjPFV66_vZ05-vO8wPATrpzAonA,9085
|
51
|
+
indexify/proto/task_scheduler_pb2.pyi,sha256=aXrB7-eNwgchy2OVlvEfPXtr9EyYoU-sgbdSRVNEI8s,11357
|
52
|
+
indexify/proto/task_scheduler_pb2_grpc.py,sha256=STtk9XrBzLbmWdLwpL55Obyf9ehUesfxxysxER32SEE,6854
|
53
|
+
indexify-0.3.16.dist-info/METADATA,sha256=3jBagszOve3WAI9dTFN7ZtT1x_0hBFhLPOFYs-KQ6AU,1158
|
54
|
+
indexify-0.3.16.dist-info/WHEEL,sha256=RaoafKOydTQ7I_I3JTrPCg6kUmTgtm4BornzOqyEfJ8,88
|
55
|
+
indexify-0.3.16.dist-info/entry_points.txt,sha256=GU9wmsgvN7nQw3N2X0PMYn1RSvF6CrhH9RuC2D8d3Gk,53
|
56
|
+
indexify-0.3.16.dist-info/RECORD,,
|
@@ -1,127 +0,0 @@
|
|
1
|
-
import asyncio
|
2
|
-
from typing import Any, List, Optional
|
3
|
-
|
4
|
-
import grpc
|
5
|
-
|
6
|
-
from indexify.task_scheduler.proto.task_scheduler_pb2 import (
|
7
|
-
AllowedFunction,
|
8
|
-
ExecutorState,
|
9
|
-
ExecutorStatus,
|
10
|
-
FunctionExecutorDescription,
|
11
|
-
)
|
12
|
-
from indexify.task_scheduler.proto.task_scheduler_pb2 import (
|
13
|
-
FunctionExecutorState as FunctionExecutorStateProto,
|
14
|
-
)
|
15
|
-
from indexify.task_scheduler.proto.task_scheduler_pb2 import (
|
16
|
-
GPUResources,
|
17
|
-
HostResources,
|
18
|
-
ReportExecutorStateRequest,
|
19
|
-
)
|
20
|
-
from indexify.task_scheduler.proto.task_scheduler_pb2_grpc import (
|
21
|
-
TaskSchedulerServiceStub,
|
22
|
-
)
|
23
|
-
|
24
|
-
from .api_objects import FunctionURI
|
25
|
-
from .function_executor.function_executor_state import FunctionExecutorState
|
26
|
-
from .function_executor.function_executor_states_container import (
|
27
|
-
FunctionExecutorStatesContainer,
|
28
|
-
)
|
29
|
-
|
30
|
-
_REPORTING_INTERVAL_SEC = 5
|
31
|
-
_REPORT_RPC_TIMEOUT_SEC = 5
|
32
|
-
|
33
|
-
|
34
|
-
class ExecutorStateReporter:
|
35
|
-
def __init__(
|
36
|
-
self,
|
37
|
-
executor_id: str,
|
38
|
-
function_allowlist: Optional[List[FunctionURI]],
|
39
|
-
function_executor_states: FunctionExecutorStatesContainer,
|
40
|
-
server_channel: grpc.aio.Channel,
|
41
|
-
logger: Any,
|
42
|
-
):
|
43
|
-
self._executor_id: str = executor_id
|
44
|
-
self._function_executor_states: FunctionExecutorStatesContainer = (
|
45
|
-
function_executor_states
|
46
|
-
)
|
47
|
-
self._stub: TaskSchedulerServiceStub = TaskSchedulerServiceStub(server_channel)
|
48
|
-
self._logger: Any = logger.bind(module=__name__)
|
49
|
-
self._is_shutdown: bool = False
|
50
|
-
self._executor_status: ExecutorStatus = ExecutorStatus.EXECUTOR_STATUS_UNKNOWN
|
51
|
-
self._allowed_functions: List[AllowedFunction] = []
|
52
|
-
|
53
|
-
for function_uri in (
|
54
|
-
function_allowlist if function_allowlist is not None else []
|
55
|
-
):
|
56
|
-
allowed_function = AllowedFunction(
|
57
|
-
namespace=function_uri.namespace,
|
58
|
-
graph_name=function_uri.compute_graph,
|
59
|
-
function_name=function_uri.compute_fn,
|
60
|
-
)
|
61
|
-
if function_uri.version is not None:
|
62
|
-
allowed_function.graph_version = function_uri.version
|
63
|
-
self._allowed_functions.append(allowed_function)
|
64
|
-
|
65
|
-
# TODO: Update Executor to call status updates.
|
66
|
-
def update_status(self, value: ExecutorStatus):
|
67
|
-
self._executor_status = value
|
68
|
-
|
69
|
-
async def run(self):
|
70
|
-
while not self._is_shutdown:
|
71
|
-
await self._report_state()
|
72
|
-
await asyncio.sleep(_REPORTING_INTERVAL_SEC)
|
73
|
-
|
74
|
-
async def _report_state(self):
|
75
|
-
state = ExecutorState(
|
76
|
-
executor_id=self._executor_id,
|
77
|
-
executor_status=self._executor_status,
|
78
|
-
host_resources=await self._fetch_host_resources(),
|
79
|
-
allowed_functions=self._allowed_functions,
|
80
|
-
function_executor_states=await self._fetch_function_executor_states(),
|
81
|
-
)
|
82
|
-
|
83
|
-
await self._stub.report_executor_state(
|
84
|
-
ReportExecutorStateRequest(executor_state=state),
|
85
|
-
timeout=_REPORT_RPC_TIMEOUT_SEC,
|
86
|
-
)
|
87
|
-
|
88
|
-
async def _fetch_host_resources(self) -> HostResources:
|
89
|
-
# We're only supporting Executors with non empty function allowlist right now.
|
90
|
-
# In this mode Server should ignore available host resources.
|
91
|
-
# This is why it's okay to report zeros right now.
|
92
|
-
return HostResources(
|
93
|
-
cpu_count=0,
|
94
|
-
memory_bytes=0,
|
95
|
-
disk_bytes=0,
|
96
|
-
gpu=GPUResources(
|
97
|
-
count=0,
|
98
|
-
model="",
|
99
|
-
),
|
100
|
-
)
|
101
|
-
|
102
|
-
async def _fetch_function_executor_states(self) -> List[FunctionExecutorStateProto]:
|
103
|
-
states = []
|
104
|
-
|
105
|
-
async for function_executor_state in self._function_executor_states:
|
106
|
-
function_executor_state: FunctionExecutorState
|
107
|
-
states.append(
|
108
|
-
FunctionExecutorStateProto(
|
109
|
-
description=FunctionExecutorDescription(
|
110
|
-
id=function_executor_state.id,
|
111
|
-
namespace=function_executor_state.namespace,
|
112
|
-
graph_name=function_executor_state.graph_name,
|
113
|
-
graph_version=function_executor_state.graph_version,
|
114
|
-
function_name=function_executor_state.function_name,
|
115
|
-
),
|
116
|
-
status=function_executor_state.status,
|
117
|
-
)
|
118
|
-
)
|
119
|
-
|
120
|
-
return states
|
121
|
-
|
122
|
-
async def shutdown(self):
|
123
|
-
"""Shuts down the state reporter.
|
124
|
-
|
125
|
-
Never raises any exceptions.
|
126
|
-
"""
|
127
|
-
self._is_shutdown = True
|
@@ -1,69 +0,0 @@
|
|
1
|
-
# -*- coding: utf-8 -*-
|
2
|
-
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
3
|
-
# NO CHECKED-IN PROTOBUF GENCODE
|
4
|
-
# source: indexify/task_scheduler/proto/task_scheduler.proto
|
5
|
-
# Protobuf Python Version: 5.29.0
|
6
|
-
"""Generated protocol buffer code."""
|
7
|
-
from google.protobuf import descriptor as _descriptor
|
8
|
-
from google.protobuf import descriptor_pool as _descriptor_pool
|
9
|
-
from google.protobuf import runtime_version as _runtime_version
|
10
|
-
from google.protobuf import symbol_database as _symbol_database
|
11
|
-
from google.protobuf.internal import builder as _builder
|
12
|
-
|
13
|
-
_runtime_version.ValidateProtobufRuntimeVersion(
|
14
|
-
_runtime_version.Domain.PUBLIC,
|
15
|
-
5,
|
16
|
-
29,
|
17
|
-
0,
|
18
|
-
"",
|
19
|
-
"indexify/task_scheduler/proto/task_scheduler.proto",
|
20
|
-
)
|
21
|
-
# @@protoc_insertion_point(imports)
|
22
|
-
|
23
|
-
_sym_db = _symbol_database.Default()
|
24
|
-
|
25
|
-
|
26
|
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
|
27
|
-
b'\n2indexify/task_scheduler/proto/task_scheduler.proto\x12\x16task_scheduler_service"l\n\x0cGPUResources\x12\x12\n\x05\x63ount\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x34\n\x05model\x18\x02 \x01(\x0e\x32 .task_scheduler_service.GPUModelH\x01\x88\x01\x01\x42\x08\n\x06_countB\x08\n\x06_model"\xc9\x01\n\rHostResources\x12\x16\n\tcpu_count\x18\x01 \x01(\rH\x00\x88\x01\x01\x12\x19\n\x0cmemory_bytes\x18\x02 \x01(\x04H\x01\x88\x01\x01\x12\x17\n\ndisk_bytes\x18\x03 \x01(\x04H\x02\x88\x01\x01\x12\x36\n\x03gpu\x18\x04 \x01(\x0b\x32$.task_scheduler_service.GPUResourcesH\x03\x88\x01\x01\x42\x0c\n\n_cpu_countB\x0f\n\r_memory_bytesB\r\n\x0b_disk_bytesB\x06\n\x04_gpu"\xbb\x01\n\x0f\x41llowedFunction\x12\x16\n\tnamespace\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ngraph_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x42\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_function_nameB\x10\n\x0e_graph_version"\x85\x02\n\x1b\x46unctionExecutorDescription\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x16\n\timage_uri\x18\x06 \x01(\tH\x05\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x0c\n\n_image_uri"\xc6\x01\n\x15\x46unctionExecutorState\x12M\n\x0b\x64\x65scription\x18\x01 \x01(\x0b\x32\x33.task_scheduler_service.FunctionExecutorDescriptionH\x00\x88\x01\x01\x12\x43\n\x06status\x18\x02 \x01(\x0e\x32..task_scheduler_service.FunctionExecutorStatusH\x01\x88\x01\x01\x42\x0e\n\x0c_descriptionB\t\n\x07_status"\xff\x02\n\rExecutorState\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x44\n\x0f\x65xecutor_status\x18\x02 \x01(\x0e\x32&.task_scheduler_service.ExecutorStatusH\x01\x88\x01\x01\x12\x42\n\x0ehost_resources\x18\x03 \x01(\x0b\x32%.task_scheduler_service.HostResourcesH\x02\x88\x01\x01\x12\x42\n\x11\x61llowed_functions\x18\x04 \x03(\x0b\x32\'.task_scheduler_service.AllowedFunction\x12O\n\x18\x66unction_executor_states\x18\x05 \x03(\x0b\x32-.task_scheduler_service.FunctionExecutorStateB\x0e\n\x0c_executor_idB\x12\n\x10_executor_statusB\x11\n\x0f_host_resources"s\n\x1aReportExecutorStateRequest\x12\x42\n\x0e\x65xecutor_state\x18\x01 \x01(\x0b\x32%.task_scheduler_service.ExecutorStateH\x00\x88\x01\x01\x42\x11\n\x0f_executor_state"\x1d\n\x1bReportExecutorStateResponse"\xe0\x02\n\x04Task\x12\x0f\n\x02id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x16\n\tnamespace\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\ngraph_name\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rgraph_version\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1a\n\rfunction_name\x18\x05 \x01(\tH\x04\x88\x01\x01\x12 \n\x13graph_invocation_id\x18\x06 \x01(\tH\x05\x88\x01\x01\x12\x16\n\tinput_key\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x1f\n\x12reducer_output_key\x18\t \x01(\tH\x07\x88\x01\x01\x42\x05\n\x03_idB\x0c\n\n_namespaceB\r\n\x0b_graph_nameB\x10\n\x0e_graph_versionB\x10\n\x0e_function_nameB\x16\n\x14_graph_invocation_idB\x0c\n\n_input_keyB\x15\n\x13_reducer_output_key"\x86\x01\n\x0eTaskAllocation\x12!\n\x14\x66unction_executor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12/\n\x04task\x18\x02 \x01(\x0b\x32\x1c.task_scheduler_service.TaskH\x01\x88\x01\x01\x42\x17\n\x15_function_executor_idB\x07\n\x05_task"K\n\x1fGetDesiredExecutorStatesRequest\x12\x18\n\x0b\x65xecutor_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x42\x0e\n\x0c_executor_id"\xc7\x01\n\x14\x44\x65siredExecutorState\x12O\n\x12\x66unction_executors\x18\x01 \x03(\x0b\x32\x33.task_scheduler_service.FunctionExecutorDescription\x12@\n\x10task_allocations\x18\x02 \x03(\x0b\x32&.task_scheduler_service.TaskAllocation\x12\x12\n\x05\x63lock\x18\x03 \x01(\x04H\x00\x88\x01\x01\x42\x08\n\x06_clock*\x86\x03\n\x08GPUModel\x12\x15\n\x11GPU_MODEL_UNKNOWN\x10\x00\x12"\n\x1eGPU_MODEL_NVIDIA_TESLA_T4_16GB\x10\n\x12$\n GPU_MODEL_NVIDIA_TESLA_V100_16GB\x10\x14\x12\x1d\n\x19GPU_MODEL_NVIDIA_A10_24GB\x10\x1e\x12\x1f\n\x1bGPU_MODEL_NVIDIA_A6000_48GB\x10(\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_40GB\x10\x32\x12#\n\x1fGPU_MODEL_NVIDIA_A100_SXM4_80GB\x10\x33\x12"\n\x1eGPU_MODEL_NVIDIA_A100_PCI_40GB\x10\x34\x12#\n\x1fGPU_MODEL_NVIDIA_H100_SXM5_80GB\x10<\x12"\n\x1eGPU_MODEL_NVIDIA_H100_PCI_80GB\x10=\x12"\n\x1eGPU_MODEL_NVIDIA_RTX_6000_24GB\x10>*\xa3\x03\n\x16\x46unctionExecutorStatus\x12$\n FUNCTION_EXECUTOR_STATUS_UNKNOWN\x10\x00\x12$\n FUNCTION_EXECUTOR_STATUS_STOPPED\x10\x01\x12(\n$FUNCTION_EXECUTOR_STATUS_STARTING_UP\x10\x02\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_CUSTOMER_ERROR\x10\x03\x12:\n6FUNCTION_EXECUTOR_STATUS_STARTUP_FAILED_PLATFORM_ERROR\x10\x04\x12!\n\x1d\x46UNCTION_EXECUTOR_STATUS_IDLE\x10\x05\x12)\n%FUNCTION_EXECUTOR_STATUS_RUNNING_TASK\x10\x06\x12&\n"FUNCTION_EXECUTOR_STATUS_UNHEALTHY\x10\x07\x12%\n!FUNCTION_EXECUTOR_STATUS_STOPPING\x10\x08*\xa8\x01\n\x0e\x45xecutorStatus\x12\x1b\n\x17\x45XECUTOR_STATUS_UNKNOWN\x10\x00\x12\x1c\n\x18\x45XECUTOR_STATUS_STARTING\x10\x01\x12\x1b\n\x17\x45XECUTOR_STATUS_RUNNING\x10\x02\x12\x1b\n\x17\x45XECUTOR_STATUS_DRAINED\x10\x03\x12!\n\x1d\x45XECUTOR_STATUS_SHUTTING_DOWN\x10\x04\x32\xa6\x02\n\x14TaskSchedulerService\x12\x82\x01\n\x15report_executor_state\x12\x32.task_scheduler_service.ReportExecutorStateRequest\x1a\x33.task_scheduler_service.ReportExecutorStateResponse"\x00\x12\x88\x01\n\x1bget_desired_executor_states\x12\x37.task_scheduler_service.GetDesiredExecutorStatesRequest\x1a,.task_scheduler_service.DesiredExecutorState"\x00\x30\x01\x62\x06proto3'
|
28
|
-
)
|
29
|
-
|
30
|
-
_globals = globals()
|
31
|
-
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
32
|
-
_builder.BuildTopDescriptorsAndMessages(
|
33
|
-
DESCRIPTOR, "indexify.task_scheduler.proto.task_scheduler_pb2", _globals
|
34
|
-
)
|
35
|
-
if not _descriptor._USE_C_DESCRIPTORS:
|
36
|
-
DESCRIPTOR._loaded_options = None
|
37
|
-
_globals["_GPUMODEL"]._serialized_start = 2353
|
38
|
-
_globals["_GPUMODEL"]._serialized_end = 2743
|
39
|
-
_globals["_FUNCTIONEXECUTORSTATUS"]._serialized_start = 2746
|
40
|
-
_globals["_FUNCTIONEXECUTORSTATUS"]._serialized_end = 3165
|
41
|
-
_globals["_EXECUTORSTATUS"]._serialized_start = 3168
|
42
|
-
_globals["_EXECUTORSTATUS"]._serialized_end = 3336
|
43
|
-
_globals["_GPURESOURCES"]._serialized_start = 78
|
44
|
-
_globals["_GPURESOURCES"]._serialized_end = 186
|
45
|
-
_globals["_HOSTRESOURCES"]._serialized_start = 189
|
46
|
-
_globals["_HOSTRESOURCES"]._serialized_end = 390
|
47
|
-
_globals["_ALLOWEDFUNCTION"]._serialized_start = 393
|
48
|
-
_globals["_ALLOWEDFUNCTION"]._serialized_end = 580
|
49
|
-
_globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_start = 583
|
50
|
-
_globals["_FUNCTIONEXECUTORDESCRIPTION"]._serialized_end = 844
|
51
|
-
_globals["_FUNCTIONEXECUTORSTATE"]._serialized_start = 847
|
52
|
-
_globals["_FUNCTIONEXECUTORSTATE"]._serialized_end = 1045
|
53
|
-
_globals["_EXECUTORSTATE"]._serialized_start = 1048
|
54
|
-
_globals["_EXECUTORSTATE"]._serialized_end = 1431
|
55
|
-
_globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_start = 1433
|
56
|
-
_globals["_REPORTEXECUTORSTATEREQUEST"]._serialized_end = 1548
|
57
|
-
_globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_start = 1550
|
58
|
-
_globals["_REPORTEXECUTORSTATERESPONSE"]._serialized_end = 1579
|
59
|
-
_globals["_TASK"]._serialized_start = 1582
|
60
|
-
_globals["_TASK"]._serialized_end = 1934
|
61
|
-
_globals["_TASKALLOCATION"]._serialized_start = 1937
|
62
|
-
_globals["_TASKALLOCATION"]._serialized_end = 2071
|
63
|
-
_globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_start = 2073
|
64
|
-
_globals["_GETDESIREDEXECUTORSTATESREQUEST"]._serialized_end = 2148
|
65
|
-
_globals["_DESIREDEXECUTORSTATE"]._serialized_start = 2151
|
66
|
-
_globals["_DESIREDEXECUTORSTATE"]._serialized_end = 2350
|
67
|
-
_globals["_TASKSCHEDULERSERVICE"]._serialized_start = 3339
|
68
|
-
_globals["_TASKSCHEDULERSERVICE"]._serialized_end = 3633
|
69
|
-
# @@protoc_insertion_point(module_scope)
|
File without changes
|
File without changes
|