indexify 0.3.31__py3-none-any.whl → 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- indexify/cli/__init__.py +18 -0
- indexify/cli/build_image.py +51 -0
- indexify/cli/deploy.py +57 -0
- indexify/cli/executor.py +205 -0
- indexify/executor/{grpc/channel_manager.py → channel_manager.py} +17 -11
- indexify/executor/executor.py +57 -313
- indexify/executor/function_allowlist.py +59 -0
- indexify/executor/function_executor/function_executor.py +12 -6
- indexify/executor/function_executor/invocation_state_client.py +25 -3
- indexify/executor/function_executor/server/function_executor_server_factory.py +3 -3
- indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py +22 -11
- indexify/executor/function_executor_controller/__init__.py +13 -0
- indexify/executor/function_executor_controller/completed_task_metrics.py +82 -0
- indexify/executor/function_executor_controller/create_function_executor.py +154 -0
- indexify/executor/function_executor_controller/debug_event_loop.py +37 -0
- indexify/executor/function_executor_controller/destroy_function_executor.py +28 -0
- indexify/executor/function_executor_controller/downloads.py +199 -0
- indexify/executor/function_executor_controller/events.py +172 -0
- indexify/executor/function_executor_controller/function_executor_controller.py +759 -0
- indexify/executor/function_executor_controller/loggers.py +57 -0
- indexify/executor/function_executor_controller/message_validators.py +65 -0
- indexify/executor/function_executor_controller/metrics/completed_task_metrics.py +68 -0
- indexify/executor/{metrics/downloader.py → function_executor_controller/metrics/downloads.py} +1 -3
- indexify/executor/function_executor_controller/metrics/function_executor_controller.py +60 -0
- indexify/executor/{function_executor/metrics/single_task_runner.py → function_executor_controller/metrics/run_task.py} +9 -3
- indexify/executor/function_executor_controller/metrics/upload_task_output.py +39 -0
- indexify/executor/function_executor_controller/prepare_task.py +38 -0
- indexify/executor/function_executor_controller/run_task.py +201 -0
- indexify/executor/function_executor_controller/task_info.py +33 -0
- indexify/executor/function_executor_controller/task_output.py +122 -0
- indexify/executor/function_executor_controller/upload_task_output.py +234 -0
- indexify/executor/host_resources/host_resources.py +20 -25
- indexify/executor/{grpc/metrics → metrics}/channel_manager.py +1 -1
- indexify/executor/metrics/executor.py +0 -47
- indexify/executor/{grpc/metrics → metrics}/state_reconciler.py +1 -1
- indexify/executor/{grpc/metrics → metrics}/state_reporter.py +1 -1
- indexify/executor/monitoring/health_checker/generic_health_checker.py +6 -59
- indexify/executor/monitoring/health_checker/health_checker.py +0 -11
- indexify/executor/{grpc/state_reconciler.py → state_reconciler.py} +139 -141
- indexify/executor/state_reporter.py +364 -0
- indexify/proto/executor_api.proto +67 -59
- indexify/proto/executor_api_pb2.py +52 -52
- indexify/proto/executor_api_pb2.pyi +125 -104
- indexify/proto/executor_api_pb2_grpc.py +0 -47
- {indexify-0.3.31.dist-info → indexify-0.4.2.dist-info}/METADATA +1 -3
- indexify-0.4.2.dist-info/RECORD +68 -0
- indexify-0.4.2.dist-info/entry_points.txt +3 -0
- indexify/cli/cli.py +0 -268
- indexify/executor/api_objects.py +0 -92
- indexify/executor/downloader.py +0 -417
- indexify/executor/executor_flavor.py +0 -7
- indexify/executor/function_executor/function_executor_state.py +0 -107
- indexify/executor/function_executor/function_executor_states_container.py +0 -93
- indexify/executor/function_executor/function_executor_status.py +0 -95
- indexify/executor/function_executor/metrics/function_executor_state.py +0 -46
- indexify/executor/function_executor/metrics/function_executor_state_container.py +0 -10
- indexify/executor/function_executor/single_task_runner.py +0 -345
- indexify/executor/function_executor/task_input.py +0 -21
- indexify/executor/function_executor/task_output.py +0 -105
- indexify/executor/grpc/function_executor_controller.py +0 -418
- indexify/executor/grpc/metrics/task_controller.py +0 -8
- indexify/executor/grpc/state_reporter.py +0 -317
- indexify/executor/grpc/task_controller.py +0 -508
- indexify/executor/metrics/task_fetcher.py +0 -21
- indexify/executor/metrics/task_reporter.py +0 -53
- indexify/executor/metrics/task_runner.py +0 -52
- indexify/executor/monitoring/function_allowlist.py +0 -25
- indexify/executor/runtime_probes.py +0 -68
- indexify/executor/task_fetcher.py +0 -96
- indexify/executor/task_reporter.py +0 -459
- indexify/executor/task_runner.py +0 -177
- indexify-0.3.31.dist-info/RECORD +0 -68
- indexify-0.3.31.dist-info/entry_points.txt +0 -3
- {indexify-0.3.31.dist-info → indexify-0.4.2.dist-info}/WHEEL +0 -0
@@ -0,0 +1,172 @@
|
|
1
|
+
from enum import Enum
|
2
|
+
from typing import Optional
|
3
|
+
|
4
|
+
from indexify.executor.function_executor.function_executor import (
|
5
|
+
FunctionError,
|
6
|
+
FunctionExecutor,
|
7
|
+
)
|
8
|
+
from indexify.proto.executor_api_pb2 import FunctionExecutorTerminationReason
|
9
|
+
|
10
|
+
from .task_info import TaskInfo
|
11
|
+
|
12
|
+
|
13
|
+
class EventType(Enum):
|
14
|
+
FUNCTION_EXECUTOR_CREATED = 1
|
15
|
+
FUNCTION_EXECUTOR_DESTROYED = 2
|
16
|
+
SHUTDOWN_INITIATED = 3
|
17
|
+
TASK_PREPARATION_FINISHED = 4
|
18
|
+
SCHEDULE_TASK_EXECUTION = 5
|
19
|
+
TASK_EXECUTION_FINISHED = 6
|
20
|
+
TASK_OUTPUT_UPLOAD_FINISHED = 7
|
21
|
+
|
22
|
+
|
23
|
+
class BaseEvent:
|
24
|
+
"""
|
25
|
+
Base class for events in the FunctionExecutorController.
|
26
|
+
This class can be extended to create specific event types.
|
27
|
+
"""
|
28
|
+
|
29
|
+
def __init__(self, event_type: EventType):
|
30
|
+
self.event_type = event_type
|
31
|
+
|
32
|
+
def __str__(self) -> str:
|
33
|
+
return f"Event(type={self.event_type.name})"
|
34
|
+
|
35
|
+
|
36
|
+
class FunctionExecutorCreated(BaseEvent):
|
37
|
+
"""
|
38
|
+
Event indicating that Function Executor got created or failed.
|
39
|
+
|
40
|
+
If the error is CustomerError, it indicates an error in customer code.
|
41
|
+
The function_executor field is None if any errors happened.
|
42
|
+
"""
|
43
|
+
|
44
|
+
def __init__(
|
45
|
+
self,
|
46
|
+
function_executor: Optional[FunctionExecutor] = None,
|
47
|
+
function_error: Optional[FunctionError] = None,
|
48
|
+
termination_reason: FunctionExecutorTerminationReason = None, # type: Optional[FunctionExecutorTerminationReason]
|
49
|
+
):
|
50
|
+
super().__init__(EventType.FUNCTION_EXECUTOR_CREATED)
|
51
|
+
self.function_executor: Optional[FunctionExecutor] = function_executor
|
52
|
+
self.function_error: Optional[FunctionError] = function_error
|
53
|
+
# Reason for FE termination if failed to create FE (.function_executor is None).
|
54
|
+
self.termination_reason = termination_reason
|
55
|
+
|
56
|
+
|
57
|
+
class FunctionExecutorDestroyed(BaseEvent):
|
58
|
+
"""
|
59
|
+
Event indicating that Function Executor has been destroyed.
|
60
|
+
"""
|
61
|
+
|
62
|
+
def __init__(
|
63
|
+
self, is_success: bool, termination_reason: FunctionExecutorTerminationReason
|
64
|
+
):
|
65
|
+
super().__init__(EventType.FUNCTION_EXECUTOR_DESTROYED)
|
66
|
+
self.is_success: bool = is_success
|
67
|
+
self.termination_reason: FunctionExecutorTerminationReason = termination_reason
|
68
|
+
|
69
|
+
def __str__(self) -> str:
|
70
|
+
return (
|
71
|
+
f"Event(type={self.event_type.name}, "
|
72
|
+
f"is_success={self.is_success}, "
|
73
|
+
f"termination_reason={FunctionExecutorTerminationReason.Name(self.termination_reason)})"
|
74
|
+
)
|
75
|
+
|
76
|
+
|
77
|
+
class ShutdownInitiated(BaseEvent):
|
78
|
+
"""
|
79
|
+
Event indicating that Function Executor shutdown has been initiated.
|
80
|
+
"""
|
81
|
+
|
82
|
+
def __init__(self, termination_reason: FunctionExecutorTerminationReason):
|
83
|
+
super().__init__(EventType.SHUTDOWN_INITIATED)
|
84
|
+
self.termination_reason: FunctionExecutorTerminationReason = termination_reason
|
85
|
+
|
86
|
+
def __str__(self) -> str:
|
87
|
+
return (
|
88
|
+
f"Event(type={self.event_type.name}, "
|
89
|
+
f"termination_reason={FunctionExecutorTerminationReason.Name(self.termination_reason)})"
|
90
|
+
)
|
91
|
+
|
92
|
+
|
93
|
+
class TaskPreparationFinished(BaseEvent):
|
94
|
+
"""
|
95
|
+
Event indicating that a task has been prepared for execution or failed to do that.
|
96
|
+
"""
|
97
|
+
|
98
|
+
def __init__(
|
99
|
+
self,
|
100
|
+
task_info: TaskInfo,
|
101
|
+
is_success: bool,
|
102
|
+
):
|
103
|
+
super().__init__(EventType.TASK_PREPARATION_FINISHED)
|
104
|
+
self.task_info: TaskInfo = task_info
|
105
|
+
self.is_success: bool = is_success
|
106
|
+
|
107
|
+
def __str__(self) -> str:
|
108
|
+
return (
|
109
|
+
f"Event(type={self.event_type.name}, "
|
110
|
+
f"task_id={self.task_info.task.id}, "
|
111
|
+
f"allocation_id={self.task_info.allocation_id}), "
|
112
|
+
f"is_success={self.is_success}"
|
113
|
+
)
|
114
|
+
|
115
|
+
|
116
|
+
class ScheduleTaskExecution(BaseEvent):
|
117
|
+
"""
|
118
|
+
Event indicating that a task execution has been scheduled.
|
119
|
+
"""
|
120
|
+
|
121
|
+
def __init__(self):
|
122
|
+
super().__init__(EventType.SCHEDULE_TASK_EXECUTION)
|
123
|
+
|
124
|
+
|
125
|
+
class TaskExecutionFinished(BaseEvent):
|
126
|
+
"""
|
127
|
+
Event indicating that a task execution has been finished on Function Executor.
|
128
|
+
"""
|
129
|
+
|
130
|
+
def __init__(
|
131
|
+
self,
|
132
|
+
task_info: TaskInfo,
|
133
|
+
function_executor_termination_reason: FunctionExecutorTerminationReason, # type: Optional[FunctionExecutorTerminationReason]
|
134
|
+
):
|
135
|
+
super().__init__(EventType.TASK_EXECUTION_FINISHED)
|
136
|
+
self.task_info: TaskInfo = task_info
|
137
|
+
# Not None if the FE needs to get destroyed after running the task.
|
138
|
+
self.function_executor_termination_reason = function_executor_termination_reason
|
139
|
+
|
140
|
+
def __str__(self) -> str:
|
141
|
+
function_executor_termination_reason_str: str = (
|
142
|
+
"None"
|
143
|
+
if self.function_executor_termination_reason is None
|
144
|
+
else FunctionExecutorTerminationReason.Name(
|
145
|
+
self.function_executor_termination_reason
|
146
|
+
)
|
147
|
+
)
|
148
|
+
return (
|
149
|
+
f"Event(type={self.event_type.name}, "
|
150
|
+
f"task_id={self.task_info.task.id}, "
|
151
|
+
f"allocation_id={self.task_info.allocation_id}), "
|
152
|
+
f"function_executor_termination_reason={function_executor_termination_reason_str}"
|
153
|
+
)
|
154
|
+
|
155
|
+
|
156
|
+
class TaskOutputUploadFinished(BaseEvent):
|
157
|
+
"""
|
158
|
+
Event indicating that a task output has been uploaded.
|
159
|
+
"""
|
160
|
+
|
161
|
+
def __init__(self, task_info: TaskInfo, is_success: bool):
|
162
|
+
super().__init__(EventType.TASK_OUTPUT_UPLOAD_FINISHED)
|
163
|
+
self.task_info: TaskInfo = task_info
|
164
|
+
self.is_success: bool = is_success
|
165
|
+
|
166
|
+
def __str__(self) -> str:
|
167
|
+
return (
|
168
|
+
f"Event(type={self.event_type.name}, "
|
169
|
+
f"task_id={self.task_info.task.id}, "
|
170
|
+
f"allocation_id={self.task_info.allocation_id}), "
|
171
|
+
f"is_success={self.is_success}"
|
172
|
+
)
|