indexify 0.3.31__py3-none-any.whl → 0.4.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. indexify/cli/__init__.py +18 -0
  2. indexify/cli/build_image.py +51 -0
  3. indexify/cli/deploy.py +57 -0
  4. indexify/cli/executor.py +205 -0
  5. indexify/executor/{grpc/channel_manager.py → channel_manager.py} +17 -11
  6. indexify/executor/executor.py +57 -313
  7. indexify/executor/function_allowlist.py +59 -0
  8. indexify/executor/function_executor/function_executor.py +12 -6
  9. indexify/executor/function_executor/invocation_state_client.py +25 -3
  10. indexify/executor/function_executor/server/function_executor_server_factory.py +3 -3
  11. indexify/executor/function_executor/server/subprocess_function_executor_server_factory.py +22 -11
  12. indexify/executor/function_executor_controller/__init__.py +13 -0
  13. indexify/executor/function_executor_controller/completed_task_metrics.py +82 -0
  14. indexify/executor/function_executor_controller/create_function_executor.py +154 -0
  15. indexify/executor/function_executor_controller/debug_event_loop.py +37 -0
  16. indexify/executor/function_executor_controller/destroy_function_executor.py +28 -0
  17. indexify/executor/function_executor_controller/downloads.py +199 -0
  18. indexify/executor/function_executor_controller/events.py +172 -0
  19. indexify/executor/function_executor_controller/function_executor_controller.py +759 -0
  20. indexify/executor/function_executor_controller/loggers.py +57 -0
  21. indexify/executor/function_executor_controller/message_validators.py +65 -0
  22. indexify/executor/function_executor_controller/metrics/completed_task_metrics.py +68 -0
  23. indexify/executor/{metrics/downloader.py → function_executor_controller/metrics/downloads.py} +1 -3
  24. indexify/executor/function_executor_controller/metrics/function_executor_controller.py +60 -0
  25. indexify/executor/{function_executor/metrics/single_task_runner.py → function_executor_controller/metrics/run_task.py} +9 -3
  26. indexify/executor/function_executor_controller/metrics/upload_task_output.py +39 -0
  27. indexify/executor/function_executor_controller/prepare_task.py +38 -0
  28. indexify/executor/function_executor_controller/run_task.py +201 -0
  29. indexify/executor/function_executor_controller/task_info.py +33 -0
  30. indexify/executor/function_executor_controller/task_output.py +122 -0
  31. indexify/executor/function_executor_controller/upload_task_output.py +234 -0
  32. indexify/executor/host_resources/host_resources.py +20 -25
  33. indexify/executor/{grpc/metrics → metrics}/channel_manager.py +1 -1
  34. indexify/executor/metrics/executor.py +0 -47
  35. indexify/executor/{grpc/metrics → metrics}/state_reconciler.py +1 -1
  36. indexify/executor/{grpc/metrics → metrics}/state_reporter.py +1 -1
  37. indexify/executor/monitoring/health_checker/generic_health_checker.py +6 -59
  38. indexify/executor/monitoring/health_checker/health_checker.py +0 -11
  39. indexify/executor/{grpc/state_reconciler.py → state_reconciler.py} +139 -141
  40. indexify/executor/state_reporter.py +364 -0
  41. indexify/proto/executor_api.proto +67 -59
  42. indexify/proto/executor_api_pb2.py +52 -52
  43. indexify/proto/executor_api_pb2.pyi +125 -104
  44. indexify/proto/executor_api_pb2_grpc.py +0 -47
  45. {indexify-0.3.31.dist-info → indexify-0.4.2.dist-info}/METADATA +1 -3
  46. indexify-0.4.2.dist-info/RECORD +68 -0
  47. indexify-0.4.2.dist-info/entry_points.txt +3 -0
  48. indexify/cli/cli.py +0 -268
  49. indexify/executor/api_objects.py +0 -92
  50. indexify/executor/downloader.py +0 -417
  51. indexify/executor/executor_flavor.py +0 -7
  52. indexify/executor/function_executor/function_executor_state.py +0 -107
  53. indexify/executor/function_executor/function_executor_states_container.py +0 -93
  54. indexify/executor/function_executor/function_executor_status.py +0 -95
  55. indexify/executor/function_executor/metrics/function_executor_state.py +0 -46
  56. indexify/executor/function_executor/metrics/function_executor_state_container.py +0 -10
  57. indexify/executor/function_executor/single_task_runner.py +0 -345
  58. indexify/executor/function_executor/task_input.py +0 -21
  59. indexify/executor/function_executor/task_output.py +0 -105
  60. indexify/executor/grpc/function_executor_controller.py +0 -418
  61. indexify/executor/grpc/metrics/task_controller.py +0 -8
  62. indexify/executor/grpc/state_reporter.py +0 -317
  63. indexify/executor/grpc/task_controller.py +0 -508
  64. indexify/executor/metrics/task_fetcher.py +0 -21
  65. indexify/executor/metrics/task_reporter.py +0 -53
  66. indexify/executor/metrics/task_runner.py +0 -52
  67. indexify/executor/monitoring/function_allowlist.py +0 -25
  68. indexify/executor/runtime_probes.py +0 -68
  69. indexify/executor/task_fetcher.py +0 -96
  70. indexify/executor/task_reporter.py +0 -459
  71. indexify/executor/task_runner.py +0 -177
  72. indexify-0.3.31.dist-info/RECORD +0 -68
  73. indexify-0.3.31.dist-info/entry_points.txt +0 -3
  74. {indexify-0.3.31.dist-info → indexify-0.4.2.dist-info}/WHEEL +0 -0
@@ -0,0 +1,172 @@
1
+ from enum import Enum
2
+ from typing import Optional
3
+
4
+ from indexify.executor.function_executor.function_executor import (
5
+ FunctionError,
6
+ FunctionExecutor,
7
+ )
8
+ from indexify.proto.executor_api_pb2 import FunctionExecutorTerminationReason
9
+
10
+ from .task_info import TaskInfo
11
+
12
+
13
+ class EventType(Enum):
14
+ FUNCTION_EXECUTOR_CREATED = 1
15
+ FUNCTION_EXECUTOR_DESTROYED = 2
16
+ SHUTDOWN_INITIATED = 3
17
+ TASK_PREPARATION_FINISHED = 4
18
+ SCHEDULE_TASK_EXECUTION = 5
19
+ TASK_EXECUTION_FINISHED = 6
20
+ TASK_OUTPUT_UPLOAD_FINISHED = 7
21
+
22
+
23
+ class BaseEvent:
24
+ """
25
+ Base class for events in the FunctionExecutorController.
26
+ This class can be extended to create specific event types.
27
+ """
28
+
29
+ def __init__(self, event_type: EventType):
30
+ self.event_type = event_type
31
+
32
+ def __str__(self) -> str:
33
+ return f"Event(type={self.event_type.name})"
34
+
35
+
36
+ class FunctionExecutorCreated(BaseEvent):
37
+ """
38
+ Event indicating that Function Executor got created or failed.
39
+
40
+ If the error is CustomerError, it indicates an error in customer code.
41
+ The function_executor field is None if any errors happened.
42
+ """
43
+
44
+ def __init__(
45
+ self,
46
+ function_executor: Optional[FunctionExecutor] = None,
47
+ function_error: Optional[FunctionError] = None,
48
+ termination_reason: FunctionExecutorTerminationReason = None, # type: Optional[FunctionExecutorTerminationReason]
49
+ ):
50
+ super().__init__(EventType.FUNCTION_EXECUTOR_CREATED)
51
+ self.function_executor: Optional[FunctionExecutor] = function_executor
52
+ self.function_error: Optional[FunctionError] = function_error
53
+ # Reason for FE termination if failed to create FE (.function_executor is None).
54
+ self.termination_reason = termination_reason
55
+
56
+
57
+ class FunctionExecutorDestroyed(BaseEvent):
58
+ """
59
+ Event indicating that Function Executor has been destroyed.
60
+ """
61
+
62
+ def __init__(
63
+ self, is_success: bool, termination_reason: FunctionExecutorTerminationReason
64
+ ):
65
+ super().__init__(EventType.FUNCTION_EXECUTOR_DESTROYED)
66
+ self.is_success: bool = is_success
67
+ self.termination_reason: FunctionExecutorTerminationReason = termination_reason
68
+
69
+ def __str__(self) -> str:
70
+ return (
71
+ f"Event(type={self.event_type.name}, "
72
+ f"is_success={self.is_success}, "
73
+ f"termination_reason={FunctionExecutorTerminationReason.Name(self.termination_reason)})"
74
+ )
75
+
76
+
77
+ class ShutdownInitiated(BaseEvent):
78
+ """
79
+ Event indicating that Function Executor shutdown has been initiated.
80
+ """
81
+
82
+ def __init__(self, termination_reason: FunctionExecutorTerminationReason):
83
+ super().__init__(EventType.SHUTDOWN_INITIATED)
84
+ self.termination_reason: FunctionExecutorTerminationReason = termination_reason
85
+
86
+ def __str__(self) -> str:
87
+ return (
88
+ f"Event(type={self.event_type.name}, "
89
+ f"termination_reason={FunctionExecutorTerminationReason.Name(self.termination_reason)})"
90
+ )
91
+
92
+
93
+ class TaskPreparationFinished(BaseEvent):
94
+ """
95
+ Event indicating that a task has been prepared for execution or failed to do that.
96
+ """
97
+
98
+ def __init__(
99
+ self,
100
+ task_info: TaskInfo,
101
+ is_success: bool,
102
+ ):
103
+ super().__init__(EventType.TASK_PREPARATION_FINISHED)
104
+ self.task_info: TaskInfo = task_info
105
+ self.is_success: bool = is_success
106
+
107
+ def __str__(self) -> str:
108
+ return (
109
+ f"Event(type={self.event_type.name}, "
110
+ f"task_id={self.task_info.task.id}, "
111
+ f"allocation_id={self.task_info.allocation_id}), "
112
+ f"is_success={self.is_success}"
113
+ )
114
+
115
+
116
+ class ScheduleTaskExecution(BaseEvent):
117
+ """
118
+ Event indicating that a task execution has been scheduled.
119
+ """
120
+
121
+ def __init__(self):
122
+ super().__init__(EventType.SCHEDULE_TASK_EXECUTION)
123
+
124
+
125
+ class TaskExecutionFinished(BaseEvent):
126
+ """
127
+ Event indicating that a task execution has been finished on Function Executor.
128
+ """
129
+
130
+ def __init__(
131
+ self,
132
+ task_info: TaskInfo,
133
+ function_executor_termination_reason: FunctionExecutorTerminationReason, # type: Optional[FunctionExecutorTerminationReason]
134
+ ):
135
+ super().__init__(EventType.TASK_EXECUTION_FINISHED)
136
+ self.task_info: TaskInfo = task_info
137
+ # Not None if the FE needs to get destroyed after running the task.
138
+ self.function_executor_termination_reason = function_executor_termination_reason
139
+
140
+ def __str__(self) -> str:
141
+ function_executor_termination_reason_str: str = (
142
+ "None"
143
+ if self.function_executor_termination_reason is None
144
+ else FunctionExecutorTerminationReason.Name(
145
+ self.function_executor_termination_reason
146
+ )
147
+ )
148
+ return (
149
+ f"Event(type={self.event_type.name}, "
150
+ f"task_id={self.task_info.task.id}, "
151
+ f"allocation_id={self.task_info.allocation_id}), "
152
+ f"function_executor_termination_reason={function_executor_termination_reason_str}"
153
+ )
154
+
155
+
156
+ class TaskOutputUploadFinished(BaseEvent):
157
+ """
158
+ Event indicating that a task output has been uploaded.
159
+ """
160
+
161
+ def __init__(self, task_info: TaskInfo, is_success: bool):
162
+ super().__init__(EventType.TASK_OUTPUT_UPLOAD_FINISHED)
163
+ self.task_info: TaskInfo = task_info
164
+ self.is_success: bool = is_success
165
+
166
+ def __str__(self) -> str:
167
+ return (
168
+ f"Event(type={self.event_type.name}, "
169
+ f"task_id={self.task_info.task.id}, "
170
+ f"allocation_id={self.task_info.allocation_id}), "
171
+ f"is_success={self.is_success}"
172
+ )