ul-api-utils 9.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. example/__init__.py +0 -0
  2. example/conf.py +35 -0
  3. example/main.py +24 -0
  4. example/models/__init__.py +0 -0
  5. example/permissions.py +6 -0
  6. example/pure_flask_example.py +65 -0
  7. example/rate_limit_load.py +10 -0
  8. example/redis_repository.py +22 -0
  9. example/routes/__init__.py +0 -0
  10. example/routes/api_some.py +335 -0
  11. example/sockets/__init__.py +0 -0
  12. example/sockets/on_connect.py +16 -0
  13. example/sockets/on_disconnect.py +14 -0
  14. example/sockets/on_json.py +10 -0
  15. example/sockets/on_message.py +13 -0
  16. example/sockets/on_open.py +16 -0
  17. example/workers/__init__.py +0 -0
  18. example/workers/worker.py +28 -0
  19. ul_api_utils/__init__.py +0 -0
  20. ul_api_utils/access/__init__.py +122 -0
  21. ul_api_utils/api_resource/__init__.py +0 -0
  22. ul_api_utils/api_resource/api_request.py +105 -0
  23. ul_api_utils/api_resource/api_resource.py +414 -0
  24. ul_api_utils/api_resource/api_resource_config.py +20 -0
  25. ul_api_utils/api_resource/api_resource_error_handling.py +21 -0
  26. ul_api_utils/api_resource/api_resource_fn_typing.py +356 -0
  27. ul_api_utils/api_resource/api_resource_type.py +16 -0
  28. ul_api_utils/api_resource/api_response.py +300 -0
  29. ul_api_utils/api_resource/api_response_db.py +26 -0
  30. ul_api_utils/api_resource/api_response_payload_alias.py +25 -0
  31. ul_api_utils/api_resource/db_types.py +9 -0
  32. ul_api_utils/api_resource/signature_check.py +41 -0
  33. ul_api_utils/commands/__init__.py +0 -0
  34. ul_api_utils/commands/cmd_enc_keys.py +172 -0
  35. ul_api_utils/commands/cmd_gen_api_user_token.py +77 -0
  36. ul_api_utils/commands/cmd_gen_new_api_user.py +106 -0
  37. ul_api_utils/commands/cmd_generate_api_docs.py +181 -0
  38. ul_api_utils/commands/cmd_start.py +110 -0
  39. ul_api_utils/commands/cmd_worker_start.py +76 -0
  40. ul_api_utils/commands/start/__init__.py +0 -0
  41. ul_api_utils/commands/start/gunicorn.conf.local.py +0 -0
  42. ul_api_utils/commands/start/gunicorn.conf.py +26 -0
  43. ul_api_utils/commands/start/wsgi.py +22 -0
  44. ul_api_utils/conf/ul-debugger-main.js +1 -0
  45. ul_api_utils/conf/ul-debugger-ui.js +1 -0
  46. ul_api_utils/conf.py +70 -0
  47. ul_api_utils/const.py +78 -0
  48. ul_api_utils/debug/__init__.py +0 -0
  49. ul_api_utils/debug/debugger.py +119 -0
  50. ul_api_utils/debug/malloc.py +93 -0
  51. ul_api_utils/debug/stat.py +444 -0
  52. ul_api_utils/encrypt/__init__.py +0 -0
  53. ul_api_utils/encrypt/encrypt_decrypt_abstract.py +15 -0
  54. ul_api_utils/encrypt/encrypt_decrypt_aes_xtea.py +59 -0
  55. ul_api_utils/errors.py +200 -0
  56. ul_api_utils/internal_api/__init__.py +0 -0
  57. ul_api_utils/internal_api/__tests__/__init__.py +0 -0
  58. ul_api_utils/internal_api/__tests__/internal_api.py +29 -0
  59. ul_api_utils/internal_api/__tests__/internal_api_content_type.py +22 -0
  60. ul_api_utils/internal_api/internal_api.py +369 -0
  61. ul_api_utils/internal_api/internal_api_check_context.py +42 -0
  62. ul_api_utils/internal_api/internal_api_error.py +17 -0
  63. ul_api_utils/internal_api/internal_api_response.py +296 -0
  64. ul_api_utils/main.py +29 -0
  65. ul_api_utils/modules/__init__.py +0 -0
  66. ul_api_utils/modules/__tests__/__init__.py +0 -0
  67. ul_api_utils/modules/__tests__/test_api_sdk_jwt.py +195 -0
  68. ul_api_utils/modules/api_sdk.py +555 -0
  69. ul_api_utils/modules/api_sdk_config.py +63 -0
  70. ul_api_utils/modules/api_sdk_jwt.py +377 -0
  71. ul_api_utils/modules/intermediate_state.py +34 -0
  72. ul_api_utils/modules/worker_context.py +35 -0
  73. ul_api_utils/modules/worker_sdk.py +109 -0
  74. ul_api_utils/modules/worker_sdk_config.py +13 -0
  75. ul_api_utils/py.typed +0 -0
  76. ul_api_utils/resources/__init__.py +0 -0
  77. ul_api_utils/resources/caching.py +196 -0
  78. ul_api_utils/resources/debugger_scripts.py +97 -0
  79. ul_api_utils/resources/health_check/__init__.py +0 -0
  80. ul_api_utils/resources/health_check/const.py +2 -0
  81. ul_api_utils/resources/health_check/health_check.py +439 -0
  82. ul_api_utils/resources/health_check/health_check_template.py +64 -0
  83. ul_api_utils/resources/health_check/resource.py +97 -0
  84. ul_api_utils/resources/not_implemented.py +25 -0
  85. ul_api_utils/resources/permissions.py +29 -0
  86. ul_api_utils/resources/rate_limitter.py +84 -0
  87. ul_api_utils/resources/socketio.py +55 -0
  88. ul_api_utils/resources/swagger.py +119 -0
  89. ul_api_utils/resources/web_forms/__init__.py +0 -0
  90. ul_api_utils/resources/web_forms/custom_fields/__init__.py +0 -0
  91. ul_api_utils/resources/web_forms/custom_fields/custom_checkbox_select.py +5 -0
  92. ul_api_utils/resources/web_forms/custom_widgets/__init__.py +0 -0
  93. ul_api_utils/resources/web_forms/custom_widgets/custom_select_widget.py +86 -0
  94. ul_api_utils/resources/web_forms/custom_widgets/custom_text_input_widget.py +42 -0
  95. ul_api_utils/resources/web_forms/uni_form.py +75 -0
  96. ul_api_utils/sentry.py +52 -0
  97. ul_api_utils/utils/__init__.py +0 -0
  98. ul_api_utils/utils/__tests__/__init__.py +0 -0
  99. ul_api_utils/utils/__tests__/api_path_version.py +16 -0
  100. ul_api_utils/utils/__tests__/unwrap_typing.py +67 -0
  101. ul_api_utils/utils/api_encoding.py +51 -0
  102. ul_api_utils/utils/api_format.py +61 -0
  103. ul_api_utils/utils/api_method.py +55 -0
  104. ul_api_utils/utils/api_pagination.py +58 -0
  105. ul_api_utils/utils/api_path_version.py +60 -0
  106. ul_api_utils/utils/api_request_info.py +6 -0
  107. ul_api_utils/utils/avro.py +131 -0
  108. ul_api_utils/utils/broker_topics_message_count.py +47 -0
  109. ul_api_utils/utils/cached_per_request.py +23 -0
  110. ul_api_utils/utils/colors.py +31 -0
  111. ul_api_utils/utils/constants.py +7 -0
  112. ul_api_utils/utils/decode_base64.py +9 -0
  113. ul_api_utils/utils/deprecated.py +19 -0
  114. ul_api_utils/utils/flags.py +29 -0
  115. ul_api_utils/utils/flask_swagger_generator/__init__.py +0 -0
  116. ul_api_utils/utils/flask_swagger_generator/conf.py +4 -0
  117. ul_api_utils/utils/flask_swagger_generator/exceptions.py +7 -0
  118. ul_api_utils/utils/flask_swagger_generator/specifiers/__init__.py +0 -0
  119. ul_api_utils/utils/flask_swagger_generator/specifiers/swagger_models.py +57 -0
  120. ul_api_utils/utils/flask_swagger_generator/specifiers/swagger_specifier.py +48 -0
  121. ul_api_utils/utils/flask_swagger_generator/specifiers/swagger_three_specifier.py +777 -0
  122. ul_api_utils/utils/flask_swagger_generator/specifiers/swagger_version.py +40 -0
  123. ul_api_utils/utils/flask_swagger_generator/utils/__init__.py +0 -0
  124. ul_api_utils/utils/flask_swagger_generator/utils/input_type.py +77 -0
  125. ul_api_utils/utils/flask_swagger_generator/utils/parameter_type.py +51 -0
  126. ul_api_utils/utils/flask_swagger_generator/utils/replace_in_dict.py +18 -0
  127. ul_api_utils/utils/flask_swagger_generator/utils/request_type.py +52 -0
  128. ul_api_utils/utils/flask_swagger_generator/utils/schema_type.py +15 -0
  129. ul_api_utils/utils/flask_swagger_generator/utils/security_type.py +39 -0
  130. ul_api_utils/utils/imports.py +16 -0
  131. ul_api_utils/utils/instance_checks.py +16 -0
  132. ul_api_utils/utils/jinja/__init__.py +0 -0
  133. ul_api_utils/utils/jinja/t_url_for.py +19 -0
  134. ul_api_utils/utils/jinja/to_pretty_json.py +11 -0
  135. ul_api_utils/utils/json_encoder.py +126 -0
  136. ul_api_utils/utils/load_modules.py +15 -0
  137. ul_api_utils/utils/memory_db/__init__.py +0 -0
  138. ul_api_utils/utils/memory_db/__tests__/__init__.py +0 -0
  139. ul_api_utils/utils/memory_db/errors.py +8 -0
  140. ul_api_utils/utils/memory_db/repository.py +102 -0
  141. ul_api_utils/utils/token_check.py +14 -0
  142. ul_api_utils/utils/token_check_through_request.py +16 -0
  143. ul_api_utils/utils/unwrap_typing.py +117 -0
  144. ul_api_utils/utils/uuid_converter.py +22 -0
  145. ul_api_utils/validators/__init__.py +0 -0
  146. ul_api_utils/validators/__tests__/__init__.py +0 -0
  147. ul_api_utils/validators/__tests__/test_custom_fields.py +32 -0
  148. ul_api_utils/validators/custom_fields.py +66 -0
  149. ul_api_utils/validators/validate_empty_object.py +10 -0
  150. ul_api_utils/validators/validate_uuid.py +11 -0
  151. ul_api_utils-9.3.0.dist-info/LICENSE +21 -0
  152. ul_api_utils-9.3.0.dist-info/METADATA +279 -0
  153. ul_api_utils-9.3.0.dist-info/RECORD +156 -0
  154. ul_api_utils-9.3.0.dist-info/WHEEL +5 -0
  155. ul_api_utils-9.3.0.dist-info/entry_points.txt +2 -0
  156. ul_api_utils-9.3.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,439 @@
1
+ import time
2
+ from enum import Enum, IntEnum
3
+ from typing import List, Callable, Any, Optional, Dict, TYPE_CHECKING, Tuple, NamedTuple, Set, Union
4
+
5
+ from pydantic import model_validator, ConfigDict, Field, BaseModel, PositiveInt, TypeAdapter
6
+ from sqlalchemy.sql import text
7
+ from ul_unipipeline.errors import UniError
8
+ from ul_unipipeline.modules.uni import Uni
9
+
10
+ from ul_api_utils.api_resource.api_response import JsonApiResponsePayload
11
+ from ul_api_utils.errors import Client4XXInternalApiError, Server5XXInternalApiError
12
+ from ul_api_utils.internal_api.internal_api import InternalApi
13
+ from ul_api_utils.internal_api.internal_api_response import InternalApiResponse
14
+ from ul_api_utils.resources.health_check.const import INTERNAL_API_HEALTH_CHECK_PATH, SERVICE_NAME_DELIMITER
15
+
16
+ if TYPE_CHECKING:
17
+ import flask_sqlalchemy
18
+ from redis import Redis
19
+
20
+
21
+ class HealthCheckResultStatus(IntEnum):
22
+ OK = 200
23
+ WARN = 400
24
+ HAS_ERRORS = 503
25
+
26
+ @property
27
+ def status_code(self) -> int:
28
+ if self is HealthCheckResultStatus.OK:
29
+ return 200
30
+ if self is HealthCheckResultStatus.WARN:
31
+ return 400
32
+ if self is HealthCheckResultStatus.HAS_ERRORS:
33
+ return 503
34
+ raise NotImplementedError()
35
+
36
+
37
+ THandlerReturn = Tuple[HealthCheckResultStatus, str, List['HealthCheckResult']]
38
+
39
+
40
+ class HealthCheckStepType(Enum):
41
+ COMMON_CHECK = "COMMON_CHECK"
42
+ INTERNAL_API_CHECK = "INTERNAL_API_CHECK"
43
+ MESSAGE_QUEUE_CHECK = "MESSAGE_QUEUE_CHECK"
44
+ INTERNAL_API_HEALTH_CHECK = "INTERNAL_API_HEALTH_CHECK"
45
+
46
+ def _handle_internal_api_check(
47
+ self,
48
+ fn: Callable[..., InternalApiResponse[JsonApiResponsePayload]],
49
+ args: Tuple[Any, ...],
50
+ kwargs: Dict[str, Any],
51
+ ) -> THandlerReturn:
52
+ try:
53
+ executed_result = fn(*args, **kwargs)
54
+ executed_result.check()
55
+ except Exception as e: # noqa: B902
56
+ return HealthCheckResultStatus.HAS_ERRORS, str(e), []
57
+ return HealthCheckResultStatus.OK, '', []
58
+
59
+ def _handle_internal_api_health_check(self, fn: Callable[..., InternalApiResponse[JsonApiResponsePayload]], args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> THandlerReturn:
60
+ try:
61
+ error = None
62
+ executed_result = fn(*args, **kwargs)
63
+ try:
64
+ executed_result.check()
65
+ except (Client4XXInternalApiError, Server5XXInternalApiError) as e:
66
+ error = e
67
+ payload = TypeAdapter(HealthCheckApiResponse).validate_python(executed_result.payload_raw).checks
68
+ if error:
69
+ if isinstance(error, Client4XXInternalApiError):
70
+ return HealthCheckResultStatus.WARN, '', payload
71
+ return HealthCheckResultStatus.HAS_ERRORS, '', payload
72
+ return HealthCheckResultStatus.OK, '', payload
73
+ except Exception as e: # noqa: B902
74
+ return HealthCheckResultStatus.HAS_ERRORS, str(e), []
75
+
76
+ def _handle_common_check(self, fn: Callable[..., None], args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> THandlerReturn:
77
+ try:
78
+ fn(*args, **kwargs)
79
+ except Exception as e: # noqa: B902
80
+ return HealthCheckResultStatus.HAS_ERRORS, str(e), []
81
+ return HealthCheckResultStatus.OK, '', []
82
+
83
+ def _handle_message_queue_check(self, fn: Callable[..., List['HealthCheckQueueStats']], args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> THandlerReturn:
84
+ try:
85
+ queues_stats = fn(*args, **kwargs)
86
+ except Exception as e: # noqa: B902
87
+ return HealthCheckResultStatus.HAS_ERRORS, str(e), []
88
+
89
+ return (
90
+ max(stat.status for stat in queues_stats),
91
+ "".join(f"{stat.info}. \n" for stat in queues_stats),
92
+ [],
93
+ )
94
+
95
+ def run(self, fn: Callable[..., Any], args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> Tuple[HealthCheckResultStatus, str, List['HealthCheckResult']]:
96
+ if self is HealthCheckStepType.COMMON_CHECK:
97
+ return self._handle_common_check(fn, args, kwargs)
98
+ if self is HealthCheckStepType.INTERNAL_API_CHECK:
99
+ return self._handle_internal_api_check(fn, args, kwargs)
100
+ if self is HealthCheckStepType.INTERNAL_API_HEALTH_CHECK:
101
+ return self._handle_internal_api_health_check(fn, args, kwargs)
102
+ if self is HealthCheckStepType.MESSAGE_QUEUE_CHECK:
103
+ return self._handle_message_queue_check(fn, args, kwargs)
104
+ raise NotImplementedError()
105
+
106
+
107
+ class HealthCheckResult(BaseModel):
108
+ name: str = Field(
109
+ ...,
110
+ title="Health check step name",
111
+ description="The health-check contains multiple steps to check, each of the steps gets a name.",
112
+ )
113
+ time_spent: float = Field(
114
+ ...,
115
+ title="Time of execution",
116
+ description="Each health-check step takes some time to be processed, "
117
+ "tells how much time took code execution for the health-check step.",
118
+ )
119
+ status: HealthCheckResultStatus
120
+ info: Optional[str] = Field(
121
+ None,
122
+ title="Information about health-check step",
123
+ description="Each successful health-check step has a status code "
124
+ "but each failed have the error description here.",
125
+ )
126
+ type: HealthCheckStepType
127
+ internal_health_check_results: List['HealthCheckResult'] = Field(default_factory=list)
128
+
129
+ @property
130
+ def ok(self) -> bool:
131
+ return self.status == HealthCheckResultStatus.OK
132
+
133
+
134
+ def get_general_status(statuses: Set[HealthCheckResultStatus]) -> HealthCheckResultStatus:
135
+ if HealthCheckResultStatus.HAS_ERRORS in statuses:
136
+ return HealthCheckResultStatus.HAS_ERRORS
137
+ if HealthCheckResultStatus.HAS_ERRORS not in statuses and HealthCheckResultStatus.WARN in statuses:
138
+ return HealthCheckResultStatus.WARN
139
+ return HealthCheckResultStatus.OK
140
+
141
+
142
+ class HealthCheckStep(NamedTuple):
143
+ name: str
144
+ type: HealthCheckStepType
145
+ executable: Callable[..., Any]
146
+ executable_args: Tuple[Any, ...] = tuple()
147
+ executable_kwargs: Any = Field(default_factory=lambda: dict())
148
+
149
+ def run(self) -> Tuple[HealthCheckResultStatus, float, str, List[HealthCheckResult]]:
150
+ start_time = time.perf_counter()
151
+ status, info, internal_health_check_results = self.type.run(self.executable, self.executable_args, self.executable_kwargs)
152
+ time_spent = time.perf_counter() - start_time
153
+ return status, time_spent, info, internal_health_check_results
154
+
155
+
156
+ class HealthCheckContext:
157
+ def __init__(self, service_name: str, request_service_names: str, db: Optional['flask_sqlalchemy.SQLAlchemy'] = None):
158
+ self._db = db
159
+ self._request_service_names = service_name + SERVICE_NAME_DELIMITER + request_service_names
160
+ self._steps: List[HealthCheckStep] = []
161
+
162
+ @property
163
+ def steps(self) -> List[HealthCheckStep]:
164
+ return self._steps
165
+
166
+ def add_step(
167
+ self,
168
+ name: str,
169
+ executable: Callable[..., Any],
170
+ type_: HealthCheckStepType = HealthCheckStepType.COMMON_CHECK,
171
+ executable_args: Optional[Tuple[Any, ...]] = None,
172
+ executable_kwargs: Optional[Dict[str, Any]] = None,
173
+ ) -> None:
174
+ """
175
+ Registers a health-check step that would be executed automatically when health-check endpoint is called.
176
+
177
+ Note:
178
+ The executable provided to this method should not handle any exceptions that might be raised
179
+ because the error handling would be applied later under the hood, when all steps are executed.
180
+ On the contrary, the executable can raise exceptions inside of it as an indicator to the health-check
181
+ that something gone wrong. Also, function should not return any value because it won't be handled.
182
+ """
183
+ if not executable_kwargs:
184
+ executable_kwargs = {}
185
+ if not executable_args:
186
+ executable_args = tuple()
187
+ assert callable(executable), "You should provide a function that performs a step of a health check"
188
+ assert isinstance(executable_kwargs, dict), "Provide keyword arguments to function"
189
+ already_existing_steps = [step.name for step in self._steps]
190
+ assert name not in already_existing_steps, "This step has already been registered"
191
+ self._steps.append(
192
+ HealthCheckStep(
193
+ name=name,
194
+ type=type_,
195
+ executable=executable,
196
+ executable_args=executable_args,
197
+ executable_kwargs=executable_kwargs,
198
+ ),
199
+ )
200
+
201
+ def check_internal_api_route(
202
+ self,
203
+ internal_api: InternalApi,
204
+ name: str,
205
+ path: str,
206
+ *,
207
+ private: bool = True,
208
+ has_std_schema: bool = True,
209
+ q: Optional[Dict[str, Any]] = None,
210
+ access_token: Optional[str] = None,
211
+ headers: Optional[Dict[str, str]] = None,
212
+ kind: HealthCheckStepType = HealthCheckStepType.INTERNAL_API_CHECK,
213
+ ) -> None:
214
+ """
215
+ Registers a wrapper for add_step but created with purpose
216
+ to distinguish Internal Api checks from other custom checks.
217
+
218
+ Note:
219
+ Supports only GET calls to Internal Api.
220
+ If you want your health-check step to perform other
221
+ methods you might need to add it here but its arguable
222
+ that you expect this behavior from health-check.
223
+
224
+ Parameters:
225
+ internal_api: Instance of self_api (internal_api) that can be found in project configuration.
226
+ name: The name of the health-check step, f.e. "All BS are online" or "GET devices".
227
+ path: The endpoint that you want to call through internal api, f.e. "/devices/info".
228
+ private: Private or not.
229
+ has_std_schema: Has schema or not.
230
+ q: Query parameters for the internal api GET request.
231
+ access_token: Token to access the internal_api endpoint, usually should use _default_auth_token.
232
+ headers: Headers for the request.
233
+ kind: Type of the check.
234
+ """
235
+ self.add_step(
236
+ name=name,
237
+ type_=kind,
238
+ executable=internal_api.request_get,
239
+ executable_args=(path,),
240
+ executable_kwargs={
241
+ "private": private,
242
+ "has_std_schema": has_std_schema,
243
+ "q": q,
244
+ "access_token": access_token,
245
+ "headers": headers,
246
+ },
247
+ )
248
+
249
+ def check_internal_api_health(self, internal_api: InternalApi, name: str) -> None:
250
+ """
251
+ Registers a wrapper for add_step but created with purpose
252
+ to distinguish self-api health-check from other custom checks.
253
+ """
254
+ self.check_internal_api_route(
255
+ internal_api,
256
+ name,
257
+ kind=HealthCheckStepType.INTERNAL_API_HEALTH_CHECK,
258
+ path=INTERNAL_API_HEALTH_CHECK_PATH,
259
+ q={"service_names": self._request_service_names},
260
+ )
261
+
262
+ def check_database_connection_exists(self) -> None:
263
+ """
264
+ Registers a wrapper for add_step but created with purpose
265
+ to distinguish database connection check from other custom checks.
266
+ """
267
+ if self._db is None:
268
+ raise NotImplementedError("This function should not be used in a health-check of a service, "
269
+ f"that doesn't have db connection, {self._db=}")
270
+ self.add_step(
271
+ name="Database connection exists",
272
+ type_=HealthCheckStepType.COMMON_CHECK,
273
+ executable=self._db_connection_exists,
274
+ )
275
+
276
+ def check_redis_connection_exists(self, redis_client: 'Union[Redis[str], Redis[bytes]]') -> None:
277
+ """
278
+ Registers a wrapper for add_step but created with purpose
279
+ to distinguish redis client connection check from other custom checks.
280
+ """
281
+ self.add_step(
282
+ name="Redis connection exists",
283
+ type_=HealthCheckStepType.COMMON_CHECK,
284
+ executable=self._redis_connection_exists,
285
+ executable_args=(redis_client,),
286
+ )
287
+
288
+ def check_message_queue_connection_exists(
289
+ self, uni: Uni,
290
+ ) -> None:
291
+ """
292
+ Registers a wrapper for add_step but created with purpose
293
+ to distinguish message queue connection check from other custom checks.
294
+ """
295
+ self.add_step(
296
+ name="Message Queue Connection",
297
+ type_=HealthCheckStepType.COMMON_CHECK,
298
+ executable=self._message_queue_connection_exists,
299
+ executable_kwargs={"uni": uni},
300
+ )
301
+
302
+ def check_message_queues_health(
303
+ self,
304
+ uni: Uni,
305
+ override_queue_limits: Optional[Dict[str, 'HealthCheckMessageQueueRange']] = None,
306
+ ) -> None:
307
+ """
308
+ Registers a wrapper for add_step but created with purpose
309
+ to distinguish message queue health checks from other custom checks.
310
+
311
+ Parameters:
312
+ uni: Instance of Uni class with .dag config, usually can be found in lib.py in every service.
313
+ override_queue_limits: Dictionary that contains queue name as a key, and HealthCheckMessageQueueRange
314
+ with attributes **OK** and **WARN** set.
315
+ """
316
+ self.add_step(
317
+ name="Message Queue Checks",
318
+ type_=HealthCheckStepType.MESSAGE_QUEUE_CHECK,
319
+ executable=self._message_queue_count_check,
320
+ executable_kwargs={
321
+ "uni": uni,
322
+ "override_queue_limits": override_queue_limits,
323
+ },
324
+ )
325
+
326
+ def _db_connection_exists(self) -> None:
327
+ """Basic SELECT 1 query to check the database connection."""
328
+ if self._db: # just for mypy
329
+ self._db.session.query(text("1")).from_statement(text("SELECT 1")).all()
330
+
331
+ def _redis_connection_exists(self, redis_client: 'Union[Redis[str], Redis[bytes]]') -> None:
332
+ connection = redis_client.ping()
333
+ if not connection:
334
+ raise ConnectionError("Couldn't establish a connection to Redis client.")
335
+
336
+ def _message_queue_connection_exists(self, uni: Uni) -> None:
337
+ """Get the broker if success than connection exists."""
338
+ workers = uni.config.workers.values()
339
+ assert workers, "Workers are not set up"
340
+ for wd in uni.config.workers.values():
341
+ broker = uni._mediator.get_broker(wd.broker.name)
342
+ broker.connect()
343
+
344
+ @staticmethod
345
+ def _message_queue_count_check(
346
+ uni: Uni,
347
+ override_queue_limits: Optional[Dict[str, 'HealthCheckMessageQueueRange']] = None,
348
+ ) -> List['HealthCheckQueueStats']:
349
+ """
350
+ Retrieves all message broker queues and processing a status for each one of them.
351
+ Checks for typos that could've been coded through **override_queue_limits** parameter.
352
+ By default, it checks message queue health by pending messages and the default configuration is defined
353
+ in HealthCheckMessageQueueRange.
354
+ """
355
+ if override_queue_limits is None:
356
+ override_queue_limits = {}
357
+
358
+ queue_stats = []
359
+ available_queues = [wd.topic for wd in uni.config.workers.values()]
360
+ for overriden_queue in override_queue_limits.keys():
361
+ if overriden_queue not in available_queues:
362
+ raise KeyError(f"Can't find queue with name {overriden_queue} in the list of available queues.")
363
+
364
+ for wd in uni.config.workers.values():
365
+ broker = uni._mediator.get_broker(wd.broker.name)
366
+ try:
367
+ message_count = broker.get_topic_approximate_messages_count(wd.topic)
368
+ except UniError:
369
+ continue
370
+ if wd.topic in override_queue_limits:
371
+ overriden_queue_limit = override_queue_limits[wd.topic]
372
+ else:
373
+ overriden_queue_limit = HealthCheckMessageQueueRange()
374
+ queue_status = overriden_queue_limit.get_status(message_count=message_count)
375
+ queue_stats.append(HealthCheckQueueStats(message_count=message_count, queue_name=wd.topic, status=queue_status))
376
+ return queue_stats
377
+
378
+
379
+ class HealthCheckApiResponse(JsonApiResponsePayload):
380
+ service_name: str
381
+ checks: List[HealthCheckResult] = Field(default_factory=list)
382
+
383
+ model_config = ConfigDict(use_enum_values=True)
384
+
385
+
386
+ class HealthCheckMessageQueueRange(BaseModel):
387
+ """
388
+ Data class that was implemented in order to classify the health of message queues.
389
+ It derives status of message queue by the number of messages the queue has in it.
390
+ Queue classification by default:
391
+ - **OK** if the queue has **0** pending messages
392
+ - **WARN** if the queue has **1-100** pending messages
393
+ - **HAS_ERRORS** if the queue has **more than 100** pending messages
394
+ Note:
395
+ To override the default behavior, please provide attributes (ok, warn).
396
+ Make sure that warn is more than ok.
397
+ """
398
+ ok: PositiveInt = Field(
399
+ 0,
400
+ title="Max OK message count",
401
+ description="Maximum number of messages for queue to have in order to be classified with status HEALTHY (200).",
402
+ )
403
+ warn: PositiveInt = Field(
404
+ 100,
405
+ title="Max WARN message count",
406
+ description="Maximum number of messages for queue to have in order to be classified with status WARN (400).",
407
+ )
408
+
409
+ @model_validator(mode='after')
410
+ def root_validate(self) -> 'HealthCheckMessageQueueRange':
411
+ ok, warn = self.ok, self.warn
412
+ if ok is not None and warn is not None:
413
+ if ok >= warn:
414
+ raise ValueError(f"OK attribute should be less than WARN attribute, but you provided {ok=}, {warn=}")
415
+ return self
416
+
417
+ def get_status(self, message_count: int) -> HealthCheckResultStatus:
418
+ if message_count <= self.ok:
419
+ return HealthCheckResultStatus.OK
420
+ if self.ok < message_count <= self.warn:
421
+ return HealthCheckResultStatus.WARN
422
+ return HealthCheckResultStatus.HAS_ERRORS
423
+
424
+
425
+ class HealthCheckQueueStats(BaseModel):
426
+ message_count: int
427
+ queue_name: str
428
+ status: HealthCheckResultStatus
429
+
430
+ @property
431
+ def info(self) -> str:
432
+ return f"Queue with name {self.queue_name} has {self.message_count} messages and status {self.status}"
433
+
434
+
435
+ class HealthCheckQueueMessageCountStatus(BaseModel):
436
+ queue_name: str
437
+ message_count_status_map: Dict[range, HealthCheckResultStatus]
438
+
439
+ model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -0,0 +1,64 @@
1
+ from datetime import datetime
2
+ from typing import List
3
+
4
+ from ul_api_utils.resources.health_check.health_check import HealthCheckResult
5
+
6
+
7
+ def render_result_row(result: HealthCheckResult, lvl: int) -> str:
8
+ table_s = ''
9
+ td_s = ''
10
+ td_s += f'<td>{"->&nbsp;" * lvl}{result.name}</td>'
11
+ td_s += f'<td>{result.status.name}</td>'
12
+ td_s += f'<td>{result.time_spent: <4.3f}s</td>'
13
+ td_s += f'<td><pre style="display: block; padding: 0; margin: 0;">' \
14
+ f'<code style="display: block; padding: 0; margin: 0;">{result.info if result.info is not None else "-"}</code></pre></td>'
15
+ table_s += f'<tr class="status-{result.status.name.lower()}">{td_s}</tr>'
16
+ if result.internal_health_check_results:
17
+ for res in result.internal_health_check_results:
18
+ table_s += render_result_row(res, lvl + 1)
19
+ return table_s
20
+
21
+
22
+ def generate_health_check_table(health_check_results: List[HealthCheckResult], service_name: str) -> str:
23
+ header_s = f'''<!doctype html>
24
+ <html lang="en">
25
+ <head>
26
+ <meta charset="UTF-8">
27
+ <meta name="viewport" content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
28
+ <meta http-equiv="X-UA-Compatible" content="ie=edge">
29
+ <title>Health Check</title>
30
+ </head>
31
+ <body>
32
+ <div class="container">
33
+ <h1>Health Check Results {service_name} ({datetime.now().strftime("%Y-%m-%d %H:%M:%S")})</h1>
34
+ '''
35
+ footer_s = '''
36
+ </div>
37
+ <style>
38
+ .container { max-width: 1024px; margin: 30vh auto 100px; font-family: monospace; font-size: 14px; line-height: 1.2; }
39
+ table { border-collapse: collapse; width: 100%; background: white; }
40
+ table th { padding: 2px 4px; }
41
+ table td { padding: 2px 4px; border: 1px solid #ccc; }
42
+ table tr.status-ok > td { background: #a7ffa7; }
43
+ table tr.status-warn > td { background: yellow; }
44
+ table tr.status-has_errors > td { background: #f7abab; }
45
+ </style>
46
+ </body>
47
+ </html>
48
+ '''
49
+
50
+ table_s = '<table>' \
51
+ '<thead>' \
52
+ '<tr>' \
53
+ '<th align="left">Name of Test</th>' \
54
+ '<th align="left">Status</th>' \
55
+ '<th align="left">Duration</th>' \
56
+ '<th align="left">Info</th>' \
57
+ '</tr>' \
58
+ '</thead>' \
59
+ '<tbody>'
60
+ for result in health_check_results:
61
+ table_s += render_result_row(result, 0)
62
+
63
+ table_s += '</tbody></table>'
64
+ return header_s + table_s + footer_s
@@ -0,0 +1,97 @@
1
+ from typing import Callable, TYPE_CHECKING, List, Tuple
2
+
3
+ from ul_api_utils.api_resource.api_request import ApiRequestQuery
4
+ from ul_api_utils.api_resource.api_resource import ApiResource
5
+ from ul_api_utils.api_resource.api_resource_config import ApiResourceConfig
6
+ from ul_api_utils.api_resource.api_response import HtmlApiResponse, JsonApiResponse
7
+ from ul_api_utils.resources.health_check.const import SERVICE_NAME_DELIMITER
8
+ from ul_api_utils.resources.health_check.health_check import HealthCheckContext, HealthCheckStep, HealthCheckResultStatus, HealthCheckResult, HealthCheckApiResponse
9
+ from ul_api_utils.resources.health_check.health_check_template import generate_health_check_table
10
+ from ul_api_utils.utils.api_method import ApiMethod
11
+
12
+ if TYPE_CHECKING:
13
+ from ul_api_utils.modules.api_sdk import ApiSdk
14
+
15
+
16
+ class HealthCheckQuery(ApiRequestQuery):
17
+ service_names: str = ''
18
+
19
+
20
+ def run_health_check_steps(steps: List[HealthCheckStep]) -> Tuple[bool, HealthCheckResultStatus, List[HealthCheckResult]]:
21
+ """
22
+ Runs all the health-check steps and populates the result list
23
+ where all results of steps execution are stored.
24
+ Catches all exceptions that are being raised inside health-check steps and writes the exception info
25
+ to the info.
26
+ Also, calculates time of execution of each health-check step.
27
+ """
28
+ results: List[HealthCheckResult] = []
29
+ for step in steps:
30
+ step_status, time_spent, info, payload = step.run()
31
+ results.append(HealthCheckResult(
32
+ type=step.type,
33
+ name=step.name,
34
+ time_spent=time_spent,
35
+ status=step_status,
36
+ info=info,
37
+ internal_health_check_results=payload,
38
+ ))
39
+ result_statuses = set(result.status for result in results)
40
+ ok = any(status != HealthCheckResultStatus.OK for status in result_statuses)
41
+ status = max(result_statuses)
42
+ return ok, status, results
43
+
44
+
45
+ def init_health_check_resource(fn: Callable[[HealthCheckContext], None], *, api_sdk: 'ApiSdk') -> None:
46
+ @api_sdk.html_view(ApiMethod.GET, path="/sys/health-check", access=api_sdk.ACCESS_PUBLIC)
47
+ def health_check_web(api_resource: ApiResource, query: HealthCheckQuery) -> HtmlApiResponse:
48
+ if api_sdk.config.service_name in query.service_names.split(SERVICE_NAME_DELIMITER):
49
+ return HtmlApiResponse(
50
+ ok=True,
51
+ content='',
52
+ status_code=200,
53
+ )
54
+
55
+ health_check_context = HealthCheckContext(
56
+ db=api_sdk.db,
57
+ service_name=api_sdk.config.service_name,
58
+ request_service_names=query.service_names,
59
+ )
60
+ fn(health_check_context)
61
+ ok, status, results = run_health_check_steps(health_check_context.steps)
62
+
63
+ return HtmlApiResponse(
64
+ ok=ok,
65
+ content=generate_health_check_table(results, api_sdk.config.service_name),
66
+ status_code=status.status_code,
67
+ )
68
+
69
+ @api_sdk.rest_api(ApiMethod.GET, path="/health-check", access=api_sdk.ACCESS_PUBLIC, config=ApiResourceConfig(swagger_disabled=True))
70
+ def health_check_api(api_resource: ApiResource, query: HealthCheckQuery) -> JsonApiResponse[HealthCheckApiResponse]:
71
+ if api_sdk.config.service_name in query.service_names.split(SERVICE_NAME_DELIMITER):
72
+ response = api_resource.response_ok(
73
+ HealthCheckApiResponse(
74
+ service_name=api_sdk.config.service_name,
75
+ ),
76
+ total_count=0,
77
+ )
78
+ response.status_code = 200
79
+ return response
80
+
81
+ health_check_context = HealthCheckContext(
82
+ db=api_sdk.db,
83
+ service_name=api_sdk.config.service_name,
84
+ request_service_names=query.service_names,
85
+ )
86
+ fn(health_check_context)
87
+ ok, status, results = run_health_check_steps(health_check_context.steps)
88
+
89
+ response = api_resource.response_ok(
90
+ HealthCheckApiResponse(
91
+ service_name=api_sdk.config.service_name,
92
+ checks=results,
93
+ ),
94
+ total_count=len(results),
95
+ )
96
+ response.status_code = status.status_code
97
+ return response
@@ -0,0 +1,25 @@
1
+ from typing import Tuple, Callable
2
+
3
+ from flask import request, Response, jsonify
4
+
5
+ from ul_api_utils.const import RESPONSE_PROP_OK, RESPONSE_PROP_PAYLOAD, RESPONSE_PROP_ERRORS, MIME__JSON
6
+ from ul_api_utils.debug.debugger import Debugger
7
+ from ul_api_utils.utils.api_method import ApiMethod
8
+
9
+
10
+ def not_implemented_handler(error: Exception, *, import_name: str, debugger_enabled: Callable[[], bool]) -> Tuple[Response, int]:
11
+ d = Debugger(import_name, debugger_enabled(), ApiMethod(request.method), request.url)
12
+
13
+ if MIME__JSON not in request.headers.get('accept', ''):
14
+ return Response('501. Not Implemented'), 501
15
+ return jsonify({
16
+ RESPONSE_PROP_OK: False,
17
+ RESPONSE_PROP_PAYLOAD: None,
18
+ RESPONSE_PROP_ERRORS: [
19
+ {
20
+ "error_type": "not-implemented",
21
+ "error_message": f"{request.method} {request.url}",
22
+ },
23
+ ],
24
+ **(d.render_dict(501) if d is not None else {}),
25
+ }), 501
@@ -0,0 +1,29 @@
1
+ from typing import List, TYPE_CHECKING, Optional, Union, Callable, Any, Dict
2
+
3
+ from ul_api_utils.access import GLOBAL_PERMISSION__PUBLIC, PermissionRegistry
4
+ from ul_api_utils.api_resource.api_resource import ApiResource
5
+ from ul_api_utils.api_resource.api_resource_config import ApiResourceConfig
6
+ from ul_api_utils.api_resource.api_response import JsonApiResponse, JsonApiResponsePayload
7
+ from ul_api_utils.const import API_PATH__PERMISSIONS
8
+
9
+ if TYPE_CHECKING:
10
+ from ul_api_utils.modules.api_sdk import ApiSdk
11
+
12
+
13
+ class ApiPermissionsResponse(JsonApiResponsePayload):
14
+ category: str
15
+ permissions: List[Dict[str, Any]]
16
+ service: str
17
+
18
+
19
+ def load_permissions(sdk: 'ApiSdk', app_name: str, permissions_registry: Optional[Union[Callable[[], PermissionRegistry], PermissionRegistry]]) -> None:
20
+ @sdk.rest_api('GET', API_PATH__PERMISSIONS, access=GLOBAL_PERMISSION__PUBLIC, config=ApiResourceConfig(swagger_group='system'))
21
+ def permissions(api_resource: ApiResource) -> JsonApiResponse[List[ApiPermissionsResponse]]:
22
+ if permissions_registry is None:
23
+ reg = PermissionRegistry(app_name, 0, 0)
24
+ elif isinstance(permissions_registry, PermissionRegistry):
25
+ reg = permissions_registry
26
+ else:
27
+ reg = permissions_registry()
28
+ resource_permissions = reg.get_categories_with_permissions()
29
+ return api_resource.response_ok(resource_permissions, len(resource_permissions))