thestage 0.6.2__py3-none-any.whl → 0.6.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. thestage/.env +4 -5
  2. thestage/__init__.py +3 -3
  3. thestage/__main__.py +9 -9
  4. thestage/cli_command.py +56 -56
  5. thestage/cli_command_helper.py +51 -51
  6. thestage/color_scheme/color_scheme.py +7 -7
  7. thestage/config/__init__.py +18 -18
  8. thestage/config/config_storage.py +5 -5
  9. thestage/config/env_base.py +7 -7
  10. thestage/controllers/__init__.py +0 -0
  11. thestage/controllers/base_controller.py +67 -67
  12. thestage/controllers/config_controller.py +137 -137
  13. thestage/controllers/container_controller.py +389 -389
  14. thestage/controllers/instance_controller.py +183 -183
  15. thestage/controllers/project_controller.py +810 -810
  16. thestage/controllers/utils_controller.py +32 -32
  17. thestage/debug_main.dist.py +28 -28
  18. thestage/entities/__init__.py +0 -0
  19. thestage/entities/container.py +17 -17
  20. thestage/entities/enums/__init__.py +0 -0
  21. thestage/entities/enums/order_direction_type.py +6 -6
  22. thestage/entities/enums/shell_type.py +7 -7
  23. thestage/entities/enums/tail_output_type.py +6 -6
  24. thestage/entities/enums/yes_no_response.py +7 -7
  25. thestage/entities/file_item.py +27 -27
  26. thestage/entities/project_inference_simulator.py +18 -18
  27. thestage/entities/project_inference_simulator_model.py +16 -16
  28. thestage/entities/project_task.py +19 -19
  29. thestage/entities/rented_instance.py +19 -19
  30. thestage/entities/self_hosted_instance.py +18 -18
  31. thestage/exceptions/__init__.py +0 -0
  32. thestage/exceptions/auth_exception.py +6 -6
  33. thestage/exceptions/base_exception.py +13 -13
  34. thestage/exceptions/business_logic_exception.py +6 -6
  35. thestage/exceptions/config_exception.py +6 -6
  36. thestage/exceptions/file_system_exception.py +6 -6
  37. thestage/exceptions/git_access_exception.py +17 -17
  38. thestage/exceptions/remote_server_exception.py +24 -24
  39. thestage/git/ProgressPrinter.py +22 -22
  40. thestage/helpers/__init__.py +0 -0
  41. thestage/helpers/error_handler.py +115 -115
  42. thestage/helpers/exception_hook.py +14 -14
  43. thestage/helpers/logger/__init__.py +0 -0
  44. thestage/helpers/logger/app_logger.py +50 -50
  45. thestage/helpers/ssh_util.py +38 -38
  46. thestage/i18n/en_GB/messages.po +947 -947
  47. thestage/i18n/translation.py +9 -9
  48. thestage/main.py +36 -36
  49. thestage/services/.env +6 -6
  50. thestage/services/__init__.py +0 -0
  51. thestage/services/abstract_mapper.py +9 -9
  52. thestage/services/abstract_service.py +87 -87
  53. thestage/services/app_config_service.py +52 -52
  54. thestage/services/clients/__init__.py +0 -0
  55. thestage/services/clients/git/__init__.py +0 -0
  56. thestage/services/clients/git/git_client.py +436 -436
  57. thestage/services/clients/thestage_api/__init__.py +0 -0
  58. thestage/services/clients/thestage_api/api_client.py +718 -718
  59. thestage/services/clients/thestage_api/core/api_client_core.py +108 -108
  60. thestage/services/clients/thestage_api/core/http_client_exception.py +12 -12
  61. thestage/services/clients/thestage_api/dtos/__init__.py +0 -0
  62. thestage/services/clients/thestage_api/dtos/base_response.py +13 -13
  63. thestage/services/clients/thestage_api/dtos/cloud_provider_region.py +19 -19
  64. thestage/services/clients/thestage_api/dtos/container_param_request.py +11 -11
  65. thestage/services/clients/thestage_api/dtos/container_response.py +67 -67
  66. thestage/services/clients/thestage_api/dtos/docker_container_assigned_device.py +10 -10
  67. thestage/services/clients/thestage_api/dtos/docker_container_controller/docker_container_list_request.py +13 -13
  68. thestage/services/clients/thestage_api/dtos/docker_container_controller/docker_container_list_response.py +13 -13
  69. thestage/services/clients/thestage_api/dtos/docker_container_mapping.py +10 -10
  70. thestage/services/clients/thestage_api/dtos/entity_filter_request.py +14 -14
  71. thestage/services/clients/thestage_api/dtos/enums/__init__.py +0 -0
  72. thestage/services/clients/thestage_api/dtos/enums/container_pending_action.py +10 -10
  73. thestage/services/clients/thestage_api/dtos/enums/container_status.py +17 -17
  74. thestage/services/clients/thestage_api/dtos/enums/cpu_type.py +8 -8
  75. thestage/services/clients/thestage_api/dtos/enums/currency_type.py +10 -10
  76. thestage/services/clients/thestage_api/dtos/enums/daemon_status.py +9 -9
  77. thestage/services/clients/thestage_api/dtos/enums/disk_type.py +7 -7
  78. thestage/services/clients/thestage_api/dtos/enums/drive_type.py +7 -7
  79. thestage/services/clients/thestage_api/dtos/enums/gpu_name.py +8 -8
  80. thestage/services/clients/thestage_api/dtos/enums/inference_model_status.py +9 -9
  81. thestage/services/clients/thestage_api/dtos/enums/inference_simulator_status.py +15 -15
  82. thestage/services/clients/thestage_api/dtos/enums/instance_rented_status.py +17 -17
  83. thestage/services/clients/thestage_api/dtos/enums/instance_type.py +7 -7
  84. thestage/services/clients/thestage_api/dtos/enums/location_region.py +11 -11
  85. thestage/services/clients/thestage_api/dtos/enums/power_status.py +10 -10
  86. thestage/services/clients/thestage_api/dtos/enums/provider_name.py +11 -11
  87. thestage/services/clients/thestage_api/dtos/enums/selfhosted_status.py +10 -10
  88. thestage/services/clients/thestage_api/dtos/enums/task_execution_status.py +12 -12
  89. thestage/services/clients/thestage_api/dtos/enums/task_status.py +12 -12
  90. thestage/services/clients/thestage_api/dtos/frontend_status.py +10 -10
  91. thestage/services/clients/thestage_api/dtos/inference_controller/deploy_inference_model_to_instance_request.py +13 -13
  92. thestage/services/clients/thestage_api/dtos/inference_controller/deploy_inference_model_to_instance_response.py +13 -13
  93. thestage/services/clients/thestage_api/dtos/inference_controller/deploy_inference_model_to_sagemaker_request.py +12 -12
  94. thestage/services/clients/thestage_api/dtos/inference_controller/deploy_inference_model_to_sagemaker_response.py +12 -12
  95. thestage/services/clients/thestage_api/dtos/inference_controller/get_inference_simulator_request.py +10 -10
  96. thestage/services/clients/thestage_api/dtos/inference_controller/get_inference_simulator_response.py +13 -13
  97. thestage/services/clients/thestage_api/dtos/inference_controller/inference_simulator_list_for_project_request.py +14 -14
  98. thestage/services/clients/thestage_api/dtos/inference_controller/inference_simulator_list_for_project_response.py +12 -12
  99. thestage/services/clients/thestage_api/dtos/inference_controller/inference_simulator_model_list_for_project_request.py +12 -12
  100. thestage/services/clients/thestage_api/dtos/inference_controller/inference_simulator_model_list_for_project_response.py +13 -13
  101. thestage/services/clients/thestage_api/dtos/inference_simulator_model_response.py +11 -11
  102. thestage/services/clients/thestage_api/dtos/inference_simulator_response.py +11 -11
  103. thestage/services/clients/thestage_api/dtos/installed_service.py +17 -17
  104. thestage/services/clients/thestage_api/dtos/instance_detected_gpus.py +20 -20
  105. thestage/services/clients/thestage_api/dtos/instance_rented_response.py +71 -71
  106. thestage/services/clients/thestage_api/dtos/logging_controller/docker_container_log_stream_request.py +7 -7
  107. thestage/services/clients/thestage_api/dtos/logging_controller/log_polling_request.py +13 -13
  108. thestage/services/clients/thestage_api/dtos/logging_controller/log_polling_response.py +14 -14
  109. thestage/services/clients/thestage_api/dtos/logging_controller/task_log_stream_request.py +7 -7
  110. thestage/services/clients/thestage_api/dtos/logging_controller/user_logs_query_request.py +21 -21
  111. thestage/services/clients/thestage_api/dtos/logging_controller/user_logs_query_response.py +14 -14
  112. thestage/services/clients/thestage_api/dtos/paginated_entity_list.py +11 -11
  113. thestage/services/clients/thestage_api/dtos/pagination_data.py +10 -10
  114. thestage/services/clients/thestage_api/dtos/price_definition.py +14 -14
  115. thestage/services/clients/thestage_api/dtos/project_controller/project_get_deploy_ssh_key_request.py +7 -7
  116. thestage/services/clients/thestage_api/dtos/project_controller/project_get_deploy_ssh_key_response.py +10 -10
  117. thestage/services/clients/thestage_api/dtos/project_controller/project_push_inference_simulator_model_request.py +8 -8
  118. thestage/services/clients/thestage_api/dtos/project_controller/project_push_inference_simulator_model_response.py +6 -6
  119. thestage/services/clients/thestage_api/dtos/project_controller/project_run_task_request.py +15 -15
  120. thestage/services/clients/thestage_api/dtos/project_controller/project_run_task_response.py +10 -10
  121. thestage/services/clients/thestage_api/dtos/project_controller/project_start_inference_simulator_request.py +13 -13
  122. thestage/services/clients/thestage_api/dtos/project_controller/project_start_inference_simulator_response.py +10 -10
  123. thestage/services/clients/thestage_api/dtos/project_response.py +32 -32
  124. thestage/services/clients/thestage_api/dtos/selfhosted_instance_response.py +56 -56
  125. thestage/services/clients/thestage_api/dtos/sftp_path_helper.py +13 -13
  126. thestage/services/clients/thestage_api/dtos/ssh_key_controller/add_ssh_key_to_user_request.py +8 -8
  127. thestage/services/clients/thestage_api/dtos/ssh_key_controller/add_ssh_key_to_user_response.py +11 -11
  128. thestage/services/clients/thestage_api/dtos/ssh_key_controller/add_ssh_public_key_to_instance_request.py +8 -8
  129. thestage/services/clients/thestage_api/dtos/ssh_key_controller/add_ssh_public_key_to_instance_response.py +11 -11
  130. thestage/services/clients/thestage_api/dtos/ssh_key_controller/is_user_has_public_ssh_key_request.py +7 -7
  131. thestage/services/clients/thestage_api/dtos/ssh_key_controller/is_user_has_public_ssh_key_response.py +12 -12
  132. thestage/services/clients/thestage_api/dtos/task_controller/task_list_for_project_request.py +10 -10
  133. thestage/services/clients/thestage_api/dtos/task_controller/task_list_for_project_response.py +12 -12
  134. thestage/services/clients/thestage_api/dtos/task_controller/task_status_localized_map_response.py +9 -9
  135. thestage/services/clients/thestage_api/dtos/task_controller/task_view_response.py +12 -12
  136. thestage/services/clients/thestage_api/dtos/user_controller/user_profile.py +12 -12
  137. thestage/services/clients/thestage_api/dtos/validate_token_response.py +11 -11
  138. thestage/services/config_provider/__init__.py +0 -0
  139. thestage/services/config_provider/config_provider.py +237 -237
  140. thestage/services/connect/connect_service.py +193 -196
  141. thestage/services/connect/dto/remote_server_config.py +9 -9
  142. thestage/services/container/__init__.py +0 -0
  143. thestage/services/container/container_service.py +374 -374
  144. thestage/services/container/mapper/__init__.py +0 -0
  145. thestage/services/container/mapper/container_mapper.py +30 -30
  146. thestage/services/core_files/config_entity.py +26 -26
  147. thestage/services/filesystem_service.py +133 -133
  148. thestage/services/instance/__init__.py +0 -0
  149. thestage/services/instance/instance_service.py +303 -303
  150. thestage/services/instance/mapper/__init__.py +0 -0
  151. thestage/services/instance/mapper/instance_mapper.py +24 -24
  152. thestage/services/instance/mapper/selfhosted_mapper.py +33 -33
  153. thestage/services/logging/byte_print_style.py +5 -5
  154. thestage/services/logging/dto/log_message.py +15 -15
  155. thestage/services/logging/dto/log_type.py +6 -6
  156. thestage/services/logging/exception/log_polling_exception.py +6 -6
  157. thestage/services/logging/logging_constants.py +3 -3
  158. thestage/services/logging/logging_service.py +367 -367
  159. thestage/services/project/__init__.py +0 -0
  160. thestage/services/project/dto/inference_simulator_dto.py +22 -22
  161. thestage/services/project/dto/inference_simulator_model_dto.py +20 -20
  162. thestage/services/project/dto/project_config.py +14 -14
  163. thestage/services/project/mapper/__init__.py +0 -0
  164. thestage/services/project/mapper/project_inference_simulator_mapper.py +21 -21
  165. thestage/services/project/mapper/project_inference_simulator_model_mapper.py +21 -21
  166. thestage/services/project/mapper/project_task_mapper.py +22 -22
  167. thestage/services/project/project_service.py +1253 -1253
  168. thestage/services/remote_server_service.py +609 -609
  169. thestage/services/service_factory.py +97 -97
  170. thestage/services/task/dto/task_dto.py +40 -40
  171. thestage/services/validation_service.py +61 -61
  172. {thestage-0.6.2.dist-info → thestage-0.6.3.dist-info}/LICENSE.txt +12 -12
  173. {thestage-0.6.2.dist-info → thestage-0.6.3.dist-info}/METADATA +3 -2
  174. thestage-0.6.3.dist-info/RECORD +176 -0
  175. {thestage-0.6.2.dist-info → thestage-0.6.3.dist-info}/WHEEL +1 -1
  176. thestage-0.6.2.dist-info/RECORD +0 -176
  177. {thestage-0.6.2.dist-info → thestage-0.6.3.dist-info}/entry_points.txt +0 -0
@@ -1,367 +1,367 @@
1
- import asyncio
2
- from _signal import SIGINT
3
- from asyncio import CancelledError, Task, StreamWriter
4
- from datetime import datetime
5
- from typing import Optional, Dict
6
-
7
- import aioconsole
8
- import typer
9
- from httpx import ReadTimeout, ConnectError, ConnectTimeout
10
-
11
- from thestage.helpers.logger.app_logger import app_logger
12
- from thestage.services.clients.thestage_api.dtos.enums.container_status import DockerContainerStatus
13
- from thestage.services.clients.thestage_api.dtos.enums.inference_simulator_status import InferenceSimulatorStatus
14
- from thestage.services.clients.thestage_api.dtos.enums.task_status import TaskStatus
15
- from thestage.services.clients.thestage_api.dtos.inference_controller.get_inference_simulator_response import \
16
- GetInferenceSimulatorResponse
17
- from thestage.services.clients.thestage_api.dtos.task_controller.task_view_response import TaskViewResponse
18
- from thestage.services.logging.byte_print_style import BytePrintStyle
19
- from thestage.services.logging.dto.log_message import LogMessage
20
- from thestage.services.logging.dto.log_type import LogType
21
- from thestage.i18n.translation import __
22
- from thestage.services.abstract_service import AbstractService
23
- from thestage.services.clients.thestage_api.dtos.container_response import DockerContainerDto
24
- from thestage.helpers.error_handler import error_handler
25
- from thestage.services.clients.thestage_api.api_client import TheStageApiClient
26
- from rich import print
27
-
28
- from thestage.services.logging.exception.log_polling_exception import LogPollingException
29
- from thestage.services.logging.logging_constants import LOG_MESSAGE_CODE_TASK_FINISHED, \
30
- LOG_MESSAGE_CODE_INFERENCE_SIMULATOR_FAILED
31
-
32
- is_logs_streaming = False
33
-
34
-
35
- class LoggingService:
36
- __thestage_api_client: TheStageApiClient = None
37
-
38
- def __init__(
39
- self,
40
- thestage_api_client: TheStageApiClient,
41
- ):
42
- self.__thestage_api_client = thestage_api_client
43
-
44
-
45
- @error_handler()
46
- def print_last_task_logs(self, task_id: int, logs_number: Optional[int]):
47
- logs = self.__thestage_api_client.query_user_logs(
48
- task_id=task_id,
49
- limit=logs_number
50
- )
51
- for log_message in reversed(logs.queryResult):
52
- self.__print_log_line_object(log_message)
53
-
54
-
55
- @error_handler()
56
- def print_last_inference_simulator_logs(self, inference_simulator_id: int, logs_number: Optional[int]):
57
- logs = self.__thestage_api_client.query_user_logs(
58
- inference_simulator_id=inference_simulator_id,
59
- limit=logs_number
60
- )
61
- for log_message in reversed(logs.queryResult):
62
- self.__print_log_line_object(log_message)
63
-
64
-
65
- @error_handler()
66
- def print_last_container_logs(self, container_uid: str, logs_number: Optional[int]):
67
- container: Optional[DockerContainerDto] = self.__thestage_api_client.get_container(
68
- container_slug=container_uid,
69
- )
70
-
71
- if not container:
72
- typer.echo("Container was not found")
73
- raise typer.Exit(1)
74
-
75
- logs = self.__thestage_api_client.query_user_logs(
76
- container_id=container.id,
77
- limit=logs_number
78
- )
79
- for log_message in reversed(logs.queryResult):
80
- self.__print_log_line_object(log_message)
81
-
82
-
83
- @error_handler()
84
- def stream_task_logs_with_controls(self, task_id: int):
85
- asyncio.run(
86
- self.__stream_task_logs_with_controls_async(task_id=task_id)
87
- )
88
-
89
-
90
- @error_handler()
91
- async def __stream_task_logs_with_controls_async(self, task_id: int):
92
- task_view_response: Optional[TaskViewResponse] = self.__thestage_api_client.get_task(task_id=task_id,)
93
-
94
- task_status_map: Dict[str, str] = self.__thestage_api_client.get_task_localized_status_map()
95
-
96
- task = task_view_response.task
97
-
98
- if task:
99
- if task.frontend_status.status_key not in [TaskStatus.RUNNING, TaskStatus.SCHEDULED]:
100
- typer.echo(__("Task must be in status '%required_status%' to stream real-time logs. Task %task_id% status: '%status%'.", {
101
- 'task_id': str(task.id),
102
- 'status': task.frontend_status.status_translation,
103
- 'required_status': task_status_map.get(TaskStatus.RUNNING) or TaskStatus.RUNNING
104
- }))
105
- raise typer.Exit(1)
106
- else:
107
- typer.echo(__("Task with ID %task_id% was not found", {'task_id': task.id}))
108
- raise typer.Exit(1)
109
-
110
- typer.echo(__(
111
- f"Log stream for task %task_id% started",
112
- {
113
- 'task_id': str(task.id),
114
- }
115
- ))
116
-
117
- typer.echo(__("CTRL+C to cancel the task. CTRL+D to disconnect from log stream."))
118
-
119
- print_logs_task = asyncio.create_task(self.print_realtime_logs(task_id=task.id))
120
- input_task = asyncio.create_task(self.read_log_stream_input())
121
-
122
- def sigint_handler():
123
- input_task.cancel()
124
-
125
- loop = asyncio.get_event_loop()
126
- for signal_item in [SIGINT]: # SIGINT == CTRL+C
127
- loop.add_signal_handler(signal_item, sigint_handler)
128
-
129
- done, pending = await asyncio.wait([print_logs_task, input_task], return_when=asyncio.FIRST_COMPLETED)
130
-
131
- if input_task in done:
132
- print_logs_task.cancel()
133
- if not input_task.result(): # result is only expected if ctrl+D triggered EOFError
134
- typer.echo(f"\rTask {task_id} will be canceled")
135
- self.__thestage_api_client.cancel_task(
136
- task_id=task.id,
137
- )
138
-
139
- @error_handler()
140
- def stream_inference_simulator_logs_with_controls(self, slug: str):
141
- asyncio.run(
142
- self.__stream_inference_simulator_logs_with_controls_async(
143
- slug=slug
144
- )
145
- )
146
-
147
- @error_handler()
148
- async def __stream_inference_simulator_logs_with_controls_async(self, slug: str):
149
- get_inference_simulator_response: Optional[GetInferenceSimulatorResponse] = self.__thestage_api_client.get_inference_simulator(
150
- slug=slug,
151
- )
152
-
153
- inference_simulator_status_map: Dict[str, str] = self.__thestage_api_client.get_inference_simulator_business_status_map()
154
-
155
- inference_simulator = get_inference_simulator_response.inferenceSimulator
156
-
157
- if inference_simulator:
158
- if inference_simulator.status not in ['SCHEDULED', 'CREATING', 'RUNNING']:
159
- typer.echo(
160
- __("Inference simulator must be in status '%required_status%' to stream real-time logs. Inference simulator status: '%status%'.",
161
- {
162
- 'status': inference_simulator.status,
163
- 'required_status': inference_simulator_status_map.get(InferenceSimulatorStatus.RUNNING) or InferenceSimulatorStatus.RUNNING
164
- }))
165
- raise typer.Exit(1)
166
- else:
167
- typer.echo(__("Inference simulator with unique ID %slug% was not found", {'slug': inference_simulator.slug}))
168
- raise typer.Exit(1)
169
-
170
- typer.echo(__(
171
- f"Log stream for inference simulator %slug% started",
172
- {
173
- 'slug': str(inference_simulator.slug),
174
- }
175
- ))
176
-
177
- typer.echo(__("CTRL+D to disconnect from log stream."))
178
-
179
- print_task_or_inference_simulator_logs = asyncio.create_task(
180
- self.print_realtime_logs(inference_simulator_id=inference_simulator.id)
181
- )
182
- input_task = asyncio.create_task(self.read_log_stream_input())
183
-
184
- done, pending = await asyncio.wait([print_task_or_inference_simulator_logs, input_task],
185
- return_when=asyncio.FIRST_COMPLETED)
186
-
187
- if input_task in done:
188
- print_task_or_inference_simulator_logs.cancel()
189
- typer.echo(__(f"Disconnected from log stream. You can try to reconnect with 'thestage project inference-simulator logs {slug}'."))
190
-
191
-
192
- @error_handler()
193
- def stream_container_logs_with_controls(self, container_uid: str):
194
- asyncio.run(
195
- self.__stream_container_logs_with_controls_async(
196
- container_uid=container_uid
197
- )
198
- )
199
-
200
-
201
- @error_handler()
202
- async def __stream_container_logs_with_controls_async(self, container_uid: str):
203
- container: Optional[DockerContainerDto] = self.__thestage_api_client.get_container(
204
- container_slug=container_uid,
205
- )
206
-
207
- if container:
208
- if container.frontend_status.status_key not in [DockerContainerStatus.RUNNING]:
209
- typer.echo(f"Container status: '{container.frontend_status.status_translation}'")
210
- else:
211
- typer.echo("Container was not found")
212
- raise typer.Exit(1)
213
-
214
- typer.echo(f"Log stream for Docker container started")
215
- typer.echo("CTRL+D to disconnect from log stream.")
216
-
217
- print_logs_task = asyncio.create_task(self.print_realtime_logs(docker_container_id=container.id))
218
- input_task = asyncio.create_task(self.read_log_stream_input())
219
-
220
- def sigint_handler():
221
- input_task.cancel()
222
-
223
- loop = asyncio.get_event_loop()
224
- for signal_item in [SIGINT]: # SIGINT == CTRL+C
225
- loop.add_signal_handler(signal_item, sigint_handler)
226
-
227
- done, pending = await asyncio.wait([print_logs_task, input_task], return_when=asyncio.FIRST_COMPLETED)
228
-
229
- if input_task in done:
230
- print_logs_task.cancel()
231
-
232
-
233
- async def read_log_stream_input(self):
234
- try:
235
- while True:
236
- input1 = await aioconsole.ainput()
237
- except EOFError:
238
- typer.echo(__("\rExited from log stream"))
239
- return True
240
- except CancelledError: # Always appears if async task is canceled and leaves huge traces
241
- pass
242
-
243
-
244
- async def print_realtime_logs(
245
- self,
246
- task_id: Optional[int] = None,
247
- inference_simulator_id: Optional[int] = None,
248
- docker_container_id: Optional[int] = None,
249
- ):
250
- polling_interval_seconds: float = 4 # also adjust polling api method timeout if changed
251
- between_logs_sleeping_coef: float = 1 # we emulate delay between logs, but if for any reason code runs for too long - delays will be controlled with this coef
252
- last_iteration_log_timestamp: Optional[str] = None # pointer to next iteration polling start (obtained from each response)
253
- last_log_id: Optional[str] = None # pointer to next iteration polling start - to exclude the log id from result (obtained from each response)
254
- consecutive_error_count: int = 0 # connectivity errors count - stream will disconnect if too many errors in a row
255
- iteration_started_at: datetime # used to control iteration duration - polling should be done at around exact rate
256
- errors_started_at: Optional[datetime] = None # time since errors started to stream disconnect
257
-
258
- is_no_more_logs = False
259
- while not is_no_more_logs:
260
- log_wait_remaining_limit: float = polling_interval_seconds # hard limit just in case
261
- iteration_started_at = datetime.utcnow()
262
- last_printed_log_timestamp: Optional[datetime] = None
263
- reader, writer = await aioconsole.get_standard_streams()
264
-
265
- # this shows (somewhat accurate) time difference between logs here and in real time. should not grow.
266
- # if last_iteration_log_timestamp:
267
- # last_log_timestamp_parsed = datetime.strptime(last_iteration_log_timestamp, '%Y-%m-%dT%H:%M:%S.%f')
268
- # stream_to_logs_diff = datetime.utcnow() - last_log_timestamp_parsed
269
- # print_nonblocking(f'TDIFF {stream_to_logs_diff.total_seconds()}', writer)
270
- try:
271
- logs_response = await self.__thestage_api_client.poll_logs_httpx(
272
- task_id=task_id,
273
- inference_simulator_id=inference_simulator_id,
274
- docker_container_id=docker_container_id,
275
- last_log_timestamp=last_iteration_log_timestamp,
276
- last_log_id=last_log_id
277
- )
278
-
279
- if not logs_response.is_success:
280
- app_logger.info(f'Polling logs error: {logs_response.message}')
281
- raise LogPollingException('')
282
-
283
-
284
- if consecutive_error_count > 0:
285
- consecutive_error_count = 0
286
- errors_started_at = None
287
- log_wait_remaining_limit = 0 # no log delays after reconnect
288
-
289
- last_iteration_log_timestamp = logs_response.lastLogTimestamp
290
- last_log_id = logs_response.lastLogId
291
-
292
- for log_item in logs_response.logs:
293
- current_log_timestamp = datetime.strptime(log_item.timestamp[:26], '%Y-%m-%dT%H:%M:%S.%f') # python does not like nanoseconds
294
- if last_printed_log_timestamp is not None and log_wait_remaining_limit > 0:
295
- logs_sleeptime = (current_log_timestamp - last_printed_log_timestamp).total_seconds() * between_logs_sleeping_coef
296
- await asyncio.sleep(logs_sleeptime)
297
- log_wait_remaining_limit -= logs_sleeptime
298
- self.__print_log_line_object_nonblocking(log_item, writer)
299
- last_printed_log_timestamp = current_log_timestamp
300
- if log_item.messageCode == LOG_MESSAGE_CODE_TASK_FINISHED or log_item.messageCode == LOG_MESSAGE_CODE_INFERENCE_SIMULATOR_FAILED:
301
- is_no_more_logs = True
302
-
303
- if is_no_more_logs:
304
- break
305
- except (ReadTimeout, ConnectError, ConnectTimeout, LogPollingException) as e:
306
- consecutive_error_count += 1
307
- if consecutive_error_count == 1:
308
- if isinstance(e, LogPollingException):
309
- print_nonblocking("Some problems raised while getting logs...", writer, BytePrintStyle.ORANGE)
310
- else:
311
- print_nonblocking("Network issues, attempting to re-establish connection...", writer, BytePrintStyle.ORANGE)
312
- if not errors_started_at:
313
- errors_started_at = datetime.utcnow()
314
-
315
- if consecutive_error_count > 7:
316
- seconds_with_error = (datetime.utcnow() - errors_started_at).total_seconds()
317
- if inference_simulator_id:
318
- print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage inference-simulator logs <inference-simulator-UID>' to reconnect.", writer)
319
- elif task_id:
320
- print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage project task logs {task_id}' to reconnect.", writer)
321
- elif docker_container_id:
322
- print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage container logs <docker-container-UID>' to reconnect.", writer)
323
- else:
324
- print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds)", writer)
325
- break
326
-
327
- # depending on iteration duration - sleep for the remaining time and adjust log sleep coefficient if needed
328
- iteration_duration = (datetime.utcnow() - iteration_started_at).total_seconds()
329
- if iteration_duration > polling_interval_seconds:
330
- between_logs_sleeping_coef *= 0.85
331
- else:
332
- await asyncio.sleep(polling_interval_seconds - iteration_duration)
333
- if between_logs_sleeping_coef < 1:
334
- between_logs_sleeping_coef = min(1.0, between_logs_sleeping_coef * 1.15)
335
-
336
-
337
- def __print_log_line(self, log_message_raw_json: str):
338
- log_message = LogMessage.model_validate_json(log_message_raw_json)
339
- if not log_message.logType and log_message.message == 'ping':
340
- return
341
- self.__print_log_line_object(log_message)
342
-
343
-
344
- @staticmethod
345
- def __print_log_line_object(log_message: LogMessage):
346
- line_color: str = "grey78"
347
-
348
- if not log_message.logType and log_message.message == 'ping':
349
- return
350
-
351
- if log_message.logType == LogType.STDERR.value:
352
- line_color = "red"
353
- if log_message.message:
354
- print(f'[{line_color}][not bold]{log_message.message}[/not bold][/{line_color}]')
355
-
356
- @staticmethod
357
- def __print_log_line_object_nonblocking(log_message: LogMessage, writer: StreamWriter):
358
- if log_message.message:
359
- line_color: str = BytePrintStyle.RESET
360
- if log_message.logType == LogType.STDERR.value:
361
- line_color = BytePrintStyle.RED
362
-
363
- print_nonblocking(f'{log_message.message}', writer, line_color)
364
-
365
-
366
- def print_nonblocking(line: str, writer: StreamWriter, color_code: str = BytePrintStyle.RESET):
367
- writer.write(str.encode(f'{color_code}{line}{BytePrintStyle.RESET}\r\n'))
1
+ import asyncio
2
+ from _signal import SIGINT
3
+ from asyncio import CancelledError, Task, StreamWriter
4
+ from datetime import datetime
5
+ from typing import Optional, Dict
6
+
7
+ import aioconsole
8
+ import typer
9
+ from httpx import ReadTimeout, ConnectError, ConnectTimeout
10
+
11
+ from thestage.helpers.logger.app_logger import app_logger
12
+ from thestage.services.clients.thestage_api.dtos.enums.container_status import DockerContainerStatus
13
+ from thestage.services.clients.thestage_api.dtos.enums.inference_simulator_status import InferenceSimulatorStatus
14
+ from thestage.services.clients.thestage_api.dtos.enums.task_status import TaskStatus
15
+ from thestage.services.clients.thestage_api.dtos.inference_controller.get_inference_simulator_response import \
16
+ GetInferenceSimulatorResponse
17
+ from thestage.services.clients.thestage_api.dtos.task_controller.task_view_response import TaskViewResponse
18
+ from thestage.services.logging.byte_print_style import BytePrintStyle
19
+ from thestage.services.logging.dto.log_message import LogMessage
20
+ from thestage.services.logging.dto.log_type import LogType
21
+ from thestage.i18n.translation import __
22
+ from thestage.services.abstract_service import AbstractService
23
+ from thestage.services.clients.thestage_api.dtos.container_response import DockerContainerDto
24
+ from thestage.helpers.error_handler import error_handler
25
+ from thestage.services.clients.thestage_api.api_client import TheStageApiClient
26
+ from rich import print
27
+
28
+ from thestage.services.logging.exception.log_polling_exception import LogPollingException
29
+ from thestage.services.logging.logging_constants import LOG_MESSAGE_CODE_TASK_FINISHED, \
30
+ LOG_MESSAGE_CODE_INFERENCE_SIMULATOR_FAILED
31
+
32
+ is_logs_streaming = False
33
+
34
+
35
+ class LoggingService:
36
+ __thestage_api_client: TheStageApiClient = None
37
+
38
+ def __init__(
39
+ self,
40
+ thestage_api_client: TheStageApiClient,
41
+ ):
42
+ self.__thestage_api_client = thestage_api_client
43
+
44
+
45
+ @error_handler()
46
+ def print_last_task_logs(self, task_id: int, logs_number: Optional[int]):
47
+ logs = self.__thestage_api_client.query_user_logs(
48
+ task_id=task_id,
49
+ limit=logs_number
50
+ )
51
+ for log_message in reversed(logs.queryResult):
52
+ self.__print_log_line_object(log_message)
53
+
54
+
55
+ @error_handler()
56
+ def print_last_inference_simulator_logs(self, inference_simulator_id: int, logs_number: Optional[int]):
57
+ logs = self.__thestage_api_client.query_user_logs(
58
+ inference_simulator_id=inference_simulator_id,
59
+ limit=logs_number
60
+ )
61
+ for log_message in reversed(logs.queryResult):
62
+ self.__print_log_line_object(log_message)
63
+
64
+
65
+ @error_handler()
66
+ def print_last_container_logs(self, container_uid: str, logs_number: Optional[int]):
67
+ container: Optional[DockerContainerDto] = self.__thestage_api_client.get_container(
68
+ container_slug=container_uid,
69
+ )
70
+
71
+ if not container:
72
+ typer.echo("Container was not found")
73
+ raise typer.Exit(1)
74
+
75
+ logs = self.__thestage_api_client.query_user_logs(
76
+ container_id=container.id,
77
+ limit=logs_number
78
+ )
79
+ for log_message in reversed(logs.queryResult):
80
+ self.__print_log_line_object(log_message)
81
+
82
+
83
+ @error_handler()
84
+ def stream_task_logs_with_controls(self, task_id: int):
85
+ asyncio.run(
86
+ self.__stream_task_logs_with_controls_async(task_id=task_id)
87
+ )
88
+
89
+
90
+ @error_handler()
91
+ async def __stream_task_logs_with_controls_async(self, task_id: int):
92
+ task_view_response: Optional[TaskViewResponse] = self.__thestage_api_client.get_task(task_id=task_id,)
93
+
94
+ task_status_map: Dict[str, str] = self.__thestage_api_client.get_task_localized_status_map()
95
+
96
+ task = task_view_response.task
97
+
98
+ if task:
99
+ if task.frontend_status.status_key not in [TaskStatus.RUNNING, TaskStatus.SCHEDULED]:
100
+ typer.echo(__("Task must be in status '%required_status%' to stream real-time logs. Task %task_id% status: '%status%'.", {
101
+ 'task_id': str(task.id),
102
+ 'status': task.frontend_status.status_translation,
103
+ 'required_status': task_status_map.get(TaskStatus.RUNNING) or TaskStatus.RUNNING
104
+ }))
105
+ raise typer.Exit(1)
106
+ else:
107
+ typer.echo(__("Task with ID %task_id% was not found", {'task_id': task.id}))
108
+ raise typer.Exit(1)
109
+
110
+ typer.echo(__(
111
+ f"Log stream for task %task_id% started",
112
+ {
113
+ 'task_id': str(task.id),
114
+ }
115
+ ))
116
+
117
+ typer.echo(__("CTRL+C to cancel the task. CTRL+D to disconnect from log stream."))
118
+
119
+ print_logs_task = asyncio.create_task(self.print_realtime_logs(task_id=task.id))
120
+ input_task = asyncio.create_task(self.read_log_stream_input())
121
+
122
+ def sigint_handler():
123
+ input_task.cancel()
124
+
125
+ loop = asyncio.get_event_loop()
126
+ for signal_item in [SIGINT]: # SIGINT == CTRL+C
127
+ loop.add_signal_handler(signal_item, sigint_handler)
128
+
129
+ done, pending = await asyncio.wait([print_logs_task, input_task], return_when=asyncio.FIRST_COMPLETED)
130
+
131
+ if input_task in done:
132
+ print_logs_task.cancel()
133
+ if not input_task.result(): # result is only expected if ctrl+D triggered EOFError
134
+ typer.echo(f"\rTask {task_id} will be canceled")
135
+ self.__thestage_api_client.cancel_task(
136
+ task_id=task.id,
137
+ )
138
+
139
+ @error_handler()
140
+ def stream_inference_simulator_logs_with_controls(self, slug: str):
141
+ asyncio.run(
142
+ self.__stream_inference_simulator_logs_with_controls_async(
143
+ slug=slug
144
+ )
145
+ )
146
+
147
+ @error_handler()
148
+ async def __stream_inference_simulator_logs_with_controls_async(self, slug: str):
149
+ get_inference_simulator_response: Optional[GetInferenceSimulatorResponse] = self.__thestage_api_client.get_inference_simulator(
150
+ slug=slug,
151
+ )
152
+
153
+ inference_simulator_status_map: Dict[str, str] = self.__thestage_api_client.get_inference_simulator_business_status_map()
154
+
155
+ inference_simulator = get_inference_simulator_response.inferenceSimulator
156
+
157
+ if inference_simulator:
158
+ if inference_simulator.status not in ['SCHEDULED', 'CREATING', 'RUNNING']:
159
+ typer.echo(
160
+ __("Inference simulator must be in status '%required_status%' to stream real-time logs. Inference simulator status: '%status%'.",
161
+ {
162
+ 'status': inference_simulator.status,
163
+ 'required_status': inference_simulator_status_map.get(InferenceSimulatorStatus.RUNNING) or InferenceSimulatorStatus.RUNNING
164
+ }))
165
+ raise typer.Exit(1)
166
+ else:
167
+ typer.echo(__("Inference simulator with unique ID %slug% was not found", {'slug': inference_simulator.slug}))
168
+ raise typer.Exit(1)
169
+
170
+ typer.echo(__(
171
+ f"Log stream for inference simulator %slug% started",
172
+ {
173
+ 'slug': str(inference_simulator.slug),
174
+ }
175
+ ))
176
+
177
+ typer.echo(__("CTRL+D to disconnect from log stream."))
178
+
179
+ print_task_or_inference_simulator_logs = asyncio.create_task(
180
+ self.print_realtime_logs(inference_simulator_id=inference_simulator.id)
181
+ )
182
+ input_task = asyncio.create_task(self.read_log_stream_input())
183
+
184
+ done, pending = await asyncio.wait([print_task_or_inference_simulator_logs, input_task],
185
+ return_when=asyncio.FIRST_COMPLETED)
186
+
187
+ if input_task in done:
188
+ print_task_or_inference_simulator_logs.cancel()
189
+ typer.echo(__(f"Disconnected from log stream. You can try to reconnect with 'thestage project inference-simulator logs {slug}'."))
190
+
191
+
192
+ @error_handler()
193
+ def stream_container_logs_with_controls(self, container_uid: str):
194
+ asyncio.run(
195
+ self.__stream_container_logs_with_controls_async(
196
+ container_uid=container_uid
197
+ )
198
+ )
199
+
200
+
201
+ @error_handler()
202
+ async def __stream_container_logs_with_controls_async(self, container_uid: str):
203
+ container: Optional[DockerContainerDto] = self.__thestage_api_client.get_container(
204
+ container_slug=container_uid,
205
+ )
206
+
207
+ if container:
208
+ if container.frontend_status.status_key not in [DockerContainerStatus.RUNNING]:
209
+ typer.echo(f"Container status: '{container.frontend_status.status_translation}'")
210
+ else:
211
+ typer.echo("Container was not found")
212
+ raise typer.Exit(1)
213
+
214
+ typer.echo(f"Log stream for Docker container started")
215
+ typer.echo("CTRL+D to disconnect from log stream.")
216
+
217
+ print_logs_task = asyncio.create_task(self.print_realtime_logs(docker_container_id=container.id))
218
+ input_task = asyncio.create_task(self.read_log_stream_input())
219
+
220
+ def sigint_handler():
221
+ input_task.cancel()
222
+
223
+ loop = asyncio.get_event_loop()
224
+ for signal_item in [SIGINT]: # SIGINT == CTRL+C
225
+ loop.add_signal_handler(signal_item, sigint_handler)
226
+
227
+ done, pending = await asyncio.wait([print_logs_task, input_task], return_when=asyncio.FIRST_COMPLETED)
228
+
229
+ if input_task in done:
230
+ print_logs_task.cancel()
231
+
232
+
233
+ async def read_log_stream_input(self):
234
+ try:
235
+ while True:
236
+ input1 = await aioconsole.ainput()
237
+ except EOFError:
238
+ typer.echo(__("\rExited from log stream"))
239
+ return True
240
+ except CancelledError: # Always appears if async task is canceled and leaves huge traces
241
+ pass
242
+
243
+
244
+ async def print_realtime_logs(
245
+ self,
246
+ task_id: Optional[int] = None,
247
+ inference_simulator_id: Optional[int] = None,
248
+ docker_container_id: Optional[int] = None,
249
+ ):
250
+ polling_interval_seconds: float = 4 # also adjust polling api method timeout if changed
251
+ between_logs_sleeping_coef: float = 1 # we emulate delay between logs, but if for any reason code runs for too long - delays will be controlled with this coef
252
+ last_iteration_log_timestamp: Optional[str] = None # pointer to next iteration polling start (obtained from each response)
253
+ last_log_id: Optional[str] = None # pointer to next iteration polling start - to exclude the log id from result (obtained from each response)
254
+ consecutive_error_count: int = 0 # connectivity errors count - stream will disconnect if too many errors in a row
255
+ iteration_started_at: datetime # used to control iteration duration - polling should be done at around exact rate
256
+ errors_started_at: Optional[datetime] = None # time since errors started to stream disconnect
257
+
258
+ is_no_more_logs = False
259
+ while not is_no_more_logs:
260
+ log_wait_remaining_limit: float = polling_interval_seconds # hard limit just in case
261
+ iteration_started_at = datetime.utcnow()
262
+ last_printed_log_timestamp: Optional[datetime] = None
263
+ reader, writer = await aioconsole.get_standard_streams()
264
+
265
+ # this shows (somewhat accurate) time difference between logs here and in real time. should not grow.
266
+ # if last_iteration_log_timestamp:
267
+ # last_log_timestamp_parsed = datetime.strptime(last_iteration_log_timestamp, '%Y-%m-%dT%H:%M:%S.%f')
268
+ # stream_to_logs_diff = datetime.utcnow() - last_log_timestamp_parsed
269
+ # print_nonblocking(f'TDIFF {stream_to_logs_diff.total_seconds()}', writer)
270
+ try:
271
+ logs_response = await self.__thestage_api_client.poll_logs_httpx(
272
+ task_id=task_id,
273
+ inference_simulator_id=inference_simulator_id,
274
+ docker_container_id=docker_container_id,
275
+ last_log_timestamp=last_iteration_log_timestamp,
276
+ last_log_id=last_log_id
277
+ )
278
+
279
+ if not logs_response.is_success:
280
+ app_logger.info(f'Polling logs error: {logs_response.message}')
281
+ raise LogPollingException('')
282
+
283
+
284
+ if consecutive_error_count > 0:
285
+ consecutive_error_count = 0
286
+ errors_started_at = None
287
+ log_wait_remaining_limit = 0 # no log delays after reconnect
288
+
289
+ last_iteration_log_timestamp = logs_response.lastLogTimestamp
290
+ last_log_id = logs_response.lastLogId
291
+
292
+ for log_item in logs_response.logs:
293
+ current_log_timestamp = datetime.strptime(log_item.timestamp[:26], '%Y-%m-%dT%H:%M:%S.%f') # python does not like nanoseconds
294
+ if last_printed_log_timestamp is not None and log_wait_remaining_limit > 0:
295
+ logs_sleeptime = (current_log_timestamp - last_printed_log_timestamp).total_seconds() * between_logs_sleeping_coef
296
+ await asyncio.sleep(logs_sleeptime)
297
+ log_wait_remaining_limit -= logs_sleeptime
298
+ self.__print_log_line_object_nonblocking(log_item, writer)
299
+ last_printed_log_timestamp = current_log_timestamp
300
+ if log_item.messageCode == LOG_MESSAGE_CODE_TASK_FINISHED or log_item.messageCode == LOG_MESSAGE_CODE_INFERENCE_SIMULATOR_FAILED:
301
+ is_no_more_logs = True
302
+
303
+ if is_no_more_logs:
304
+ break
305
+ except (ReadTimeout, ConnectError, ConnectTimeout, LogPollingException) as e:
306
+ consecutive_error_count += 1
307
+ if consecutive_error_count == 1:
308
+ if isinstance(e, LogPollingException):
309
+ print_nonblocking("Some problems raised while getting logs...", writer, BytePrintStyle.ORANGE)
310
+ else:
311
+ print_nonblocking("Network issues, attempting to re-establish connection...", writer, BytePrintStyle.ORANGE)
312
+ if not errors_started_at:
313
+ errors_started_at = datetime.utcnow()
314
+
315
+ if consecutive_error_count > 7:
316
+ seconds_with_error = (datetime.utcnow() - errors_started_at).total_seconds()
317
+ if inference_simulator_id:
318
+ print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage inference-simulator logs <inference-simulator-UID>' to reconnect.", writer)
319
+ elif task_id:
320
+ print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage project task logs {task_id}' to reconnect.", writer)
321
+ elif docker_container_id:
322
+ print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage container logs <docker-container-UID>' to reconnect.", writer)
323
+ else:
324
+ print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds)", writer)
325
+ break
326
+
327
+ # depending on iteration duration - sleep for the remaining time and adjust log sleep coefficient if needed
328
+ iteration_duration = (datetime.utcnow() - iteration_started_at).total_seconds()
329
+ if iteration_duration > polling_interval_seconds:
330
+ between_logs_sleeping_coef *= 0.85
331
+ else:
332
+ await asyncio.sleep(polling_interval_seconds - iteration_duration)
333
+ if between_logs_sleeping_coef < 1:
334
+ between_logs_sleeping_coef = min(1.0, between_logs_sleeping_coef * 1.15)
335
+
336
+
337
+ def __print_log_line(self, log_message_raw_json: str):
338
+ log_message = LogMessage.model_validate_json(log_message_raw_json)
339
+ if not log_message.logType and log_message.message == 'ping':
340
+ return
341
+ self.__print_log_line_object(log_message)
342
+
343
+
344
+ @staticmethod
345
+ def __print_log_line_object(log_message: LogMessage):
346
+ line_color: str = "grey78"
347
+
348
+ if not log_message.logType and log_message.message == 'ping':
349
+ return
350
+
351
+ if log_message.logType == LogType.STDERR.value:
352
+ line_color = "red"
353
+ if log_message.message:
354
+ print(f'[{line_color}][not bold]{log_message.message}[/not bold][/{line_color}]')
355
+
356
+ @staticmethod
357
+ def __print_log_line_object_nonblocking(log_message: LogMessage, writer: StreamWriter):
358
+ if log_message.message:
359
+ line_color: str = BytePrintStyle.RESET
360
+ if log_message.logType == LogType.STDERR.value:
361
+ line_color = BytePrintStyle.RED
362
+
363
+ print_nonblocking(f'{log_message.message}', writer, line_color)
364
+
365
+
366
+ def print_nonblocking(line: str, writer: StreamWriter, color_code: str = BytePrintStyle.RESET):
367
+ writer.write(str.encode(f'{color_code}{line}{BytePrintStyle.RESET}\r\n'))