thestage 0.5.47__py3-none-any.whl → 0.5.49__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- thestage/.env +5 -5
- thestage/__init__.py +3 -4
- thestage/__main__.py +9 -9
- thestage/cli_command.py +56 -56
- thestage/cli_command_helper.py +51 -51
- thestage/color_scheme/color_scheme.py +7 -7
- thestage/config/__init__.py +18 -18
- thestage/config/config_storage.py +5 -3
- thestage/config/env_base.py +7 -7
- thestage/controllers/__init__.py +0 -0
- thestage/controllers/base_controller.py +67 -67
- thestage/controllers/config_controller.py +137 -137
- thestage/controllers/container_controller.py +389 -389
- thestage/controllers/instance_controller.py +183 -183
- thestage/controllers/project_controller.py +802 -783
- thestage/controllers/utils_controller.py +32 -32
- thestage/debug_main.dist.py +28 -28
- thestage/entities/__init__.py +0 -0
- thestage/entities/container.py +17 -17
- thestage/entities/enums/__init__.py +0 -0
- thestage/entities/enums/order_direction_type.py +6 -6
- thestage/entities/enums/shell_type.py +7 -7
- thestage/entities/enums/tail_output_type.py +6 -6
- thestage/entities/enums/yes_no_response.py +7 -7
- thestage/entities/file_item.py +27 -27
- thestage/entities/project_inference_simulator.py +18 -18
- thestage/entities/project_inference_simulator_model.py +16 -16
- thestage/entities/project_task.py +19 -19
- thestage/entities/rented_instance.py +19 -19
- thestage/entities/self_hosted_instance.py +18 -18
- thestage/exceptions/__init__.py +0 -0
- thestage/exceptions/auth_exception.py +6 -6
- thestage/exceptions/base_exception.py +13 -13
- thestage/exceptions/business_logic_exception.py +6 -6
- thestage/exceptions/config_exception.py +6 -6
- thestage/exceptions/file_system_exception.py +6 -6
- thestage/exceptions/git_access_exception.py +17 -17
- thestage/exceptions/remote_server_exception.py +24 -24
- thestage/git/ProgressPrinter.py +22 -22
- thestage/helpers/__init__.py +0 -0
- thestage/helpers/error_handler.py +115 -115
- thestage/helpers/exception_hook.py +14 -14
- thestage/helpers/logger/__init__.py +0 -0
- thestage/helpers/logger/app_logger.py +50 -50
- thestage/helpers/ssh_util.py +38 -38
- thestage/i18n/en_GB/messages.po +947 -947
- thestage/i18n/translation.py +9 -9
- thestage/main.py +36 -36
- thestage/services/.env +6 -6
- thestage/services/__init__.py +0 -0
- thestage/services/abstract_mapper.py +9 -9
- thestage/services/abstract_service.py +87 -87
- thestage/services/app_config_service.py +52 -52
- thestage/services/clients/__init__.py +0 -0
- thestage/services/clients/git/__init__.py +0 -0
- thestage/services/clients/git/git_client.py +433 -331
- thestage/services/clients/thestage_api/__init__.py +0 -0
- thestage/services/clients/thestage_api/api_client.py +718 -720
- thestage/services/clients/thestage_api/core/api_client_core.py +108 -108
- thestage/services/clients/thestage_api/core/http_client_exception.py +12 -12
- thestage/services/clients/thestage_api/dtos/__init__.py +0 -0
- thestage/services/clients/thestage_api/dtos/base_response.py +13 -13
- thestage/services/clients/thestage_api/dtos/cloud_provider_region.py +19 -19
- thestage/services/clients/thestage_api/dtos/container_param_request.py +11 -11
- thestage/services/clients/thestage_api/dtos/container_response.py +67 -67
- thestage/services/clients/thestage_api/dtos/docker_container_assigned_device.py +10 -10
- thestage/services/clients/thestage_api/dtos/docker_container_controller/docker_container_list_request.py +13 -13
- thestage/services/clients/thestage_api/dtos/docker_container_controller/docker_container_list_response.py +13 -13
- thestage/services/clients/thestage_api/dtos/docker_container_mapping.py +10 -10
- thestage/services/clients/thestage_api/dtos/entity_filter_request.py +14 -14
- thestage/services/clients/thestage_api/dtos/enums/__init__.py +0 -0
- thestage/services/clients/thestage_api/dtos/enums/container_pending_action.py +10 -10
- thestage/services/clients/thestage_api/dtos/enums/container_status.py +17 -17
- thestage/services/clients/thestage_api/dtos/enums/cpu_type.py +8 -8
- thestage/services/clients/thestage_api/dtos/enums/currency_type.py +10 -10
- thestage/services/clients/thestage_api/dtos/enums/daemon_status.py +9 -9
- thestage/services/clients/thestage_api/dtos/enums/disk_type.py +7 -7
- thestage/services/clients/thestage_api/dtos/enums/drive_type.py +7 -7
- thestage/services/clients/thestage_api/dtos/enums/gpu_name.py +8 -8
- thestage/services/clients/thestage_api/dtos/enums/inference_model_status.py +9 -9
- thestage/services/clients/thestage_api/dtos/enums/inference_simulator_status.py +15 -15
- thestage/services/clients/thestage_api/dtos/enums/instance_rented_status.py +17 -17
- thestage/services/clients/thestage_api/dtos/enums/instance_type.py +7 -7
- thestage/services/clients/thestage_api/dtos/enums/location_region.py +11 -11
- thestage/services/clients/thestage_api/dtos/enums/power_status.py +10 -10
- thestage/services/clients/thestage_api/dtos/enums/provider_name.py +11 -11
- thestage/services/clients/thestage_api/dtos/enums/selfhosted_status.py +10 -10
- thestage/services/clients/thestage_api/dtos/enums/task_execution_status.py +12 -12
- thestage/services/clients/thestage_api/dtos/enums/task_status.py +12 -12
- thestage/services/clients/thestage_api/dtos/frontend_status.py +10 -10
- thestage/services/clients/thestage_api/dtos/inference_controller/deploy_inference_model_to_instance_request.py +13 -13
- thestage/services/clients/thestage_api/dtos/inference_controller/deploy_inference_model_to_instance_response.py +13 -13
- thestage/services/clients/thestage_api/dtos/inference_controller/deploy_inference_model_to_sagemaker_request.py +12 -12
- thestage/services/clients/thestage_api/dtos/inference_controller/deploy_inference_model_to_sagemaker_response.py +12 -12
- thestage/services/clients/thestage_api/dtos/inference_controller/get_inference_simulator_request.py +10 -10
- thestage/services/clients/thestage_api/dtos/inference_controller/get_inference_simulator_response.py +13 -13
- thestage/services/clients/thestage_api/dtos/inference_controller/inference_simulator_list_for_project_request.py +14 -14
- thestage/services/clients/thestage_api/dtos/inference_controller/inference_simulator_list_for_project_response.py +12 -12
- thestage/services/clients/thestage_api/dtos/inference_controller/inference_simulator_model_list_for_project_request.py +12 -12
- thestage/services/clients/thestage_api/dtos/inference_controller/inference_simulator_model_list_for_project_response.py +13 -13
- thestage/services/clients/thestage_api/dtos/inference_simulator_model_response.py +11 -11
- thestage/services/clients/thestage_api/dtos/inference_simulator_response.py +11 -11
- thestage/services/clients/thestage_api/dtos/installed_service.py +17 -17
- thestage/services/clients/thestage_api/dtos/instance_detected_gpus.py +20 -20
- thestage/services/clients/thestage_api/dtos/instance_rented_response.py +71 -71
- thestage/services/clients/thestage_api/dtos/logging_controller/docker_container_log_stream_request.py +7 -7
- thestage/services/clients/thestage_api/dtos/logging_controller/log_polling_request.py +13 -13
- thestage/services/clients/thestage_api/dtos/logging_controller/log_polling_response.py +14 -14
- thestage/services/clients/thestage_api/dtos/logging_controller/task_log_stream_request.py +7 -7
- thestage/services/clients/thestage_api/dtos/logging_controller/user_logs_query_request.py +21 -21
- thestage/services/clients/thestage_api/dtos/logging_controller/user_logs_query_response.py +14 -14
- thestage/services/clients/thestage_api/dtos/paginated_entity_list.py +11 -11
- thestage/services/clients/thestage_api/dtos/pagination_data.py +10 -10
- thestage/services/clients/thestage_api/dtos/price_definition.py +14 -14
- thestage/services/clients/thestage_api/dtos/project_controller/project_get_deploy_ssh_key_request.py +7 -7
- thestage/services/clients/thestage_api/dtos/project_controller/project_get_deploy_ssh_key_response.py +10 -10
- thestage/services/clients/thestage_api/dtos/project_controller/project_push_inference_simulator_model_request.py +8 -8
- thestage/services/clients/thestage_api/dtos/project_controller/project_push_inference_simulator_model_response.py +6 -6
- thestage/services/clients/thestage_api/dtos/project_controller/project_run_task_request.py +15 -15
- thestage/services/clients/thestage_api/dtos/project_controller/project_run_task_response.py +10 -10
- thestage/services/clients/thestage_api/dtos/project_controller/project_start_inference_simulator_request.py +13 -14
- thestage/services/clients/thestage_api/dtos/project_controller/project_start_inference_simulator_response.py +10 -10
- thestage/services/clients/thestage_api/dtos/project_response.py +32 -32
- thestage/services/clients/thestage_api/dtos/selfhosted_instance_response.py +56 -56
- thestage/services/clients/thestage_api/dtos/sftp_path_helper.py +13 -13
- thestage/services/clients/thestage_api/dtos/ssh_key_controller/add_ssh_key_to_user_request.py +8 -8
- thestage/services/clients/thestage_api/dtos/ssh_key_controller/add_ssh_key_to_user_response.py +11 -11
- thestage/services/clients/thestage_api/dtos/ssh_key_controller/add_ssh_public_key_to_instance_request.py +8 -8
- thestage/services/clients/thestage_api/dtos/ssh_key_controller/add_ssh_public_key_to_instance_response.py +11 -11
- thestage/services/clients/thestage_api/dtos/ssh_key_controller/is_user_has_public_ssh_key_request.py +7 -7
- thestage/services/clients/thestage_api/dtos/ssh_key_controller/is_user_has_public_ssh_key_response.py +12 -12
- thestage/services/clients/thestage_api/dtos/task_controller/task_list_for_project_request.py +10 -10
- thestage/services/clients/thestage_api/dtos/task_controller/task_list_for_project_response.py +12 -12
- thestage/services/clients/thestage_api/dtos/task_controller/task_status_localized_map_response.py +9 -9
- thestage/services/clients/thestage_api/dtos/task_controller/task_view_response.py +12 -12
- thestage/services/clients/thestage_api/dtos/user_controller/user_profile.py +12 -12
- thestage/services/clients/thestage_api/dtos/validate_token_response.py +11 -11
- thestage/services/config_provider/__init__.py +0 -0
- thestage/services/config_provider/config_provider.py +237 -237
- thestage/services/connect/connect_service.py +196 -196
- thestage/services/connect/dto/remote_server_config.py +9 -9
- thestage/services/container/__init__.py +0 -0
- thestage/services/container/container_service.py +374 -374
- thestage/services/container/mapper/__init__.py +0 -0
- thestage/services/container/mapper/container_mapper.py +30 -30
- thestage/services/core_files/config_entity.py +26 -26
- thestage/services/filesystem_service.py +133 -133
- thestage/services/instance/__init__.py +0 -0
- thestage/services/instance/instance_service.py +303 -303
- thestage/services/instance/mapper/__init__.py +0 -0
- thestage/services/instance/mapper/instance_mapper.py +24 -24
- thestage/services/instance/mapper/selfhosted_mapper.py +33 -33
- thestage/services/logging/byte_print_style.py +5 -5
- thestage/services/logging/dto/log_message.py +15 -15
- thestage/services/logging/dto/log_type.py +6 -6
- thestage/services/logging/exception/log_polling_exception.py +6 -6
- thestage/services/logging/logging_constants.py +3 -3
- thestage/services/logging/logging_service.py +367 -367
- thestage/services/project/__init__.py +0 -0
- thestage/services/project/dto/inference_simulator_dto.py +22 -22
- thestage/services/project/dto/inference_simulator_model_dto.py +20 -20
- thestage/services/project/dto/project_config.py +14 -14
- thestage/services/project/mapper/__init__.py +0 -0
- thestage/services/project/mapper/project_inference_simulator_mapper.py +21 -21
- thestage/services/project/mapper/project_inference_simulator_model_mapper.py +21 -21
- thestage/services/project/mapper/project_task_mapper.py +22 -22
- thestage/services/project/project_service.py +1260 -1241
- thestage/services/remote_server_service.py +609 -609
- thestage/services/service_factory.py +97 -97
- thestage/services/task/dto/task_dto.py +40 -40
- thestage/services/validation_service.py +61 -61
- {thestage-0.5.47.dist-info → thestage-0.5.49.dist-info}/LICENSE.txt +12 -12
- {thestage-0.5.47.dist-info → thestage-0.5.49.dist-info}/METADATA +1 -1
- thestage-0.5.49.dist-info/RECORD +176 -0
- {thestage-0.5.47.dist-info → thestage-0.5.49.dist-info}/WHEEL +1 -1
- thestage/debug_tests.py +0 -12
- thestage/services/clients/.DS_Store +0 -0
- thestage-0.5.47.dist-info/RECORD +0 -178
- {thestage-0.5.47.dist-info → thestage-0.5.49.dist-info}/entry_points.txt +0 -0
|
@@ -1,367 +1,367 @@
|
|
|
1
|
-
import asyncio
|
|
2
|
-
from _signal import SIGINT
|
|
3
|
-
from asyncio import CancelledError, Task, StreamWriter
|
|
4
|
-
from datetime import datetime
|
|
5
|
-
from typing import Optional, Dict
|
|
6
|
-
|
|
7
|
-
import aioconsole
|
|
8
|
-
import typer
|
|
9
|
-
from httpx import ReadTimeout, ConnectError, ConnectTimeout
|
|
10
|
-
|
|
11
|
-
from thestage.helpers.logger.app_logger import app_logger
|
|
12
|
-
from thestage.services.clients.thestage_api.dtos.enums.container_status import DockerContainerStatus
|
|
13
|
-
from thestage.services.clients.thestage_api.dtos.enums.inference_simulator_status import InferenceSimulatorStatus
|
|
14
|
-
from thestage.services.clients.thestage_api.dtos.enums.task_status import TaskStatus
|
|
15
|
-
from thestage.services.clients.thestage_api.dtos.inference_controller.get_inference_simulator_response import \
|
|
16
|
-
GetInferenceSimulatorResponse
|
|
17
|
-
from thestage.services.clients.thestage_api.dtos.task_controller.task_view_response import TaskViewResponse
|
|
18
|
-
from thestage.services.logging.byte_print_style import BytePrintStyle
|
|
19
|
-
from thestage.services.logging.dto.log_message import LogMessage
|
|
20
|
-
from thestage.services.logging.dto.log_type import LogType
|
|
21
|
-
from thestage.i18n.translation import __
|
|
22
|
-
from thestage.services.abstract_service import AbstractService
|
|
23
|
-
from thestage.services.clients.thestage_api.dtos.container_response import DockerContainerDto
|
|
24
|
-
from thestage.helpers.error_handler import error_handler
|
|
25
|
-
from thestage.services.clients.thestage_api.api_client import TheStageApiClient
|
|
26
|
-
from rich import print
|
|
27
|
-
|
|
28
|
-
from thestage.services.logging.exception.log_polling_exception import LogPollingException
|
|
29
|
-
from thestage.services.logging.logging_constants import LOG_MESSAGE_CODE_TASK_FINISHED, \
|
|
30
|
-
LOG_MESSAGE_CODE_INFERENCE_SIMULATOR_FAILED
|
|
31
|
-
|
|
32
|
-
is_logs_streaming = False
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
class LoggingService:
|
|
36
|
-
__thestage_api_client: TheStageApiClient = None
|
|
37
|
-
|
|
38
|
-
def __init__(
|
|
39
|
-
self,
|
|
40
|
-
thestage_api_client: TheStageApiClient,
|
|
41
|
-
):
|
|
42
|
-
self.__thestage_api_client = thestage_api_client
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
@error_handler()
|
|
46
|
-
def print_last_task_logs(self, task_id: int, logs_number: Optional[int]):
|
|
47
|
-
logs = self.__thestage_api_client.query_user_logs(
|
|
48
|
-
task_id=task_id,
|
|
49
|
-
limit=logs_number
|
|
50
|
-
)
|
|
51
|
-
for log_message in reversed(logs.queryResult):
|
|
52
|
-
self.__print_log_line_object(log_message)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
@error_handler()
|
|
56
|
-
def print_last_inference_simulator_logs(self, inference_simulator_id: int, logs_number: Optional[int]):
|
|
57
|
-
logs = self.__thestage_api_client.query_user_logs(
|
|
58
|
-
inference_simulator_id=inference_simulator_id,
|
|
59
|
-
limit=logs_number
|
|
60
|
-
)
|
|
61
|
-
for log_message in reversed(logs.queryResult):
|
|
62
|
-
self.__print_log_line_object(log_message)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
@error_handler()
|
|
66
|
-
def print_last_container_logs(self, container_uid: str, logs_number: Optional[int]):
|
|
67
|
-
container: Optional[DockerContainerDto] = self.__thestage_api_client.get_container(
|
|
68
|
-
container_slug=container_uid,
|
|
69
|
-
)
|
|
70
|
-
|
|
71
|
-
if not container:
|
|
72
|
-
typer.echo("Container was not found")
|
|
73
|
-
raise typer.Exit(1)
|
|
74
|
-
|
|
75
|
-
logs = self.__thestage_api_client.query_user_logs(
|
|
76
|
-
container_id=container.id,
|
|
77
|
-
limit=logs_number
|
|
78
|
-
)
|
|
79
|
-
for log_message in reversed(logs.queryResult):
|
|
80
|
-
self.__print_log_line_object(log_message)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
@error_handler()
|
|
84
|
-
def stream_task_logs_with_controls(self, task_id: int):
|
|
85
|
-
asyncio.run(
|
|
86
|
-
self.__stream_task_logs_with_controls_async(task_id=task_id)
|
|
87
|
-
)
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
@error_handler()
|
|
91
|
-
async def __stream_task_logs_with_controls_async(self, task_id: int):
|
|
92
|
-
task_view_response: Optional[TaskViewResponse] = self.__thestage_api_client.get_task(task_id=task_id,)
|
|
93
|
-
|
|
94
|
-
task_status_map: Dict[str, str] = self.__thestage_api_client.get_task_localized_status_map()
|
|
95
|
-
|
|
96
|
-
task = task_view_response.task
|
|
97
|
-
|
|
98
|
-
if task:
|
|
99
|
-
if task.frontend_status.status_key not in [TaskStatus.RUNNING, TaskStatus.SCHEDULED]:
|
|
100
|
-
typer.echo(__("Task must be in status '%required_status%' to stream real-time logs. Task %task_id% status: '%status%'.", {
|
|
101
|
-
'task_id': str(task.id),
|
|
102
|
-
'status': task.frontend_status.status_translation,
|
|
103
|
-
'required_status': task_status_map.get(TaskStatus.RUNNING) or TaskStatus.RUNNING
|
|
104
|
-
}))
|
|
105
|
-
raise typer.Exit(1)
|
|
106
|
-
else:
|
|
107
|
-
typer.echo(__("Task with ID %task_id% was not found", {'task_id': task.id}))
|
|
108
|
-
raise typer.Exit(1)
|
|
109
|
-
|
|
110
|
-
typer.echo(__(
|
|
111
|
-
f"Log stream for task %task_id% started",
|
|
112
|
-
{
|
|
113
|
-
'task_id': str(task.id),
|
|
114
|
-
}
|
|
115
|
-
))
|
|
116
|
-
|
|
117
|
-
typer.echo(__("CTRL+C to cancel the task. CTRL+D to disconnect from log stream."))
|
|
118
|
-
|
|
119
|
-
print_logs_task = asyncio.create_task(self.print_realtime_logs(task_id=task.id))
|
|
120
|
-
input_task = asyncio.create_task(self.read_log_stream_input())
|
|
121
|
-
|
|
122
|
-
def sigint_handler():
|
|
123
|
-
input_task.cancel()
|
|
124
|
-
|
|
125
|
-
loop = asyncio.get_event_loop()
|
|
126
|
-
for signal_item in [SIGINT]: # SIGINT == CTRL+C
|
|
127
|
-
loop.add_signal_handler(signal_item, sigint_handler)
|
|
128
|
-
|
|
129
|
-
done, pending = await asyncio.wait([print_logs_task, input_task], return_when=asyncio.FIRST_COMPLETED)
|
|
130
|
-
|
|
131
|
-
if input_task in done:
|
|
132
|
-
print_logs_task.cancel()
|
|
133
|
-
if not input_task.result(): # result is only expected if ctrl+D triggered EOFError
|
|
134
|
-
typer.echo(f"\rTask {task_id} will be canceled")
|
|
135
|
-
self.__thestage_api_client.cancel_task(
|
|
136
|
-
task_id=task.id,
|
|
137
|
-
)
|
|
138
|
-
|
|
139
|
-
@error_handler()
|
|
140
|
-
def stream_inference_simulator_logs_with_controls(self, slug: str):
|
|
141
|
-
asyncio.run(
|
|
142
|
-
self.__stream_inference_simulator_logs_with_controls_async(
|
|
143
|
-
slug=slug
|
|
144
|
-
)
|
|
145
|
-
)
|
|
146
|
-
|
|
147
|
-
@error_handler()
|
|
148
|
-
async def __stream_inference_simulator_logs_with_controls_async(self, slug: str):
|
|
149
|
-
get_inference_simulator_response: Optional[GetInferenceSimulatorResponse] = self.__thestage_api_client.get_inference_simulator(
|
|
150
|
-
slug=slug,
|
|
151
|
-
)
|
|
152
|
-
|
|
153
|
-
inference_simulator_status_map: Dict[str, str] = self.__thestage_api_client.get_inference_simulator_business_status_map()
|
|
154
|
-
|
|
155
|
-
inference_simulator = get_inference_simulator_response.inferenceSimulator
|
|
156
|
-
|
|
157
|
-
if inference_simulator:
|
|
158
|
-
if inference_simulator.status not in ['SCHEDULED', 'CREATING', 'RUNNING']:
|
|
159
|
-
typer.echo(
|
|
160
|
-
__("Inference simulator must be in status '%required_status%' to stream real-time logs. Inference simulator status: '%status%'.",
|
|
161
|
-
{
|
|
162
|
-
'status': inference_simulator.status,
|
|
163
|
-
'required_status': inference_simulator_status_map.get(InferenceSimulatorStatus.RUNNING) or InferenceSimulatorStatus.RUNNING
|
|
164
|
-
}))
|
|
165
|
-
raise typer.Exit(1)
|
|
166
|
-
else:
|
|
167
|
-
typer.echo(__("Inference simulator with unique ID %slug% was not found", {'slug': inference_simulator.slug}))
|
|
168
|
-
raise typer.Exit(1)
|
|
169
|
-
|
|
170
|
-
typer.echo(__(
|
|
171
|
-
f"Log stream for inference simulator %slug% started",
|
|
172
|
-
{
|
|
173
|
-
'slug': str(inference_simulator.slug),
|
|
174
|
-
}
|
|
175
|
-
))
|
|
176
|
-
|
|
177
|
-
typer.echo(__("CTRL+D to disconnect from log stream."))
|
|
178
|
-
|
|
179
|
-
print_task_or_inference_simulator_logs = asyncio.create_task(
|
|
180
|
-
self.print_realtime_logs(inference_simulator_id=inference_simulator.id)
|
|
181
|
-
)
|
|
182
|
-
input_task = asyncio.create_task(self.read_log_stream_input())
|
|
183
|
-
|
|
184
|
-
done, pending = await asyncio.wait([print_task_or_inference_simulator_logs, input_task],
|
|
185
|
-
return_when=asyncio.FIRST_COMPLETED)
|
|
186
|
-
|
|
187
|
-
if input_task in done:
|
|
188
|
-
print_task_or_inference_simulator_logs.cancel()
|
|
189
|
-
typer.echo(__(f"Disconnected from log stream. You can try to reconnect with 'thestage project inference-simulator logs {slug}'."))
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
@error_handler()
|
|
193
|
-
def stream_container_logs_with_controls(self, container_uid: str):
|
|
194
|
-
asyncio.run(
|
|
195
|
-
self.__stream_container_logs_with_controls_async(
|
|
196
|
-
container_uid=container_uid
|
|
197
|
-
)
|
|
198
|
-
)
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
@error_handler()
|
|
202
|
-
async def __stream_container_logs_with_controls_async(self, container_uid: str):
|
|
203
|
-
container: Optional[DockerContainerDto] = self.__thestage_api_client.get_container(
|
|
204
|
-
container_slug=container_uid,
|
|
205
|
-
)
|
|
206
|
-
|
|
207
|
-
if container:
|
|
208
|
-
if container.frontend_status.status_key not in [DockerContainerStatus.RUNNING]:
|
|
209
|
-
typer.echo(f"Container status: '{container.frontend_status.status_translation}'")
|
|
210
|
-
else:
|
|
211
|
-
typer.echo("Container was not found")
|
|
212
|
-
raise typer.Exit(1)
|
|
213
|
-
|
|
214
|
-
typer.echo(f"Log stream for Docker container started")
|
|
215
|
-
typer.echo("CTRL+D to disconnect from log stream.")
|
|
216
|
-
|
|
217
|
-
print_logs_task = asyncio.create_task(self.print_realtime_logs(docker_container_id=container.id))
|
|
218
|
-
input_task = asyncio.create_task(self.read_log_stream_input())
|
|
219
|
-
|
|
220
|
-
def sigint_handler():
|
|
221
|
-
input_task.cancel()
|
|
222
|
-
|
|
223
|
-
loop = asyncio.get_event_loop()
|
|
224
|
-
for signal_item in [SIGINT]: # SIGINT == CTRL+C
|
|
225
|
-
loop.add_signal_handler(signal_item, sigint_handler)
|
|
226
|
-
|
|
227
|
-
done, pending = await asyncio.wait([print_logs_task, input_task], return_when=asyncio.FIRST_COMPLETED)
|
|
228
|
-
|
|
229
|
-
if input_task in done:
|
|
230
|
-
print_logs_task.cancel()
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
async def read_log_stream_input(self):
|
|
234
|
-
try:
|
|
235
|
-
while True:
|
|
236
|
-
input1 = await aioconsole.ainput()
|
|
237
|
-
except EOFError:
|
|
238
|
-
typer.echo(__("\rExited from log stream"))
|
|
239
|
-
return True
|
|
240
|
-
except CancelledError: # Always appears if async task is canceled and leaves huge traces
|
|
241
|
-
pass
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
async def print_realtime_logs(
|
|
245
|
-
self,
|
|
246
|
-
task_id: Optional[int] = None,
|
|
247
|
-
inference_simulator_id: Optional[int] = None,
|
|
248
|
-
docker_container_id: Optional[int] = None,
|
|
249
|
-
):
|
|
250
|
-
polling_interval_seconds: float = 4 # also adjust polling api method timeout if changed
|
|
251
|
-
between_logs_sleeping_coef: float = 1 # we emulate delay between logs, but if for any reason code runs for too long - delays will be controlled with this coef
|
|
252
|
-
last_iteration_log_timestamp: Optional[str] = None # pointer to next iteration polling start (obtained from each response)
|
|
253
|
-
last_log_id: Optional[str] = None # pointer to next iteration polling start - to exclude the log id from result (obtained from each response)
|
|
254
|
-
consecutive_error_count: int = 0 # connectivity errors count - stream will disconnect if too many errors in a row
|
|
255
|
-
iteration_started_at: datetime # used to control iteration duration - polling should be done at around exact rate
|
|
256
|
-
errors_started_at: Optional[datetime] = None # time since errors started to stream disconnect
|
|
257
|
-
|
|
258
|
-
is_no_more_logs = False
|
|
259
|
-
while not is_no_more_logs:
|
|
260
|
-
log_wait_remaining_limit: float = polling_interval_seconds # hard limit just in case
|
|
261
|
-
iteration_started_at = datetime.utcnow()
|
|
262
|
-
last_printed_log_timestamp: Optional[datetime] = None
|
|
263
|
-
reader, writer = await aioconsole.get_standard_streams()
|
|
264
|
-
|
|
265
|
-
# this shows (somewhat accurate) time difference between logs here and in real time. should not grow.
|
|
266
|
-
# if last_iteration_log_timestamp:
|
|
267
|
-
# last_log_timestamp_parsed = datetime.strptime(last_iteration_log_timestamp, '%Y-%m-%dT%H:%M:%S.%f')
|
|
268
|
-
# stream_to_logs_diff = datetime.utcnow() - last_log_timestamp_parsed
|
|
269
|
-
# print_nonblocking(f'TDIFF {stream_to_logs_diff.total_seconds()}', writer)
|
|
270
|
-
try:
|
|
271
|
-
logs_response = await self.__thestage_api_client.poll_logs_httpx(
|
|
272
|
-
task_id=task_id,
|
|
273
|
-
inference_simulator_id=inference_simulator_id,
|
|
274
|
-
docker_container_id=docker_container_id,
|
|
275
|
-
last_log_timestamp=last_iteration_log_timestamp,
|
|
276
|
-
last_log_id=last_log_id
|
|
277
|
-
)
|
|
278
|
-
|
|
279
|
-
if not logs_response.is_success:
|
|
280
|
-
app_logger.info(f'Polling logs error: {logs_response.message}')
|
|
281
|
-
raise LogPollingException('')
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
if consecutive_error_count > 0:
|
|
285
|
-
consecutive_error_count = 0
|
|
286
|
-
errors_started_at = None
|
|
287
|
-
log_wait_remaining_limit = 0 # no log delays after reconnect
|
|
288
|
-
|
|
289
|
-
last_iteration_log_timestamp = logs_response.lastLogTimestamp
|
|
290
|
-
last_log_id = logs_response.lastLogId
|
|
291
|
-
|
|
292
|
-
for log_item in logs_response.logs:
|
|
293
|
-
current_log_timestamp = datetime.strptime(log_item.timestamp[:26], '%Y-%m-%dT%H:%M:%S.%f') # python does not like nanoseconds
|
|
294
|
-
if last_printed_log_timestamp is not None and log_wait_remaining_limit > 0:
|
|
295
|
-
logs_sleeptime = (current_log_timestamp - last_printed_log_timestamp).total_seconds() * between_logs_sleeping_coef
|
|
296
|
-
await asyncio.sleep(logs_sleeptime)
|
|
297
|
-
log_wait_remaining_limit -= logs_sleeptime
|
|
298
|
-
self.__print_log_line_object_nonblocking(log_item, writer)
|
|
299
|
-
last_printed_log_timestamp = current_log_timestamp
|
|
300
|
-
if log_item.messageCode == LOG_MESSAGE_CODE_TASK_FINISHED or log_item.messageCode == LOG_MESSAGE_CODE_INFERENCE_SIMULATOR_FAILED:
|
|
301
|
-
is_no_more_logs = True
|
|
302
|
-
|
|
303
|
-
if is_no_more_logs:
|
|
304
|
-
break
|
|
305
|
-
except (ReadTimeout, ConnectError, ConnectTimeout, LogPollingException) as e:
|
|
306
|
-
consecutive_error_count += 1
|
|
307
|
-
if consecutive_error_count == 1:
|
|
308
|
-
if isinstance(e, LogPollingException):
|
|
309
|
-
print_nonblocking("Some problems raised while getting logs...", writer, BytePrintStyle.ORANGE)
|
|
310
|
-
else:
|
|
311
|
-
print_nonblocking("Network issues, attempting to re-establish connection...", writer, BytePrintStyle.ORANGE)
|
|
312
|
-
if not errors_started_at:
|
|
313
|
-
errors_started_at = datetime.utcnow()
|
|
314
|
-
|
|
315
|
-
if consecutive_error_count > 7:
|
|
316
|
-
seconds_with_error = (datetime.utcnow() - errors_started_at).total_seconds()
|
|
317
|
-
if inference_simulator_id:
|
|
318
|
-
print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage inference-simulator logs <inference-simulator-UID>' to reconnect.", writer)
|
|
319
|
-
elif task_id:
|
|
320
|
-
print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage project task logs {task_id}' to reconnect.", writer)
|
|
321
|
-
elif docker_container_id:
|
|
322
|
-
print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage container logs <docker-container-UID>' to reconnect.", writer)
|
|
323
|
-
else:
|
|
324
|
-
print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds)", writer)
|
|
325
|
-
break
|
|
326
|
-
|
|
327
|
-
# depending on iteration duration - sleep for the remaining time and adjust log sleep coefficient if needed
|
|
328
|
-
iteration_duration = (datetime.utcnow() - iteration_started_at).total_seconds()
|
|
329
|
-
if iteration_duration > polling_interval_seconds:
|
|
330
|
-
between_logs_sleeping_coef *= 0.85
|
|
331
|
-
else:
|
|
332
|
-
await asyncio.sleep(polling_interval_seconds - iteration_duration)
|
|
333
|
-
if between_logs_sleeping_coef < 1:
|
|
334
|
-
between_logs_sleeping_coef = min(1.0, between_logs_sleeping_coef * 1.15)
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
def __print_log_line(self, log_message_raw_json: str):
|
|
338
|
-
log_message = LogMessage.model_validate_json(log_message_raw_json)
|
|
339
|
-
if not log_message.logType and log_message.message == 'ping':
|
|
340
|
-
return
|
|
341
|
-
self.__print_log_line_object(log_message)
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
@staticmethod
|
|
345
|
-
def __print_log_line_object(log_message: LogMessage):
|
|
346
|
-
line_color: str = "grey78"
|
|
347
|
-
|
|
348
|
-
if not log_message.logType and log_message.message == 'ping':
|
|
349
|
-
return
|
|
350
|
-
|
|
351
|
-
if log_message.logType == LogType.STDERR.value:
|
|
352
|
-
line_color = "red"
|
|
353
|
-
if log_message.message:
|
|
354
|
-
print(f'[{line_color}][not bold]{log_message.message}[/not bold][/{line_color}]')
|
|
355
|
-
|
|
356
|
-
@staticmethod
|
|
357
|
-
def __print_log_line_object_nonblocking(log_message: LogMessage, writer: StreamWriter):
|
|
358
|
-
if log_message.message:
|
|
359
|
-
line_color: str = BytePrintStyle.RESET
|
|
360
|
-
if log_message.logType == LogType.STDERR.value:
|
|
361
|
-
line_color = BytePrintStyle.RED
|
|
362
|
-
|
|
363
|
-
print_nonblocking(f'{log_message.message}', writer, line_color)
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
def print_nonblocking(line: str, writer: StreamWriter, color_code: str = BytePrintStyle.RESET):
|
|
367
|
-
writer.write(str.encode(f'{color_code}{line}{BytePrintStyle.RESET}\r\n'))
|
|
1
|
+
import asyncio
|
|
2
|
+
from _signal import SIGINT
|
|
3
|
+
from asyncio import CancelledError, Task, StreamWriter
|
|
4
|
+
from datetime import datetime
|
|
5
|
+
from typing import Optional, Dict
|
|
6
|
+
|
|
7
|
+
import aioconsole
|
|
8
|
+
import typer
|
|
9
|
+
from httpx import ReadTimeout, ConnectError, ConnectTimeout
|
|
10
|
+
|
|
11
|
+
from thestage.helpers.logger.app_logger import app_logger
|
|
12
|
+
from thestage.services.clients.thestage_api.dtos.enums.container_status import DockerContainerStatus
|
|
13
|
+
from thestage.services.clients.thestage_api.dtos.enums.inference_simulator_status import InferenceSimulatorStatus
|
|
14
|
+
from thestage.services.clients.thestage_api.dtos.enums.task_status import TaskStatus
|
|
15
|
+
from thestage.services.clients.thestage_api.dtos.inference_controller.get_inference_simulator_response import \
|
|
16
|
+
GetInferenceSimulatorResponse
|
|
17
|
+
from thestage.services.clients.thestage_api.dtos.task_controller.task_view_response import TaskViewResponse
|
|
18
|
+
from thestage.services.logging.byte_print_style import BytePrintStyle
|
|
19
|
+
from thestage.services.logging.dto.log_message import LogMessage
|
|
20
|
+
from thestage.services.logging.dto.log_type import LogType
|
|
21
|
+
from thestage.i18n.translation import __
|
|
22
|
+
from thestage.services.abstract_service import AbstractService
|
|
23
|
+
from thestage.services.clients.thestage_api.dtos.container_response import DockerContainerDto
|
|
24
|
+
from thestage.helpers.error_handler import error_handler
|
|
25
|
+
from thestage.services.clients.thestage_api.api_client import TheStageApiClient
|
|
26
|
+
from rich import print
|
|
27
|
+
|
|
28
|
+
from thestage.services.logging.exception.log_polling_exception import LogPollingException
|
|
29
|
+
from thestage.services.logging.logging_constants import LOG_MESSAGE_CODE_TASK_FINISHED, \
|
|
30
|
+
LOG_MESSAGE_CODE_INFERENCE_SIMULATOR_FAILED
|
|
31
|
+
|
|
32
|
+
is_logs_streaming = False
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class LoggingService:
|
|
36
|
+
__thestage_api_client: TheStageApiClient = None
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
thestage_api_client: TheStageApiClient,
|
|
41
|
+
):
|
|
42
|
+
self.__thestage_api_client = thestage_api_client
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@error_handler()
|
|
46
|
+
def print_last_task_logs(self, task_id: int, logs_number: Optional[int]):
|
|
47
|
+
logs = self.__thestage_api_client.query_user_logs(
|
|
48
|
+
task_id=task_id,
|
|
49
|
+
limit=logs_number
|
|
50
|
+
)
|
|
51
|
+
for log_message in reversed(logs.queryResult):
|
|
52
|
+
self.__print_log_line_object(log_message)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@error_handler()
|
|
56
|
+
def print_last_inference_simulator_logs(self, inference_simulator_id: int, logs_number: Optional[int]):
|
|
57
|
+
logs = self.__thestage_api_client.query_user_logs(
|
|
58
|
+
inference_simulator_id=inference_simulator_id,
|
|
59
|
+
limit=logs_number
|
|
60
|
+
)
|
|
61
|
+
for log_message in reversed(logs.queryResult):
|
|
62
|
+
self.__print_log_line_object(log_message)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@error_handler()
|
|
66
|
+
def print_last_container_logs(self, container_uid: str, logs_number: Optional[int]):
|
|
67
|
+
container: Optional[DockerContainerDto] = self.__thestage_api_client.get_container(
|
|
68
|
+
container_slug=container_uid,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
if not container:
|
|
72
|
+
typer.echo("Container was not found")
|
|
73
|
+
raise typer.Exit(1)
|
|
74
|
+
|
|
75
|
+
logs = self.__thestage_api_client.query_user_logs(
|
|
76
|
+
container_id=container.id,
|
|
77
|
+
limit=logs_number
|
|
78
|
+
)
|
|
79
|
+
for log_message in reversed(logs.queryResult):
|
|
80
|
+
self.__print_log_line_object(log_message)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@error_handler()
|
|
84
|
+
def stream_task_logs_with_controls(self, task_id: int):
|
|
85
|
+
asyncio.run(
|
|
86
|
+
self.__stream_task_logs_with_controls_async(task_id=task_id)
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@error_handler()
|
|
91
|
+
async def __stream_task_logs_with_controls_async(self, task_id: int):
|
|
92
|
+
task_view_response: Optional[TaskViewResponse] = self.__thestage_api_client.get_task(task_id=task_id,)
|
|
93
|
+
|
|
94
|
+
task_status_map: Dict[str, str] = self.__thestage_api_client.get_task_localized_status_map()
|
|
95
|
+
|
|
96
|
+
task = task_view_response.task
|
|
97
|
+
|
|
98
|
+
if task:
|
|
99
|
+
if task.frontend_status.status_key not in [TaskStatus.RUNNING, TaskStatus.SCHEDULED]:
|
|
100
|
+
typer.echo(__("Task must be in status '%required_status%' to stream real-time logs. Task %task_id% status: '%status%'.", {
|
|
101
|
+
'task_id': str(task.id),
|
|
102
|
+
'status': task.frontend_status.status_translation,
|
|
103
|
+
'required_status': task_status_map.get(TaskStatus.RUNNING) or TaskStatus.RUNNING
|
|
104
|
+
}))
|
|
105
|
+
raise typer.Exit(1)
|
|
106
|
+
else:
|
|
107
|
+
typer.echo(__("Task with ID %task_id% was not found", {'task_id': task.id}))
|
|
108
|
+
raise typer.Exit(1)
|
|
109
|
+
|
|
110
|
+
typer.echo(__(
|
|
111
|
+
f"Log stream for task %task_id% started",
|
|
112
|
+
{
|
|
113
|
+
'task_id': str(task.id),
|
|
114
|
+
}
|
|
115
|
+
))
|
|
116
|
+
|
|
117
|
+
typer.echo(__("CTRL+C to cancel the task. CTRL+D to disconnect from log stream."))
|
|
118
|
+
|
|
119
|
+
print_logs_task = asyncio.create_task(self.print_realtime_logs(task_id=task.id))
|
|
120
|
+
input_task = asyncio.create_task(self.read_log_stream_input())
|
|
121
|
+
|
|
122
|
+
def sigint_handler():
|
|
123
|
+
input_task.cancel()
|
|
124
|
+
|
|
125
|
+
loop = asyncio.get_event_loop()
|
|
126
|
+
for signal_item in [SIGINT]: # SIGINT == CTRL+C
|
|
127
|
+
loop.add_signal_handler(signal_item, sigint_handler)
|
|
128
|
+
|
|
129
|
+
done, pending = await asyncio.wait([print_logs_task, input_task], return_when=asyncio.FIRST_COMPLETED)
|
|
130
|
+
|
|
131
|
+
if input_task in done:
|
|
132
|
+
print_logs_task.cancel()
|
|
133
|
+
if not input_task.result(): # result is only expected if ctrl+D triggered EOFError
|
|
134
|
+
typer.echo(f"\rTask {task_id} will be canceled")
|
|
135
|
+
self.__thestage_api_client.cancel_task(
|
|
136
|
+
task_id=task.id,
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
@error_handler()
|
|
140
|
+
def stream_inference_simulator_logs_with_controls(self, slug: str):
|
|
141
|
+
asyncio.run(
|
|
142
|
+
self.__stream_inference_simulator_logs_with_controls_async(
|
|
143
|
+
slug=slug
|
|
144
|
+
)
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
@error_handler()
|
|
148
|
+
async def __stream_inference_simulator_logs_with_controls_async(self, slug: str):
|
|
149
|
+
get_inference_simulator_response: Optional[GetInferenceSimulatorResponse] = self.__thestage_api_client.get_inference_simulator(
|
|
150
|
+
slug=slug,
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
inference_simulator_status_map: Dict[str, str] = self.__thestage_api_client.get_inference_simulator_business_status_map()
|
|
154
|
+
|
|
155
|
+
inference_simulator = get_inference_simulator_response.inferenceSimulator
|
|
156
|
+
|
|
157
|
+
if inference_simulator:
|
|
158
|
+
if inference_simulator.status not in ['SCHEDULED', 'CREATING', 'RUNNING']:
|
|
159
|
+
typer.echo(
|
|
160
|
+
__("Inference simulator must be in status '%required_status%' to stream real-time logs. Inference simulator status: '%status%'.",
|
|
161
|
+
{
|
|
162
|
+
'status': inference_simulator.status,
|
|
163
|
+
'required_status': inference_simulator_status_map.get(InferenceSimulatorStatus.RUNNING) or InferenceSimulatorStatus.RUNNING
|
|
164
|
+
}))
|
|
165
|
+
raise typer.Exit(1)
|
|
166
|
+
else:
|
|
167
|
+
typer.echo(__("Inference simulator with unique ID %slug% was not found", {'slug': inference_simulator.slug}))
|
|
168
|
+
raise typer.Exit(1)
|
|
169
|
+
|
|
170
|
+
typer.echo(__(
|
|
171
|
+
f"Log stream for inference simulator %slug% started",
|
|
172
|
+
{
|
|
173
|
+
'slug': str(inference_simulator.slug),
|
|
174
|
+
}
|
|
175
|
+
))
|
|
176
|
+
|
|
177
|
+
typer.echo(__("CTRL+D to disconnect from log stream."))
|
|
178
|
+
|
|
179
|
+
print_task_or_inference_simulator_logs = asyncio.create_task(
|
|
180
|
+
self.print_realtime_logs(inference_simulator_id=inference_simulator.id)
|
|
181
|
+
)
|
|
182
|
+
input_task = asyncio.create_task(self.read_log_stream_input())
|
|
183
|
+
|
|
184
|
+
done, pending = await asyncio.wait([print_task_or_inference_simulator_logs, input_task],
|
|
185
|
+
return_when=asyncio.FIRST_COMPLETED)
|
|
186
|
+
|
|
187
|
+
if input_task in done:
|
|
188
|
+
print_task_or_inference_simulator_logs.cancel()
|
|
189
|
+
typer.echo(__(f"Disconnected from log stream. You can try to reconnect with 'thestage project inference-simulator logs {slug}'."))
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
@error_handler()
|
|
193
|
+
def stream_container_logs_with_controls(self, container_uid: str):
|
|
194
|
+
asyncio.run(
|
|
195
|
+
self.__stream_container_logs_with_controls_async(
|
|
196
|
+
container_uid=container_uid
|
|
197
|
+
)
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@error_handler()
|
|
202
|
+
async def __stream_container_logs_with_controls_async(self, container_uid: str):
|
|
203
|
+
container: Optional[DockerContainerDto] = self.__thestage_api_client.get_container(
|
|
204
|
+
container_slug=container_uid,
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
if container:
|
|
208
|
+
if container.frontend_status.status_key not in [DockerContainerStatus.RUNNING]:
|
|
209
|
+
typer.echo(f"Container status: '{container.frontend_status.status_translation}'")
|
|
210
|
+
else:
|
|
211
|
+
typer.echo("Container was not found")
|
|
212
|
+
raise typer.Exit(1)
|
|
213
|
+
|
|
214
|
+
typer.echo(f"Log stream for Docker container started")
|
|
215
|
+
typer.echo("CTRL+D to disconnect from log stream.")
|
|
216
|
+
|
|
217
|
+
print_logs_task = asyncio.create_task(self.print_realtime_logs(docker_container_id=container.id))
|
|
218
|
+
input_task = asyncio.create_task(self.read_log_stream_input())
|
|
219
|
+
|
|
220
|
+
def sigint_handler():
|
|
221
|
+
input_task.cancel()
|
|
222
|
+
|
|
223
|
+
loop = asyncio.get_event_loop()
|
|
224
|
+
for signal_item in [SIGINT]: # SIGINT == CTRL+C
|
|
225
|
+
loop.add_signal_handler(signal_item, sigint_handler)
|
|
226
|
+
|
|
227
|
+
done, pending = await asyncio.wait([print_logs_task, input_task], return_when=asyncio.FIRST_COMPLETED)
|
|
228
|
+
|
|
229
|
+
if input_task in done:
|
|
230
|
+
print_logs_task.cancel()
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
async def read_log_stream_input(self):
|
|
234
|
+
try:
|
|
235
|
+
while True:
|
|
236
|
+
input1 = await aioconsole.ainput()
|
|
237
|
+
except EOFError:
|
|
238
|
+
typer.echo(__("\rExited from log stream"))
|
|
239
|
+
return True
|
|
240
|
+
except CancelledError: # Always appears if async task is canceled and leaves huge traces
|
|
241
|
+
pass
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
async def print_realtime_logs(
|
|
245
|
+
self,
|
|
246
|
+
task_id: Optional[int] = None,
|
|
247
|
+
inference_simulator_id: Optional[int] = None,
|
|
248
|
+
docker_container_id: Optional[int] = None,
|
|
249
|
+
):
|
|
250
|
+
polling_interval_seconds: float = 4 # also adjust polling api method timeout if changed
|
|
251
|
+
between_logs_sleeping_coef: float = 1 # we emulate delay between logs, but if for any reason code runs for too long - delays will be controlled with this coef
|
|
252
|
+
last_iteration_log_timestamp: Optional[str] = None # pointer to next iteration polling start (obtained from each response)
|
|
253
|
+
last_log_id: Optional[str] = None # pointer to next iteration polling start - to exclude the log id from result (obtained from each response)
|
|
254
|
+
consecutive_error_count: int = 0 # connectivity errors count - stream will disconnect if too many errors in a row
|
|
255
|
+
iteration_started_at: datetime # used to control iteration duration - polling should be done at around exact rate
|
|
256
|
+
errors_started_at: Optional[datetime] = None # time since errors started to stream disconnect
|
|
257
|
+
|
|
258
|
+
is_no_more_logs = False
|
|
259
|
+
while not is_no_more_logs:
|
|
260
|
+
log_wait_remaining_limit: float = polling_interval_seconds # hard limit just in case
|
|
261
|
+
iteration_started_at = datetime.utcnow()
|
|
262
|
+
last_printed_log_timestamp: Optional[datetime] = None
|
|
263
|
+
reader, writer = await aioconsole.get_standard_streams()
|
|
264
|
+
|
|
265
|
+
# this shows (somewhat accurate) time difference between logs here and in real time. should not grow.
|
|
266
|
+
# if last_iteration_log_timestamp:
|
|
267
|
+
# last_log_timestamp_parsed = datetime.strptime(last_iteration_log_timestamp, '%Y-%m-%dT%H:%M:%S.%f')
|
|
268
|
+
# stream_to_logs_diff = datetime.utcnow() - last_log_timestamp_parsed
|
|
269
|
+
# print_nonblocking(f'TDIFF {stream_to_logs_diff.total_seconds()}', writer)
|
|
270
|
+
try:
|
|
271
|
+
logs_response = await self.__thestage_api_client.poll_logs_httpx(
|
|
272
|
+
task_id=task_id,
|
|
273
|
+
inference_simulator_id=inference_simulator_id,
|
|
274
|
+
docker_container_id=docker_container_id,
|
|
275
|
+
last_log_timestamp=last_iteration_log_timestamp,
|
|
276
|
+
last_log_id=last_log_id
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
if not logs_response.is_success:
|
|
280
|
+
app_logger.info(f'Polling logs error: {logs_response.message}')
|
|
281
|
+
raise LogPollingException('')
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
if consecutive_error_count > 0:
|
|
285
|
+
consecutive_error_count = 0
|
|
286
|
+
errors_started_at = None
|
|
287
|
+
log_wait_remaining_limit = 0 # no log delays after reconnect
|
|
288
|
+
|
|
289
|
+
last_iteration_log_timestamp = logs_response.lastLogTimestamp
|
|
290
|
+
last_log_id = logs_response.lastLogId
|
|
291
|
+
|
|
292
|
+
for log_item in logs_response.logs:
|
|
293
|
+
current_log_timestamp = datetime.strptime(log_item.timestamp[:26], '%Y-%m-%dT%H:%M:%S.%f') # python does not like nanoseconds
|
|
294
|
+
if last_printed_log_timestamp is not None and log_wait_remaining_limit > 0:
|
|
295
|
+
logs_sleeptime = (current_log_timestamp - last_printed_log_timestamp).total_seconds() * between_logs_sleeping_coef
|
|
296
|
+
await asyncio.sleep(logs_sleeptime)
|
|
297
|
+
log_wait_remaining_limit -= logs_sleeptime
|
|
298
|
+
self.__print_log_line_object_nonblocking(log_item, writer)
|
|
299
|
+
last_printed_log_timestamp = current_log_timestamp
|
|
300
|
+
if log_item.messageCode == LOG_MESSAGE_CODE_TASK_FINISHED or log_item.messageCode == LOG_MESSAGE_CODE_INFERENCE_SIMULATOR_FAILED:
|
|
301
|
+
is_no_more_logs = True
|
|
302
|
+
|
|
303
|
+
if is_no_more_logs:
|
|
304
|
+
break
|
|
305
|
+
except (ReadTimeout, ConnectError, ConnectTimeout, LogPollingException) as e:
|
|
306
|
+
consecutive_error_count += 1
|
|
307
|
+
if consecutive_error_count == 1:
|
|
308
|
+
if isinstance(e, LogPollingException):
|
|
309
|
+
print_nonblocking("Some problems raised while getting logs...", writer, BytePrintStyle.ORANGE)
|
|
310
|
+
else:
|
|
311
|
+
print_nonblocking("Network issues, attempting to re-establish connection...", writer, BytePrintStyle.ORANGE)
|
|
312
|
+
if not errors_started_at:
|
|
313
|
+
errors_started_at = datetime.utcnow()
|
|
314
|
+
|
|
315
|
+
if consecutive_error_count > 7:
|
|
316
|
+
seconds_with_error = (datetime.utcnow() - errors_started_at).total_seconds()
|
|
317
|
+
if inference_simulator_id:
|
|
318
|
+
print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage inference-simulator logs <inference-simulator-UID>' to reconnect.", writer)
|
|
319
|
+
elif task_id:
|
|
320
|
+
print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage project task logs {task_id}' to reconnect.", writer)
|
|
321
|
+
elif docker_container_id:
|
|
322
|
+
print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds). Try 'thestage container logs <docker-container-UID>' to reconnect.", writer)
|
|
323
|
+
else:
|
|
324
|
+
print_nonblocking(f"Log stream: disconnected from server (connectivity issues for {seconds_with_error} seconds)", writer)
|
|
325
|
+
break
|
|
326
|
+
|
|
327
|
+
# depending on iteration duration - sleep for the remaining time and adjust log sleep coefficient if needed
|
|
328
|
+
iteration_duration = (datetime.utcnow() - iteration_started_at).total_seconds()
|
|
329
|
+
if iteration_duration > polling_interval_seconds:
|
|
330
|
+
between_logs_sleeping_coef *= 0.85
|
|
331
|
+
else:
|
|
332
|
+
await asyncio.sleep(polling_interval_seconds - iteration_duration)
|
|
333
|
+
if between_logs_sleeping_coef < 1:
|
|
334
|
+
between_logs_sleeping_coef = min(1.0, between_logs_sleeping_coef * 1.15)
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def __print_log_line(self, log_message_raw_json: str):
|
|
338
|
+
log_message = LogMessage.model_validate_json(log_message_raw_json)
|
|
339
|
+
if not log_message.logType and log_message.message == 'ping':
|
|
340
|
+
return
|
|
341
|
+
self.__print_log_line_object(log_message)
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
@staticmethod
|
|
345
|
+
def __print_log_line_object(log_message: LogMessage):
|
|
346
|
+
line_color: str = "grey78"
|
|
347
|
+
|
|
348
|
+
if not log_message.logType and log_message.message == 'ping':
|
|
349
|
+
return
|
|
350
|
+
|
|
351
|
+
if log_message.logType == LogType.STDERR.value:
|
|
352
|
+
line_color = "red"
|
|
353
|
+
if log_message.message:
|
|
354
|
+
print(f'[{line_color}][not bold]{log_message.message}[/not bold][/{line_color}]')
|
|
355
|
+
|
|
356
|
+
@staticmethod
|
|
357
|
+
def __print_log_line_object_nonblocking(log_message: LogMessage, writer: StreamWriter):
|
|
358
|
+
if log_message.message:
|
|
359
|
+
line_color: str = BytePrintStyle.RESET
|
|
360
|
+
if log_message.logType == LogType.STDERR.value:
|
|
361
|
+
line_color = BytePrintStyle.RED
|
|
362
|
+
|
|
363
|
+
print_nonblocking(f'{log_message.message}', writer, line_color)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def print_nonblocking(line: str, writer: StreamWriter, color_code: str = BytePrintStyle.RESET):
|
|
367
|
+
writer.write(str.encode(f'{color_code}{line}{BytePrintStyle.RESET}\r\n'))
|