trainwave-cli 0.2.5__tar.gz → 0.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/PKG-INFO +1 -1
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/pyproject.toml +1 -1
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/api.py +12 -12
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/cli/jobs.py +3 -53
- trainwave_cli-0.2.6/trainwave_cli/cli/logs.py +270 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/README.md +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/__init__.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/cli/auth.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/cli/offers.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/cli/secrets.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/cli/setup.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/config/config.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/format_utils.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/io.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/main.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/utils.py +0 -0
- {trainwave_cli-0.2.5 → trainwave_cli-0.2.6}/trainwave_cli/xceptions.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "trainwave-cli"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.6"
|
|
4
4
|
description = "Trainwave CLI - A command-line interface for managing Trainwave resources"
|
|
5
5
|
authors = ["Johan Backman <johan@trainwave.ai>"]
|
|
6
6
|
maintainers = ["Johan Backman <johan@trainwave.ai>"]
|
|
@@ -34,12 +34,12 @@ class JobStatus(Enum):
|
|
|
34
34
|
|
|
35
35
|
@dataclass(frozen=True)
|
|
36
36
|
class CloudOffer:
|
|
37
|
-
cpus: int
|
|
38
|
-
memory_mb: int
|
|
39
|
-
compliance_soc2: bool
|
|
40
|
-
gpu_type: str
|
|
41
|
-
gpu_memory_mb: int
|
|
42
|
-
gpus: int
|
|
37
|
+
cpus: int = 0
|
|
38
|
+
memory_mb: int = 0
|
|
39
|
+
compliance_soc2: bool = False
|
|
40
|
+
gpu_type: str | None = None
|
|
41
|
+
gpu_memory_mb: int = 0
|
|
42
|
+
gpus: int = 0
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
@dataclass
|
|
@@ -47,19 +47,19 @@ class JobConfig:
|
|
|
47
47
|
id: str
|
|
48
48
|
rid: str
|
|
49
49
|
name: str
|
|
50
|
-
expires_at: int
|
|
51
|
-
cpus: int
|
|
52
|
-
gpus: int
|
|
53
|
-
gpu_type: str
|
|
50
|
+
expires_at: int = -1
|
|
51
|
+
cpus: int = 0
|
|
52
|
+
gpus: int = 0
|
|
53
|
+
gpu_type: str | None = None
|
|
54
54
|
|
|
55
55
|
|
|
56
56
|
@dataclass(frozen=True)
|
|
57
57
|
class TrainWaveUser:
|
|
58
58
|
id: str
|
|
59
59
|
rid: str
|
|
60
|
-
first_name: str
|
|
61
|
-
last_name: str
|
|
62
60
|
email: str
|
|
61
|
+
first_name: str | None = None
|
|
62
|
+
last_name: str | None = None
|
|
63
63
|
|
|
64
64
|
|
|
65
65
|
@dataclass(frozen=True)
|
|
@@ -1,22 +1,19 @@
|
|
|
1
|
-
import json
|
|
2
1
|
import os
|
|
3
|
-
import sys
|
|
4
2
|
import traceback
|
|
5
3
|
from dataclasses import asdict
|
|
6
4
|
from datetime import datetime, timezone
|
|
7
5
|
from pathlib import Path
|
|
8
6
|
from typing import Annotated
|
|
9
|
-
from urllib.parse import urlparse
|
|
10
7
|
|
|
11
8
|
import human_readable
|
|
12
9
|
import pytimeparse
|
|
13
10
|
import typer
|
|
14
|
-
import websockets
|
|
15
11
|
from loguru import logger
|
|
16
12
|
from tabulate import tabulate
|
|
17
13
|
from typer_config.decorators import use_toml_config
|
|
18
14
|
|
|
19
|
-
from trainwave_cli.api import Api
|
|
15
|
+
from trainwave_cli.api import Api
|
|
16
|
+
from trainwave_cli.cli.logs import stream_logs
|
|
20
17
|
from trainwave_cli.config.config import config
|
|
21
18
|
from trainwave_cli.format_utils import cents_to_dollars_str
|
|
22
19
|
from trainwave_cli.io import create_tarball
|
|
@@ -219,51 +216,4 @@ async def logs(
|
|
|
219
216
|
job_id: str,
|
|
220
217
|
) -> None:
|
|
221
218
|
"""Get the logs of a job."""
|
|
222
|
-
|
|
223
|
-
job = await api_client.get_job(job_id)
|
|
224
|
-
|
|
225
|
-
if job.state in (JobStatus.SUBMIT_CODE, JobStatus.LAUNCHING):
|
|
226
|
-
typer.echo(f"Job {job_id} has not started yet.")
|
|
227
|
-
typer.Exit(1)
|
|
228
|
-
|
|
229
|
-
logger.remove()
|
|
230
|
-
logger.add(
|
|
231
|
-
sys.stdout,
|
|
232
|
-
format="<green>{extra[log_time]}</green> | <white>{message}</white>",
|
|
233
|
-
colorize=True,
|
|
234
|
-
serialize=False,
|
|
235
|
-
level="DEBUG",
|
|
236
|
-
enqueue=True,
|
|
237
|
-
catch=True,
|
|
238
|
-
)
|
|
239
|
-
parsed = urlparse(config.endpoint)
|
|
240
|
-
scheme = "wss" if parsed.scheme == "https" else "ws"
|
|
241
|
-
port = ""
|
|
242
|
-
if parsed.port:
|
|
243
|
-
port = f":{parsed.port}"
|
|
244
|
-
uri = (
|
|
245
|
-
f"{scheme}://{parsed.hostname}{port}/ws/logs/{job_id}/?api_key={config.api_key}"
|
|
246
|
-
)
|
|
247
|
-
|
|
248
|
-
def format_ts(ts: str | None) -> str:
|
|
249
|
-
if ts is None:
|
|
250
|
-
return ""
|
|
251
|
-
timestamp = int(ts) / 1e9
|
|
252
|
-
dt = datetime.fromtimestamp(timestamp)
|
|
253
|
-
return dt.strftime("%Y-%m-%d %H:%M:%S.%f")
|
|
254
|
-
|
|
255
|
-
ssl_context = None
|
|
256
|
-
if "trainwave.dev" in uri:
|
|
257
|
-
import ssl
|
|
258
|
-
|
|
259
|
-
ssl_context = ssl.SSLContext()
|
|
260
|
-
ssl_context.verify_mode = ssl.CERT_NONE
|
|
261
|
-
opts = {"ssl": ssl_context} if ssl_context else {}
|
|
262
|
-
|
|
263
|
-
async with websockets.connect(uri, **opts) as websocket:
|
|
264
|
-
try:
|
|
265
|
-
while response := await websocket.recv():
|
|
266
|
-
data = json.loads(response)
|
|
267
|
-
logger.info(data["message"], log_time=format_ts(data.get("ts")))
|
|
268
|
-
except websockets.exceptions.ConnectionClosedOK:
|
|
269
|
-
pass
|
|
219
|
+
await stream_logs(job_id)
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import json
|
|
3
|
+
import signal
|
|
4
|
+
import ssl
|
|
5
|
+
import sys
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from urllib.parse import urlparse
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
import websockets
|
|
11
|
+
from loguru import logger
|
|
12
|
+
|
|
13
|
+
from trainwave_cli.api import Api, JobStatus
|
|
14
|
+
from trainwave_cli.config.config import config
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def format_timestamp(ts: str | None) -> str:
|
|
18
|
+
"""
|
|
19
|
+
Convert a nanosecond timestamp to a human-readable datetime string.
|
|
20
|
+
|
|
21
|
+
Args:
|
|
22
|
+
----
|
|
23
|
+
ts: Timestamp in nanoseconds as a string, or None
|
|
24
|
+
|
|
25
|
+
Returns:
|
|
26
|
+
-------
|
|
27
|
+
Formatted datetime string in YYYY-MM-DD HH:MM:SS.ffffff format, or empty string if ts is None
|
|
28
|
+
|
|
29
|
+
"""
|
|
30
|
+
if ts is None:
|
|
31
|
+
return ""
|
|
32
|
+
timestamp = int(ts) / 1e9
|
|
33
|
+
dt = datetime.fromtimestamp(timestamp)
|
|
34
|
+
return dt.strftime("%Y-%m-%d %H:%M:%S.%f")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def setup_logger():
|
|
38
|
+
"""
|
|
39
|
+
Configure the logger with appropriate formatting for the logs command.
|
|
40
|
+
|
|
41
|
+
Removes any existing handlers and adds a new handler that outputs to stdout
|
|
42
|
+
with colorized timestamps and messages.
|
|
43
|
+
"""
|
|
44
|
+
logger.remove()
|
|
45
|
+
logger.add(
|
|
46
|
+
sys.stdout,
|
|
47
|
+
format="<green>{extra[log_time]}</green> | <white>{message}</white>",
|
|
48
|
+
colorize=True,
|
|
49
|
+
serialize=False,
|
|
50
|
+
level="DEBUG",
|
|
51
|
+
enqueue=True,
|
|
52
|
+
catch=True,
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def build_websocket_uri(job_id: str) -> str:
|
|
57
|
+
"""
|
|
58
|
+
Construct the WebSocket URI for connecting to job logs.
|
|
59
|
+
|
|
60
|
+
Args:
|
|
61
|
+
----
|
|
62
|
+
job_id: The ID of the job to connect to
|
|
63
|
+
|
|
64
|
+
Returns:
|
|
65
|
+
-------
|
|
66
|
+
Complete WebSocket URI including protocol, host, port, path, and API key
|
|
67
|
+
|
|
68
|
+
"""
|
|
69
|
+
parsed = urlparse(config.endpoint)
|
|
70
|
+
scheme = "wss" if parsed.scheme == "https" else "ws"
|
|
71
|
+
port = f":{parsed.port}" if parsed.port else ""
|
|
72
|
+
return (
|
|
73
|
+
f"{scheme}://{parsed.hostname}{port}/ws/logs/{job_id}/?api_key={config.api_key}"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def get_ssl_context(uri: str) -> ssl.SSLContext | None:
|
|
78
|
+
"""
|
|
79
|
+
Create an SSL context for secure WebSocket connections.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
----
|
|
83
|
+
uri: The WebSocket URI to check
|
|
84
|
+
|
|
85
|
+
Returns:
|
|
86
|
+
-------
|
|
87
|
+
SSL context with certificate verification disabled for trainwave.dev domains,
|
|
88
|
+
or None if no SSL context is needed
|
|
89
|
+
|
|
90
|
+
"""
|
|
91
|
+
if "trainwave.dev" in uri:
|
|
92
|
+
ssl_context = ssl.SSLContext()
|
|
93
|
+
ssl_context.verify_mode = ssl.CERT_NONE
|
|
94
|
+
return ssl_context
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
class LogsConnection:
|
|
99
|
+
"""
|
|
100
|
+
Manages a WebSocket connection for streaming job logs.
|
|
101
|
+
|
|
102
|
+
This class handles the lifecycle of a WebSocket connection, including:
|
|
103
|
+
- Setting up signal handlers for graceful shutdown
|
|
104
|
+
- Establishing and maintaining the connection
|
|
105
|
+
- Processing incoming log messages
|
|
106
|
+
- Cleaning up resources when the connection is terminated
|
|
107
|
+
"""
|
|
108
|
+
|
|
109
|
+
def __init__(self, job_id: str):
|
|
110
|
+
"""
|
|
111
|
+
Initialize a new logs connection.
|
|
112
|
+
|
|
113
|
+
Args:
|
|
114
|
+
----
|
|
115
|
+
job_id: The ID of the job to connect to
|
|
116
|
+
|
|
117
|
+
"""
|
|
118
|
+
self.job_id = job_id
|
|
119
|
+
self.shutdown_event = asyncio.Event()
|
|
120
|
+
self.websocket = None
|
|
121
|
+
self.original_sigint = None
|
|
122
|
+
self.original_sigterm = None
|
|
123
|
+
|
|
124
|
+
def setup_signal_handlers(self):
|
|
125
|
+
"""
|
|
126
|
+
Register signal handlers for graceful shutdown.
|
|
127
|
+
|
|
128
|
+
Saves the original handlers and sets up new handlers for SIGINT and SIGTERM
|
|
129
|
+
that will trigger the shutdown event when received.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
def handle_sigint(sig, frame):
|
|
133
|
+
self.shutdown_event.set()
|
|
134
|
+
|
|
135
|
+
def handle_sigterm(sig, frame):
|
|
136
|
+
self.shutdown_event.set()
|
|
137
|
+
|
|
138
|
+
self.original_sigint = signal.getsignal(signal.SIGINT)
|
|
139
|
+
self.original_sigterm = signal.getsignal(signal.SIGTERM)
|
|
140
|
+
signal.signal(signal.SIGINT, handle_sigint)
|
|
141
|
+
signal.signal(signal.SIGTERM, handle_sigterm)
|
|
142
|
+
|
|
143
|
+
def restore_signal_handlers(self):
|
|
144
|
+
"""
|
|
145
|
+
Restore the original signal handlers.
|
|
146
|
+
|
|
147
|
+
Called during cleanup to ensure the application's signal handling
|
|
148
|
+
returns to its original state.
|
|
149
|
+
"""
|
|
150
|
+
if self.original_sigint:
|
|
151
|
+
signal.signal(signal.SIGINT, self.original_sigint)
|
|
152
|
+
if self.original_sigterm:
|
|
153
|
+
signal.signal(signal.SIGTERM, self.original_sigterm)
|
|
154
|
+
|
|
155
|
+
async def close_websocket(self):
|
|
156
|
+
"""
|
|
157
|
+
Safely close the WebSocket connection.
|
|
158
|
+
|
|
159
|
+
Attempts to close the connection if it exists and is not already closed,
|
|
160
|
+
catching and ignoring any exceptions that might occur during closure.
|
|
161
|
+
"""
|
|
162
|
+
if self.websocket and not self.websocket.closed:
|
|
163
|
+
try:
|
|
164
|
+
await self.websocket.close()
|
|
165
|
+
except Exception:
|
|
166
|
+
pass
|
|
167
|
+
|
|
168
|
+
async def process_message(self, message: str):
|
|
169
|
+
"""
|
|
170
|
+
Process and display a log message.
|
|
171
|
+
|
|
172
|
+
Parses the JSON message, extracts the log content and timestamp,
|
|
173
|
+
and outputs it to the logger with appropriate formatting.
|
|
174
|
+
|
|
175
|
+
Args:
|
|
176
|
+
----
|
|
177
|
+
message: The raw JSON message received from the WebSocket
|
|
178
|
+
|
|
179
|
+
"""
|
|
180
|
+
data = json.loads(message)
|
|
181
|
+
logger.info(data["message"], log_time=format_timestamp(data.get("ts")))
|
|
182
|
+
|
|
183
|
+
async def connect_and_listen(self):
|
|
184
|
+
"""
|
|
185
|
+
Establish a WebSocket connection and listen for log messages.
|
|
186
|
+
|
|
187
|
+
Connects to the job logs endpoint, sets up a message processing loop,
|
|
188
|
+
and handles various connection states and errors. The loop continues
|
|
189
|
+
until the shutdown event is set or the connection is closed by the server.
|
|
190
|
+
"""
|
|
191
|
+
uri = build_websocket_uri(self.job_id)
|
|
192
|
+
ssl_context = get_ssl_context(uri)
|
|
193
|
+
opts = {"ssl": ssl_context} if ssl_context else {}
|
|
194
|
+
|
|
195
|
+
try:
|
|
196
|
+
async with websockets.connect(uri, **opts) as ws:
|
|
197
|
+
self.websocket = ws
|
|
198
|
+
typer.echo(f"Connected to logs for job {self.job_id}")
|
|
199
|
+
|
|
200
|
+
while not self.shutdown_event.is_set():
|
|
201
|
+
try:
|
|
202
|
+
# Use a timeout to periodically check if we should shut down
|
|
203
|
+
response = await asyncio.wait_for(ws.recv(), timeout=1.0)
|
|
204
|
+
await self.process_message(response)
|
|
205
|
+
except asyncio.TimeoutError:
|
|
206
|
+
# This is expected, just continue to check shutdown_event
|
|
207
|
+
continue
|
|
208
|
+
except websockets.exceptions.ConnectionClosed:
|
|
209
|
+
typer.echo("Connection closed by server.")
|
|
210
|
+
break
|
|
211
|
+
except Exception as e:
|
|
212
|
+
typer.echo(f"Error in logs connection: {e}")
|
|
213
|
+
finally:
|
|
214
|
+
await self.close_websocket()
|
|
215
|
+
typer.echo("Logs connection terminated.")
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
async def check_job_status(job_id: str) -> bool:
|
|
219
|
+
"""
|
|
220
|
+
Verify if a job is in a state that allows log streaming.
|
|
221
|
+
|
|
222
|
+
Queries the API to get the current state of the job and checks if
|
|
223
|
+
it has progressed beyond the initial setup phases.
|
|
224
|
+
|
|
225
|
+
Args:
|
|
226
|
+
----
|
|
227
|
+
job_id: The ID of the job to check
|
|
228
|
+
|
|
229
|
+
Returns:
|
|
230
|
+
-------
|
|
231
|
+
True if the job is ready for logs, False otherwise
|
|
232
|
+
|
|
233
|
+
"""
|
|
234
|
+
api_client = Api(config.api_key, config.endpoint)
|
|
235
|
+
job = await api_client.get_job(job_id)
|
|
236
|
+
|
|
237
|
+
if job.state in (JobStatus.SUBMIT_CODE, JobStatus.LAUNCHING):
|
|
238
|
+
typer.echo(f"Job {job_id} has not started yet.")
|
|
239
|
+
return False
|
|
240
|
+
|
|
241
|
+
return True
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
async def stream_logs(job_id: str):
|
|
245
|
+
"""
|
|
246
|
+
Stream logs for a specified job.
|
|
247
|
+
|
|
248
|
+
Orchestrates the entire logs streaming process:
|
|
249
|
+
1. Verifies the job is in a state that allows log streaming
|
|
250
|
+
2. Sets up the logger with appropriate formatting
|
|
251
|
+
3. Creates and manages a WebSocket connection
|
|
252
|
+
4. Handles graceful shutdown when interrupted
|
|
253
|
+
|
|
254
|
+
Args:
|
|
255
|
+
----
|
|
256
|
+
job_id: The ID of the job to stream logs for
|
|
257
|
+
|
|
258
|
+
"""
|
|
259
|
+
if not await check_job_status(job_id):
|
|
260
|
+
typer.Exit(1)
|
|
261
|
+
|
|
262
|
+
setup_logger()
|
|
263
|
+
|
|
264
|
+
logs_connection = LogsConnection(job_id)
|
|
265
|
+
logs_connection.setup_signal_handlers()
|
|
266
|
+
|
|
267
|
+
try:
|
|
268
|
+
await logs_connection.connect_and_listen()
|
|
269
|
+
finally:
|
|
270
|
+
logs_connection.restore_signal_handlers()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|