trainwave-cli 0.2.5__tar.gz → 0.2.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: trainwave-cli
3
- Version: 0.2.5
3
+ Version: 0.2.7
4
4
  Summary: Trainwave CLI - A command-line interface for managing Trainwave resources
5
5
  License: MIT
6
6
  Keywords: cli,trainwave,ai,machine-learning
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "trainwave-cli"
3
- version = "0.2.5"
3
+ version = "0.2.7"
4
4
  description = "Trainwave CLI - A command-line interface for managing Trainwave resources"
5
5
  authors = ["Johan Backman <johan@trainwave.ai>"]
6
6
  maintainers = ["Johan Backman <johan@trainwave.ai>"]
@@ -34,12 +34,12 @@ class JobStatus(Enum):
34
34
 
35
35
  @dataclass(frozen=True)
36
36
  class CloudOffer:
37
- cpus: int
38
- memory_mb: int
39
- compliance_soc2: bool
40
- gpu_type: str
41
- gpu_memory_mb: int
42
- gpus: int
37
+ cpus: int = 0
38
+ memory_mb: int = 0
39
+ compliance_soc2: bool = False
40
+ gpu_type: str | None = None
41
+ gpu_memory_mb: int = 0
42
+ gpus: int = 0
43
43
 
44
44
 
45
45
  @dataclass
@@ -47,19 +47,19 @@ class JobConfig:
47
47
  id: str
48
48
  rid: str
49
49
  name: str
50
- expires_at: int
51
- cpus: int
52
- gpus: int
53
- gpu_type: str
50
+ expires_at: int = -1
51
+ cpus: int = 0
52
+ gpus: int = 0
53
+ gpu_type: str | None = None
54
54
 
55
55
 
56
56
  @dataclass(frozen=True)
57
57
  class TrainWaveUser:
58
58
  id: str
59
59
  rid: str
60
- first_name: str
61
- last_name: str
62
60
  email: str
61
+ first_name: str | None = None
62
+ last_name: str | None = None
63
63
 
64
64
 
65
65
  @dataclass(frozen=True)
@@ -133,7 +133,7 @@ class Api:
133
133
  ):
134
134
  verify_ssl = "trainwave.dev" not in endpoint
135
135
  endpoint = endpoint.rstrip("/")
136
- self.client = httpx.AsyncClient(base_url=endpoint, verify=verify_ssl)
136
+ self.client = httpx.AsyncClient(base_url=endpoint, verify=verify_ssl, timeout=15)
137
137
  self.api_key = api_key
138
138
  self.project = project
139
139
  self.organization = organization
@@ -1,22 +1,19 @@
1
- import json
2
1
  import os
3
- import sys
4
2
  import traceback
5
3
  from dataclasses import asdict
6
4
  from datetime import datetime, timezone
7
5
  from pathlib import Path
8
6
  from typing import Annotated
9
- from urllib.parse import urlparse
10
7
 
11
8
  import human_readable
12
9
  import pytimeparse
13
10
  import typer
14
- import websockets
15
11
  from loguru import logger
16
12
  from tabulate import tabulate
17
13
  from typer_config.decorators import use_toml_config
18
14
 
19
- from trainwave_cli.api import Api, JobStatus
15
+ from trainwave_cli.api import Api
16
+ from trainwave_cli.cli.logs import stream_logs
20
17
  from trainwave_cli.config.config import config
21
18
  from trainwave_cli.format_utils import cents_to_dollars_str
22
19
  from trainwave_cli.io import create_tarball
@@ -219,51 +216,4 @@ async def logs(
219
216
  job_id: str,
220
217
  ) -> None:
221
218
  """Get the logs of a job."""
222
- api_client = Api(config.api_key, config.endpoint)
223
- job = await api_client.get_job(job_id)
224
-
225
- if job.state in (JobStatus.SUBMIT_CODE, JobStatus.LAUNCHING):
226
- typer.echo(f"Job {job_id} has not started yet.")
227
- typer.Exit(1)
228
-
229
- logger.remove()
230
- logger.add(
231
- sys.stdout,
232
- format="<green>{extra[log_time]}</green> | <white>{message}</white>",
233
- colorize=True,
234
- serialize=False,
235
- level="DEBUG",
236
- enqueue=True,
237
- catch=True,
238
- )
239
- parsed = urlparse(config.endpoint)
240
- scheme = "wss" if parsed.scheme == "https" else "ws"
241
- port = ""
242
- if parsed.port:
243
- port = f":{parsed.port}"
244
- uri = (
245
- f"{scheme}://{parsed.hostname}{port}/ws/logs/{job_id}/?api_key={config.api_key}"
246
- )
247
-
248
- def format_ts(ts: str | None) -> str:
249
- if ts is None:
250
- return ""
251
- timestamp = int(ts) / 1e9
252
- dt = datetime.fromtimestamp(timestamp)
253
- return dt.strftime("%Y-%m-%d %H:%M:%S.%f")
254
-
255
- ssl_context = None
256
- if "trainwave.dev" in uri:
257
- import ssl
258
-
259
- ssl_context = ssl.SSLContext()
260
- ssl_context.verify_mode = ssl.CERT_NONE
261
- opts = {"ssl": ssl_context} if ssl_context else {}
262
-
263
- async with websockets.connect(uri, **opts) as websocket:
264
- try:
265
- while response := await websocket.recv():
266
- data = json.loads(response)
267
- logger.info(data["message"], log_time=format_ts(data.get("ts")))
268
- except websockets.exceptions.ConnectionClosedOK:
269
- pass
219
+ await stream_logs(job_id)
@@ -0,0 +1,270 @@
1
+ import asyncio
2
+ import json
3
+ import signal
4
+ import ssl
5
+ import sys
6
+ from datetime import datetime
7
+ from urllib.parse import urlparse
8
+
9
+ import typer
10
+ import websockets
11
+ from loguru import logger
12
+
13
+ from trainwave_cli.api import Api, JobStatus
14
+ from trainwave_cli.config.config import config
15
+
16
+
17
+ def format_timestamp(ts: str | None) -> str:
18
+ """
19
+ Convert a nanosecond timestamp to a human-readable datetime string.
20
+
21
+ Args:
22
+ ----
23
+ ts: Timestamp in nanoseconds as a string, or None
24
+
25
+ Returns:
26
+ -------
27
+ Formatted datetime string in YYYY-MM-DD HH:MM:SS.ffffff format, or empty string if ts is None
28
+
29
+ """
30
+ if ts is None:
31
+ return ""
32
+ timestamp = int(ts) / 1e9
33
+ dt = datetime.fromtimestamp(timestamp)
34
+ return dt.strftime("%Y-%m-%d %H:%M:%S.%f")
35
+
36
+
37
+ def setup_logger():
38
+ """
39
+ Configure the logger with appropriate formatting for the logs command.
40
+
41
+ Removes any existing handlers and adds a new handler that outputs to stdout
42
+ with colorized timestamps and messages.
43
+ """
44
+ logger.remove()
45
+ logger.add(
46
+ sys.stdout,
47
+ format="<green>{extra[log_time]}</green> | <white>{message}</white>",
48
+ colorize=True,
49
+ serialize=False,
50
+ level="DEBUG",
51
+ enqueue=True,
52
+ catch=True,
53
+ )
54
+
55
+
56
+ def build_websocket_uri(job_id: str) -> str:
57
+ """
58
+ Construct the WebSocket URI for connecting to job logs.
59
+
60
+ Args:
61
+ ----
62
+ job_id: The ID of the job to connect to
63
+
64
+ Returns:
65
+ -------
66
+ Complete WebSocket URI including protocol, host, port, path, and API key
67
+
68
+ """
69
+ parsed = urlparse(config.endpoint)
70
+ scheme = "wss" if parsed.scheme == "https" else "ws"
71
+ port = f":{parsed.port}" if parsed.port else ""
72
+ return (
73
+ f"{scheme}://{parsed.hostname}{port}/ws/logs/{job_id}/?api_key={config.api_key}"
74
+ )
75
+
76
+
77
+ def get_ssl_context(uri: str) -> ssl.SSLContext | None:
78
+ """
79
+ Create an SSL context for secure WebSocket connections.
80
+
81
+ Args:
82
+ ----
83
+ uri: The WebSocket URI to check
84
+
85
+ Returns:
86
+ -------
87
+ SSL context with certificate verification disabled for trainwave.dev domains,
88
+ or None if no SSL context is needed
89
+
90
+ """
91
+ if "trainwave.dev" in uri:
92
+ ssl_context = ssl.SSLContext()
93
+ ssl_context.verify_mode = ssl.CERT_NONE
94
+ return ssl_context
95
+ return None
96
+
97
+
98
+ class LogsConnection:
99
+ """
100
+ Manages a WebSocket connection for streaming job logs.
101
+
102
+ This class handles the lifecycle of a WebSocket connection, including:
103
+ - Setting up signal handlers for graceful shutdown
104
+ - Establishing and maintaining the connection
105
+ - Processing incoming log messages
106
+ - Cleaning up resources when the connection is terminated
107
+ """
108
+
109
+ def __init__(self, job_id: str):
110
+ """
111
+ Initialize a new logs connection.
112
+
113
+ Args:
114
+ ----
115
+ job_id: The ID of the job to connect to
116
+
117
+ """
118
+ self.job_id = job_id
119
+ self.shutdown_event = asyncio.Event()
120
+ self.websocket = None
121
+ self.original_sigint = None
122
+ self.original_sigterm = None
123
+
124
+ def setup_signal_handlers(self):
125
+ """
126
+ Register signal handlers for graceful shutdown.
127
+
128
+ Saves the original handlers and sets up new handlers for SIGINT and SIGTERM
129
+ that will trigger the shutdown event when received.
130
+ """
131
+
132
+ def handle_sigint(sig, frame):
133
+ self.shutdown_event.set()
134
+
135
+ def handle_sigterm(sig, frame):
136
+ self.shutdown_event.set()
137
+
138
+ self.original_sigint = signal.getsignal(signal.SIGINT)
139
+ self.original_sigterm = signal.getsignal(signal.SIGTERM)
140
+ signal.signal(signal.SIGINT, handle_sigint)
141
+ signal.signal(signal.SIGTERM, handle_sigterm)
142
+
143
+ def restore_signal_handlers(self):
144
+ """
145
+ Restore the original signal handlers.
146
+
147
+ Called during cleanup to ensure the application's signal handling
148
+ returns to its original state.
149
+ """
150
+ if self.original_sigint:
151
+ signal.signal(signal.SIGINT, self.original_sigint)
152
+ if self.original_sigterm:
153
+ signal.signal(signal.SIGTERM, self.original_sigterm)
154
+
155
+ async def close_websocket(self):
156
+ """
157
+ Safely close the WebSocket connection.
158
+
159
+ Attempts to close the connection if it exists and is not already closed,
160
+ catching and ignoring any exceptions that might occur during closure.
161
+ """
162
+ if self.websocket and not self.websocket.closed:
163
+ try:
164
+ await self.websocket.close()
165
+ except Exception:
166
+ pass
167
+
168
+ async def process_message(self, message: str):
169
+ """
170
+ Process and display a log message.
171
+
172
+ Parses the JSON message, extracts the log content and timestamp,
173
+ and outputs it to the logger with appropriate formatting.
174
+
175
+ Args:
176
+ ----
177
+ message: The raw JSON message received from the WebSocket
178
+
179
+ """
180
+ data = json.loads(message)
181
+ logger.info(data["message"], log_time=format_timestamp(data.get("ts")))
182
+
183
+ async def connect_and_listen(self):
184
+ """
185
+ Establish a WebSocket connection and listen for log messages.
186
+
187
+ Connects to the job logs endpoint, sets up a message processing loop,
188
+ and handles various connection states and errors. The loop continues
189
+ until the shutdown event is set or the connection is closed by the server.
190
+ """
191
+ uri = build_websocket_uri(self.job_id)
192
+ ssl_context = get_ssl_context(uri)
193
+ opts = {"ssl": ssl_context} if ssl_context else {}
194
+
195
+ try:
196
+ async with websockets.connect(uri, **opts) as ws:
197
+ self.websocket = ws
198
+ typer.echo(f"Connected to logs for job {self.job_id}")
199
+
200
+ while not self.shutdown_event.is_set():
201
+ try:
202
+ # Use a timeout to periodically check if we should shut down
203
+ response = await asyncio.wait_for(ws.recv(), timeout=1.0)
204
+ await self.process_message(response)
205
+ except asyncio.TimeoutError:
206
+ # This is expected, just continue to check shutdown_event
207
+ continue
208
+ except websockets.exceptions.ConnectionClosed:
209
+ typer.echo("Connection closed by server.")
210
+ break
211
+ except Exception as e:
212
+ typer.echo(f"Error in logs connection: {e}")
213
+ finally:
214
+ await self.close_websocket()
215
+ typer.echo("Logs connection terminated.")
216
+
217
+
218
+ async def check_job_status(job_id: str) -> bool:
219
+ """
220
+ Verify if a job is in a state that allows log streaming.
221
+
222
+ Queries the API to get the current state of the job and checks if
223
+ it has progressed beyond the initial setup phases.
224
+
225
+ Args:
226
+ ----
227
+ job_id: The ID of the job to check
228
+
229
+ Returns:
230
+ -------
231
+ True if the job is ready for logs, False otherwise
232
+
233
+ """
234
+ api_client = Api(config.api_key, config.endpoint)
235
+ job = await api_client.get_job(job_id)
236
+
237
+ if job.state in (JobStatus.SUBMIT_CODE, JobStatus.LAUNCHING):
238
+ typer.echo(f"Job {job_id} has not started yet.")
239
+ return False
240
+
241
+ return True
242
+
243
+
244
+ async def stream_logs(job_id: str):
245
+ """
246
+ Stream logs for a specified job.
247
+
248
+ Orchestrates the entire logs streaming process:
249
+ 1. Verifies the job is in a state that allows log streaming
250
+ 2. Sets up the logger with appropriate formatting
251
+ 3. Creates and manages a WebSocket connection
252
+ 4. Handles graceful shutdown when interrupted
253
+
254
+ Args:
255
+ ----
256
+ job_id: The ID of the job to stream logs for
257
+
258
+ """
259
+ if not await check_job_status(job_id):
260
+ typer.Exit(1)
261
+
262
+ setup_logger()
263
+
264
+ logs_connection = LogsConnection(job_id)
265
+ logs_connection.setup_signal_handlers()
266
+
267
+ try:
268
+ await logs_connection.connect_and_listen()
269
+ finally:
270
+ logs_connection.restore_signal_handlers()
File without changes