trainwave-cli 0.3.2__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/PKG-INFO +1 -1
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/example/trainwave.toml +3 -3
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/pyproject.toml +1 -1
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/api.py +47 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/cli/jobs.py +91 -2
- trainwave_cli-0.4.0/trainwave_cli/cli/ssh_keys.py +112 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/main.py +2 -1
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/uv.lock +1 -1
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/.dockerignore +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/.gitignore +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/.gitlab-ci.yml +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/Makefile +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/README.md +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/example/requirements.txt +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/example/run +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/example/run.sh +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/example/setup.sh +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/example/train.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/pyrightconfig.json +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/ruff.toml +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/tests/cli/test_auth.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/tests/cli/test_jobs.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/tests/cli/test_secrets.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/tests/cli/test_setup.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/tests/conftest.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/tests/test_api.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/tests/test_io.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/tests/test_main.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/tests/test_utils.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/__init__.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/cli/auth.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/cli/logs.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/cli/offers.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/cli/secrets.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/cli/setup.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/config/config.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/format_utils.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/io.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/utils.py +0 -0
- {trainwave_cli-0.3.2 → trainwave_cli-0.4.0}/trainwave_cli/xceptions.py +0 -0
|
@@ -3,10 +3,10 @@ expires = "1h"
|
|
|
3
3
|
setup_command = "bash setup.sh"
|
|
4
4
|
run_command = "bash run.sh"
|
|
5
5
|
image = "trainwave/pytorch:2.3.1"
|
|
6
|
-
framework = "PyTorch"
|
|
7
6
|
project = "p-kgzxmqte"
|
|
8
|
-
compliance =
|
|
9
|
-
|
|
7
|
+
compliance = true
|
|
8
|
+
gpu_type = "NVIDIA-RTX-A5000-24GB"
|
|
9
|
+
gpus = 1
|
|
10
10
|
|
|
11
11
|
[env_vars]
|
|
12
12
|
TESTING1 = "1"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "trainwave-cli"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.4.0"
|
|
4
4
|
description = "Trainwave CLI - A command-line interface for managing Trainwave resources"
|
|
5
5
|
authors = [{ name = "Johan Backman", email = "johan@trainwave.ai" }]
|
|
6
6
|
maintainers = [{ name = "Johan Backman", email = "johan@trainwave.ai" }]
|
|
@@ -51,6 +51,33 @@ class JobConfig:
|
|
|
51
51
|
cpus: int = 0
|
|
52
52
|
gpus: int = 0
|
|
53
53
|
gpu_type: str | None = None
|
|
54
|
+
interactive: bool = False
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(frozen=True)
|
|
58
|
+
class JobInstance:
|
|
59
|
+
id: str
|
|
60
|
+
rid: str
|
|
61
|
+
state: str
|
|
62
|
+
provider: str | None = None
|
|
63
|
+
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class UserSSHKey:
|
|
68
|
+
id: str
|
|
69
|
+
rid: str
|
|
70
|
+
name: str
|
|
71
|
+
public_key: str
|
|
72
|
+
key_type: str
|
|
73
|
+
fingerprint_sha256: str
|
|
74
|
+
created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class SSHProxyInfo:
|
|
79
|
+
host: str
|
|
80
|
+
port: int
|
|
54
81
|
|
|
55
82
|
|
|
56
83
|
@dataclass(frozen=True)
|
|
@@ -91,6 +118,7 @@ class Job:
|
|
|
91
118
|
total_cost: int = 0
|
|
92
119
|
upload_url: str = ""
|
|
93
120
|
url: str | None = None
|
|
121
|
+
instances: list[JobInstance] = field(default_factory=list)
|
|
94
122
|
|
|
95
123
|
|
|
96
124
|
@dataclass(frozen=True)
|
|
@@ -274,6 +302,25 @@ class Api:
|
|
|
274
302
|
for secret in secrets:
|
|
275
303
|
await self.request("DELETE", f"/api/v1/organizations/{org_id}/secrets/{secret}/")
|
|
276
304
|
|
|
305
|
+
async def list_ssh_keys(self) -> list[UserSSHKey]:
|
|
306
|
+
res = await self.request("GET", "/api/v1/ssh-keys/")
|
|
307
|
+
return [from_dict(UserSSHKey, k) for k in res.json()["results"]]
|
|
308
|
+
|
|
309
|
+
async def add_ssh_key(self, name: str, public_key: str) -> UserSSHKey:
|
|
310
|
+
res = await self.request(
|
|
311
|
+
"POST",
|
|
312
|
+
"/api/v1/ssh-keys/",
|
|
313
|
+
json={"name": name, "public_key": public_key},
|
|
314
|
+
)
|
|
315
|
+
return from_dict(UserSSHKey, res.json())
|
|
316
|
+
|
|
317
|
+
async def remove_ssh_key(self, key_id: str) -> None:
|
|
318
|
+
await self.request("DELETE", f"/api/v1/ssh-keys/{key_id}/")
|
|
319
|
+
|
|
320
|
+
async def get_ssh_proxy_info(self) -> SSHProxyInfo:
|
|
321
|
+
res = await self.request("GET", "/api/v1/ssh-proxy/")
|
|
322
|
+
return from_dict(SSHProxyInfo, res.json())
|
|
323
|
+
|
|
277
324
|
async def code_submission(self, job: Job) -> None:
|
|
278
325
|
await self.request("POST", f"/api/v1/jobs/{job.id}/code_submission/")
|
|
279
326
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import json as _json
|
|
2
2
|
import os
|
|
3
|
+
import sys
|
|
3
4
|
import traceback
|
|
4
5
|
from dataclasses import asdict
|
|
5
6
|
from datetime import datetime, timezone
|
|
@@ -13,7 +14,7 @@ from loguru import logger
|
|
|
13
14
|
from tabulate import tabulate
|
|
14
15
|
from typer_config.decorators import use_toml_config
|
|
15
16
|
|
|
16
|
-
from trainwave_cli.api import Api
|
|
17
|
+
from trainwave_cli.api import Api, JobStatus
|
|
17
18
|
from trainwave_cli.cli.logs import stream_logs
|
|
18
19
|
from trainwave_cli.config.config import config
|
|
19
20
|
from trainwave_cli.format_utils import cents_to_dollars_str
|
|
@@ -45,7 +46,8 @@ async def launch( # noqa
|
|
|
45
46
|
exclude_gitignore: Annotated[
|
|
46
47
|
bool,
|
|
47
48
|
typer.Option(
|
|
48
|
-
|
|
49
|
+
"--exclude-gitignore/--no-exclude-gitignore",
|
|
50
|
+
help="If set the gitignore file will be used to ignore files when packaging the job",
|
|
49
51
|
),
|
|
50
52
|
] = True,
|
|
51
53
|
exclude_regex: Annotated[
|
|
@@ -62,6 +64,13 @@ async def launch( # noqa
|
|
|
62
64
|
compliance_soc2: Annotated[
|
|
63
65
|
bool | None, typer.Option("--compliance-soc2", help="Only use SOC2 compliant machines")
|
|
64
66
|
] = None,
|
|
67
|
+
interactive: Annotated[
|
|
68
|
+
bool,
|
|
69
|
+
typer.Option(
|
|
70
|
+
"--interactive",
|
|
71
|
+
help="Hold the workload on boot so you can SSH in and run it manually.",
|
|
72
|
+
),
|
|
73
|
+
] = False,
|
|
65
74
|
json: bool = False,
|
|
66
75
|
) -> None:
|
|
67
76
|
"""Launch a job on Trainwave using the current configuration."""
|
|
@@ -90,6 +99,8 @@ async def launch( # noqa
|
|
|
90
99
|
}
|
|
91
100
|
if compliance_soc2 is not None:
|
|
92
101
|
conf["compliance_soc2"] = compliance_soc2
|
|
102
|
+
if interactive:
|
|
103
|
+
conf["interactive"] = True
|
|
93
104
|
|
|
94
105
|
if conf["gpu_type"] != "" and conf["gpus"] == 0:
|
|
95
106
|
conf["gpus"] = 1 # default to 1 gpu if gpu_type is set
|
|
@@ -139,6 +150,11 @@ async def launch( # noqa
|
|
|
139
150
|
typer.echo("\nMatched machine\n" + tabulate(tab_values, headers="keys") + "\n")
|
|
140
151
|
typer.echo(f"Cost per hour for this job: ${job.cost_per_hour:.2f}")
|
|
141
152
|
typer.echo(f"Job created: {job.rid} ({job.id}) -> {job.url}")
|
|
153
|
+
if interactive:
|
|
154
|
+
typer.echo(
|
|
155
|
+
"\nInteractive mode: once the instance is running, connect with:"
|
|
156
|
+
)
|
|
157
|
+
typer.echo(f" trainwave jobs ssh {job.rid}")
|
|
142
158
|
else:
|
|
143
159
|
print(_json.dumps(to_dict_recursive(job), indent=4))
|
|
144
160
|
except Exception as e:
|
|
@@ -211,3 +227,76 @@ async def logs(
|
|
|
211
227
|
) -> None:
|
|
212
228
|
"""Get the logs of a job."""
|
|
213
229
|
await stream_logs(job_id)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
@app.command()
|
|
233
|
+
@async_command
|
|
234
|
+
@ensure_api_key
|
|
235
|
+
async def ssh(
|
|
236
|
+
job_id: Annotated[str, typer.Argument(help="Job ID or RID to SSH into")],
|
|
237
|
+
instance: Annotated[
|
|
238
|
+
str | None,
|
|
239
|
+
typer.Option(
|
|
240
|
+
help="Instance RID. Required only when the job has more than one running instance.",
|
|
241
|
+
),
|
|
242
|
+
] = None,
|
|
243
|
+
extra: Annotated[
|
|
244
|
+
list[str] | None,
|
|
245
|
+
typer.Argument(help="Extra arguments forwarded to ssh (e.g. a command to run)."),
|
|
246
|
+
] = None,
|
|
247
|
+
) -> None:
|
|
248
|
+
"""Open an SSH session to the running instance of a job."""
|
|
249
|
+
api_client = Api(config.api_key, config.endpoint)
|
|
250
|
+
try:
|
|
251
|
+
job = await api_client.get_job(job_id)
|
|
252
|
+
except ValueError as e:
|
|
253
|
+
typer.echo(f"Error: {e}")
|
|
254
|
+
sys.exit(1)
|
|
255
|
+
|
|
256
|
+
if job.state != JobStatus.RUNNING:
|
|
257
|
+
typer.echo(
|
|
258
|
+
f"Job {job.rid} is in state {job.state.value}; SSH is only available while RUNNING."
|
|
259
|
+
)
|
|
260
|
+
sys.exit(1)
|
|
261
|
+
|
|
262
|
+
running = [i for i in job.instances if i.state == "RUNNING"]
|
|
263
|
+
if not running:
|
|
264
|
+
typer.echo(
|
|
265
|
+
f"Job {job.rid} has no running instance yet. Try again once it's up."
|
|
266
|
+
)
|
|
267
|
+
sys.exit(1)
|
|
268
|
+
|
|
269
|
+
if instance:
|
|
270
|
+
match = next((i for i in running if instance in (i.rid, i.id)), None)
|
|
271
|
+
if match is None:
|
|
272
|
+
typer.echo(f"Instance {instance} not found among running instances of {job.rid}.")
|
|
273
|
+
sys.exit(1)
|
|
274
|
+
target = match
|
|
275
|
+
elif len(running) > 1:
|
|
276
|
+
rids = ", ".join(i.rid for i in running)
|
|
277
|
+
typer.echo(
|
|
278
|
+
f"Job {job.rid} has multiple running instances ({rids}). "
|
|
279
|
+
"Pick one with --instance <rid>."
|
|
280
|
+
)
|
|
281
|
+
sys.exit(1)
|
|
282
|
+
else:
|
|
283
|
+
target = running[0]
|
|
284
|
+
|
|
285
|
+
try:
|
|
286
|
+
proxy = await api_client.get_ssh_proxy_info()
|
|
287
|
+
except ValueError as e:
|
|
288
|
+
typer.echo(f"Error fetching SSH proxy info: {e}")
|
|
289
|
+
sys.exit(1)
|
|
290
|
+
|
|
291
|
+
username = f"{job.rid}-{target.rid}"
|
|
292
|
+
typer.echo(f"Connecting to {username}@{proxy.host}:{proxy.port}...")
|
|
293
|
+
|
|
294
|
+
argv = [
|
|
295
|
+
"ssh",
|
|
296
|
+
"-p",
|
|
297
|
+
str(proxy.port),
|
|
298
|
+
f"{username}@{proxy.host}",
|
|
299
|
+
]
|
|
300
|
+
if extra:
|
|
301
|
+
argv.extend(extra)
|
|
302
|
+
os.execvp("ssh", argv)
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import socket
|
|
2
|
+
import sys
|
|
3
|
+
from datetime import datetime, timezone
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import human_readable
|
|
8
|
+
import typer
|
|
9
|
+
from tabulate import tabulate
|
|
10
|
+
|
|
11
|
+
from trainwave_cli.api import Api
|
|
12
|
+
from trainwave_cli.config.config import config
|
|
13
|
+
from trainwave_cli.utils import async_command, ensure_api_key
|
|
14
|
+
|
|
15
|
+
app = typer.Typer()
|
|
16
|
+
|
|
17
|
+
_DEFAULT_KEY_CANDIDATES = (
|
|
18
|
+
"id_ed25519.pub",
|
|
19
|
+
"id_ecdsa.pub",
|
|
20
|
+
"id_rsa.pub",
|
|
21
|
+
"id_dsa.pub",
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _autofind_public_key() -> Path | None:
|
|
26
|
+
ssh_dir = Path.home() / ".ssh"
|
|
27
|
+
for candidate in _DEFAULT_KEY_CANDIDATES:
|
|
28
|
+
path = ssh_dir / candidate
|
|
29
|
+
if path.is_file():
|
|
30
|
+
return path
|
|
31
|
+
return None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@app.command(name="list")
|
|
35
|
+
@async_command
|
|
36
|
+
@ensure_api_key
|
|
37
|
+
async def list_keys() -> None:
|
|
38
|
+
"""List SSH keys registered on your account."""
|
|
39
|
+
api_client = Api(config.api_key, config.endpoint)
|
|
40
|
+
keys = await api_client.list_ssh_keys()
|
|
41
|
+
if not keys:
|
|
42
|
+
typer.echo("No SSH keys registered. Add one with `trainwave ssh-keys add`.")
|
|
43
|
+
return
|
|
44
|
+
headers = ["ID", "NAME", "TYPE", "FINGERPRINT", "ADDED"]
|
|
45
|
+
table = [
|
|
46
|
+
[
|
|
47
|
+
k.rid,
|
|
48
|
+
k.name,
|
|
49
|
+
k.key_type,
|
|
50
|
+
f"SHA256:{k.fingerprint_sha256}",
|
|
51
|
+
human_readable.date_time(datetime.now(timezone.utc) - k.created_at),
|
|
52
|
+
]
|
|
53
|
+
for k in keys
|
|
54
|
+
]
|
|
55
|
+
typer.echo(tabulate(table, headers=headers, tablefmt="simple"))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
@app.command()
|
|
59
|
+
@async_command
|
|
60
|
+
@ensure_api_key
|
|
61
|
+
async def add(
|
|
62
|
+
path: Annotated[
|
|
63
|
+
Path | None,
|
|
64
|
+
typer.Argument(
|
|
65
|
+
help="Path to a .pub file. If omitted, common keys in ~/.ssh are auto-detected.",
|
|
66
|
+
),
|
|
67
|
+
] = None,
|
|
68
|
+
name: Annotated[
|
|
69
|
+
str | None,
|
|
70
|
+
typer.Option(help="Label for the key. Defaults to '<hostname>:<filename>'."),
|
|
71
|
+
] = None,
|
|
72
|
+
) -> None:
|
|
73
|
+
"""Register an SSH public key on your account."""
|
|
74
|
+
if path is None:
|
|
75
|
+
path = _autofind_public_key()
|
|
76
|
+
if path is None:
|
|
77
|
+
typer.echo(
|
|
78
|
+
"No SSH public key found in ~/.ssh. Generate one with "
|
|
79
|
+
"`ssh-keygen -t ed25519` or pass the path explicitly."
|
|
80
|
+
)
|
|
81
|
+
sys.exit(1)
|
|
82
|
+
typer.echo(f"Using {path}")
|
|
83
|
+
|
|
84
|
+
if not path.is_file():
|
|
85
|
+
typer.echo(f"File not found: {path}")
|
|
86
|
+
sys.exit(1)
|
|
87
|
+
|
|
88
|
+
public_key = path.read_text().strip()
|
|
89
|
+
if not public_key:
|
|
90
|
+
typer.echo(f"File is empty: {path}")
|
|
91
|
+
sys.exit(1)
|
|
92
|
+
|
|
93
|
+
label = name or f"{socket.gethostname()}:{path.name}"
|
|
94
|
+
api_client = Api(config.api_key, config.endpoint)
|
|
95
|
+
try:
|
|
96
|
+
key = await api_client.add_ssh_key(label, public_key)
|
|
97
|
+
except ValueError as e:
|
|
98
|
+
typer.echo(f"Error: {e}")
|
|
99
|
+
sys.exit(1)
|
|
100
|
+
typer.echo(f"Added SSH key {key.rid} ({key.name}) -> SHA256:{key.fingerprint_sha256}")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@app.command()
|
|
104
|
+
@async_command
|
|
105
|
+
@ensure_api_key
|
|
106
|
+
async def remove(
|
|
107
|
+
key_id: Annotated[str, typer.Argument(help="ID or RID of the key to remove")],
|
|
108
|
+
) -> None:
|
|
109
|
+
"""Remove an SSH key from your account."""
|
|
110
|
+
api_client = Api(config.api_key, config.endpoint)
|
|
111
|
+
await api_client.remove_ssh_key(key_id)
|
|
112
|
+
typer.echo(f"Removed SSH key {key_id}")
|
|
@@ -4,13 +4,14 @@ import sys
|
|
|
4
4
|
import typer
|
|
5
5
|
from loguru import logger
|
|
6
6
|
|
|
7
|
-
from trainwave_cli.cli import auth, jobs, secrets, setup
|
|
7
|
+
from trainwave_cli.cli import auth, jobs, secrets, setup, ssh_keys
|
|
8
8
|
|
|
9
9
|
app = typer.Typer()
|
|
10
10
|
app.add_typer(jobs.app, name="jobs", help="Manage training jobs")
|
|
11
11
|
app.add_typer(auth.app, name="auth", help="Authenticate with Trainwave")
|
|
12
12
|
app.add_typer(setup.app, name="config")
|
|
13
13
|
app.add_typer(secrets.app, name="secrets", help="Manage job secrets")
|
|
14
|
+
app.add_typer(ssh_keys.app, name="ssh-keys", help="Manage SSH keys for interactive jobs")
|
|
14
15
|
|
|
15
16
|
|
|
16
17
|
def get_package_name() -> str:
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|