trainwave-cli 0.2.7__tar.gz → 0.2.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/PKG-INFO +1 -1
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/pyproject.toml +2 -1
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/api.py +19 -31
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/cli/jobs.py +23 -29
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/io.py +15 -13
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/utils.py +17 -1
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/README.md +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/__init__.py +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/cli/auth.py +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/cli/logs.py +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/cli/offers.py +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/cli/secrets.py +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/cli/setup.py +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/config/config.py +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/format_utils.py +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/main.py +0 -0
- {trainwave_cli-0.2.7 → trainwave_cli-0.2.8}/trainwave_cli/xceptions.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "trainwave-cli"
|
|
3
|
-
version = "0.2.
|
|
3
|
+
version = "0.2.8"
|
|
4
4
|
description = "Trainwave CLI - A command-line interface for managing Trainwave resources"
|
|
5
5
|
authors = ["Johan Backman <johan@trainwave.ai>"]
|
|
6
6
|
maintainers = ["Johan Backman <johan@trainwave.ai>"]
|
|
@@ -57,6 +57,7 @@ pytest-asyncio = "^0.25.3"
|
|
|
57
57
|
pytest-cov = "^6.0.0"
|
|
58
58
|
pytest-httpx = "^0.35.0"
|
|
59
59
|
pytest-mock = "^3.14.0"
|
|
60
|
+
ruff = "^0.11.7"
|
|
60
61
|
|
|
61
62
|
[tool.poetry.scripts]
|
|
62
63
|
wave = "trainwave_cli.main:entrypoint"
|
|
@@ -151,18 +151,12 @@ class Api:
|
|
|
151
151
|
raise ValueError(f"Error: {res.text}")
|
|
152
152
|
return res
|
|
153
153
|
|
|
154
|
-
async def unauthenticated_request(
|
|
155
|
-
self, method: str, path: str, **kwargs
|
|
156
|
-
) -> httpx.Response:
|
|
154
|
+
async def unauthenticated_request(self, method: str, path: str, **kwargs) -> httpx.Response:
|
|
157
155
|
res = await self.client.request(method, path, **kwargs)
|
|
158
156
|
return self._ensure_no_errors(res)
|
|
159
157
|
|
|
160
|
-
async def create_cli_config_session(
|
|
161
|
-
self,
|
|
162
|
-
) -> tuple[str, str]:
|
|
163
|
-
res = await self.request(
|
|
164
|
-
"POST", "/api/v1/cli/config_session/", json=existing_config
|
|
165
|
-
)
|
|
158
|
+
async def create_cli_config_session(self, existing_config: dict[str, Any]) -> tuple[str, str]:
|
|
159
|
+
res = await self.request("POST", "/api/v1/cli/config_session/", json=existing_config)
|
|
166
160
|
json_body = res.json()
|
|
167
161
|
return str(json_body["url"]), str(json_body["token"])
|
|
168
162
|
|
|
@@ -181,9 +175,7 @@ class Api:
|
|
|
181
175
|
json_body = res.json()
|
|
182
176
|
return str(json_body["url"]), str(json_body["token"])
|
|
183
177
|
|
|
184
|
-
async def check_cli_auth_session_status(
|
|
185
|
-
self, token: str
|
|
186
|
-
) -> tuple[CLIAuthStatus, str | None]:
|
|
178
|
+
async def check_cli_auth_session_status(self, token: str) -> tuple[CLIAuthStatus, str | None]:
|
|
187
179
|
res = await self.unauthenticated_request(
|
|
188
180
|
"POST", f"/api/v1/cli/session_status/", json={"token": token}
|
|
189
181
|
)
|
|
@@ -198,10 +190,7 @@ class Api:
|
|
|
198
190
|
async def check_api_key(self) -> bool:
|
|
199
191
|
try:
|
|
200
192
|
res = await self.request("GET", "/api/v1/organizations/")
|
|
201
|
-
return (
|
|
202
|
-
res.status_code == HTTPStatus.OK.value
|
|
203
|
-
and len(res.json()["results"]) > 0
|
|
204
|
-
)
|
|
193
|
+
return res.status_code == HTTPStatus.OK.value and len(res.json()["results"]) > 0
|
|
205
194
|
except ValueError:
|
|
206
195
|
return False
|
|
207
196
|
|
|
@@ -213,9 +202,7 @@ class Api:
|
|
|
213
202
|
async def list_projects(self, organization_id: str) -> list[TrainwaveProject]:
|
|
214
203
|
res = await self.request("GET", "/api/v1/projects/")
|
|
215
204
|
res.raise_for_status()
|
|
216
|
-
return [
|
|
217
|
-
from_dict(TrainwaveProject, project) for project in res.json()["results"]
|
|
218
|
-
]
|
|
205
|
+
return [from_dict(TrainwaveProject, project) for project in res.json()["results"]]
|
|
219
206
|
|
|
220
207
|
async def create_project(self, organization_id: str, name: str) -> TrainwaveProject:
|
|
221
208
|
res = await self.request(
|
|
@@ -277,29 +264,29 @@ class Api:
|
|
|
277
264
|
async def set_secrets(
|
|
278
265
|
self, org_id: str, secrets: dict[str, str], project: str | None = None
|
|
279
266
|
) -> None:
|
|
280
|
-
data: dict[str, Any] = {
|
|
281
|
-
"secrets": [{"name": k, "value": v} for k, v in secrets.items()]
|
|
282
|
-
}
|
|
267
|
+
data: dict[str, Any] = {"secrets": [{"name": k, "value": v} for k, v in secrets.items()]}
|
|
283
268
|
if project:
|
|
284
269
|
data["project"] = project
|
|
285
270
|
|
|
286
|
-
await self.request(
|
|
287
|
-
"POST", f"/api/v1/organizations/{org_id}/secrets/", json=data
|
|
288
|
-
)
|
|
271
|
+
await self.request("POST", f"/api/v1/organizations/{org_id}/secrets/", json=data)
|
|
289
272
|
|
|
290
273
|
async def unset_secrets(self, org_id: str, secrets: list[str]) -> None:
|
|
291
274
|
for secret in secrets:
|
|
292
|
-
await self.request(
|
|
293
|
-
"DELETE", f"/api/v1/organizations/{org_id}/secrets/{secret}/"
|
|
294
|
-
)
|
|
275
|
+
await self.request("DELETE", f"/api/v1/organizations/{org_id}/secrets/{secret}/")
|
|
295
276
|
|
|
296
277
|
async def code_submission(self, job: Job) -> None:
|
|
297
278
|
await self.request("POST", f"/api/v1/jobs/{job.id}/code_submission/")
|
|
298
279
|
|
|
299
|
-
async def upload_code(
|
|
280
|
+
async def upload_code(
|
|
281
|
+
self, tarball: Path, presigned_url: str, show_progress_bar: bool = True
|
|
282
|
+
) -> None:
|
|
300
283
|
size = tarball.stat().st_size
|
|
301
284
|
|
|
302
|
-
progress_bar =
|
|
285
|
+
progress_bar = (
|
|
286
|
+
tqdm(total=size, unit="B", unit_scale=True, desc="Uploading")
|
|
287
|
+
if show_progress_bar
|
|
288
|
+
else None
|
|
289
|
+
)
|
|
303
290
|
|
|
304
291
|
async def file_chunk_iterator(filename):
|
|
305
292
|
async with aiofiles.open(filename, "rb") as file:
|
|
@@ -307,7 +294,8 @@ class Api:
|
|
|
307
294
|
chunk = await file.read(64 * 1024)
|
|
308
295
|
if not chunk:
|
|
309
296
|
break
|
|
310
|
-
|
|
297
|
+
if show_progress_bar:
|
|
298
|
+
progress_bar.update(len(chunk))
|
|
311
299
|
yield chunk
|
|
312
300
|
|
|
313
301
|
async with aiohttp.ClientSession() as session:
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import json as _json
|
|
1
2
|
import os
|
|
2
3
|
import traceback
|
|
3
4
|
from dataclasses import asdict
|
|
@@ -17,7 +18,7 @@ from trainwave_cli.cli.logs import stream_logs
|
|
|
17
18
|
from trainwave_cli.config.config import config
|
|
18
19
|
from trainwave_cli.format_utils import cents_to_dollars_str
|
|
19
20
|
from trainwave_cli.io import create_tarball
|
|
20
|
-
from trainwave_cli.utils import async_command, ensure_api_key, truncate_string
|
|
21
|
+
from trainwave_cli.utils import async_command, ensure_api_key, to_dict_recursive, truncate_string
|
|
21
22
|
|
|
22
23
|
app = typer.Typer()
|
|
23
24
|
|
|
@@ -30,14 +31,10 @@ async def launch( # noqa
|
|
|
30
31
|
# Basic job parameters
|
|
31
32
|
name: Annotated[str, typer.Option(help="The name of the job")],
|
|
32
33
|
project: Annotated[str, typer.Option(help="The project ID or RID")],
|
|
33
|
-
setup_command: Annotated[
|
|
34
|
-
str, typer.Option(help="The command to run before the job")
|
|
35
|
-
],
|
|
34
|
+
setup_command: Annotated[str, typer.Option(help="The command to run before the job")],
|
|
36
35
|
run_command: Annotated[str, typer.Option(help="The command to run the job")],
|
|
37
36
|
# Optional params
|
|
38
|
-
organization: Annotated[
|
|
39
|
-
str | None, typer.Option(help="ID or RID of organization")
|
|
40
|
-
] = None,
|
|
37
|
+
organization: Annotated[str | None, typer.Option(help="ID or RID of organization")] = None,
|
|
41
38
|
expires: Annotated[
|
|
42
39
|
str,
|
|
43
40
|
typer.Option(
|
|
@@ -53,26 +50,19 @@ async def launch( # noqa
|
|
|
53
50
|
] = True,
|
|
54
51
|
exclude_regex: Annotated[
|
|
55
52
|
str | None,
|
|
56
|
-
typer.Option(
|
|
57
|
-
help="Files matching this pattern will be ignored when packaging up the job"
|
|
58
|
-
),
|
|
53
|
+
typer.Option(help="Files matching this pattern will be ignored when packaging up the job"),
|
|
59
54
|
] = None,
|
|
60
55
|
# Harware parameters
|
|
61
|
-
image: Annotated[
|
|
62
|
-
str, typer.Option(help="The docker image to use as base for the job")
|
|
63
|
-
] = "",
|
|
56
|
+
image: Annotated[str, typer.Option(help="The docker image to use as base for the job")] = "",
|
|
64
57
|
hdd_size_mb: Annotated[int, typer.Option(help="Hard drive size in MB")] = 20,
|
|
65
58
|
memory_mb: Annotated[int, typer.Option(help="Memory in MB")] = 0,
|
|
66
59
|
cpus: Annotated[int, typer.Option(help="Number of CPUs")] = 0,
|
|
67
|
-
gpus: Annotated[
|
|
68
|
-
|
|
69
|
-
] = 0,
|
|
70
|
-
gpu_type: Annotated[
|
|
71
|
-
str, typer.Option(help="Canonical name of the GPU to use")
|
|
72
|
-
] = "",
|
|
60
|
+
gpus: Annotated[int, typer.Option(help="Number of GPUs to allocate of the `gpu_type`")] = 0,
|
|
61
|
+
gpu_type: Annotated[str, typer.Option(help="Canonical name of the GPU to use")] = "",
|
|
73
62
|
compliance_soc2: Annotated[
|
|
74
63
|
bool | None, typer.Option(help="Only use SOC2 compliant machines")
|
|
75
64
|
] = None,
|
|
65
|
+
json: bool = False,
|
|
76
66
|
) -> None:
|
|
77
67
|
"""Launch a job on Trainwave using the current configuration."""
|
|
78
68
|
api_client = Api(config.api_key, config.endpoint, project)
|
|
@@ -126,14 +116,17 @@ async def launch( # noqa
|
|
|
126
116
|
"name": tab_keys,
|
|
127
117
|
"value": [render(k, conf[k]) for k in tab_keys],
|
|
128
118
|
}
|
|
129
|
-
|
|
130
|
-
|
|
119
|
+
if not json:
|
|
120
|
+
typer.echo(tabulate(tab_values, headers="keys"))
|
|
121
|
+
typer.echo("\n")
|
|
131
122
|
|
|
132
123
|
tarball = None
|
|
133
124
|
try:
|
|
134
125
|
job = await api_client.create_job(conf)
|
|
135
|
-
tarball = create_tarball(
|
|
136
|
-
|
|
126
|
+
tarball = create_tarball(
|
|
127
|
+
Path().cwd(), exclude_gitignore, exclude_regex, show_progress_bar=not json
|
|
128
|
+
)
|
|
129
|
+
await api_client.upload_code(Path(tarball.name), job.upload_url, show_progress_bar=not json)
|
|
137
130
|
await api_client.code_submission(job)
|
|
138
131
|
|
|
139
132
|
co = asdict(job.cloud_offer)
|
|
@@ -142,9 +135,12 @@ async def launch( # noqa
|
|
|
142
135
|
"name": tab_keys,
|
|
143
136
|
"value": [truncate_string(str(co[k]), 50) for k in tab_keys],
|
|
144
137
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
138
|
+
if not json:
|
|
139
|
+
typer.echo("\nMatched machine\n" + tabulate(tab_values, headers="keys") + "\n")
|
|
140
|
+
typer.echo(f"Cost per hour for this job: ${job.cost_per_hour:.2f}")
|
|
141
|
+
typer.echo(f"Job created: {job.rid} ({job.id}) -> {job.url}")
|
|
142
|
+
else:
|
|
143
|
+
print(_json.dumps(to_dict_recursive(job), indent=4))
|
|
148
144
|
except Exception as e:
|
|
149
145
|
traceback.print_exc()
|
|
150
146
|
logger.error(f"An error occurred: {e}")
|
|
@@ -162,9 +158,7 @@ async def list_jobs(
|
|
|
162
158
|
project: Annotated[str, typer.Option(help="Project ID or RID")] = "",
|
|
163
159
|
) -> None:
|
|
164
160
|
"""List all jobs in the current project."""
|
|
165
|
-
api_client = Api(
|
|
166
|
-
config.api_key, config.endpoint, project=project, organization=organization
|
|
167
|
-
)
|
|
161
|
+
api_client = Api(config.api_key, config.endpoint, project=project, organization=organization)
|
|
168
162
|
jobs = await api_client.list_jobs()
|
|
169
163
|
|
|
170
164
|
headers = ["ID", "NAME", "STATUS", "GPUS", "COST", "CREATED"]
|
|
@@ -3,6 +3,7 @@ import re
|
|
|
3
3
|
import tarfile
|
|
4
4
|
import tempfile
|
|
5
5
|
import typing
|
|
6
|
+
from contextlib import nullcontext
|
|
6
7
|
from pathlib import Path
|
|
7
8
|
|
|
8
9
|
import pathspec
|
|
@@ -13,15 +14,14 @@ WARN_FILE_SIZE = 50 * 1024 * 1024 # 100 MB
|
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
def get_all_files_in_dir(directory):
|
|
16
|
-
return [
|
|
17
|
-
os.path.join(root, file)
|
|
18
|
-
for root, dirs, files in os.walk(directory)
|
|
19
|
-
for file in files
|
|
20
|
-
]
|
|
17
|
+
return [os.path.join(root, file) for root, dirs, files in os.walk(directory) for file in files]
|
|
21
18
|
|
|
22
19
|
|
|
23
20
|
def create_tarball(
|
|
24
|
-
source_dir: Path,
|
|
21
|
+
source_dir: Path,
|
|
22
|
+
exclude_gitignore: bool,
|
|
23
|
+
exclude_regex: str | None,
|
|
24
|
+
show_progress_bar: bool = True,
|
|
25
25
|
) -> typing.IO[typing.Any]:
|
|
26
26
|
temp = tempfile.NamedTemporaryFile(suffix=".tar")
|
|
27
27
|
spec = None
|
|
@@ -42,17 +42,19 @@ def create_tarball(
|
|
|
42
42
|
files = {file for file in files if not re.search(exclude_regex, file)}
|
|
43
43
|
|
|
44
44
|
total_size = sum(os.path.getsize(file) for file in files)
|
|
45
|
+
progress_bar = (
|
|
46
|
+
tqdm(total=total_size, unit="B", unit_scale=True, desc="Creating archive")
|
|
47
|
+
if show_progress_bar
|
|
48
|
+
else nullcontext()
|
|
49
|
+
)
|
|
45
50
|
with tarfile.open(temp.name, "w:gz") as tar:
|
|
46
|
-
with
|
|
47
|
-
total=total_size, unit="B", unit_scale=True, desc="Creating archive"
|
|
48
|
-
) as progress_bar:
|
|
51
|
+
with progress_bar:
|
|
49
52
|
for file in files:
|
|
50
53
|
if os.path.getsize(file) > WARN_FILE_SIZE:
|
|
51
|
-
logger.warning(
|
|
52
|
-
f"{file} is larger than {WARN_FILE_SIZE/(1024*1024)}MB"
|
|
53
|
-
)
|
|
54
|
+
logger.warning(f"{file} is larger than {WARN_FILE_SIZE / (1024 * 1024)}MB")
|
|
54
55
|
arcname = os.path.relpath(file, source_dir)
|
|
55
56
|
tar.add(file, arcname=arcname)
|
|
56
|
-
|
|
57
|
+
if show_progress_bar:
|
|
58
|
+
progress_bar.update(os.path.getsize(file))
|
|
57
59
|
|
|
58
60
|
return temp
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
from dataclasses import fields, is_dataclass
|
|
2
3
|
from datetime import datetime
|
|
3
4
|
from enum import Enum
|
|
4
5
|
from functools import wraps
|
|
5
|
-
from typing import TypeVar
|
|
6
|
+
from typing import Any, TypeVar
|
|
6
7
|
|
|
7
8
|
import typer
|
|
8
9
|
from dacite import from_dict as dacite_from_dict
|
|
@@ -76,3 +77,18 @@ def from_dict(data_class: type[T], data: dict) -> T:
|
|
|
76
77
|
data,
|
|
77
78
|
config=Config(cast=[Enum], type_hooks={datetime: datetime.fromisoformat}),
|
|
78
79
|
)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def to_dict_recursive(obj: Any) -> Any:
|
|
83
|
+
if is_dataclass(obj):
|
|
84
|
+
result = {}
|
|
85
|
+
for f in fields(obj):
|
|
86
|
+
value = getattr(obj, f.name)
|
|
87
|
+
result[f.name] = to_dict_recursive(value)
|
|
88
|
+
return result
|
|
89
|
+
elif isinstance(obj, (list, tuple)):
|
|
90
|
+
return [to_dict_recursive(item) for item in obj]
|
|
91
|
+
elif isinstance(obj, dict):
|
|
92
|
+
return {key: to_dict_recursive(value) for key, value in obj.items()}
|
|
93
|
+
else:
|
|
94
|
+
return str(obj)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|