trainwave-cli 0.1.4__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: trainwave-cli
3
- Version: 0.1.4
3
+ Version: 0.1.5
4
4
  Summary:
5
5
  Author: Johan Backman
6
6
  Author-email: johan@trainwave.ai
@@ -11,8 +11,10 @@ Classifier: Programming Language :: Python :: 3.11
11
11
  Classifier: Programming Language :: Python :: 3.12
12
12
  Requires-Dist: aiofiles (>=24.1.0,<25.0.0)
13
13
  Requires-Dist: aiohttp (>=3.10,<4.0)
14
+ Requires-Dist: beaupy (>=3.9.2,<4.0.0)
14
15
  Requires-Dist: boto3 (>=1.30)
15
16
  Requires-Dist: httpx (==0.27.0)
17
+ Requires-Dist: human-readable (>=1.3.4,<2.0.0)
16
18
  Requires-Dist: loguru (>=0.7.2,<0.8.0)
17
19
  Requires-Dist: pathspec (>=0.12.1,<0.13.0)
18
20
  Requires-Dist: pytimeparse (>=1.1.8,<2.0.0)
@@ -1,26 +1,28 @@
1
1
  [tool.poetry]
2
2
  name = "trainwave-cli"
3
- version = "0.1.4"
3
+ version = "0.1.5"
4
4
  description = ""
5
5
  authors = ["Johan Backman <johan@trainwave.ai>"]
6
6
  readme = "README.md"
7
7
  packages = [{ include = "trainwave_cli" }]
8
8
 
9
9
  [tool.poetry.dependencies]
10
+ aiofiles = "^24.1.0"
10
11
  aiohttp = "^3.10"
12
+ beaupy = "^3.9.2"
11
13
  boto3 = ">=1.30"
12
14
  httpx = "0.27.0"
15
+ human-readable = "^1.3.4"
13
16
  loguru = "^0.7.2"
17
+ pathspec = "^0.12.1"
14
18
  python = "^3.10"
15
19
  pytimeparse = "^1.1.8"
20
+ tabulate = "^0.9.0"
16
21
  toml = "^0.10.2"
17
22
  tqdm = "^4.66.4"
18
23
  typer = "^0.12.3"
19
24
  typer-config = { extras = ["all"], version = "^1.4.0" }
20
25
  websockets = "^12.0"
21
- aiofiles = "^24.1.0"
22
- pathspec = "^0.12.1"
23
- tabulate = "^0.9.0"
24
26
 
25
27
  [tool.poetry.scripts]
26
28
  wave = "trainwave_cli.main:entrypoint"
@@ -1,7 +1,6 @@
1
- import os
2
1
  import socket
3
- import tempfile
4
- from dataclasses import dataclass
2
+ from dataclasses import dataclass, field
3
+ from datetime import datetime, timezone
5
4
  from enum import Enum
6
5
  from http import HTTPStatus
7
6
  from pathlib import Path
@@ -18,7 +17,6 @@ from trainwave_cli.utils import from_dict
18
17
  @dataclass(frozen=True)
19
18
  class CloudOffer:
20
19
  cpus: int
21
- cpu_type: str
22
20
  memory_mb: int
23
21
  compliance_soc2: bool
24
22
  gpu_type: str
@@ -26,6 +24,17 @@ class CloudOffer:
26
24
  gpus: int
27
25
 
28
26
 
27
+ @dataclass
28
+ class JobConfig:
29
+ id: str
30
+ rid: str
31
+ name: str
32
+ expires_at: int
33
+ cpus: int
34
+ gpus: int
35
+ gpu_type: str
36
+
37
+
29
38
  @dataclass
30
39
  class Job:
31
40
  id: str
@@ -33,9 +42,13 @@ class Job:
33
42
  state: str
34
43
  s3_url: str
35
44
  project: str
36
- upload_url: str
37
45
  cloud_offer: CloudOffer
38
46
  cost_per_hour: float
47
+ config: JobConfig
48
+ created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
49
+ total_cost: int = 0
50
+ upload_url: str = ""
51
+ url: str | None = None
39
52
 
40
53
 
41
54
  @dataclass(frozen=True)
@@ -47,6 +60,29 @@ class TrainWaveUser:
47
60
  email: str
48
61
 
49
62
 
63
+ @dataclass(frozen=True)
64
+ class TrainwaveOrganization:
65
+ id: str
66
+ rid: str
67
+ name: str
68
+ computed_credit_balance: int
69
+
70
+
71
+ @dataclass(frozen=True)
72
+ class TrainwaveProject:
73
+ id: str
74
+ rid: str
75
+ name: str
76
+
77
+
78
+ @dataclass(frozen=True)
79
+ class DjangoPagination:
80
+ count: int
81
+ next: str | None
82
+ previous: str | None
83
+ results: list[dict[str, Any]]
84
+
85
+
50
86
  class CLIAuthStatus(Enum):
51
87
  NOT_FOUND = "NOT_FOUND"
52
88
  NOT_COMPLETED = "NOT_COMPLETED"
@@ -71,11 +107,13 @@ class Api:
71
107
  api_key: str | None,
72
108
  endpoint: str,
73
109
  project: str = "",
110
+ organization: str = "",
74
111
  ):
75
112
  verify_ssl = "trainwave.dev" not in endpoint
76
113
  self.client = httpx.AsyncClient(base_url=endpoint, verify=verify_ssl)
77
114
  self.api_key = api_key
78
115
  self.project = project
116
+ self.organization = organization
79
117
 
80
118
  async def request(self, method, path, **kwargs):
81
119
  headers = kwargs.pop("headers", {})
@@ -120,6 +158,25 @@ class Api:
120
158
  return True
121
159
  return False
122
160
 
161
+ async def list_organizations(self) -> list[TrainwaveOrganization]:
162
+ res = await self.request("GET", "/api/v1/organizations/")
163
+ res.raise_for_status()
164
+ return [from_dict(TrainwaveOrganization, org) for org in res.json()["results"]]
165
+
166
+ async def list_projects(self, organization_id: str) -> list[TrainwaveProject]:
167
+ res = await self.request("GET", "/api/v1/projects/")
168
+ res.raise_for_status()
169
+ return [from_dict(TrainwaveProject, project) for project in res.json()["results"]]
170
+
171
+ async def create_project(self, organization_id: str, name: str) -> TrainwaveProject:
172
+ res = await self.request(
173
+ "POST",
174
+ "/api/v1/projects/",
175
+ json={"name": name, "organization": organization_id},
176
+ )
177
+ res.raise_for_status()
178
+ return from_dict(TrainwaveProject, res.json())
179
+
123
180
  async def get_myself(self) -> TrainWaveUser:
124
181
  res = await self.request("GET", "/api/v1/users/me/")
125
182
  res.raise_for_status()
@@ -132,6 +189,14 @@ class Api:
132
189
  email=json_body.get("email"),
133
190
  )
134
191
 
192
+ async def list_jobs(self) -> list[Job]:
193
+ path = f"/api/v1/jobs/?org_id={self.organization}"
194
+ res = await self.request(
195
+ "GET",
196
+ path,
197
+ )
198
+ return [from_dict(Job, job) for job in res.json()["results"]]
199
+
135
200
  async def create_job(self, config: dict[str, Any]) -> Job:
136
201
  res = await self.request(
137
202
  "POST",
@@ -25,8 +25,8 @@ async def login() -> None:
25
25
  api_client = Api(None, config.endpoint)
26
26
  session_url, session_token = await api_client.create_cli_auth_session()
27
27
 
28
- print("Opening your browser to complete the login.")
29
- print(f"{_BOLD}URL:{_RESET} {session_url} \n")
28
+ typer.echo("Opening your browser to complete the login.\n")
29
+ typer.echo(f"{_BOLD}URL:{_RESET} {session_url} \n")
30
30
 
31
31
  webbrowser.open_new_tab(session_url)
32
32
 
@@ -54,13 +54,14 @@ async def login() -> None:
54
54
  api = Api(api_token, config.endpoint)
55
55
  user = await api.get_myself()
56
56
 
57
- print("✅ Success!")
58
- print(f"Logged in as: {_BOLD}{user.email}{_RESET}")
57
+ typer.echo("\n")
58
+ typer.echo("✅ Success!\n")
59
+ typer.echo(f"Logged in as: {_BOLD}{user.email}{_RESET}")
59
60
 
60
61
  config.api_key = api_token
61
62
  config_manager.save()
62
63
  else:
63
- print("❌ Something went wrong. Try again.")
64
+ typer.echo("❌ Something went wrong. Try again.")
64
65
 
65
66
 
66
67
  @app.command()
@@ -77,7 +78,7 @@ async def logout() -> None:
77
78
  async def whoami() -> None:
78
79
  api = Api(config.api_key, config.endpoint)
79
80
  res = await api.get_myself()
80
- print(res.email)
81
+ typer.echo(res.email)
81
82
 
82
83
 
83
84
  @app.command()
@@ -98,7 +99,7 @@ async def set_token(api_key: str):
98
99
  @async_command
99
100
  @ensure_api_key
100
101
  async def token() -> None:
101
- print(config.api_key)
102
+ typer.echo(config.api_key)
102
103
 
103
104
 
104
105
  @app.command()
@@ -5,18 +5,21 @@ import traceback
5
5
  from dataclasses import asdict
6
6
  from datetime import datetime, timezone
7
7
  from pathlib import Path
8
- from typing import Annotated, Optional
8
+ from typing import Annotated
9
9
  from urllib.parse import urlparse
10
10
 
11
+ import human_readable
11
12
  import pytimeparse
12
13
  import typer
13
14
  import websockets
15
+ from dateutil import parser
14
16
  from loguru import logger
15
17
  from tabulate import tabulate
16
18
  from typer_config.decorators import use_toml_config
17
19
 
18
20
  from trainwave_cli.api import Api
19
21
  from trainwave_cli.config.config import config
22
+ from trainwave_cli.format_utils import cents_to_dollars_str
20
23
  from trainwave_cli.io import create_tarball
21
24
  from trainwave_cli.utils import async_command, ensure_api_key, truncate_string
22
25
 
@@ -34,10 +37,11 @@ async def launch( # noqa
34
37
  setup_command: Annotated[str, typer.Option()],
35
38
  run_command: Annotated[str, typer.Option()],
36
39
  # Optional params
40
+ organization: Annotated[str, typer.Option()] | None = None,
37
41
  expires: Annotated[str, typer.Option()] = "",
38
42
  env_vars={}, # dict[str, str] not supported by typer yet
39
43
  exclude_gitignore: Annotated[bool, typer.Option()] = True,
40
- exclude_regex: Optional[Annotated[str, typer.Option()]] = None,
44
+ exclude_regex: Annotated[str, typer.Option()] | None = None,
41
45
  # Harware parameters
42
46
  image: Annotated[str, typer.Option()] = "",
43
47
  hdd_size_mb: Annotated[int, typer.Option()] = 20,
@@ -45,7 +49,7 @@ async def launch( # noqa
45
49
  cpus: Annotated[int, typer.Option()] = 0,
46
50
  gpus: Annotated[int, typer.Option()] = 0,
47
51
  gpu_type: Annotated[str, typer.Option()] = "",
48
- compliance_soc2: Annotated[bool, typer.Option()] = None,
52
+ compliance_soc2: Annotated[bool, typer.Option()] | None = None,
49
53
  ) -> None:
50
54
  api_client = Api(config.api_key, config.endpoint, project)
51
55
  # Interpolate environe variables
@@ -73,6 +77,9 @@ async def launch( # noqa
73
77
  if compliance_soc2 is not None:
74
78
  conf["compliance_soc2"] = compliance_soc2
75
79
 
80
+ if conf["gpu_type"] != "" and conf["gpus"] == 0:
81
+ conf["gpus"] = 1 # default to 1 gpu if gpu_type is set
82
+
76
83
  if expires != "":
77
84
  expires_time = pytimeparse.parse(expires)
78
85
  if expires_time is None:
@@ -113,7 +120,7 @@ async def launch( # noqa
113
120
  }
114
121
  logger.info("\nMatched machine\n" + tabulate(tab_values, headers="keys") + "\n")
115
122
  logger.info(f"Cost per hour for this job: ${job.cost_per_hour:.2f}")
116
- logger.info(f"Job created: {job.rid} ({job.id})")
123
+ logger.info(f"Job created: {job.rid} ({job.id}) -> {job.url}")
117
124
  except Exception as e:
118
125
  traceback.print_exc()
119
126
  logger.error(f"An error occurred: {e}")
@@ -122,6 +129,36 @@ async def launch( # noqa
122
129
  tarball.close()
123
130
 
124
131
 
132
+ @app.command(name="list")
133
+ @async_command
134
+ @use_toml_config(default_value="trainwave.toml")
135
+ @ensure_api_key
136
+ async def list_jobs(
137
+ organization: Annotated[str, typer.Option()],
138
+ project: Annotated[str, typer.Option()] = "",
139
+ ) -> None:
140
+ api_client = Api(
141
+ config.api_key, config.endpoint, project=project, organization=organization
142
+ )
143
+ jobs = await api_client.list_jobs()
144
+
145
+ headers = ["ID", "NAME", "STATUS", "CREATED", "GPUS", "COST"]
146
+ table = [
147
+ [
148
+ j.rid,
149
+ j.config.name,
150
+ j.state,
151
+ human_readable.date_time(
152
+ datetime.now(timezone.utc) - parser.parse(j.created_at)
153
+ ),
154
+ f"{j.config.gpus} ({j.cloud_offer.gpu_type})",
155
+ f"{cents_to_dollars_str(j.total_cost)} (${j.cost_per_hour}/h)",
156
+ ]
157
+ for j in jobs
158
+ ]
159
+ typer.echo(tabulate(table, headers=headers, tablefmt="simple"))
160
+
161
+
125
162
  @app.command()
126
163
  @async_command
127
164
  @use_toml_config(default_value="trainwave.toml")
@@ -168,7 +205,9 @@ async def logs(
168
205
  port = ""
169
206
  if parsed.port:
170
207
  port = f":{parsed.port}"
171
- uri = f"{scheme}://{parsed.hostname}{port}/ws/logs/{job_id}/?api_key={config.api_key}"
208
+ uri = (
209
+ f"{scheme}://{parsed.hostname}{port}/ws/logs/{job_id}/?api_key={config.api_key}"
210
+ )
172
211
 
173
212
  def format_ts(ts: str | None) -> str:
174
213
  if ts is None:
@@ -1,5 +1,5 @@
1
1
  import typer
2
- from beaupy import confirm, prompt, select, select_multiple
2
+ from beaupy import select
3
3
  from rich.console import Console
4
4
 
5
5
  from trainwave_cli.api import Api
@@ -21,15 +21,13 @@ CURSOR = "➡️"
21
21
  @ensure_api_key
22
22
  async def list():
23
23
  api_client = Api(config.api_key, config.endpoint)
24
- offers = await api_client.offers()
24
+ _offers = await api_client.offers()
25
25
 
26
26
  console = Console()
27
27
  console.print("Select Hardware")
28
28
  hw = select(HARDWARE_OPTIONS, cursor=CURSOR)
29
29
 
30
30
  if hw == "GPU":
31
-
31
+ pass
32
32
  else:
33
33
  pass
34
-
35
-
@@ -0,0 +1,114 @@
1
+ from typing import Annotated, Any
2
+
3
+ import toml
4
+ import typer
5
+ from beaupy import confirm, prompt, select
6
+ from typer_config import conf_callback_factory
7
+ from typer_config.decorators import use_config
8
+ from typer_config.loaders import toml_loader
9
+
10
+ from trainwave_cli.api import Api
11
+ from trainwave_cli.config.config import config
12
+ from trainwave_cli.utils import async_command, ensure_api_key
13
+
14
+ app = typer.Typer()
15
+
16
+
17
+ def pyproject_loader(param_value: str) -> dict[str, Any]:
18
+ if not param_value: # set a default path to read from
19
+ param_value = "trainwave.toml"
20
+ try:
21
+ pyproject = toml_loader(param_value)
22
+ except FileNotFoundError:
23
+ return {}
24
+ return pyproject
25
+
26
+
27
+ pyproject_callback = conf_callback_factory(pyproject_loader)
28
+
29
+
30
+ _CREATE_NEW_PROJECT_OPTION = "( Create a new project )"
31
+
32
+
33
+ async def _run_setup_guide(
34
+ existing_org: str,
35
+ existing_project: str,
36
+ ) -> dict[str, str | int] | None:
37
+ typer.echo("Running setup\n")
38
+
39
+ api_client = Api(config.api_key, config.endpoint)
40
+
41
+ # Step 1: Select an organization that user is part of
42
+ organizations = await api_client.list_organizations()
43
+ if not organizations:
44
+ typer.echo("You are not part of any organization.\n")
45
+ typer.echo("Please visit the Trainwave web app to create an organization.\n")
46
+ return
47
+
48
+ formatted_org_selection = [f"{org.name}" for org in organizations]
49
+
50
+ typer.echo("1. Select which organization to use.\n")
51
+ select_org = select(formatted_org_selection)
52
+
53
+ selected_org = next(org for org in organizations if org.name == select_org)
54
+
55
+ # Step 2: Select or create a project to use
56
+
57
+ projects = await api_client.list_projects(selected_org.id)
58
+ project_options = [f"{project.name}" for project in projects]
59
+ project_options.append(_CREATE_NEW_PROJECT_OPTION)
60
+
61
+ typer.echo("2. Select which project to use.\n")
62
+ typer.echo(
63
+ "A project groups together all the jobs you run and allows you to enforce spending limits and default configurations for all jobs run in the project.\n"
64
+ )
65
+
66
+ selected_project = select(project_options)
67
+
68
+ if selected_project == _CREATE_NEW_PROJECT_OPTION:
69
+ project_name = prompt("Enter the name of the new project")
70
+ project = await api_client.create_project(selected_org.id, project_name)
71
+ else:
72
+ project = next(
73
+ project for project in projects if project.name == selected_project
74
+ )
75
+
76
+ typer.echo("Give your job a name. Example: 'Training a LLM'\n")
77
+
78
+ name = prompt("Enter the name of the job")
79
+
80
+ # TODO: Ask for GPU type and count
81
+ # SETUP COMMAND and RUN COMMAND
82
+ return {
83
+ "organization": selected_org.rid,
84
+ "project": project.rid,
85
+ "name": name,
86
+ }
87
+
88
+
89
+ @app.callback(invoke_without_command=True)
90
+ @async_command
91
+ @use_config(pyproject_callback)
92
+ @ensure_api_key
93
+ async def default(
94
+ organization: Annotated[str, typer.Option()] = "",
95
+ project: Annotated[str, typer.Option()] = "",
96
+ name: Annotated[str, typer.Option()] = "",
97
+ ) -> None:
98
+ # Stuff is not set
99
+ if not organization:
100
+ typer.echo("No config file found.\n")
101
+
102
+ if confirm(
103
+ "Would you like to create a new Trainwave configuration (trainwave.toml)?"
104
+ ):
105
+ new_config = await _run_setup_guide(organization, project, name)
106
+
107
+ typer.echo(new_config)
108
+
109
+ with open("trainwave.toml", "w") as f:
110
+ toml.dump(new_config, f)
111
+
112
+ # Ask user if they want to switch org or project
113
+ else:
114
+ pass
@@ -38,7 +38,9 @@ class ConfigManager:
38
38
  with open(self.CONFIG_PATH) as f:
39
39
  data = toml.load(f)
40
40
  self.config = Config(
41
- **{k: v for k, v in data.items() if k in Config.__dataclass_fields__.keys()},
41
+ **{
42
+ k: v for k, v in data.items() if k in Config.__dataclass_fields__.keys()
43
+ },
42
44
  )
43
45
 
44
46
  def _setup_config(self) -> None:
@@ -0,0 +1,3 @@
1
+ def cents_to_dollars_str(cents: int) -> str:
2
+ dollars = cents / 100
3
+ return f"${dollars:,.2f}"
@@ -13,11 +13,11 @@ WARN_FILE_SIZE = 50 * 1024 * 1024 # 100 MB
13
13
 
14
14
 
15
15
  def get_all_files_in_dir(directory):
16
- file_list = []
17
- for root, dirs, files in os.walk(directory):
18
- for file in files:
19
- file_list.append(os.path.join(root, file))
20
- return file_list
16
+ return [
17
+ os.path.join(root, file)
18
+ for root, dirs, files in os.walk(directory)
19
+ for file in files
20
+ ]
21
21
 
22
22
 
23
23
  def create_tarball(
@@ -48,7 +48,9 @@ def create_tarball(
48
48
  ) as progress_bar:
49
49
  for file in files:
50
50
  if os.path.getsize(file) > WARN_FILE_SIZE:
51
- logger.warning(f"{file} is larger than {WARN_FILE_SIZE/(1024*1024)}MB")
51
+ logger.warning(
52
+ f"{file} is larger than {WARN_FILE_SIZE/(1024*1024)}MB"
53
+ )
52
54
  arcname = os.path.relpath(file, source_dir)
53
55
  tar.add(file, arcname=arcname)
54
56
  progress_bar.update(os.path.getsize(file))
@@ -3,11 +3,12 @@ import sys
3
3
  import typer
4
4
  from loguru import logger
5
5
 
6
- from trainwave_cli.cli import auth, jobs
6
+ from trainwave_cli.cli import auth, jobs, setup
7
7
 
8
8
  app = typer.Typer()
9
9
  app.add_typer(jobs.app, name="jobs")
10
10
  app.add_typer(auth.app, name="auth")
11
+ app.add_typer(setup.app, name="setup")
11
12
 
12
13
 
13
14
  def entrypoint() -> None:
@@ -1,7 +1,7 @@
1
1
  import asyncio
2
2
  from dataclasses import fields, is_dataclass
3
3
  from functools import wraps
4
- from typing import Type, TypeVar
4
+ from typing import TypeVar
5
5
 
6
6
  import typer
7
7
 
@@ -31,7 +31,7 @@ def ensure_api_key(f):
31
31
  @wraps(f)
32
32
  def wrapper(*args, **kwargs):
33
33
  if config.api_key in {"", None}:
34
- print(
34
+ typer.echo(
35
35
  "No access token found. Please run `trainwave auth login` to login to your account."
36
36
  )
37
37
  raise typer.Exit(1)
@@ -49,7 +49,7 @@ def truncate_string(s: str, max_len: int) -> str:
49
49
  T = TypeVar("T")
50
50
 
51
51
 
52
- def from_dict(data_class: Type[T], data: dict) -> T:
52
+ def from_dict(data_class: type[T], data: dict) -> T:
53
53
  """Recursively constructs a dataclass from a nested dictionary, skipping unmatched fields."""
54
54
  field_names = {f.name for f in fields(data_class)}
55
55
  filtered_data = {k: v for k, v in data.items() if k in field_names}
File without changes