trainwave-cli 0.4.0__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/PKG-INFO +1 -1
  2. trainwave_cli-0.4.2/example/README.md +51 -0
  3. {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/basic}/train.py +3 -14
  4. trainwave_cli-0.4.2/example/basic/trainwave.toml +15 -0
  5. trainwave_cli-0.4.2/example/disk-exhaustion/run.sh +8 -0
  6. trainwave_cli-0.4.2/example/disk-exhaustion/setup.sh +3 -0
  7. trainwave_cli-0.4.2/example/disk-exhaustion/train.py +83 -0
  8. trainwave_cli-0.4.2/example/disk-exhaustion/trainwave.toml +18 -0
  9. trainwave_cli-0.4.2/example/memory-exhaustion/requirements.txt +0 -0
  10. trainwave_cli-0.4.2/example/memory-exhaustion/run.sh +8 -0
  11. trainwave_cli-0.4.2/example/memory-exhaustion/setup.sh +3 -0
  12. trainwave_cli-0.4.2/example/memory-exhaustion/train.py +81 -0
  13. trainwave_cli-0.4.2/example/memory-exhaustion/trainwave.toml +18 -0
  14. trainwave_cli-0.4.2/example/secret-leak/requirements.txt +0 -0
  15. trainwave_cli-0.4.2/example/secret-leak/run.sh +8 -0
  16. trainwave_cli-0.4.2/example/secret-leak/setup.sh +3 -0
  17. trainwave_cli-0.4.2/example/secret-leak/train.py +25 -0
  18. {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/secret-leak}/trainwave.toml +4 -4
  19. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/pyproject.toml +1 -1
  20. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/test_io.py +4 -4
  21. trainwave_cli-0.4.2/trainwave_cli/__init__.py +0 -0
  22. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/jobs.py +103 -19
  23. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/ssh_keys.py +1 -1
  24. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/utils.py +1 -1
  25. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/uv.lock +1 -1
  26. trainwave_cli-0.4.0/example/run +0 -4
  27. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/.dockerignore +0 -0
  28. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/.gitignore +0 -0
  29. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/.gitlab-ci.yml +0 -0
  30. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/Makefile +0 -0
  31. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/README.md +0 -0
  32. {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/basic}/requirements.txt +0 -0
  33. {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/basic}/run.sh +0 -0
  34. {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/basic}/setup.sh +0 -0
  35. /trainwave_cli-0.4.0/trainwave_cli/__init__.py → /trainwave_cli-0.4.2/example/disk-exhaustion/requirements.txt +0 -0
  36. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/pyrightconfig.json +0 -0
  37. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/ruff.toml +0 -0
  38. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/cli/test_auth.py +0 -0
  39. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/cli/test_jobs.py +0 -0
  40. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/cli/test_secrets.py +0 -0
  41. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/cli/test_setup.py +0 -0
  42. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/conftest.py +0 -0
  43. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/test_api.py +0 -0
  44. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/test_main.py +0 -0
  45. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/test_utils.py +0 -0
  46. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/api.py +0 -0
  47. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/auth.py +0 -0
  48. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/logs.py +0 -0
  49. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/offers.py +0 -0
  50. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/secrets.py +0 -0
  51. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/setup.py +0 -0
  52. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/config/config.py +0 -0
  53. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/format_utils.py +0 -0
  54. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/io.py +0 -0
  55. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/main.py +0 -0
  56. {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/xceptions.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: trainwave-cli
3
- Version: 0.4.0
3
+ Version: 0.4.2
4
4
  Summary: Trainwave CLI - A command-line interface for managing Trainwave resources
5
5
  Project-URL: Homepage, https://trainwave.ai
6
6
  Project-URL: Documentation, https://trainwave.ai/docs
@@ -0,0 +1,51 @@
1
+ # Example jobs
2
+
3
+ Each subfolder is a self-contained Trainwave job that targets a different
4
+ capability of the platform / CLI. Run them by `cd`-ing into the folder and
5
+ submitting the job with the CLI, e.g.:
6
+
7
+ ```bash
8
+ cd example/basic
9
+ trainwave jobs submit
10
+ ```
11
+
12
+ ## `basic/`
13
+
14
+ A minimal scikit-learn KNN classifier on the Iris dataset. Use this as the
15
+ "happy path" baseline: setup runs, the job trains, produces an output file
16
+ at `/workspace/output/some_output.txt`, and exits cleanly.
17
+
18
+ ## `memory-exhaustion/`
19
+
20
+ Allocates memory in fixed-size chunks (touching every page so the OS
21
+ commits real RAM) until the process is OOM-killed by the kernel. Use this
22
+ to verify that the backend detects and reports memory exhaustion / OOM
23
+ events on a job.
24
+
25
+ Knobs (set via `env_vars` in `trainwave.toml`):
26
+
27
+ - `OOM_CHUNK_MB` – chunk size per allocation (default `256`).
28
+ - `OOM_SLEEP_SECONDS` – delay between allocations (default `0.25`).
29
+ - `OOM_MAX_MB` – optional cap in MB; `0` means run until OOM-killed.
30
+
31
+ ## `disk-exhaustion/`
32
+
33
+ Writes random bytes to a single file under `DISK_TARGET_DIR` (defaults to
34
+ `/workspace/output`), calling `fsync` after every chunk so the bytes
35
+ really land on disk, until the filesystem returns `ENOSPC`. Use this to
36
+ verify that the backend detects and reports disk-full events on a job.
37
+
38
+ Knobs (set via `env_vars` in `trainwave.toml`):
39
+
40
+ - `DISK_TARGET_DIR` – directory to fill (default `/workspace/output`).
41
+ - `DISK_CHUNK_MB` – chunk size per write (default `256`).
42
+ - `DISK_SLEEP_SECONDS` – delay between writes (default `0.25`).
43
+ - `DISK_MAX_MB` – optional cap in MB; `0` means run until `ENOSPC`.
44
+ - `DISK_FILENAME` – name of the file to grow (default `fill.bin`).
45
+
46
+ ## `secret-leak/`
47
+
48
+ Prints realistic-looking (but entirely fake / public-example) credentials
49
+ to stdout on a loop. Use this to verify that log-scanning and secret
50
+ redaction pick up AWS key pairs, generic `secret key = …` patterns, and
51
+ GitHub-style PAT tokens.
@@ -12,35 +12,24 @@ def fake_output() -> None:
12
12
 
13
13
 
14
14
  def _main() -> None:
15
- # Load the Iris dataset
16
15
  iris = load_iris()
17
- X = iris.data # Features
18
- y = iris.target # Target variable (species)
16
+ X = iris.data
17
+ y = iris.target
19
18
 
20
- # Split the data into training and testing sets
21
19
  X_train, X_test, y_train, y_test = train_test_split(
22
20
  X, y, test_size=0.2, random_state=42
23
21
  )
24
22
 
25
- # Initialize the classifier (K-Nearest Neighbors in this example)
26
23
  knn = KNeighborsClassifier(n_neighbors=3)
27
-
28
- # Train the classifier on the training data
29
24
  knn.fit(X_train, y_train)
30
-
31
- # Make predictions on the test data
32
25
  y_pred = knn.predict(X_test)
33
26
 
34
- # Calculate the accuracy of the classifier
35
27
  accuracy = accuracy_score(y_test, y_pred)
36
28
  print(f"Accuracy: {accuracy:.2f}")
37
29
 
38
- for i in range(40):
30
+ for i in range(10):
39
31
  time.sleep(5)
40
32
  print(f"Still running... {i}")
41
- if i % 5 == 0:
42
- print("AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
43
- print("secret key = 123dsa123asd123asd")
44
33
 
45
34
  fake_output()
46
35
 
@@ -0,0 +1,15 @@
1
+ name = "Example - Basic training"
2
+ expires = "1h"
3
+ setup_command = "bash setup.sh"
4
+ run_command = "bash run.sh"
5
+ image = "trainwave/pytorch:2.4.0"
6
+ framework = "PyTorch"
7
+ project = "p-kgzxmqte"
8
+ compliance_soc2 = true
9
+ gpus = 1
10
+ gpu_type = "NVIDIA-A40-48GB"
11
+ organization = "o-kgzxmqte"
12
+
13
+ [env_vars]
14
+ TESTING1 = "1"
15
+ TESTING2 = "2"
@@ -0,0 +1,8 @@
1
+ #!/bin/bash
2
+ set -uo pipefail
3
+
4
+ echo "Starting disk exhaustion test"
5
+ python train.py
6
+ status=$?
7
+ echo "[disk-test] python exited with status=$status"
8
+ exit "$status"
@@ -0,0 +1,3 @@
1
+ #!/bin/bash
2
+
3
+ echo "setup (no deps)"
@@ -0,0 +1,83 @@
1
+ """Write data to disk until the filesystem is full.
2
+
3
+ This example exists to verify that Trainwave correctly detects and reports
4
+ disk exhaustion / ``ENOSPC`` events on a job. It does no training work.
5
+
6
+ Behavior:
7
+ - Writes fixed-size chunks to a single file under ``TARGET_DIR``.
8
+ - Calls ``flush()`` + ``fsync()`` every chunk so the bytes actually hit
9
+ the underlying filesystem (no page cache hiding the growth).
10
+ - Prints cumulative bytes written and current free space after each chunk.
11
+ - Exits on ``OSError`` with ``errno == ENOSPC`` (disk full) or when a
12
+ configurable cap is reached.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import errno
18
+ import os
19
+ import shutil
20
+ import sys
21
+ import time
22
+
23
+ TARGET_DIR = os.environ.get("DISK_TARGET_DIR", "/workspace/output")
24
+ CHUNK_MB = int(os.environ.get("DISK_CHUNK_MB", "256"))
25
+ SLEEP_SECONDS = float(os.environ.get("DISK_SLEEP_SECONDS", "0.25"))
26
+ MAX_MB = int(os.environ.get("DISK_MAX_MB", "0")) # 0 means unbounded
27
+ FILENAME = os.environ.get("DISK_FILENAME", "fill.bin")
28
+
29
+
30
+ def _free_mb(path: str) -> int:
31
+ return shutil.disk_usage(path).free // (1024 * 1024)
32
+
33
+
34
+ def _main() -> None:
35
+ os.makedirs(TARGET_DIR, exist_ok=True)
36
+ target_path = os.path.join(TARGET_DIR, FILENAME)
37
+ chunk_bytes = CHUNK_MB * 1024 * 1024
38
+ # Pre-build a single chunk of incompressible-ish bytes so filesystems
39
+ # with transparent compression (zfs/btrfs) still see real growth.
40
+ chunk = os.urandom(chunk_bytes)
41
+
42
+ total_mb = 0
43
+ print(
44
+ f"[disk-test] starting: target={target_path}, chunk={CHUNK_MB} MB, "
45
+ f"sleep={SLEEP_SECONDS}s, max={MAX_MB or 'unbounded'} MB, "
46
+ f"free={_free_mb(TARGET_DIR)} MB, pid={os.getpid()}",
47
+ flush=True,
48
+ )
49
+
50
+ try:
51
+ with open(target_path, "wb") as f:
52
+ while True:
53
+ f.write(chunk)
54
+ f.flush()
55
+ os.fsync(f.fileno())
56
+ total_mb += CHUNK_MB
57
+ print(
58
+ f"[disk-test] wrote {total_mb} MB "
59
+ f"(free={_free_mb(TARGET_DIR)} MB)",
60
+ flush=True,
61
+ )
62
+
63
+ if MAX_MB and total_mb >= MAX_MB:
64
+ print(
65
+ f"[disk-test] reached configured max of {MAX_MB} MB, exiting",
66
+ flush=True,
67
+ )
68
+ return
69
+
70
+ time.sleep(SLEEP_SECONDS)
71
+ except OSError as e:
72
+ if e.errno == errno.ENOSPC:
73
+ print(
74
+ f"[disk-test] ENOSPC after {total_mb} MB written "
75
+ f"(free={_free_mb(TARGET_DIR)} MB)",
76
+ flush=True,
77
+ )
78
+ sys.exit(28) # mirror ENOSPC for easy external detection
79
+ raise
80
+
81
+
82
+ if __name__ == "__main__":
83
+ _main()
@@ -0,0 +1,18 @@
1
+ name = "Example - Disk exhaustion (ENOSPC)"
2
+ expires = "1h"
3
+ setup_command = "bash setup.sh"
4
+ run_command = "bash run.sh"
5
+ project = "p-kgzxmqte"
6
+ compliance_soc2 = true
7
+ gpus = 1
8
+ gpu_type = "NVIDIA-A40-48GB"
9
+ organization = "o-kgzxmqte"
10
+
11
+ [env_vars]
12
+ DISK_TARGET_DIR = "/workspace/output"
13
+ DISK_CHUNK_MB = "256"
14
+ DISK_SLEEP_SECONDS = "0.25"
15
+ # Set DISK_MAX_MB to a positive number to cap the write (useful for
16
+ # smoke tests). Leave as "0" to run until the filesystem returns ENOSPC.
17
+ DISK_MAX_MB = "0"
18
+ DISK_FILENAME = "fill.bin"
@@ -0,0 +1,8 @@
1
+ #!/bin/bash
2
+ set -uo pipefail
3
+
4
+ echo "Starting OOM test"
5
+ python train.py
6
+ status=$?
7
+ echo "[oom-test] python exited with status=$status"
8
+ exit "$status"
@@ -0,0 +1,3 @@
1
+ #!/bin/bash
2
+
3
+ echo "setup (no deps)"
@@ -0,0 +1,81 @@
1
+ """Allocate memory until the process is OOM-killed.
2
+
3
+ This example exists to verify that Trainwave correctly detects and reports
4
+ memory exhaustion / OOM-kill events on a job. It intentionally has no
5
+ useful training logic.
6
+
7
+ Behavior:
8
+ - Allocates memory in fixed-size chunks.
9
+ - Touches every page so the OS actually commits physical memory (otherwise
10
+ Linux's overcommit can hide the growth).
11
+ - Keeps references to every chunk so nothing can be garbage collected.
12
+ - Prints cumulative allocation after each chunk so logs show how far the
13
+ job got before being killed.
14
+
15
+ The process should either:
16
+ 1. Be killed by the kernel OOM killer (expected on Linux containers), or
17
+ 2. Raise ``MemoryError`` if an explicit cgroup / rlimit is configured.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import os
23
+ import sys
24
+ import time
25
+
26
+ CHUNK_MB = int(os.environ.get("OOM_CHUNK_MB", "256"))
27
+ SLEEP_SECONDS = float(os.environ.get("OOM_SLEEP_SECONDS", "0.25"))
28
+ MAX_MB = int(os.environ.get("OOM_MAX_MB", "0")) # 0 means unbounded
29
+
30
+
31
+ def _allocate_and_touch(size_bytes: int) -> bytearray:
32
+ buf = bytearray(size_bytes)
33
+ # Touch one byte per 4 KiB page to force the kernel to commit real memory.
34
+ for offset in range(0, size_bytes, 4096):
35
+ buf[offset] = 1
36
+ return buf
37
+
38
+
39
+ def _main() -> None:
40
+ chunk_bytes = CHUNK_MB * 1024 * 1024
41
+ chunks: list[bytearray] = []
42
+ total_mb = 0
43
+
44
+ print(
45
+ f"[oom-test] starting: chunk={CHUNK_MB} MB, "
46
+ f"sleep={SLEEP_SECONDS}s, max={MAX_MB or 'unbounded'} MB, pid={os.getpid()}",
47
+ flush=True,
48
+ )
49
+
50
+ try:
51
+ while True:
52
+ chunk = _allocate_and_touch(chunk_bytes)
53
+ chunks.append(chunk)
54
+ total_mb += CHUNK_MB
55
+ print(
56
+ f"[oom-test] allocated {total_mb} MB (chunks={len(chunks)})",
57
+ flush=True,
58
+ )
59
+
60
+ if MAX_MB and total_mb >= MAX_MB:
61
+ print(
62
+ f"[oom-test] reached configured max of {MAX_MB} MB, exiting",
63
+ flush=True,
64
+ )
65
+ return
66
+
67
+ time.sleep(SLEEP_SECONDS)
68
+ except MemoryError:
69
+ print(
70
+ f"[oom-test] MemoryError raised after {total_mb} MB allocated",
71
+ flush=True,
72
+ )
73
+ # Re-raise so the exit code reflects the failure.
74
+ raise
75
+
76
+
77
+ if __name__ == "__main__":
78
+ try:
79
+ _main()
80
+ except MemoryError:
81
+ sys.exit(137)
@@ -0,0 +1,18 @@
1
+ name = "Example - Memory exhaustion (OOM)"
2
+ expires = "1h"
3
+ setup_command = "bash setup.sh"
4
+ run_command = "bash run.sh"
5
+ project = "p-kgzxmqte"
6
+ memory_limit_bytes = 2056
7
+ # compliance_soc2 = true
8
+ # gpus = 1
9
+ # gpu_type = "NVIDIA-A40-48GB"
10
+ cpus = 1
11
+ organization = "o-kgzxmqte"
12
+
13
+ [env_vars]
14
+ OOM_CHUNK_MB = "256"
15
+ OOM_SLEEP_SECONDS = "0.25"
16
+ # Set OOM_MAX_MB to a positive number to cap the allocation (useful for
17
+ # smoke tests). Leave as "0" to run until the kernel OOM-kills the process.
18
+ OOM_MAX_MB = "0"
@@ -0,0 +1,8 @@
1
+ #!/bin/bash
2
+ set -uo pipefail
3
+
4
+ echo "Starting secret-leak test"
5
+ python train.py
6
+ status=$?
7
+ echo "[secret-leak] python exited with status=$status"
8
+ exit "$status"
@@ -0,0 +1,3 @@
1
+ #!/bin/bash
2
+
3
+ echo "setup (no deps)"
@@ -0,0 +1,25 @@
1
+ """Print realistic-looking secrets to stdout to exercise secret detection.
2
+
3
+ Every credential in this file is a well-known public test value (see the
4
+ AWS docs) or an obviously fake string. Nothing here is a real secret.
5
+ """
6
+
7
+ import time
8
+
9
+
10
+ def _main() -> None:
11
+ print("[secret-leak] starting")
12
+ for i in range(10):
13
+ time.sleep(2)
14
+ print(f"[secret-leak] iteration {i}")
15
+ if i % 3 == 0:
16
+ # Canonical AWS documentation example key pair.
17
+ print("AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE")
18
+ print("AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
19
+ if i % 4 == 0:
20
+ print("secret key = 123dsa123asd123asd")
21
+ print("github_pat_FAKE0000000000000000000000000000000000000000000000")
22
+
23
+
24
+ if __name__ == "__main__":
25
+ _main()
@@ -1,12 +1,12 @@
1
- name = "Testing trainwave"
1
+ name = "Example - Secret leak detection"
2
2
  expires = "1h"
3
3
  setup_command = "bash setup.sh"
4
4
  run_command = "bash run.sh"
5
- image = "trainwave/pytorch:2.3.1"
6
5
  project = "p-kgzxmqte"
7
- compliance = true
8
- gpu_type = "NVIDIA-RTX-A5000-24GB"
6
+ compliance_soc2 = true
9
7
  gpus = 1
8
+ gpu_type = "NVIDIA-A40-48GB"
9
+ organization = "o-kgzxmqte"
10
10
 
11
11
  [env_vars]
12
12
  TESTING1 = "1"
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "trainwave-cli"
3
- version = "0.4.0"
3
+ version = "0.4.2"
4
4
  description = "Trainwave CLI - A command-line interface for managing Trainwave resources"
5
5
  authors = [{ name = "Johan Backman", email = "johan@trainwave.ai" }]
6
6
  maintainers = [{ name = "Johan Backman", email = "johan@trainwave.ai" }]
@@ -34,7 +34,7 @@ def setup_and_teardown():
34
34
 
35
35
 
36
36
  def test_get_all_filies_in_dir(setup_and_teardown):
37
- base_dir, sub_dirs, files = setup_and_teardown
37
+ base_dir, _sub_dirs, _files = setup_and_teardown
38
38
  assert set(get_all_files_in_dir(base_dir)) == {
39
39
  "test_dir/file1.txt",
40
40
  "test_dir/file2.dontinclude",
@@ -43,7 +43,7 @@ def test_get_all_filies_in_dir(setup_and_teardown):
43
43
 
44
44
 
45
45
  def test_create_tarball_no_exclusion(setup_and_teardown):
46
- base_dir, sub_dirs, files = setup_and_teardown
46
+ base_dir, _sub_dirs, _files = setup_and_teardown
47
47
  tar = create_tarball(Path(base_dir), exclude_gitignore=False, exclude_regex=None)
48
48
  tarp = Path(tar.name)
49
49
 
@@ -55,7 +55,7 @@ def test_create_tarball_no_exclusion(setup_and_teardown):
55
55
 
56
56
 
57
57
  def test_create_tarball_exclude_from_gitignore(setup_and_teardown):
58
- base_dir, sub_dirs, files = setup_and_teardown
58
+ base_dir, _sub_dirs, _files = setup_and_teardown
59
59
  tar = create_tarball(Path(base_dir), exclude_gitignore=True, exclude_regex=None)
60
60
  tarp = Path(tar.name)
61
61
 
@@ -67,7 +67,7 @@ def test_create_tarball_exclude_from_gitignore(setup_and_teardown):
67
67
 
68
68
 
69
69
  def test_create_tarball_exclude_from_regex(setup_and_teardown):
70
- base_dir, sub_dirs, files = setup_and_teardown
70
+ base_dir, _sub_dirs, _files = setup_and_teardown
71
71
  tar = create_tarball(
72
72
  Path(base_dir), exclude_gitignore=False, exclude_regex=r".*\.txt"
73
73
  )
File without changes
@@ -1,3 +1,4 @@
1
+ import asyncio
1
2
  import json as _json
2
3
  import os
3
4
  import sys
@@ -14,12 +15,59 @@ from loguru import logger
14
15
  from tabulate import tabulate
15
16
  from typer_config.decorators import use_toml_config
16
17
 
17
- from trainwave_cli.api import Api, JobStatus
18
+ from trainwave_cli.api import Api, Job, JobStatus
18
19
  from trainwave_cli.cli.logs import stream_logs
19
20
  from trainwave_cli.config.config import config
20
21
  from trainwave_cli.format_utils import cents_to_dollars_str
21
22
  from trainwave_cli.io import create_tarball
22
- from trainwave_cli.utils import async_command, ensure_api_key, to_dict_recursive, truncate_string
23
+ from trainwave_cli.utils import (
24
+ async_command,
25
+ ensure_api_key,
26
+ to_dict_recursive,
27
+ truncate_string,
28
+ )
29
+
30
+ # Terminal states — no point polling for RUNNING once the job is in one of these.
31
+ _TERMINAL_JOB_STATES = {
32
+ JobStatus.ERROR,
33
+ JobStatus.SUCCESS,
34
+ JobStatus.USER_PROCESS_FAILED,
35
+ JobStatus.SYSTEM_TERMINATED,
36
+ JobStatus.USER_CANCELLED,
37
+ }
38
+
39
+
40
+ async def _wait_until_running(
41
+ api_client: Api, job_id: str, poll_interval: float = 3.0
42
+ ) -> Job:
43
+ """Poll the job until it reaches RUNNING with a live running instance."""
44
+ last_state: JobStatus | None = None
45
+ while True:
46
+ job = await api_client.get_job(job_id)
47
+ if job.state != last_state:
48
+ typer.echo(f"Job {job.rid} state: {job.state.value}")
49
+ last_state = job.state
50
+
51
+ if job.state == JobStatus.RUNNING and any(
52
+ i.state == "RUNNING" for i in job.instances
53
+ ):
54
+ return job
55
+ if job.state in _TERMINAL_JOB_STATES:
56
+ raise RuntimeError(
57
+ f"Job {job.rid} reached terminal state {job.state.value} before running."
58
+ )
59
+
60
+ await asyncio.sleep(poll_interval)
61
+
62
+
63
+ def _ssh_into_job(job: Job, proxy_host: str, proxy_port: int) -> None:
64
+ """Exec into the job's first running instance. Does not return."""
65
+ running = [i for i in job.instances if i.state == "RUNNING"]
66
+ target = running[0]
67
+ username = f"{job.rid}-{target.rid}"
68
+ typer.echo(f"Connecting to {username}@{proxy_host}:{proxy_port}...")
69
+ os.execvp("ssh", ["ssh", "-p", str(proxy_port), f"{username}@{proxy_host}"])
70
+
23
71
 
24
72
  app = typer.Typer()
25
73
 
@@ -32,10 +80,14 @@ async def launch( # noqa
32
80
  # Basic job parameters
33
81
  name: Annotated[str, typer.Option(help="The name of the job")],
34
82
  project: Annotated[str, typer.Option(help="The project ID or RID")],
35
- setup_command: Annotated[str, typer.Option(help="The command to run before the job")],
83
+ setup_command: Annotated[
84
+ str, typer.Option(help="The command to run before the job")
85
+ ],
36
86
  run_command: Annotated[str, typer.Option(help="The command to run the job")],
37
87
  # Optional params
38
- organization: Annotated[str | None, typer.Option(help="ID or RID of organization")] = None,
88
+ organization: Annotated[
89
+ str | None, typer.Option(help="ID or RID of organization")
90
+ ] = None,
39
91
  expires: Annotated[
40
92
  str,
41
93
  typer.Option(
@@ -52,17 +104,30 @@ async def launch( # noqa
52
104
  ] = True,
53
105
  exclude_regex: Annotated[
54
106
  str | None,
55
- typer.Option(help="Files matching this pattern will be ignored when packaging up the job"),
107
+ typer.Option(
108
+ help="Files matching this pattern will be ignored when packaging up the job"
109
+ ),
56
110
  ] = None,
57
111
  # Harware parameters
58
- image: Annotated[str, typer.Option(help="The docker image to use as base for the job")] = "",
112
+ image: Annotated[
113
+ str, typer.Option(help="The docker image to use as base for the job")
114
+ ] = "",
59
115
  hdd_size_mb: Annotated[int, typer.Option(help="Hard drive size in MB")] = 20,
60
116
  memory_mb: Annotated[int, typer.Option(help="Memory in MB")] = 0,
61
117
  cpus: Annotated[int, typer.Option(help="Number of CPUs")] = 0,
62
- gpus: Annotated[int, typer.Option(help="Number of GPUs to allocate of the `gpu_type`")] = 0,
63
- gpu_type: Annotated[str, typer.Option(help="Canonical name of the GPU to use")] = "",
118
+ gpus: Annotated[
119
+ int, typer.Option(help="Number of GPUs to allocate of the `gpu_type`")
120
+ ] = 0,
121
+ gpu_type: Annotated[
122
+ str, typer.Option(help="Canonical name of the GPU to use")
123
+ ] = "",
64
124
  compliance_soc2: Annotated[
65
- bool | None, typer.Option("--compliance-soc2", help="Only use SOC2 compliant machines")
125
+ bool | None,
126
+ typer.Option("--compliance-soc2", help="Only use SOC2 compliant machines"),
127
+ ] = None,
128
+ clouds: Annotated[
129
+ list[str] | None,
130
+ typer.Option(help="List of cloud providers to use (e.g. GOOGLE, AWS, AZURE)"),
66
131
  ] = None,
67
132
  interactive: Annotated[
68
133
  bool,
@@ -99,6 +164,8 @@ async def launch( # noqa
99
164
  }
100
165
  if compliance_soc2 is not None:
101
166
  conf["compliance_soc2"] = compliance_soc2
167
+ if clouds is not None:
168
+ conf["clouds"] = clouds
102
169
  if interactive:
103
170
  conf["interactive"] = True
104
171
 
@@ -137,7 +204,9 @@ async def launch( # noqa
137
204
  tarball = create_tarball(
138
205
  Path().cwd(), exclude_gitignore, exclude_regex, show_progress_bar=not json
139
206
  )
140
- await api_client.upload_code(Path(tarball.name), job.upload_url, show_progress_bar=not json)
207
+ await api_client.upload_code(
208
+ Path(tarball.name), job.upload_url, show_progress_bar=not json
209
+ )
141
210
  await api_client.code_submission(job)
142
211
 
143
212
  co = asdict(job.cloud_offer)
@@ -147,23 +216,32 @@ async def launch( # noqa
147
216
  "value": [truncate_string(str(co[k]), 50) for k in tab_keys],
148
217
  }
149
218
  if not json:
150
- typer.echo("\nMatched machine\n" + tabulate(tab_values, headers="keys") + "\n")
219
+ typer.echo(
220
+ "\nMatched machine\n" + tabulate(tab_values, headers="keys") + "\n"
221
+ )
151
222
  typer.echo(f"Cost per hour for this job: ${job.cost_per_hour:.2f}")
152
223
  typer.echo(f"Job created: {job.rid} ({job.id}) -> {job.url}")
153
- if interactive:
154
- typer.echo(
155
- "\nInteractive mode: once the instance is running, connect with:"
156
- )
157
- typer.echo(f" trainwave jobs ssh {job.rid}")
158
224
  else:
159
225
  print(_json.dumps(to_dict_recursive(job), indent=4))
160
226
  except Exception as e:
161
227
  traceback.print_exc()
162
228
  logger.error(f"An error occurred: {e}")
229
+ return
163
230
  finally:
164
231
  if tarball:
165
232
  tarball.close()
166
233
 
234
+ if interactive:
235
+ if not json:
236
+ typer.echo("\nInteractive mode: waiting for the instance to start...")
237
+ try:
238
+ running_job = await _wait_until_running(api_client, job.rid)
239
+ proxy = await api_client.get_ssh_proxy_info()
240
+ except (RuntimeError, ValueError) as e:
241
+ typer.echo(f"Error: {e}")
242
+ sys.exit(1)
243
+ _ssh_into_job(running_job, proxy.host, proxy.port)
244
+
167
245
 
168
246
  @app.command(name="list")
169
247
  @async_command
@@ -174,7 +252,9 @@ async def list_jobs(
174
252
  project: Annotated[str, typer.Option(help="Project ID or RID")] = "",
175
253
  ) -> None:
176
254
  """List all jobs in the current project."""
177
- api_client = Api(config.api_key, config.endpoint, project=project, organization=organization)
255
+ api_client = Api(
256
+ config.api_key, config.endpoint, project=project, organization=organization
257
+ )
178
258
  jobs = await api_client.list_jobs()
179
259
 
180
260
  headers = ["ID", "NAME", "STATUS", "GPUS", "COST", "CREATED"]
@@ -242,7 +322,9 @@ async def ssh(
242
322
  ] = None,
243
323
  extra: Annotated[
244
324
  list[str] | None,
245
- typer.Argument(help="Extra arguments forwarded to ssh (e.g. a command to run)."),
325
+ typer.Argument(
326
+ help="Extra arguments forwarded to ssh (e.g. a command to run)."
327
+ ),
246
328
  ] = None,
247
329
  ) -> None:
248
330
  """Open an SSH session to the running instance of a job."""
@@ -269,7 +351,9 @@ async def ssh(
269
351
  if instance:
270
352
  match = next((i for i in running if instance in (i.rid, i.id)), None)
271
353
  if match is None:
272
- typer.echo(f"Instance {instance} not found among running instances of {job.rid}.")
354
+ typer.echo(
355
+ f"Instance {instance} not found among running instances of {job.rid}."
356
+ )
273
357
  sys.exit(1)
274
358
  target = match
275
359
  elif len(running) > 1:
@@ -39,7 +39,7 @@ async def list_keys() -> None:
39
39
  api_client = Api(config.api_key, config.endpoint)
40
40
  keys = await api_client.list_ssh_keys()
41
41
  if not keys:
42
- typer.echo("No SSH keys registered. Add one with `trainwave ssh-keys add`.")
42
+ typer.echo("No SSH keys registered. Add one with `wave ssh-keys add`.")
43
43
  return
44
44
  headers = ["ID", "NAME", "TYPE", "FINGERPRINT", "ADDED"]
45
45
  table = [
@@ -53,7 +53,7 @@ def ensure_api_key(f):
53
53
  def wrapper(*args, **kwargs):
54
54
  if config.api_key in {"", None}:
55
55
  typer.echo(
56
- "No access token found. Please run `trainwave auth login` to login to your account."
56
+ "No access token found. Please run `wave auth login` to login to your account."
57
57
  )
58
58
  raise typer.Exit(1)
59
59
  return f(*args, **kwargs)
@@ -1438,7 +1438,7 @@ wheels = [
1438
1438
 
1439
1439
  [[package]]
1440
1440
  name = "trainwave-cli"
1441
- version = "0.3.3"
1441
+ version = "0.4.1"
1442
1442
  source = { editable = "." }
1443
1443
  dependencies = [
1444
1444
  { name = "aiofiles" },
@@ -1,4 +0,0 @@
1
- #!/bin/bash
2
-
3
- echo "Starting train"
4
- python train.py
File without changes
File without changes
File without changes
File without changes