trainwave-cli 0.4.0__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/PKG-INFO +1 -1
- trainwave_cli-0.4.2/example/README.md +51 -0
- {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/basic}/train.py +3 -14
- trainwave_cli-0.4.2/example/basic/trainwave.toml +15 -0
- trainwave_cli-0.4.2/example/disk-exhaustion/run.sh +8 -0
- trainwave_cli-0.4.2/example/disk-exhaustion/setup.sh +3 -0
- trainwave_cli-0.4.2/example/disk-exhaustion/train.py +83 -0
- trainwave_cli-0.4.2/example/disk-exhaustion/trainwave.toml +18 -0
- trainwave_cli-0.4.2/example/memory-exhaustion/requirements.txt +0 -0
- trainwave_cli-0.4.2/example/memory-exhaustion/run.sh +8 -0
- trainwave_cli-0.4.2/example/memory-exhaustion/setup.sh +3 -0
- trainwave_cli-0.4.2/example/memory-exhaustion/train.py +81 -0
- trainwave_cli-0.4.2/example/memory-exhaustion/trainwave.toml +18 -0
- trainwave_cli-0.4.2/example/secret-leak/requirements.txt +0 -0
- trainwave_cli-0.4.2/example/secret-leak/run.sh +8 -0
- trainwave_cli-0.4.2/example/secret-leak/setup.sh +3 -0
- trainwave_cli-0.4.2/example/secret-leak/train.py +25 -0
- {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/secret-leak}/trainwave.toml +4 -4
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/pyproject.toml +1 -1
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/test_io.py +4 -4
- trainwave_cli-0.4.2/trainwave_cli/__init__.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/jobs.py +103 -19
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/ssh_keys.py +1 -1
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/utils.py +1 -1
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/uv.lock +1 -1
- trainwave_cli-0.4.0/example/run +0 -4
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/.dockerignore +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/.gitignore +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/.gitlab-ci.yml +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/Makefile +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/README.md +0 -0
- {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/basic}/requirements.txt +0 -0
- {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/basic}/run.sh +0 -0
- {trainwave_cli-0.4.0/example → trainwave_cli-0.4.2/example/basic}/setup.sh +0 -0
- /trainwave_cli-0.4.0/trainwave_cli/__init__.py → /trainwave_cli-0.4.2/example/disk-exhaustion/requirements.txt +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/pyrightconfig.json +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/ruff.toml +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/cli/test_auth.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/cli/test_jobs.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/cli/test_secrets.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/cli/test_setup.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/conftest.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/test_api.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/test_main.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/tests/test_utils.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/api.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/auth.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/logs.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/offers.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/secrets.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/cli/setup.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/config/config.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/format_utils.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/io.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/main.py +0 -0
- {trainwave_cli-0.4.0 → trainwave_cli-0.4.2}/trainwave_cli/xceptions.py +0 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# Example jobs
|
|
2
|
+
|
|
3
|
+
Each subfolder is a self-contained Trainwave job that targets a different
|
|
4
|
+
capability of the platform / CLI. Run them by `cd`-ing into the folder and
|
|
5
|
+
submitting the job with the CLI, e.g.:
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
cd example/basic
|
|
9
|
+
trainwave jobs submit
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## `basic/`
|
|
13
|
+
|
|
14
|
+
A minimal scikit-learn KNN classifier on the Iris dataset. Use this as the
|
|
15
|
+
"happy path" baseline: setup runs, the job trains, produces an output file
|
|
16
|
+
at `/workspace/output/some_output.txt`, and exits cleanly.
|
|
17
|
+
|
|
18
|
+
## `memory-exhaustion/`
|
|
19
|
+
|
|
20
|
+
Allocates memory in fixed-size chunks (touching every page so the OS
|
|
21
|
+
commits real RAM) until the process is OOM-killed by the kernel. Use this
|
|
22
|
+
to verify that the backend detects and reports memory exhaustion / OOM
|
|
23
|
+
events on a job.
|
|
24
|
+
|
|
25
|
+
Knobs (set via `env_vars` in `trainwave.toml`):
|
|
26
|
+
|
|
27
|
+
- `OOM_CHUNK_MB` – chunk size per allocation (default `256`).
|
|
28
|
+
- `OOM_SLEEP_SECONDS` – delay between allocations (default `0.25`).
|
|
29
|
+
- `OOM_MAX_MB` – optional cap in MB; `0` means run until OOM-killed.
|
|
30
|
+
|
|
31
|
+
## `disk-exhaustion/`
|
|
32
|
+
|
|
33
|
+
Writes random bytes to a single file under `DISK_TARGET_DIR` (defaults to
|
|
34
|
+
`/workspace/output`), calling `fsync` after every chunk so the bytes
|
|
35
|
+
really land on disk, until the filesystem returns `ENOSPC`. Use this to
|
|
36
|
+
verify that the backend detects and reports disk-full events on a job.
|
|
37
|
+
|
|
38
|
+
Knobs (set via `env_vars` in `trainwave.toml`):
|
|
39
|
+
|
|
40
|
+
- `DISK_TARGET_DIR` – directory to fill (default `/workspace/output`).
|
|
41
|
+
- `DISK_CHUNK_MB` – chunk size per write (default `256`).
|
|
42
|
+
- `DISK_SLEEP_SECONDS` – delay between writes (default `0.25`).
|
|
43
|
+
- `DISK_MAX_MB` – optional cap in MB; `0` means run until `ENOSPC`.
|
|
44
|
+
- `DISK_FILENAME` – name of the file to grow (default `fill.bin`).
|
|
45
|
+
|
|
46
|
+
## `secret-leak/`
|
|
47
|
+
|
|
48
|
+
Prints realistic-looking (but entirely fake / public-example) credentials
|
|
49
|
+
to stdout on a loop. Use this to verify that log-scanning and secret
|
|
50
|
+
redaction pick up AWS key pairs, generic `secret key = …` patterns, and
|
|
51
|
+
GitHub-style PAT tokens.
|
|
@@ -12,35 +12,24 @@ def fake_output() -> None:
|
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
def _main() -> None:
|
|
15
|
-
# Load the Iris dataset
|
|
16
15
|
iris = load_iris()
|
|
17
|
-
X = iris.data
|
|
18
|
-
y = iris.target
|
|
16
|
+
X = iris.data
|
|
17
|
+
y = iris.target
|
|
19
18
|
|
|
20
|
-
# Split the data into training and testing sets
|
|
21
19
|
X_train, X_test, y_train, y_test = train_test_split(
|
|
22
20
|
X, y, test_size=0.2, random_state=42
|
|
23
21
|
)
|
|
24
22
|
|
|
25
|
-
# Initialize the classifier (K-Nearest Neighbors in this example)
|
|
26
23
|
knn = KNeighborsClassifier(n_neighbors=3)
|
|
27
|
-
|
|
28
|
-
# Train the classifier on the training data
|
|
29
24
|
knn.fit(X_train, y_train)
|
|
30
|
-
|
|
31
|
-
# Make predictions on the test data
|
|
32
25
|
y_pred = knn.predict(X_test)
|
|
33
26
|
|
|
34
|
-
# Calculate the accuracy of the classifier
|
|
35
27
|
accuracy = accuracy_score(y_test, y_pred)
|
|
36
28
|
print(f"Accuracy: {accuracy:.2f}")
|
|
37
29
|
|
|
38
|
-
for i in range(
|
|
30
|
+
for i in range(10):
|
|
39
31
|
time.sleep(5)
|
|
40
32
|
print(f"Still running... {i}")
|
|
41
|
-
if i % 5 == 0:
|
|
42
|
-
print("AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
|
43
|
-
print("secret key = 123dsa123asd123asd")
|
|
44
33
|
|
|
45
34
|
fake_output()
|
|
46
35
|
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
name = "Example - Basic training"
|
|
2
|
+
expires = "1h"
|
|
3
|
+
setup_command = "bash setup.sh"
|
|
4
|
+
run_command = "bash run.sh"
|
|
5
|
+
image = "trainwave/pytorch:2.4.0"
|
|
6
|
+
framework = "PyTorch"
|
|
7
|
+
project = "p-kgzxmqte"
|
|
8
|
+
compliance_soc2 = true
|
|
9
|
+
gpus = 1
|
|
10
|
+
gpu_type = "NVIDIA-A40-48GB"
|
|
11
|
+
organization = "o-kgzxmqte"
|
|
12
|
+
|
|
13
|
+
[env_vars]
|
|
14
|
+
TESTING1 = "1"
|
|
15
|
+
TESTING2 = "2"
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""Write data to disk until the filesystem is full.
|
|
2
|
+
|
|
3
|
+
This example exists to verify that Trainwave correctly detects and reports
|
|
4
|
+
disk exhaustion / ``ENOSPC`` events on a job. It does no training work.
|
|
5
|
+
|
|
6
|
+
Behavior:
|
|
7
|
+
- Writes fixed-size chunks to a single file under ``TARGET_DIR``.
|
|
8
|
+
- Calls ``flush()`` + ``fsync()`` every chunk so the bytes actually hit
|
|
9
|
+
the underlying filesystem (no page cache hiding the growth).
|
|
10
|
+
- Prints cumulative bytes written and current free space after each chunk.
|
|
11
|
+
- Exits on ``OSError`` with ``errno == ENOSPC`` (disk full) or when a
|
|
12
|
+
configurable cap is reached.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import errno
|
|
18
|
+
import os
|
|
19
|
+
import shutil
|
|
20
|
+
import sys
|
|
21
|
+
import time
|
|
22
|
+
|
|
23
|
+
TARGET_DIR = os.environ.get("DISK_TARGET_DIR", "/workspace/output")
|
|
24
|
+
CHUNK_MB = int(os.environ.get("DISK_CHUNK_MB", "256"))
|
|
25
|
+
SLEEP_SECONDS = float(os.environ.get("DISK_SLEEP_SECONDS", "0.25"))
|
|
26
|
+
MAX_MB = int(os.environ.get("DISK_MAX_MB", "0")) # 0 means unbounded
|
|
27
|
+
FILENAME = os.environ.get("DISK_FILENAME", "fill.bin")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _free_mb(path: str) -> int:
|
|
31
|
+
return shutil.disk_usage(path).free // (1024 * 1024)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _main() -> None:
|
|
35
|
+
os.makedirs(TARGET_DIR, exist_ok=True)
|
|
36
|
+
target_path = os.path.join(TARGET_DIR, FILENAME)
|
|
37
|
+
chunk_bytes = CHUNK_MB * 1024 * 1024
|
|
38
|
+
# Pre-build a single chunk of incompressible-ish bytes so filesystems
|
|
39
|
+
# with transparent compression (zfs/btrfs) still see real growth.
|
|
40
|
+
chunk = os.urandom(chunk_bytes)
|
|
41
|
+
|
|
42
|
+
total_mb = 0
|
|
43
|
+
print(
|
|
44
|
+
f"[disk-test] starting: target={target_path}, chunk={CHUNK_MB} MB, "
|
|
45
|
+
f"sleep={SLEEP_SECONDS}s, max={MAX_MB or 'unbounded'} MB, "
|
|
46
|
+
f"free={_free_mb(TARGET_DIR)} MB, pid={os.getpid()}",
|
|
47
|
+
flush=True,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
try:
|
|
51
|
+
with open(target_path, "wb") as f:
|
|
52
|
+
while True:
|
|
53
|
+
f.write(chunk)
|
|
54
|
+
f.flush()
|
|
55
|
+
os.fsync(f.fileno())
|
|
56
|
+
total_mb += CHUNK_MB
|
|
57
|
+
print(
|
|
58
|
+
f"[disk-test] wrote {total_mb} MB "
|
|
59
|
+
f"(free={_free_mb(TARGET_DIR)} MB)",
|
|
60
|
+
flush=True,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
if MAX_MB and total_mb >= MAX_MB:
|
|
64
|
+
print(
|
|
65
|
+
f"[disk-test] reached configured max of {MAX_MB} MB, exiting",
|
|
66
|
+
flush=True,
|
|
67
|
+
)
|
|
68
|
+
return
|
|
69
|
+
|
|
70
|
+
time.sleep(SLEEP_SECONDS)
|
|
71
|
+
except OSError as e:
|
|
72
|
+
if e.errno == errno.ENOSPC:
|
|
73
|
+
print(
|
|
74
|
+
f"[disk-test] ENOSPC after {total_mb} MB written "
|
|
75
|
+
f"(free={_free_mb(TARGET_DIR)} MB)",
|
|
76
|
+
flush=True,
|
|
77
|
+
)
|
|
78
|
+
sys.exit(28) # mirror ENOSPC for easy external detection
|
|
79
|
+
raise
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
if __name__ == "__main__":
|
|
83
|
+
_main()
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
name = "Example - Disk exhaustion (ENOSPC)"
|
|
2
|
+
expires = "1h"
|
|
3
|
+
setup_command = "bash setup.sh"
|
|
4
|
+
run_command = "bash run.sh"
|
|
5
|
+
project = "p-kgzxmqte"
|
|
6
|
+
compliance_soc2 = true
|
|
7
|
+
gpus = 1
|
|
8
|
+
gpu_type = "NVIDIA-A40-48GB"
|
|
9
|
+
organization = "o-kgzxmqte"
|
|
10
|
+
|
|
11
|
+
[env_vars]
|
|
12
|
+
DISK_TARGET_DIR = "/workspace/output"
|
|
13
|
+
DISK_CHUNK_MB = "256"
|
|
14
|
+
DISK_SLEEP_SECONDS = "0.25"
|
|
15
|
+
# Set DISK_MAX_MB to a positive number to cap the write (useful for
|
|
16
|
+
# smoke tests). Leave as "0" to run until the filesystem returns ENOSPC.
|
|
17
|
+
DISK_MAX_MB = "0"
|
|
18
|
+
DISK_FILENAME = "fill.bin"
|
|
File without changes
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Allocate memory until the process is OOM-killed.
|
|
2
|
+
|
|
3
|
+
This example exists to verify that Trainwave correctly detects and reports
|
|
4
|
+
memory exhaustion / OOM-kill events on a job. It intentionally has no
|
|
5
|
+
useful training logic.
|
|
6
|
+
|
|
7
|
+
Behavior:
|
|
8
|
+
- Allocates memory in fixed-size chunks.
|
|
9
|
+
- Touches every page so the OS actually commits physical memory (otherwise
|
|
10
|
+
Linux's overcommit can hide the growth).
|
|
11
|
+
- Keeps references to every chunk so nothing can be garbage collected.
|
|
12
|
+
- Prints cumulative allocation after each chunk so logs show how far the
|
|
13
|
+
job got before being killed.
|
|
14
|
+
|
|
15
|
+
The process should either:
|
|
16
|
+
1. Be killed by the kernel OOM killer (expected on Linux containers), or
|
|
17
|
+
2. Raise ``MemoryError`` if an explicit cgroup / rlimit is configured.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import os
|
|
23
|
+
import sys
|
|
24
|
+
import time
|
|
25
|
+
|
|
26
|
+
CHUNK_MB = int(os.environ.get("OOM_CHUNK_MB", "256"))
|
|
27
|
+
SLEEP_SECONDS = float(os.environ.get("OOM_SLEEP_SECONDS", "0.25"))
|
|
28
|
+
MAX_MB = int(os.environ.get("OOM_MAX_MB", "0")) # 0 means unbounded
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _allocate_and_touch(size_bytes: int) -> bytearray:
|
|
32
|
+
buf = bytearray(size_bytes)
|
|
33
|
+
# Touch one byte per 4 KiB page to force the kernel to commit real memory.
|
|
34
|
+
for offset in range(0, size_bytes, 4096):
|
|
35
|
+
buf[offset] = 1
|
|
36
|
+
return buf
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _main() -> None:
|
|
40
|
+
chunk_bytes = CHUNK_MB * 1024 * 1024
|
|
41
|
+
chunks: list[bytearray] = []
|
|
42
|
+
total_mb = 0
|
|
43
|
+
|
|
44
|
+
print(
|
|
45
|
+
f"[oom-test] starting: chunk={CHUNK_MB} MB, "
|
|
46
|
+
f"sleep={SLEEP_SECONDS}s, max={MAX_MB or 'unbounded'} MB, pid={os.getpid()}",
|
|
47
|
+
flush=True,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
try:
|
|
51
|
+
while True:
|
|
52
|
+
chunk = _allocate_and_touch(chunk_bytes)
|
|
53
|
+
chunks.append(chunk)
|
|
54
|
+
total_mb += CHUNK_MB
|
|
55
|
+
print(
|
|
56
|
+
f"[oom-test] allocated {total_mb} MB (chunks={len(chunks)})",
|
|
57
|
+
flush=True,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
if MAX_MB and total_mb >= MAX_MB:
|
|
61
|
+
print(
|
|
62
|
+
f"[oom-test] reached configured max of {MAX_MB} MB, exiting",
|
|
63
|
+
flush=True,
|
|
64
|
+
)
|
|
65
|
+
return
|
|
66
|
+
|
|
67
|
+
time.sleep(SLEEP_SECONDS)
|
|
68
|
+
except MemoryError:
|
|
69
|
+
print(
|
|
70
|
+
f"[oom-test] MemoryError raised after {total_mb} MB allocated",
|
|
71
|
+
flush=True,
|
|
72
|
+
)
|
|
73
|
+
# Re-raise so the exit code reflects the failure.
|
|
74
|
+
raise
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
if __name__ == "__main__":
|
|
78
|
+
try:
|
|
79
|
+
_main()
|
|
80
|
+
except MemoryError:
|
|
81
|
+
sys.exit(137)
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
name = "Example - Memory exhaustion (OOM)"
|
|
2
|
+
expires = "1h"
|
|
3
|
+
setup_command = "bash setup.sh"
|
|
4
|
+
run_command = "bash run.sh"
|
|
5
|
+
project = "p-kgzxmqte"
|
|
6
|
+
memory_limit_bytes = 2056
|
|
7
|
+
# compliance_soc2 = true
|
|
8
|
+
# gpus = 1
|
|
9
|
+
# gpu_type = "NVIDIA-A40-48GB"
|
|
10
|
+
cpus = 1
|
|
11
|
+
organization = "o-kgzxmqte"
|
|
12
|
+
|
|
13
|
+
[env_vars]
|
|
14
|
+
OOM_CHUNK_MB = "256"
|
|
15
|
+
OOM_SLEEP_SECONDS = "0.25"
|
|
16
|
+
# Set OOM_MAX_MB to a positive number to cap the allocation (useful for
|
|
17
|
+
# smoke tests). Leave as "0" to run until the kernel OOM-kills the process.
|
|
18
|
+
OOM_MAX_MB = "0"
|
|
File without changes
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Print realistic-looking secrets to stdout to exercise secret detection.
|
|
2
|
+
|
|
3
|
+
Every credential in this file is a well-known public test value (see the
|
|
4
|
+
AWS docs) or an obviously fake string. Nothing here is a real secret.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import time
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _main() -> None:
|
|
11
|
+
print("[secret-leak] starting")
|
|
12
|
+
for i in range(10):
|
|
13
|
+
time.sleep(2)
|
|
14
|
+
print(f"[secret-leak] iteration {i}")
|
|
15
|
+
if i % 3 == 0:
|
|
16
|
+
# Canonical AWS documentation example key pair.
|
|
17
|
+
print("AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE")
|
|
18
|
+
print("AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY")
|
|
19
|
+
if i % 4 == 0:
|
|
20
|
+
print("secret key = 123dsa123asd123asd")
|
|
21
|
+
print("github_pat_FAKE0000000000000000000000000000000000000000000000")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
if __name__ == "__main__":
|
|
25
|
+
_main()
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
name = "
|
|
1
|
+
name = "Example - Secret leak detection"
|
|
2
2
|
expires = "1h"
|
|
3
3
|
setup_command = "bash setup.sh"
|
|
4
4
|
run_command = "bash run.sh"
|
|
5
|
-
image = "trainwave/pytorch:2.3.1"
|
|
6
5
|
project = "p-kgzxmqte"
|
|
7
|
-
|
|
8
|
-
gpu_type = "NVIDIA-RTX-A5000-24GB"
|
|
6
|
+
compliance_soc2 = true
|
|
9
7
|
gpus = 1
|
|
8
|
+
gpu_type = "NVIDIA-A40-48GB"
|
|
9
|
+
organization = "o-kgzxmqte"
|
|
10
10
|
|
|
11
11
|
[env_vars]
|
|
12
12
|
TESTING1 = "1"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "trainwave-cli"
|
|
3
|
-
version = "0.4.
|
|
3
|
+
version = "0.4.2"
|
|
4
4
|
description = "Trainwave CLI - A command-line interface for managing Trainwave resources"
|
|
5
5
|
authors = [{ name = "Johan Backman", email = "johan@trainwave.ai" }]
|
|
6
6
|
maintainers = [{ name = "Johan Backman", email = "johan@trainwave.ai" }]
|
|
@@ -34,7 +34,7 @@ def setup_and_teardown():
|
|
|
34
34
|
|
|
35
35
|
|
|
36
36
|
def test_get_all_filies_in_dir(setup_and_teardown):
|
|
37
|
-
base_dir,
|
|
37
|
+
base_dir, _sub_dirs, _files = setup_and_teardown
|
|
38
38
|
assert set(get_all_files_in_dir(base_dir)) == {
|
|
39
39
|
"test_dir/file1.txt",
|
|
40
40
|
"test_dir/file2.dontinclude",
|
|
@@ -43,7 +43,7 @@ def test_get_all_filies_in_dir(setup_and_teardown):
|
|
|
43
43
|
|
|
44
44
|
|
|
45
45
|
def test_create_tarball_no_exclusion(setup_and_teardown):
|
|
46
|
-
base_dir,
|
|
46
|
+
base_dir, _sub_dirs, _files = setup_and_teardown
|
|
47
47
|
tar = create_tarball(Path(base_dir), exclude_gitignore=False, exclude_regex=None)
|
|
48
48
|
tarp = Path(tar.name)
|
|
49
49
|
|
|
@@ -55,7 +55,7 @@ def test_create_tarball_no_exclusion(setup_and_teardown):
|
|
|
55
55
|
|
|
56
56
|
|
|
57
57
|
def test_create_tarball_exclude_from_gitignore(setup_and_teardown):
|
|
58
|
-
base_dir,
|
|
58
|
+
base_dir, _sub_dirs, _files = setup_and_teardown
|
|
59
59
|
tar = create_tarball(Path(base_dir), exclude_gitignore=True, exclude_regex=None)
|
|
60
60
|
tarp = Path(tar.name)
|
|
61
61
|
|
|
@@ -67,7 +67,7 @@ def test_create_tarball_exclude_from_gitignore(setup_and_teardown):
|
|
|
67
67
|
|
|
68
68
|
|
|
69
69
|
def test_create_tarball_exclude_from_regex(setup_and_teardown):
|
|
70
|
-
base_dir,
|
|
70
|
+
base_dir, _sub_dirs, _files = setup_and_teardown
|
|
71
71
|
tar = create_tarball(
|
|
72
72
|
Path(base_dir), exclude_gitignore=False, exclude_regex=r".*\.txt"
|
|
73
73
|
)
|
|
File without changes
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import asyncio
|
|
1
2
|
import json as _json
|
|
2
3
|
import os
|
|
3
4
|
import sys
|
|
@@ -14,12 +15,59 @@ from loguru import logger
|
|
|
14
15
|
from tabulate import tabulate
|
|
15
16
|
from typer_config.decorators import use_toml_config
|
|
16
17
|
|
|
17
|
-
from trainwave_cli.api import Api, JobStatus
|
|
18
|
+
from trainwave_cli.api import Api, Job, JobStatus
|
|
18
19
|
from trainwave_cli.cli.logs import stream_logs
|
|
19
20
|
from trainwave_cli.config.config import config
|
|
20
21
|
from trainwave_cli.format_utils import cents_to_dollars_str
|
|
21
22
|
from trainwave_cli.io import create_tarball
|
|
22
|
-
from trainwave_cli.utils import
|
|
23
|
+
from trainwave_cli.utils import (
|
|
24
|
+
async_command,
|
|
25
|
+
ensure_api_key,
|
|
26
|
+
to_dict_recursive,
|
|
27
|
+
truncate_string,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
# Terminal states — no point polling for RUNNING once the job is in one of these.
|
|
31
|
+
_TERMINAL_JOB_STATES = {
|
|
32
|
+
JobStatus.ERROR,
|
|
33
|
+
JobStatus.SUCCESS,
|
|
34
|
+
JobStatus.USER_PROCESS_FAILED,
|
|
35
|
+
JobStatus.SYSTEM_TERMINATED,
|
|
36
|
+
JobStatus.USER_CANCELLED,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
async def _wait_until_running(
|
|
41
|
+
api_client: Api, job_id: str, poll_interval: float = 3.0
|
|
42
|
+
) -> Job:
|
|
43
|
+
"""Poll the job until it reaches RUNNING with a live running instance."""
|
|
44
|
+
last_state: JobStatus | None = None
|
|
45
|
+
while True:
|
|
46
|
+
job = await api_client.get_job(job_id)
|
|
47
|
+
if job.state != last_state:
|
|
48
|
+
typer.echo(f"Job {job.rid} state: {job.state.value}")
|
|
49
|
+
last_state = job.state
|
|
50
|
+
|
|
51
|
+
if job.state == JobStatus.RUNNING and any(
|
|
52
|
+
i.state == "RUNNING" for i in job.instances
|
|
53
|
+
):
|
|
54
|
+
return job
|
|
55
|
+
if job.state in _TERMINAL_JOB_STATES:
|
|
56
|
+
raise RuntimeError(
|
|
57
|
+
f"Job {job.rid} reached terminal state {job.state.value} before running."
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
await asyncio.sleep(poll_interval)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _ssh_into_job(job: Job, proxy_host: str, proxy_port: int) -> None:
|
|
64
|
+
"""Exec into the job's first running instance. Does not return."""
|
|
65
|
+
running = [i for i in job.instances if i.state == "RUNNING"]
|
|
66
|
+
target = running[0]
|
|
67
|
+
username = f"{job.rid}-{target.rid}"
|
|
68
|
+
typer.echo(f"Connecting to {username}@{proxy_host}:{proxy_port}...")
|
|
69
|
+
os.execvp("ssh", ["ssh", "-p", str(proxy_port), f"{username}@{proxy_host}"])
|
|
70
|
+
|
|
23
71
|
|
|
24
72
|
app = typer.Typer()
|
|
25
73
|
|
|
@@ -32,10 +80,14 @@ async def launch( # noqa
|
|
|
32
80
|
# Basic job parameters
|
|
33
81
|
name: Annotated[str, typer.Option(help="The name of the job")],
|
|
34
82
|
project: Annotated[str, typer.Option(help="The project ID or RID")],
|
|
35
|
-
setup_command: Annotated[
|
|
83
|
+
setup_command: Annotated[
|
|
84
|
+
str, typer.Option(help="The command to run before the job")
|
|
85
|
+
],
|
|
36
86
|
run_command: Annotated[str, typer.Option(help="The command to run the job")],
|
|
37
87
|
# Optional params
|
|
38
|
-
organization: Annotated[
|
|
88
|
+
organization: Annotated[
|
|
89
|
+
str | None, typer.Option(help="ID or RID of organization")
|
|
90
|
+
] = None,
|
|
39
91
|
expires: Annotated[
|
|
40
92
|
str,
|
|
41
93
|
typer.Option(
|
|
@@ -52,17 +104,30 @@ async def launch( # noqa
|
|
|
52
104
|
] = True,
|
|
53
105
|
exclude_regex: Annotated[
|
|
54
106
|
str | None,
|
|
55
|
-
typer.Option(
|
|
107
|
+
typer.Option(
|
|
108
|
+
help="Files matching this pattern will be ignored when packaging up the job"
|
|
109
|
+
),
|
|
56
110
|
] = None,
|
|
57
111
|
# Harware parameters
|
|
58
|
-
image: Annotated[
|
|
112
|
+
image: Annotated[
|
|
113
|
+
str, typer.Option(help="The docker image to use as base for the job")
|
|
114
|
+
] = "",
|
|
59
115
|
hdd_size_mb: Annotated[int, typer.Option(help="Hard drive size in MB")] = 20,
|
|
60
116
|
memory_mb: Annotated[int, typer.Option(help="Memory in MB")] = 0,
|
|
61
117
|
cpus: Annotated[int, typer.Option(help="Number of CPUs")] = 0,
|
|
62
|
-
gpus: Annotated[
|
|
63
|
-
|
|
118
|
+
gpus: Annotated[
|
|
119
|
+
int, typer.Option(help="Number of GPUs to allocate of the `gpu_type`")
|
|
120
|
+
] = 0,
|
|
121
|
+
gpu_type: Annotated[
|
|
122
|
+
str, typer.Option(help="Canonical name of the GPU to use")
|
|
123
|
+
] = "",
|
|
64
124
|
compliance_soc2: Annotated[
|
|
65
|
-
bool | None,
|
|
125
|
+
bool | None,
|
|
126
|
+
typer.Option("--compliance-soc2", help="Only use SOC2 compliant machines"),
|
|
127
|
+
] = None,
|
|
128
|
+
clouds: Annotated[
|
|
129
|
+
list[str] | None,
|
|
130
|
+
typer.Option(help="List of cloud providers to use (e.g. GOOGLE, AWS, AZURE)"),
|
|
66
131
|
] = None,
|
|
67
132
|
interactive: Annotated[
|
|
68
133
|
bool,
|
|
@@ -99,6 +164,8 @@ async def launch( # noqa
|
|
|
99
164
|
}
|
|
100
165
|
if compliance_soc2 is not None:
|
|
101
166
|
conf["compliance_soc2"] = compliance_soc2
|
|
167
|
+
if clouds is not None:
|
|
168
|
+
conf["clouds"] = clouds
|
|
102
169
|
if interactive:
|
|
103
170
|
conf["interactive"] = True
|
|
104
171
|
|
|
@@ -137,7 +204,9 @@ async def launch( # noqa
|
|
|
137
204
|
tarball = create_tarball(
|
|
138
205
|
Path().cwd(), exclude_gitignore, exclude_regex, show_progress_bar=not json
|
|
139
206
|
)
|
|
140
|
-
await api_client.upload_code(
|
|
207
|
+
await api_client.upload_code(
|
|
208
|
+
Path(tarball.name), job.upload_url, show_progress_bar=not json
|
|
209
|
+
)
|
|
141
210
|
await api_client.code_submission(job)
|
|
142
211
|
|
|
143
212
|
co = asdict(job.cloud_offer)
|
|
@@ -147,23 +216,32 @@ async def launch( # noqa
|
|
|
147
216
|
"value": [truncate_string(str(co[k]), 50) for k in tab_keys],
|
|
148
217
|
}
|
|
149
218
|
if not json:
|
|
150
|
-
typer.echo(
|
|
219
|
+
typer.echo(
|
|
220
|
+
"\nMatched machine\n" + tabulate(tab_values, headers="keys") + "\n"
|
|
221
|
+
)
|
|
151
222
|
typer.echo(f"Cost per hour for this job: ${job.cost_per_hour:.2f}")
|
|
152
223
|
typer.echo(f"Job created: {job.rid} ({job.id}) -> {job.url}")
|
|
153
|
-
if interactive:
|
|
154
|
-
typer.echo(
|
|
155
|
-
"\nInteractive mode: once the instance is running, connect with:"
|
|
156
|
-
)
|
|
157
|
-
typer.echo(f" trainwave jobs ssh {job.rid}")
|
|
158
224
|
else:
|
|
159
225
|
print(_json.dumps(to_dict_recursive(job), indent=4))
|
|
160
226
|
except Exception as e:
|
|
161
227
|
traceback.print_exc()
|
|
162
228
|
logger.error(f"An error occurred: {e}")
|
|
229
|
+
return
|
|
163
230
|
finally:
|
|
164
231
|
if tarball:
|
|
165
232
|
tarball.close()
|
|
166
233
|
|
|
234
|
+
if interactive:
|
|
235
|
+
if not json:
|
|
236
|
+
typer.echo("\nInteractive mode: waiting for the instance to start...")
|
|
237
|
+
try:
|
|
238
|
+
running_job = await _wait_until_running(api_client, job.rid)
|
|
239
|
+
proxy = await api_client.get_ssh_proxy_info()
|
|
240
|
+
except (RuntimeError, ValueError) as e:
|
|
241
|
+
typer.echo(f"Error: {e}")
|
|
242
|
+
sys.exit(1)
|
|
243
|
+
_ssh_into_job(running_job, proxy.host, proxy.port)
|
|
244
|
+
|
|
167
245
|
|
|
168
246
|
@app.command(name="list")
|
|
169
247
|
@async_command
|
|
@@ -174,7 +252,9 @@ async def list_jobs(
|
|
|
174
252
|
project: Annotated[str, typer.Option(help="Project ID or RID")] = "",
|
|
175
253
|
) -> None:
|
|
176
254
|
"""List all jobs in the current project."""
|
|
177
|
-
api_client = Api(
|
|
255
|
+
api_client = Api(
|
|
256
|
+
config.api_key, config.endpoint, project=project, organization=organization
|
|
257
|
+
)
|
|
178
258
|
jobs = await api_client.list_jobs()
|
|
179
259
|
|
|
180
260
|
headers = ["ID", "NAME", "STATUS", "GPUS", "COST", "CREATED"]
|
|
@@ -242,7 +322,9 @@ async def ssh(
|
|
|
242
322
|
] = None,
|
|
243
323
|
extra: Annotated[
|
|
244
324
|
list[str] | None,
|
|
245
|
-
typer.Argument(
|
|
325
|
+
typer.Argument(
|
|
326
|
+
help="Extra arguments forwarded to ssh (e.g. a command to run)."
|
|
327
|
+
),
|
|
246
328
|
] = None,
|
|
247
329
|
) -> None:
|
|
248
330
|
"""Open an SSH session to the running instance of a job."""
|
|
@@ -269,7 +351,9 @@ async def ssh(
|
|
|
269
351
|
if instance:
|
|
270
352
|
match = next((i for i in running if instance in (i.rid, i.id)), None)
|
|
271
353
|
if match is None:
|
|
272
|
-
typer.echo(
|
|
354
|
+
typer.echo(
|
|
355
|
+
f"Instance {instance} not found among running instances of {job.rid}."
|
|
356
|
+
)
|
|
273
357
|
sys.exit(1)
|
|
274
358
|
target = match
|
|
275
359
|
elif len(running) > 1:
|
|
@@ -39,7 +39,7 @@ async def list_keys() -> None:
|
|
|
39
39
|
api_client = Api(config.api_key, config.endpoint)
|
|
40
40
|
keys = await api_client.list_ssh_keys()
|
|
41
41
|
if not keys:
|
|
42
|
-
typer.echo("No SSH keys registered. Add one with `
|
|
42
|
+
typer.echo("No SSH keys registered. Add one with `wave ssh-keys add`.")
|
|
43
43
|
return
|
|
44
44
|
headers = ["ID", "NAME", "TYPE", "FINGERPRINT", "ADDED"]
|
|
45
45
|
table = [
|
|
@@ -53,7 +53,7 @@ def ensure_api_key(f):
|
|
|
53
53
|
def wrapper(*args, **kwargs):
|
|
54
54
|
if config.api_key in {"", None}:
|
|
55
55
|
typer.echo(
|
|
56
|
-
"No access token found. Please run `
|
|
56
|
+
"No access token found. Please run `wave auth login` to login to your account."
|
|
57
57
|
)
|
|
58
58
|
raise typer.Exit(1)
|
|
59
59
|
return f(*args, **kwargs)
|
trainwave_cli-0.4.0/example/run
DELETED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|