envloop-cli 0.4.0__tar.gz → 1.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/.gitignore +3 -0
- envloop_cli-1.7.0/PKG-INFO +195 -0
- envloop_cli-1.7.0/README.pypi.md +182 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/agent_runs.py +36 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/cli.py +78 -20
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/cli_errors.py +10 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/client.py +70 -42
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/http.py +5 -17
- envloop_cli-1.7.0/envloop/inference.py +35 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/profile_updates.py +23 -3
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/resources.py +0 -4
- envloop_cli-1.7.0/envloop/rsi_cli.py +49 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/train.py +25 -2
- envloop_cli-1.7.0/envloop/train_runner_cli.py +64 -0
- envloop_cli-1.7.0/envloop/training/__init__.py +4 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/pyproject.toml +2 -5
- envloop_cli-0.4.0/PKG-INFO +0 -122
- envloop_cli-0.4.0/README.pypi.md +0 -109
- envloop_cli-0.4.0/envloop/training/__init__.py +0 -123
- envloop_cli-0.4.0/envloop/training/credentials.py +0 -107
- envloop_cli-0.4.0/envloop/training/sampling_client.py +0 -58
- envloop_cli-0.4.0/envloop/training/service_client.py +0 -165
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/__init__.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/auth_cli.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/auth_login.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/auth_workspace.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/cli_output.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/config.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/job_logs.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/lifecycle_cli.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/navigation_resources.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/profile_secrets.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/resource_cli.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/sandbox.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/train_log_cli.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/utils/__init__.py +0 -0
- {envloop_cli-0.4.0 → envloop_cli-1.7.0}/envloop/utils/logging.py +0 -0
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: envloop-cli
|
|
3
|
+
Version: 1.7.0
|
|
4
|
+
Summary: EnvLoop CLI and Python SDK for jobs, training, checkpoints and inference
|
|
5
|
+
Project-URL: Homepage, https://envloop.ai
|
|
6
|
+
Project-URL: Documentation, https://docs.envloop.ai
|
|
7
|
+
Project-URL: Repository, https://github.com/EnvLoop/envloop-cli
|
|
8
|
+
Project-URL: Issues, https://github.com/EnvLoop/envloop-cli/issues
|
|
9
|
+
Project-URL: Agent instructions, https://docs.envloop.ai/skill.md
|
|
10
|
+
Requires-Python: >=3.11
|
|
11
|
+
Requires-Dist: tinker==0.34.0
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# envloop-cli: EnvLoop CLI and Python SDK
|
|
15
|
+
|
|
16
|
+
Install the public client to run agent Jobs, inspect Trials, manage training and
|
|
17
|
+
checkpoints, and deploy or call inference endpoints. The `el` and `envloop`
|
|
18
|
+
commands are equivalent. The Python import package is `envloop`.
|
|
19
|
+
|
|
20
|
+
## Version 1.7
|
|
21
|
+
|
|
22
|
+
- Use published Runner resource shortcuts for Train Jobs and fixed Checkpoint inputs for Inference Jobs.
|
|
23
|
+
- Pass `--json` explicitly to `job submit`; failures retain safe error codes and recovery guidance.
|
|
24
|
+
- Submit training through Train Jobs. The retired interactive `envloop.training` interface is removed.
|
|
25
|
+
|
|
26
|
+
After upgrading, check `el --version` and `el job submit --help`. Execution also
|
|
27
|
+
requires a matching platform, project permissions and published Runners; package
|
|
28
|
+
installation alone does not establish those prerequisites. See the
|
|
29
|
+
[PyPI release history](https://pypi.org/project/envloop-cli/#history) for published versions.
|
|
30
|
+
|
|
31
|
+
## Install
|
|
32
|
+
|
|
33
|
+
Use a Python virtual environment:
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
python -m pip install --upgrade envloop-cli
|
|
37
|
+
el --help
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The standard installation includes the client dependencies for managed training.
|
|
41
|
+
Check that the SDK can be imported:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
python -c 'from envloop import Client; print("EnvLoop SDK ready")'
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
When you see `EnvLoop SDK ready`, continue to Connect. This check imports the
|
|
48
|
+
client without creating a Job. Submit training through `client.training_jobs`;
|
|
49
|
+
follow the [managed training guide](https://docs.envloop.ai/training/cli-hello-world).
|
|
50
|
+
|
|
51
|
+
Install the client in your Python environment; the platform provides the execution resources.
|
|
52
|
+
|
|
53
|
+
## Connect
|
|
54
|
+
|
|
55
|
+
Use an authorized EnvLoop account and project. Browser authorization:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
el auth login
|
|
59
|
+
el auth status --check
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
For an existing project API key, use the masked configuration wizard:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
el auth config
|
|
66
|
+
el auth status --check
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
CI can read credentials from `ENVLOOP_API_KEY` and connection settings from
|
|
70
|
+
`ENVLOOP_BASE_URL`, `ENVLOOP_WORKSPACE_ID` and `ENVLOOP_PROJECT_ID`. Keep keys in
|
|
71
|
+
your secret manager. Public API access is scoped to the authorized project.
|
|
72
|
+
|
|
73
|
+
Credentials and settings live in `~/.envloop` (override with an absolute
|
|
74
|
+
`ENVLOOP_HOME`). On POSIX the directory must be private and owned by you. Use the
|
|
75
|
+
default path when the optional environment variable is unset:
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
mkdir -p "${ENVLOOP_HOME:-$HOME/.envloop}"
|
|
79
|
+
chmod 700 "${ENVLOOP_HOME:-$HOME/.envloop}"
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
An insecure directory causes `el` to fail with `CONFIG_INSECURE_PATH`.
|
|
83
|
+
|
|
84
|
+
## Explore your project
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
el job list --limit 5
|
|
88
|
+
el job list --type train --limit 5
|
|
89
|
+
el trail list --limit 5
|
|
90
|
+
el trail list --type train --job-id JOB_ID
|
|
91
|
+
el checkpoint list --limit 5
|
|
92
|
+
el job list --type inference
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Query a returned Job ID with `el job get JOB_ID` and `el job result JOB_ID`.
|
|
96
|
+
Trial commands use the spelling `trail`, for example
|
|
97
|
+
`el trail artifact list TRIAL_ID`. Use each command's `--help` for options.
|
|
98
|
+
|
|
99
|
+
For the Python SDK:
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
from envloop import Client
|
|
103
|
+
|
|
104
|
+
with Client.remote() as client:
|
|
105
|
+
print(client.jobs.list(limit=5))
|
|
106
|
+
print(client.trials.list(type="train", limit=5))
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Submitting work, training a model or starting inference can allocate billable
|
|
110
|
+
resources. Follow the task-specific guide and retain the outputs you need before
|
|
111
|
+
cleaning up temporary resources.
|
|
112
|
+
|
|
113
|
+
## Sandbox jobs
|
|
114
|
+
|
|
115
|
+
With a matching platform and a published `envhub/base` ANS Runner, submit a
|
|
116
|
+
Sandbox Job without a Dataset:
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
el job submit --type sandbox --runner base --command 'python --version'
|
|
120
|
+
el job submit --type sandbox --runner base --image python:3.11 --wait
|
|
121
|
+
el job list --type sandbox
|
|
122
|
+
el job log JOB_ID --follow
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
The platform resolves `base` to the registered `envhub/base@main` source ref and
|
|
126
|
+
freezes its immutable Revision. The Sandbox image defaults to `python:3.11`;
|
|
127
|
+
`--image` accepts an image name, tag or digest. The execution provider is configured
|
|
128
|
+
by the server. Job logs retain the actual captured output and collection metadata.
|
|
129
|
+
Failures include a code, readable explanation, HTTP status when available, and
|
|
130
|
+
recovery guidance without echoing arbitrary response bodies or credentials.
|
|
131
|
+
|
|
132
|
+
## Existing installations
|
|
133
|
+
|
|
134
|
+
This distribution provides both the CLI and the `envloop` import package. Use a
|
|
135
|
+
fresh environment if you previously installed the separate source distribution
|
|
136
|
+
named `envloop`, so two distributions do not own the same Python module files.
|
|
137
|
+
Existing client configuration is reused; installing a package does not change
|
|
138
|
+
your account permissions or move remote resources. Check `el --help` when
|
|
139
|
+
upgrading because command groups may differ from your earlier installation.
|
|
140
|
+
|
|
141
|
+
## Documentation
|
|
142
|
+
|
|
143
|
+
- [English documentation](https://docs.envloop.ai/)
|
|
144
|
+
- [中文文档](https://docs.envloop.ai/zh-cn/)
|
|
145
|
+
- [Installation](https://docs.envloop.ai/cli-install)
|
|
146
|
+
- [CLI reference](https://docs.envloop.ai/reference/cli)
|
|
147
|
+
- [Training and deployment](https://docs.envloop.ai/training/train-deploy)
|
|
148
|
+
- [Agent entry: skill.md](https://docs.envloop.ai/skill.md)
|
|
149
|
+
|
|
150
|
+
## Submit managed training
|
|
151
|
+
|
|
152
|
+
Prepare a recipe-supported `REQUEST.json` using the [SFT quickstart](https://docs.envloop.ai/training/quickstart).
|
|
153
|
+
Training uses billable resources. Check your project permissions, balance and data format, then submit once:
|
|
154
|
+
|
|
155
|
+
```bash
|
|
156
|
+
el job submit --type train --file REQUEST.json
|
|
157
|
+
el job get JOB_ID
|
|
158
|
+
el job log JOB_ID --follow
|
|
159
|
+
el job result JOB_ID
|
|
160
|
+
el checkpoint list --job-id JOB_ID
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
Replace `JOB_ID` with the receipt's value. Read the training status and logs, then find the registered checkpoint after success.
|
|
164
|
+
Ctrl-C ends log reading; to stop training, use `el job stop JOB_ID` and check its state and resource cleanup.
|
|
165
|
+
See [Checkpoints](https://docs.envloop.ai/training/checkpoints) to download or delete saved weights.
|
|
166
|
+
|
|
167
|
+
The built-in SFT recipe does not support custom loss, RL or optimizer-state resume.
|
|
168
|
+
For an older interactive training script, follow the [managed training migration](https://docs.envloop.ai/training/sdk-compatibility).
|
|
169
|
+
|
|
170
|
+
## Run a GPU Runner
|
|
171
|
+
|
|
172
|
+
Choose a published Runner readable in your project. Save this request as `gpu-run.json`, replacing `FULL_COMMIT` with the full Revision from `el runner list --json`:
|
|
173
|
+
|
|
174
|
+
```json
|
|
175
|
+
{
|
|
176
|
+
"request_id": "gpu-run-001",
|
|
177
|
+
"trials": [
|
|
178
|
+
{"name": "first", "spec": {"runner": "envhub/bash-runner@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_sec": 600}},
|
|
179
|
+
{"name": "second", "spec": {"runner": "envhub/bash-runner@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_sec": 600}}
|
|
180
|
+
]
|
|
181
|
+
}
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
```bash
|
|
185
|
+
el job submit --type train --file gpu-run.json
|
|
186
|
+
el job get JOB_ID
|
|
187
|
+
el job log JOB_ID --follow
|
|
188
|
+
el job result JOB_ID
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
Replace `JOB_ID` with the returned value. The example has two independent Trials, each requesting four H100 GPUs: concurrent execution needs eight GPUs. Capacity and budget determine actual concurrency. Startup and execution are billable; check the Runner requirements, project `training.run` permission and [training prices](https://docs.envloop.ai/pricing) first. Reduce the Trial or GPU count when your program supports a smaller first run.
|
|
192
|
+
|
|
193
|
+
The request file does not depend on GPU shortcut flags. If `el job submit --help` lists both `--trails` and `--gput-type`, you can instead use the shortcuts in [GPU Runner training](https://docs.envloop.ai/training/runner). Choose one submission method to avoid duplicate runs. The SDK accepts the same dictionary through `client.training_jobs.submit(request)`.
|
|
194
|
+
|
|
195
|
+
The platform starts the published Revision’s `main.sh` / `main.py` using the Runner image; it does not upload local code. An optional command can only supply a declared `run_command` parameter. If this mode or GPU type is unavailable, contact your project administrator. Use `el job stop JOB_ID` to stop every Trial, then inspect each Trial’s `cleanup_confirmed` and download the uploaded artifacts you need.
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# envloop-cli: EnvLoop CLI and Python SDK
|
|
2
|
+
|
|
3
|
+
Install the public client to run agent Jobs, inspect Trials, manage training and
|
|
4
|
+
checkpoints, and deploy or call inference endpoints. The `el` and `envloop`
|
|
5
|
+
commands are equivalent. The Python import package is `envloop`.
|
|
6
|
+
|
|
7
|
+
## Version 1.7
|
|
8
|
+
|
|
9
|
+
- Use published Runner resource shortcuts for Train Jobs and fixed Checkpoint inputs for Inference Jobs.
|
|
10
|
+
- Pass `--json` explicitly to `job submit`; failures retain safe error codes and recovery guidance.
|
|
11
|
+
- Submit training through Train Jobs. The retired interactive `envloop.training` interface is removed.
|
|
12
|
+
|
|
13
|
+
After upgrading, check `el --version` and `el job submit --help`. Execution also
|
|
14
|
+
requires a matching platform, project permissions and published Runners; package
|
|
15
|
+
installation alone does not establish those prerequisites. See the
|
|
16
|
+
[PyPI release history](https://pypi.org/project/envloop-cli/#history) for published versions.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
Use a Python virtual environment:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
python -m pip install --upgrade envloop-cli
|
|
24
|
+
el --help
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
The standard installation includes the client dependencies for managed training.
|
|
28
|
+
Check that the SDK can be imported:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
python -c 'from envloop import Client; print("EnvLoop SDK ready")'
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
When you see `EnvLoop SDK ready`, continue to Connect. This check imports the
|
|
35
|
+
client without creating a Job. Submit training through `client.training_jobs`;
|
|
36
|
+
follow the [managed training guide](https://docs.envloop.ai/training/cli-hello-world).
|
|
37
|
+
|
|
38
|
+
Install the client in your Python environment; the platform provides the execution resources.
|
|
39
|
+
|
|
40
|
+
## Connect
|
|
41
|
+
|
|
42
|
+
Use an authorized EnvLoop account and project. Browser authorization:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
el auth login
|
|
46
|
+
el auth status --check
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
For an existing project API key, use the masked configuration wizard:
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
el auth config
|
|
53
|
+
el auth status --check
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
CI can read credentials from `ENVLOOP_API_KEY` and connection settings from
|
|
57
|
+
`ENVLOOP_BASE_URL`, `ENVLOOP_WORKSPACE_ID` and `ENVLOOP_PROJECT_ID`. Keep keys in
|
|
58
|
+
your secret manager. Public API access is scoped to the authorized project.
|
|
59
|
+
|
|
60
|
+
Credentials and settings live in `~/.envloop` (override with an absolute
|
|
61
|
+
`ENVLOOP_HOME`). On POSIX the directory must be private and owned by you. Use the
|
|
62
|
+
default path when the optional environment variable is unset:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
mkdir -p "${ENVLOOP_HOME:-$HOME/.envloop}"
|
|
66
|
+
chmod 700 "${ENVLOOP_HOME:-$HOME/.envloop}"
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
An insecure directory causes `el` to fail with `CONFIG_INSECURE_PATH`.
|
|
70
|
+
|
|
71
|
+
## Explore your project
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
el job list --limit 5
|
|
75
|
+
el job list --type train --limit 5
|
|
76
|
+
el trail list --limit 5
|
|
77
|
+
el trail list --type train --job-id JOB_ID
|
|
78
|
+
el checkpoint list --limit 5
|
|
79
|
+
el job list --type inference
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Query a returned Job ID with `el job get JOB_ID` and `el job result JOB_ID`.
|
|
83
|
+
Trial commands use the spelling `trail`, for example
|
|
84
|
+
`el trail artifact list TRIAL_ID`. Use each command's `--help` for options.
|
|
85
|
+
|
|
86
|
+
For the Python SDK:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
from envloop import Client
|
|
90
|
+
|
|
91
|
+
with Client.remote() as client:
|
|
92
|
+
print(client.jobs.list(limit=5))
|
|
93
|
+
print(client.trials.list(type="train", limit=5))
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Submitting work, training a model or starting inference can allocate billable
|
|
97
|
+
resources. Follow the task-specific guide and retain the outputs you need before
|
|
98
|
+
cleaning up temporary resources.
|
|
99
|
+
|
|
100
|
+
## Sandbox jobs
|
|
101
|
+
|
|
102
|
+
With a matching platform and a published `envhub/base` ANS Runner, submit a
|
|
103
|
+
Sandbox Job without a Dataset:
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
el job submit --type sandbox --runner base --command 'python --version'
|
|
107
|
+
el job submit --type sandbox --runner base --image python:3.11 --wait
|
|
108
|
+
el job list --type sandbox
|
|
109
|
+
el job log JOB_ID --follow
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The platform resolves `base` to the registered `envhub/base@main` source ref and
|
|
113
|
+
freezes its immutable Revision. The Sandbox image defaults to `python:3.11`;
|
|
114
|
+
`--image` accepts an image name, tag or digest. The execution provider is configured
|
|
115
|
+
by the server. Job logs retain the actual captured output and collection metadata.
|
|
116
|
+
Failures include a code, readable explanation, HTTP status when available, and
|
|
117
|
+
recovery guidance without echoing arbitrary response bodies or credentials.
|
|
118
|
+
|
|
119
|
+
## Existing installations
|
|
120
|
+
|
|
121
|
+
This distribution provides both the CLI and the `envloop` import package. Use a
|
|
122
|
+
fresh environment if you previously installed the separate source distribution
|
|
123
|
+
named `envloop`, so two distributions do not own the same Python module files.
|
|
124
|
+
Existing client configuration is reused; installing a package does not change
|
|
125
|
+
your account permissions or move remote resources. Check `el --help` when
|
|
126
|
+
upgrading because command groups may differ from your earlier installation.
|
|
127
|
+
|
|
128
|
+
## Documentation
|
|
129
|
+
|
|
130
|
+
- [English documentation](https://docs.envloop.ai/)
|
|
131
|
+
- [中文文档](https://docs.envloop.ai/zh-cn/)
|
|
132
|
+
- [Installation](https://docs.envloop.ai/cli-install)
|
|
133
|
+
- [CLI reference](https://docs.envloop.ai/reference/cli)
|
|
134
|
+
- [Training and deployment](https://docs.envloop.ai/training/train-deploy)
|
|
135
|
+
- [Agent entry: skill.md](https://docs.envloop.ai/skill.md)
|
|
136
|
+
|
|
137
|
+
## Submit managed training
|
|
138
|
+
|
|
139
|
+
Prepare a recipe-supported `REQUEST.json` using the [SFT quickstart](https://docs.envloop.ai/training/quickstart).
|
|
140
|
+
Training uses billable resources. Check your project permissions, balance and data format, then submit once:
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
el job submit --type train --file REQUEST.json
|
|
144
|
+
el job get JOB_ID
|
|
145
|
+
el job log JOB_ID --follow
|
|
146
|
+
el job result JOB_ID
|
|
147
|
+
el checkpoint list --job-id JOB_ID
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Replace `JOB_ID` with the receipt's value. Read the training status and logs, then find the registered checkpoint after success.
|
|
151
|
+
Ctrl-C ends log reading; to stop training, use `el job stop JOB_ID` and check its state and resource cleanup.
|
|
152
|
+
See [Checkpoints](https://docs.envloop.ai/training/checkpoints) to download or delete saved weights.
|
|
153
|
+
|
|
154
|
+
The built-in SFT recipe does not support custom loss, RL or optimizer-state resume.
|
|
155
|
+
For an older interactive training script, follow the [managed training migration](https://docs.envloop.ai/training/sdk-compatibility).
|
|
156
|
+
|
|
157
|
+
## Run a GPU Runner
|
|
158
|
+
|
|
159
|
+
Choose a published Runner readable in your project. Save this request as `gpu-run.json`, replacing `FULL_COMMIT` with the full Revision from `el runner list --json`:
|
|
160
|
+
|
|
161
|
+
```json
|
|
162
|
+
{
|
|
163
|
+
"request_id": "gpu-run-001",
|
|
164
|
+
"trials": [
|
|
165
|
+
{"name": "first", "spec": {"runner": "envhub/bash-runner@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_sec": 600}},
|
|
166
|
+
{"name": "second", "spec": {"runner": "envhub/bash-runner@FULL_COMMIT", "resources": {"gpu_type": "H100", "gpus": 4}, "timeout_sec": 600}}
|
|
167
|
+
]
|
|
168
|
+
}
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
el job submit --type train --file gpu-run.json
|
|
173
|
+
el job get JOB_ID
|
|
174
|
+
el job log JOB_ID --follow
|
|
175
|
+
el job result JOB_ID
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
Replace `JOB_ID` with the returned value. The example has two independent Trials, each requesting four H100 GPUs: concurrent execution needs eight GPUs. Capacity and budget determine actual concurrency. Startup and execution are billable; check the Runner requirements, project `training.run` permission and [training prices](https://docs.envloop.ai/pricing) first. Reduce the Trial or GPU count when your program supports a smaller first run.
|
|
179
|
+
|
|
180
|
+
The request file does not depend on GPU shortcut flags. If `el job submit --help` lists both `--trails` and `--gput-type`, you can instead use the shortcuts in [GPU Runner training](https://docs.envloop.ai/training/runner). Choose one submission method to avoid duplicate runs. The SDK accepts the same dictionary through `client.training_jobs.submit(request)`.
|
|
181
|
+
|
|
182
|
+
The platform starts the published Revision’s `main.sh` / `main.py` using the Runner image; it does not upload local code. An optional command can only supply a declared `run_command` parameter. If this mode or GPU type is unavailable, contact your project administrator. Use `el job stop JOB_ID` to stop every Trial, then inspect each Trial’s `cleanup_confirmed` and download the uploaded artifacts you need.
|
|
@@ -7,6 +7,7 @@ using the same rules as the Console's agent runs page.
|
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
from datetime import datetime
|
|
10
|
+
from urllib.parse import quote, urlencode
|
|
10
11
|
|
|
11
12
|
from .train import TrainJobs
|
|
12
13
|
|
|
@@ -206,3 +207,38 @@ class AgentRuns:
|
|
|
206
207
|
if run not in runs:
|
|
207
208
|
raise AgentRunError("AGENT_RUN_NOT_FOUND")
|
|
208
209
|
return {**_detail(run, runs[run]), "observed_at": observed_at}
|
|
210
|
+
|
|
211
|
+
# The RSI loop service behind the platform (``/v1/agent-runs/rsi``). Starting spends money, so
|
|
212
|
+
# ``start`` needs a key with ``training.run``; ``control(stop)`` also needs ``training.stop``.
|
|
213
|
+
def _rsi(self, method: str, path: str, body=None) -> dict:
|
|
214
|
+
return self._train_jobs.transport.request(method, "/v1/agent-runs/rsi" + path, body)
|
|
215
|
+
|
|
216
|
+
def start(self, run: str, brief: str, *, model_usd_cap: float, usd_cap: float | None = None,
|
|
217
|
+
model_request_cap: int | None = None, domain: str | None = None, rounds: int | None = None,
|
|
218
|
+
dry_run: bool = False, student_model: str | None = None) -> dict:
|
|
219
|
+
"""Start a new RSI run; the platform validates every field. ``dry_run`` uses no GPU."""
|
|
220
|
+
limits = {"model_usd_cap": model_usd_cap}
|
|
221
|
+
for key, value in (("usd_cap", usd_cap), ("model_request_cap", model_request_cap)):
|
|
222
|
+
if value is not None:
|
|
223
|
+
limits[key] = value
|
|
224
|
+
body = {"run": run, "brief": brief, "limits": limits}
|
|
225
|
+
for key, value in (("domain", domain), ("rounds", rounds), ("student_model", student_model)):
|
|
226
|
+
if value is not None:
|
|
227
|
+
body[key] = value
|
|
228
|
+
if dry_run:
|
|
229
|
+
body["dry_run"] = True
|
|
230
|
+
return self._rsi("POST", "/runs", body)
|
|
231
|
+
|
|
232
|
+
def live(self, run: str, since: int | None = None) -> dict:
|
|
233
|
+
"""What a run does now; pass the previous ``next_cursor`` as ``since`` for newer events."""
|
|
234
|
+
query = "?" + urlencode({"since": since}) if since is not None else ""
|
|
235
|
+
return self._rsi("GET", f"/runs/{quote(run, safe='')}/live{query}")
|
|
236
|
+
|
|
237
|
+
def steer(self, run: str, text: str) -> dict:
|
|
238
|
+
"""Queue a message for the running operator (1 to 4000 characters)."""
|
|
239
|
+
return self._rsi("POST", f"/runs/{quote(run, safe='')}/steer", {"text": text})
|
|
240
|
+
|
|
241
|
+
def control(self, run: str, action: str, rounds: int | None = None) -> dict:
|
|
242
|
+
"""``extend`` (needs ``rounds``), ``stop`` (after the current round) or ``resume``."""
|
|
243
|
+
return self._rsi("POST", f"/runs/{quote(run, safe='')}/control",
|
|
244
|
+
{"action": action, **({} if rounds is None else {"rounds": rounds})})
|
|
@@ -133,11 +133,11 @@ def log_page_size(value):
|
|
|
133
133
|
|
|
134
134
|
def runner_arguments(submit):
|
|
135
135
|
submit.add_argument("--runner", metavar="NAME[@REVISION]",
|
|
136
|
-
help="Agent:
|
|
136
|
+
help="Agent/Train: namespace/name@REVISION; Sandbox: registered Runner name (for example base)")
|
|
137
137
|
submit.add_argument("--job-id", help="caller-selected Job ID")
|
|
138
138
|
submit.add_argument("--tag", help="run label")
|
|
139
139
|
submit.add_argument("--priority", type=int, help="waiting priority; larger values run first (default: 0)")
|
|
140
|
-
submit.add_argument("--command", help="shell
|
|
140
|
+
submit.add_argument("--command", help="Agent/Sandbox: shell override; Train: declared run_command parameter passed to the published Runner entrypoint; preserves multiline text")
|
|
141
141
|
|
|
142
142
|
|
|
143
143
|
def parser():
|
|
@@ -153,10 +153,20 @@ def parser():
|
|
|
153
153
|
from .auth_cli import add_auth_commands
|
|
154
154
|
|
|
155
155
|
jobs = groups.add_parser("job", help="Submit and manage Agent, Sandbox, Train and Inference Jobs").add_subparsers(dest="action", required=True)
|
|
156
|
-
job_submit = jobs.add_parser("submit", help="Submit an Agent, Sandbox, Train or Inference Job")
|
|
156
|
+
job_submit = jobs.add_parser("submit", help="Submit an Agent, Sandbox, Train or Inference Job", allow_abbrev=False)
|
|
157
157
|
job_submit.add_argument("--type", choices=("agent", "sandbox", "train", "inference"), default="agent", help="Job type (default: agent)")
|
|
158
|
-
job_submit.add_argument("--image", metavar="IMAGE", help="Sandbox
|
|
159
|
-
|
|
158
|
+
job_submit.add_argument("--image", metavar="IMAGE", help="Sandbox image override (server default: python:3.11); Train Runner: public spelling of --docker")
|
|
159
|
+
from .train_runner_cli import memory_mebibytes, positive_integer
|
|
160
|
+
|
|
161
|
+
job_submit.add_argument("--docker", metavar="IMAGE", help="Train Runner Docker image override; omitted uses the Runner declaration")
|
|
162
|
+
job_submit.add_argument("--trails", type=positive_integer, metavar="COUNT", help="Train Runner: 1–16 independent Trials (default: 1)")
|
|
163
|
+
job_submit.add_argument("--gput-type", metavar="MODEL", help="Train Runner: GPU model for each Trial (for example H100)")
|
|
164
|
+
job_submit.add_argument("--gpu", type=positive_integer, metavar="COUNT", help="Train Runner: positive GPU count per Trial")
|
|
165
|
+
job_submit.add_argument("--cpu", type=positive_integer, metavar="COUNT", help="Train Runner: positive CPU count per Trial")
|
|
166
|
+
for option in ('memory', 'disk', 'shm-size'):
|
|
167
|
+
job_submit.add_argument('--' + option, type=memory_mebibytes, metavar="SIZE",
|
|
168
|
+
help="Train Runner: per-Trial size in Mi/Gi/Ti (for example 64Gi)")
|
|
169
|
+
job_submit.add_argument("--file", type=Path, help="Managed Train or Inference Job JSON (spec or Train trials array; interactive training is unsupported)")
|
|
160
170
|
job_submit.add_argument("--source", type=Path, help="two-file directory for a single-task Job")
|
|
161
171
|
job_submit.add_argument("--task-id", help="select one task within the dataset")
|
|
162
172
|
job_submit.add_argument(
|
|
@@ -175,10 +185,11 @@ def parser():
|
|
|
175
185
|
help="named runtime Profile; omitted version requires an explicit default")
|
|
176
186
|
job_submit.add_argument("--seed", type=int)
|
|
177
187
|
job_submit.add_argument("--timeout-sec", type=wait_seconds,
|
|
178
|
-
help="Agent/Sandbox execution timeout (whole seconds); Train wait timeout (default: 600)")
|
|
188
|
+
help="Agent/Sandbox or Train Runner execution timeout (whole seconds); Train --file wait timeout (default: 600)")
|
|
179
189
|
job_submit.add_argument("--delete-delay", type=nonnegative_seconds, metavar="SECONDS",
|
|
180
190
|
help="keep each sandbox (E2B or Fly) after completion for SECONDS (default: 0); cancellation/timeout delete immediately")
|
|
181
191
|
job_submit.add_argument("--wait", action="store_true")
|
|
192
|
+
job_submit.add_argument("--json", action="store_true", help="Output the complete response as JSON (the default for submit; accepted for consistency with list commands)")
|
|
182
193
|
for action in ("get", "stop", "result", "pause", "resume", "wake", "event"):
|
|
183
194
|
command = jobs.add_parser(action)
|
|
184
195
|
command.add_argument("job_id")
|
|
@@ -208,6 +219,27 @@ def parser():
|
|
|
208
219
|
runs_get = runs.add_parser("get", help="Show a run's rounds, Train Jobs and checkpoints")
|
|
209
220
|
runs_get.add_argument("run", metavar="RUN", help="run label: user_metadata.rsi_run or agent_run")
|
|
210
221
|
runs_get.add_argument("--json", action="store_true", help="Output the complete run as JSON (default: tables)")
|
|
222
|
+
runs_start = runs.add_parser("start", help="Start a new RSI loop run (paid; needs training.run)")
|
|
223
|
+
runs_start.add_argument("run", metavar="RUN", help="new run id: lower-case letters, digits and dashes")
|
|
224
|
+
brief = runs_start.add_mutually_exclusive_group(required=True)
|
|
225
|
+
brief.add_argument("--brief", help="the goal of the run (up to 4000 characters)")
|
|
226
|
+
brief.add_argument("--brief-file", metavar="PATH", help="read the goal from a file, or - for standard input")
|
|
227
|
+
runs_start.add_argument("--model-usd-cap", type=float, required=True, help="strong-model spend cap in USD (the hard stop)")
|
|
228
|
+
runs_start.add_argument("--usd-cap", type=float, help="cap in USD for model plus GPU training together")
|
|
229
|
+
runs_start.add_argument("--model-request-cap", type=int, help="cap on model requests")
|
|
230
|
+
runs_start.add_argument("--domain", help="task family (default: the service's default; an unknown value is rejected)")
|
|
231
|
+
runs_start.add_argument("--rounds", type=int, help="target rounds (default: the service's default)")
|
|
232
|
+
runs_start.add_argument("--dry-run", action="store_true", help="GPU-free run: trains nothing; exercises the loop, spends model budget only")
|
|
233
|
+
runs_start.add_argument("--student-model", help="gateway student model of a dry run")
|
|
234
|
+
runs_live = runs.add_parser("live", help="Show what a run does now; --follow streams new events")
|
|
235
|
+
runs_live.add_argument("run", metavar="RUN")
|
|
236
|
+
runs_live.add_argument("--since", type=int, help="only events after this cursor (the previous next_cursor)")
|
|
237
|
+
runs_live.add_argument("--follow", action="store_true", help="poll and print new events until the run ends; Ctrl-C exits without stopping it")
|
|
238
|
+
runs_steer = runs.add_parser("steer", help="Message the running operator now (needs training.run)")
|
|
239
|
+
runs_steer.add_argument("run", metavar="RUN")
|
|
240
|
+
runs_steer.add_argument("text", metavar="TEXT", help="1 to 4000 characters")
|
|
241
|
+
runs_stop = runs.add_parser("stop", help="Stop a run after its current round (needs training.run and training.stop)")
|
|
242
|
+
runs_stop.add_argument("run", metavar="RUN")
|
|
211
243
|
tasks = groups.add_parser("trail", help="Inspect and control existing trials").add_subparsers(dest="action", required=True)
|
|
212
244
|
listing = tasks.add_parser("list", help="List Agent, Sandbox, Train and Inference worker Trials")
|
|
213
245
|
listing.add_argument("--type", choices=("agent", "sandbox", "train", "inference"), help="Filter by Trial type")
|
|
@@ -230,11 +262,11 @@ def parser():
|
|
|
230
262
|
download.add_argument("trial_id")
|
|
231
263
|
download.add_argument("directory", type=Path)
|
|
232
264
|
profiles = groups.add_parser("profile", help="Manage platform runtime profiles").add_subparsers(dest="action", required=True)
|
|
233
|
-
put = profiles.add_parser("create", help="Create an immutable Profile version from JSON")
|
|
265
|
+
put = profiles.add_parser("create", help="Create an immutable Profile version from TOML or JSON")
|
|
234
266
|
put.add_argument("name")
|
|
235
267
|
put.add_argument("version")
|
|
236
|
-
put.add_argument("--file", type=Path, required=True, help="
|
|
237
|
-
put.add_argument("--runner", metavar="RUNNER", help="Bind
|
|
268
|
+
put.add_argument("--file", type=Path, required=True, help="Profile TOML or JSON file")
|
|
269
|
+
put.add_argument("--runner", metavar="RUNNER", help="Bind a Runner revision to this version")
|
|
238
270
|
secret_input = put.add_mutually_exclusive_group()
|
|
239
271
|
secret_input.add_argument("--prompt-model-api-key", action="store_true", help="Read the model API key from a non-echoing prompt")
|
|
240
272
|
secret_input.add_argument("--model-api-key-stdin", action="store_true", help="Read the model API key from one line of standard input")
|
|
@@ -242,11 +274,16 @@ def parser():
|
|
|
242
274
|
update.add_argument("name")
|
|
243
275
|
update.add_argument("version", help="New version; existing versions cannot be overwritten")
|
|
244
276
|
update.add_argument("--from-version", help="Source version; omit to use the explicit default")
|
|
245
|
-
update.add_argument("--file", type=Path, required=True, help="JSON patch: merge
|
|
246
|
-
update.add_argument("--runner", metavar="RUNNER", help="Bind a Runner
|
|
277
|
+
update.add_argument("--file", type=Path, required=True, help="TOML or JSON patch: merge params keys, replace supplied metadata; preserve omitted fields")
|
|
278
|
+
update.add_argument("--runner", metavar="RUNNER", help="Bind a Runner revision; v1 requires a full commit; omit to preserve the source binding")
|
|
247
279
|
update_secret = update.add_mutually_exclusive_group()
|
|
248
280
|
update_secret.add_argument("--prompt-model-api-key", action="store_true", help="Read an encrypted model connection key from a non-echoing prompt")
|
|
249
281
|
update_secret.add_argument("--model-api-key-stdin", action="store_true", help="Read an encrypted model connection key from one line of standard input")
|
|
282
|
+
fork = profiles.add_parser('fork', help='Fork an existing Profile into a new immutable identity')
|
|
283
|
+
fork.add_argument('source_name')
|
|
284
|
+
fork.add_argument('source_version')
|
|
285
|
+
fork.add_argument('name')
|
|
286
|
+
fork.add_argument('version')
|
|
250
287
|
get = profiles.add_parser("get")
|
|
251
288
|
get.add_argument("name")
|
|
252
289
|
get.add_argument("--version")
|
|
@@ -368,16 +405,27 @@ def main(argv=None):
|
|
|
368
405
|
print(format_error(exc, args=args, program=cli.prog), file=sys.stderr)
|
|
369
406
|
return 1
|
|
370
407
|
if args.group == "job" and args.action == "submit":
|
|
408
|
+
from .train_runner_cli import SHORTCUT_FIELDS, runner_request
|
|
409
|
+
|
|
410
|
+
runner_mode = args.type == 'train' and args.file is None
|
|
411
|
+
if not runner_mode and any(getattr(args, key) is not None for key in SHORTCUT_FIELDS):
|
|
412
|
+
cli.error('Train Runner shortcuts require --type train --runner and cannot be combined with --file')
|
|
371
413
|
if args.type in {"train", "inference"}:
|
|
372
|
-
if args.file is None:
|
|
414
|
+
if args.file is None and not runner_mode:
|
|
373
415
|
cli.error(f"--type {args.type} requires --file REQUEST.json")
|
|
374
416
|
if args.type == 'inference' and (args.wait or args.timeout_sec is not None):
|
|
375
417
|
cli.error('Inference is a continuous service; --wait and --timeout-sec do not apply')
|
|
376
418
|
agent_fields = ("dataset", "dataset_option", "source", "task_id", "config", "split", "revision",
|
|
377
|
-
"
|
|
378
|
-
|
|
419
|
+
"job_id", "tag", "priority", "attempts", "profile", "seed", "delete_delay")
|
|
420
|
+
if not runner_mode:
|
|
421
|
+
agent_fields += ('runner', 'command', 'request_id', 'image')
|
|
379
422
|
if any(getattr(args, key) is not None for key in agent_fields):
|
|
380
|
-
cli.error(f"--type {args.type}
|
|
423
|
+
cli.error(f"--type {args.type} cannot be combined with Agent submission options or mixed --file/Runner input")
|
|
424
|
+
if runner_mode:
|
|
425
|
+
try:
|
|
426
|
+
args.train_request = runner_request(args)
|
|
427
|
+
except ValueError as exc:
|
|
428
|
+
cli.error(str(exc))
|
|
381
429
|
elif args.type == 'sandbox':
|
|
382
430
|
from .sandbox import sandbox_runner_selector, validate_image
|
|
383
431
|
|
|
@@ -406,7 +454,8 @@ def main(argv=None):
|
|
|
406
454
|
args.runner = runner_selector(args.runner)
|
|
407
455
|
except ValueError as exc:
|
|
408
456
|
cli.error(str(exc))
|
|
409
|
-
|
|
457
|
+
selections = sum(value is not None for value in (args.dataset, args.dataset_option, args.source))
|
|
458
|
+
if selections != 1 and not (selections == 0 and args.profile):
|
|
410
459
|
cli.error("provide exactly one positional dataset, --dataset or --source")
|
|
411
460
|
args.dataset = args.dataset if args.dataset is not None else args.dataset_option
|
|
412
461
|
if args.source is not None and any(getattr(args, key) is not None for key in ("runner", "task_id", "config", "split", "revision")):
|
|
@@ -430,7 +479,7 @@ def main(argv=None):
|
|
|
430
479
|
|
|
431
480
|
raw = args.file.read_bytes()
|
|
432
481
|
patch = read_profile_patch(raw)
|
|
433
|
-
if args.runner is not None:
|
|
482
|
+
if args.action == 'update' and args.runner is not None:
|
|
434
483
|
patch = with_runner(patch, args.runner)
|
|
435
484
|
raw = json.dumps(patch).encode('utf-8')
|
|
436
485
|
secrets = None
|
|
@@ -450,8 +499,10 @@ def main(argv=None):
|
|
|
450
499
|
patch=patch, from_version=args.from_version, secrets=secrets)
|
|
451
500
|
else:
|
|
452
501
|
result = client.profiles.import_spec(
|
|
453
|
-
raw, name=args.name, version=args.version, secrets=secrets
|
|
502
|
+
raw, name=args.name, version=args.version, secrets=secrets, runner=args.runner
|
|
454
503
|
)
|
|
504
|
+
elif args.action == 'fork':
|
|
505
|
+
result = client.profiles.fork(args.source_name, args.source_version, name=args.name, version=args.version)
|
|
455
506
|
elif args.action == "get":
|
|
456
507
|
result = client.profiles.get(args.name, args.version)
|
|
457
508
|
elif args.action == "list":
|
|
@@ -477,14 +528,15 @@ def main(argv=None):
|
|
|
477
528
|
if args.type == 'inference':
|
|
478
529
|
result = client.jobs.submit_inference(json.loads(args.file.read_text(encoding='utf-8')))
|
|
479
530
|
elif args.type == "train":
|
|
480
|
-
|
|
531
|
+
request = args.train_request if args.file is None else json.loads(args.file.read_text(encoding="utf-8"))
|
|
532
|
+
result = client.training_jobs.submit(request)
|
|
481
533
|
else:
|
|
482
534
|
result = client.jobs.submit(args.dataset, type=args.type, source=args.source, task_id=args.task_id,
|
|
483
535
|
attempts=args.attempts if args.attempts is not None else 1,
|
|
484
536
|
request_id=args.request_id, config=args.config, split=args.split,
|
|
485
537
|
revision=args.revision, **submission_options(args))
|
|
486
538
|
if args.wait:
|
|
487
|
-
wait_options = {"timeout": args.timeout_sec} if args.type == "train" and args.timeout_sec is not None else {}
|
|
539
|
+
wait_options = {"timeout": args.timeout_sec} if args.type == "train" and args.file is not None and args.timeout_sec is not None else {}
|
|
488
540
|
result = client.jobs.wait(result["job_id"], **wait_options)
|
|
489
541
|
elif args.action == 'event':
|
|
490
542
|
result = client.jobs.events(args.job_id)
|
|
@@ -495,6 +547,12 @@ def main(argv=None):
|
|
|
495
547
|
from .train_log_cli import read_logs
|
|
496
548
|
|
|
497
549
|
return read_logs(client.jobs, args)
|
|
550
|
+
elif args.action == "run" and args.run_action in {"start", "live", "steer", "stop"}:
|
|
551
|
+
from .rsi_cli import run_rsi
|
|
552
|
+
|
|
553
|
+
result = run_rsi(client.agent_runs, args)
|
|
554
|
+
if result is None:
|
|
555
|
+
return 0
|
|
498
556
|
elif args.action == "run":
|
|
499
557
|
result = client.agent_runs.list() if args.run_action == "list" else client.agent_runs.get(args.run)
|
|
500
558
|
elif args.action == "list":
|